Subversion Repositories Kolibri OS

Rev

Rev 3037 | Rev 3243 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright © 2008 Intel Corporation
  3.  *
  4.  * Permission is hereby granted, free of charge, to any person obtaining a
  5.  * copy of this software and associated documentation files (the "Software"),
  6.  * to deal in the Software without restriction, including without limitation
  7.  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8.  * and/or sell copies of the Software, and to permit persons to whom the
  9.  * Software is furnished to do so, subject to the following conditions:
  10.  *
  11.  * The above copyright notice and this permission notice (including the next
  12.  * paragraph) shall be included in all copies or substantial portions of the
  13.  * Software.
  14.  *
  15.  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16.  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17.  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18.  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19.  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20.  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21.  * IN THE SOFTWARE.
  22.  *
  23.  * Authors:
  24.  *    Eric Anholt <eric@anholt.net>
  25.  *
  26.  */
  27.  
  28. #include <drm/drmP.h>
  29. #include <drm/i915_drm.h>
  30. #include "i915_drv.h"
  31. #include "i915_trace.h"
  32. #include "intel_drv.h"
  33. //#include <linux/shmem_fs.h>
  34. #include <linux/slab.h>
  35. //#include <linux/swap.h>
  36. #include <linux/pci.h>
  37.  
  38. extern int x86_clflush_size;
  39.  
  40. #undef mb
  41. #undef rmb
  42. #undef wmb
  43. #define mb() asm volatile("mfence")
  44. #define rmb() asm volatile ("lfence")
  45. #define wmb() asm volatile ("sfence")
  46.  
  47. static inline void clflush(volatile void *__p)
  48. {
  49.     asm volatile("clflush %0" : "+m" (*(volatile char*)__p));
  50. }
  51.  
  52. #define MAX_ERRNO       4095
  53.  
  54. #define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
  55.  
  56. static inline long IS_ERR(const void *ptr)
  57. {
  58.     return IS_ERR_VALUE((unsigned long)ptr);
  59. }
  60.  
  61. static inline void *ERR_PTR(long error)
  62. {
  63.     return (void *) error;
  64. }
  65.  
  66. static inline long PTR_ERR(const void *ptr)
  67. {
  68.     return (long) ptr;
  69. }
  70.  
  71. void
  72. drm_gem_object_free(struct kref *kref)
  73. {
  74.     struct drm_gem_object *obj = (struct drm_gem_object *) kref;
  75.     struct drm_device *dev = obj->dev;
  76.  
  77.     BUG_ON(!mutex_is_locked(&dev->struct_mutex));
  78.  
  79.     i915_gem_free_object(obj);
  80. }
  81.  
  82. /**
  83.  * Initialize an already allocated GEM object of the specified size with
  84.  * shmfs backing store.
  85.  */
  86. int drm_gem_object_init(struct drm_device *dev,
  87.             struct drm_gem_object *obj, size_t size)
  88. {
  89.     BUG_ON((size & (PAGE_SIZE - 1)) != 0);
  90.  
  91.     obj->dev = dev;
  92.     kref_init(&obj->refcount);
  93.     atomic_set(&obj->handle_count, 0);
  94.     obj->size = size;
  95.  
  96.     return 0;
  97. }
  98.  
  99. void
  100. drm_gem_object_release(struct drm_gem_object *obj)
  101. { }
  102.  
  103.  
  104. #define I915_EXEC_CONSTANTS_MASK        (3<<6)
  105. #define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */
  106. #define I915_EXEC_CONSTANTS_ABSOLUTE    (1<<6)
  107. #define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */
  108.  
  109. static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  110. static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  111. static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  112.                                                     unsigned alignment,
  113.                                                     bool map_and_fenceable,
  114.                                                     bool nonblocking);
  115. static int i915_gem_phys_pwrite(struct drm_device *dev,
  116.                                 struct drm_i915_gem_object *obj,
  117.                                 struct drm_i915_gem_pwrite *args,
  118.                                 struct drm_file *file);
  119.  
  120. static void i915_gem_write_fence(struct drm_device *dev, int reg,
  121.                                  struct drm_i915_gem_object *obj);
  122. static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
  123.                                          struct drm_i915_fence_reg *fence,
  124.                                          bool enable);
  125.  
  126. static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
  127. static void i915_gem_shrink_all(struct drm_i915_private *dev_priv);
  128. static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
  129.  
  130. static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
  131. {
  132.         if (obj->tiling_mode)
  133.                 i915_gem_release_mmap(obj);
  134.  
  135.         /* As we do not have an associated fence register, we will force
  136.          * a tiling change if we ever need to acquire one.
  137.          */
  138.         obj->fence_dirty = false;
  139.         obj->fence_reg = I915_FENCE_REG_NONE;
  140. }
  141.  
  142. /* some bookkeeping */
  143. static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  144.                                   size_t size)
  145. {
  146.         dev_priv->mm.object_count++;
  147.         dev_priv->mm.object_memory += size;
  148. }
  149.  
  150. static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  151.                                      size_t size)
  152. {
  153.         dev_priv->mm.object_count--;
  154.         dev_priv->mm.object_memory -= size;
  155. }
  156.  
  157. #if 0
  158.  
  159. static int
  160. i915_gem_wait_for_error(struct drm_device *dev)
  161. {
  162.         struct drm_i915_private *dev_priv = dev->dev_private;
  163.         struct completion *x = &dev_priv->error_completion;
  164.         unsigned long flags;
  165.         int ret;
  166.  
  167.         if (!atomic_read(&dev_priv->mm.wedged))
  168.                 return 0;
  169.  
  170.         /*
  171.          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
  172.          * userspace. If it takes that long something really bad is going on and
  173.          * we should simply try to bail out and fail as gracefully as possible.
  174.          */
  175.         ret = wait_for_completion_interruptible_timeout(x, 10*HZ);
  176.         if (ret == 0) {
  177.                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
  178.                 return -EIO;
  179.         } else if (ret < 0) {
  180.                 return ret;
  181.         }
  182.  
  183.         if (atomic_read(&dev_priv->mm.wedged)) {
  184.                 /* GPU is hung, bump the completion count to account for
  185.                  * the token we just consumed so that we never hit zero and
  186.                  * end up waiting upon a subsequent completion event that
  187.                  * will never happen.
  188.                  */
  189.                 spin_lock_irqsave(&x->wait.lock, flags);
  190.                 x->done++;
  191.                 spin_unlock_irqrestore(&x->wait.lock, flags);
  192.         }
  193.         return 0;
  194. }
  195.  
  196. int i915_mutex_lock_interruptible(struct drm_device *dev)
  197. {
  198.         int ret;
  199.  
  200.         ret = i915_gem_wait_for_error(dev);
  201.         if (ret)
  202.                 return ret;
  203.  
  204.         ret = mutex_lock_interruptible(&dev->struct_mutex);
  205.         if (ret)
  206.                 return ret;
  207.  
  208.         WARN_ON(i915_verify_lists(dev));
  209.         return 0;
  210. }
  211. #endif
  212.  
  213. static inline bool
  214. i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
  215. {
  216.         return obj->gtt_space && !obj->active;
  217. }
  218.  
  219.  
  220. #if 0
  221.  
  222. int
  223. i915_gem_init_ioctl(struct drm_device *dev, void *data,
  224.                     struct drm_file *file)
  225. {
  226.         struct drm_i915_gem_init *args = data;
  227.  
  228.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  229.                 return -ENODEV;
  230.  
  231.         if (args->gtt_start >= args->gtt_end ||
  232.             (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
  233.                 return -EINVAL;
  234.  
  235.         /* GEM with user mode setting was never supported on ilk and later. */
  236.         if (INTEL_INFO(dev)->gen >= 5)
  237.                 return -ENODEV;
  238.  
  239.         mutex_lock(&dev->struct_mutex);
  240.         i915_gem_init_global_gtt(dev, args->gtt_start,
  241.                                  args->gtt_end, args->gtt_end);
  242.         mutex_unlock(&dev->struct_mutex);
  243.  
  244.         return 0;
  245. }
  246. #endif
  247.  
  248. int
  249. i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  250.                             struct drm_file *file)
  251. {
  252.         struct drm_i915_private *dev_priv = dev->dev_private;
  253.         struct drm_i915_gem_get_aperture *args = data;
  254.         struct drm_i915_gem_object *obj;
  255.         size_t pinned;
  256.  
  257.         pinned = 0;
  258.         mutex_lock(&dev->struct_mutex);
  259.         list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
  260.                 if (obj->pin_count)
  261.                 pinned += obj->gtt_space->size;
  262.         mutex_unlock(&dev->struct_mutex);
  263.  
  264.         args->aper_size = dev_priv->mm.gtt_total;
  265.         args->aper_available_size = args->aper_size - pinned;
  266.  
  267.         return 0;
  268. }
  269.  
  270. #if 0
  271. static int
  272. i915_gem_create(struct drm_file *file,
  273.                 struct drm_device *dev,
  274.                 uint64_t size,
  275.                 uint32_t *handle_p)
  276. {
  277.         struct drm_i915_gem_object *obj;
  278.         int ret;
  279.         u32 handle;
  280.  
  281.         size = roundup(size, PAGE_SIZE);
  282.         if (size == 0)
  283.                 return -EINVAL;
  284.  
  285.         /* Allocate the new object */
  286.         obj = i915_gem_alloc_object(dev, size);
  287.         if (obj == NULL)
  288.                 return -ENOMEM;
  289.  
  290.         ret = drm_gem_handle_create(file, &obj->base, &handle);
  291.         if (ret) {
  292.                 drm_gem_object_release(&obj->base);
  293.                 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
  294.                 kfree(obj);
  295.                 return ret;
  296.         }
  297.  
  298.         /* drop reference from allocate - handle holds it now */
  299.         drm_gem_object_unreference(&obj->base);
  300.         trace_i915_gem_object_create(obj);
  301.  
  302.         *handle_p = handle;
  303.         return 0;
  304. }
  305.  
  306. int
  307. i915_gem_dumb_create(struct drm_file *file,
  308.                      struct drm_device *dev,
  309.                      struct drm_mode_create_dumb *args)
  310. {
  311.         /* have to work out size/pitch and return them */
  312.         args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
  313.         args->size = args->pitch * args->height;
  314.         return i915_gem_create(file, dev,
  315.                                args->size, &args->handle);
  316. }
  317.  
  318. int i915_gem_dumb_destroy(struct drm_file *file,
  319.                           struct drm_device *dev,
  320.                           uint32_t handle)
  321. {
  322.         return drm_gem_handle_delete(file, handle);
  323. }
  324.  
  325. /**
  326.  * Creates a new mm object and returns a handle to it.
  327.  */
  328. int
  329. i915_gem_create_ioctl(struct drm_device *dev, void *data,
  330.                       struct drm_file *file)
  331. {
  332.         struct drm_i915_gem_create *args = data;
  333.  
  334.         return i915_gem_create(file, dev,
  335.                                args->size, &args->handle);
  336. }
  337.  
  338. static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
  339. {
  340.         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
  341.  
  342.         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
  343.                 obj->tiling_mode != I915_TILING_NONE;
  344. }
  345.  
  346. static inline int
  347. __copy_to_user_swizzled(char __user *cpu_vaddr,
  348.                         const char *gpu_vaddr, int gpu_offset,
  349.                 int length)
  350. {
  351.         int ret, cpu_offset = 0;
  352.  
  353.         while (length > 0) {
  354.                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
  355.                 int this_length = min(cacheline_end - gpu_offset, length);
  356.                 int swizzled_gpu_offset = gpu_offset ^ 64;
  357.  
  358.                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
  359.                                      gpu_vaddr + swizzled_gpu_offset,
  360.                                      this_length);
  361.                 if (ret)
  362.                         return ret + length;
  363.  
  364.                 cpu_offset += this_length;
  365.                 gpu_offset += this_length;
  366.                 length -= this_length;
  367.         }
  368.  
  369.         return 0;
  370. }
  371.  
  372. static inline int
  373. __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
  374.                           const char __user *cpu_vaddr,
  375.                           int length)
  376. {
  377.         int ret, cpu_offset = 0;
  378.  
  379.         while (length > 0) {
  380.                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
  381.                 int this_length = min(cacheline_end - gpu_offset, length);
  382.                 int swizzled_gpu_offset = gpu_offset ^ 64;
  383.  
  384.                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
  385.                                cpu_vaddr + cpu_offset,
  386.                                this_length);
  387.                 if (ret)
  388.                         return ret + length;
  389.  
  390.                 cpu_offset += this_length;
  391.                 gpu_offset += this_length;
  392.                 length -= this_length;
  393.         }
  394.  
  395.         return 0;
  396. }
  397.  
  398. /* Per-page copy function for the shmem pread fastpath.
  399.  * Flushes invalid cachelines before reading the target if
  400.  * needs_clflush is set. */
  401. static int
  402. shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
  403.                  char __user *user_data,
  404.                  bool page_do_bit17_swizzling, bool needs_clflush)
  405. {
  406.                 char *vaddr;
  407.                 int ret;
  408.  
  409.         if (unlikely(page_do_bit17_swizzling))
  410.                 return -EINVAL;
  411.  
  412.                 vaddr = kmap_atomic(page);
  413.         if (needs_clflush)
  414.                 drm_clflush_virt_range(vaddr + shmem_page_offset,
  415.                                        page_length);
  416.                 ret = __copy_to_user_inatomic(user_data,
  417.                                       vaddr + shmem_page_offset,
  418.                                               page_length);
  419.                 kunmap_atomic(vaddr);
  420.  
  421.         return ret ? -EFAULT : 0;
  422. }
  423.  
  424. static void
  425. shmem_clflush_swizzled_range(char *addr, unsigned long length,
  426.                              bool swizzled)
  427. {
  428.         if (unlikely(swizzled)) {
  429.                 unsigned long start = (unsigned long) addr;
  430.                 unsigned long end = (unsigned long) addr + length;
  431.  
  432.                 /* For swizzling simply ensure that we always flush both
  433.                  * channels. Lame, but simple and it works. Swizzled
  434.                  * pwrite/pread is far from a hotpath - current userspace
  435.                  * doesn't use it at all. */
  436.                 start = round_down(start, 128);
  437.                 end = round_up(end, 128);
  438.  
  439.                 drm_clflush_virt_range((void *)start, end - start);
  440.         } else {
  441.                 drm_clflush_virt_range(addr, length);
  442.         }
  443.  
  444. }
  445.  
  446. /* Only difference to the fast-path function is that this can handle bit17
  447.  * and uses non-atomic copy and kmap functions. */
  448. static int
  449. shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
  450.                  char __user *user_data,
  451.                  bool page_do_bit17_swizzling, bool needs_clflush)
  452. {
  453.         char *vaddr;
  454.         int ret;
  455.  
  456.         vaddr = kmap(page);
  457.         if (needs_clflush)
  458.                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  459.                                              page_length,
  460.                                              page_do_bit17_swizzling);
  461.  
  462.         if (page_do_bit17_swizzling)
  463.                 ret = __copy_to_user_swizzled(user_data,
  464.                                               vaddr, shmem_page_offset,
  465.                                               page_length);
  466.         else
  467.                 ret = __copy_to_user(user_data,
  468.                                      vaddr + shmem_page_offset,
  469.                                      page_length);
  470.         kunmap(page);
  471.  
  472.         return ret ? - EFAULT : 0;
  473. }
  474.  
  475. static int
  476. i915_gem_shmem_pread(struct drm_device *dev,
  477.                           struct drm_i915_gem_object *obj,
  478.                           struct drm_i915_gem_pread *args,
  479.                           struct drm_file *file)
  480. {
  481.         char __user *user_data;
  482.         ssize_t remain;
  483.         loff_t offset;
  484.         int shmem_page_offset, page_length, ret = 0;
  485.         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
  486.         int hit_slowpath = 0;
  487.         int prefaulted = 0;
  488.         int needs_clflush = 0;
  489.         struct scatterlist *sg;
  490.         int i;
  491.  
  492.         user_data = (char __user *) (uintptr_t) args->data_ptr;
  493.         remain = args->size;
  494.  
  495.         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
  496.  
  497.         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
  498.                 /* If we're not in the cpu read domain, set ourself into the gtt
  499.                  * read domain and manually flush cachelines (if required). This
  500.                  * optimizes for the case when the gpu will dirty the data
  501.                  * anyway again before the next pread happens. */
  502.                 if (obj->cache_level == I915_CACHE_NONE)
  503.                         needs_clflush = 1;
  504.                 if (obj->gtt_space) {
  505.                         ret = i915_gem_object_set_to_gtt_domain(obj, false);
  506.                         if (ret)
  507.                                 return ret;
  508.                 }
  509.         }
  510.  
  511.         ret = i915_gem_object_get_pages(obj);
  512.         if (ret)
  513.                 return ret;
  514.  
  515.         i915_gem_object_pin_pages(obj);
  516.  
  517.         offset = args->offset;
  518.  
  519.         for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
  520.                 struct page *page;
  521.  
  522.                 if (i < offset >> PAGE_SHIFT)
  523.                         continue;
  524.  
  525.                 if (remain <= 0)
  526.                         break;
  527.  
  528.                 /* Operation in this page
  529.                  *
  530.                  * shmem_page_offset = offset within page in shmem file
  531.                  * page_length = bytes to copy for this page
  532.                  */
  533.                 shmem_page_offset = offset_in_page(offset);
  534.                 page_length = remain;
  535.                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
  536.                         page_length = PAGE_SIZE - shmem_page_offset;
  537.  
  538.                 page = sg_page(sg);
  539.                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
  540.                         (page_to_phys(page) & (1 << 17)) != 0;
  541.  
  542.                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
  543.                                        user_data, page_do_bit17_swizzling,
  544.                                        needs_clflush);
  545.                 if (ret == 0)
  546.                         goto next_page;
  547.  
  548.                 hit_slowpath = 1;
  549.                 mutex_unlock(&dev->struct_mutex);
  550.  
  551.                 if (!prefaulted) {
  552.                         ret = fault_in_multipages_writeable(user_data, remain);
  553.                         /* Userspace is tricking us, but we've already clobbered
  554.                          * its pages with the prefault and promised to write the
  555.                          * data up to the first fault. Hence ignore any errors
  556.                          * and just continue. */
  557.                         (void)ret;
  558.                         prefaulted = 1;
  559.                 }
  560.  
  561.                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
  562.                                        user_data, page_do_bit17_swizzling,
  563.                                        needs_clflush);
  564.  
  565.                 mutex_lock(&dev->struct_mutex);
  566.  
  567. next_page:
  568.                 mark_page_accessed(page);
  569.  
  570.                 if (ret)
  571.                         goto out;
  572.  
  573.                 remain -= page_length;
  574.                 user_data += page_length;
  575.                 offset += page_length;
  576.         }
  577.  
  578. out:
  579.         i915_gem_object_unpin_pages(obj);
  580.  
  581.         if (hit_slowpath) {
  582.                 /* Fixup: Kill any reinstated backing storage pages */
  583.                 if (obj->madv == __I915_MADV_PURGED)
  584.                         i915_gem_object_truncate(obj);
  585.         }
  586.  
  587.         return ret;
  588. }
  589.  
  590. /**
  591.  * Reads data from the object referenced by handle.
  592.  *
  593.  * On error, the contents of *data are undefined.
  594.  */
  595. int
  596. i915_gem_pread_ioctl(struct drm_device *dev, void *data,
  597.                      struct drm_file *file)
  598. {
  599.         struct drm_i915_gem_pread *args = data;
  600.         struct drm_i915_gem_object *obj;
  601.         int ret = 0;
  602.  
  603.         if (args->size == 0)
  604.                 return 0;
  605.  
  606.         if (!access_ok(VERIFY_WRITE,
  607.                        (char __user *)(uintptr_t)args->data_ptr,
  608.                        args->size))
  609.                 return -EFAULT;
  610.  
  611.         ret = i915_mutex_lock_interruptible(dev);
  612.         if (ret)
  613.                 return ret;
  614.  
  615.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  616.         if (&obj->base == NULL) {
  617.                 ret = -ENOENT;
  618.                 goto unlock;
  619.         }
  620.  
  621.         /* Bounds check source.  */
  622.         if (args->offset > obj->base.size ||
  623.             args->size > obj->base.size - args->offset) {
  624.                 ret = -EINVAL;
  625.                 goto out;
  626.         }
  627.  
  628.         /* prime objects have no backing filp to GEM pread/pwrite
  629.          * pages from.
  630.          */
  631.         if (!obj->base.filp) {
  632.                 ret = -EINVAL;
  633.                 goto out;
  634.         }
  635.  
  636.         trace_i915_gem_object_pread(obj, args->offset, args->size);
  637.  
  638.         ret = i915_gem_shmem_pread(dev, obj, args, file);
  639.  
  640. out:
  641.         drm_gem_object_unreference(&obj->base);
  642. unlock:
  643.         mutex_unlock(&dev->struct_mutex);
  644.         return ret;
  645. }
  646.  
  647. /* This is the fast write path which cannot handle
  648.  * page faults in the source data
  649.  */
  650.  
  651. static inline int
  652. fast_user_write(struct io_mapping *mapping,
  653.                 loff_t page_base, int page_offset,
  654.                 char __user *user_data,
  655.                 int length)
  656. {
  657.         void __iomem *vaddr_atomic;
  658.         void *vaddr;
  659.         unsigned long unwritten;
  660.  
  661.         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
  662.         /* We can use the cpu mem copy function because this is X86. */
  663.         vaddr = (void __force*)vaddr_atomic + page_offset;
  664.         unwritten = __copy_from_user_inatomic_nocache(vaddr,
  665.                                                       user_data, length);
  666.         io_mapping_unmap_atomic(vaddr_atomic);
  667.         return unwritten;
  668. }
  669.  
  670. /**
  671.  * This is the fast pwrite path, where we copy the data directly from the
  672.  * user into the GTT, uncached.
  673.  */
  674. static int
  675. i915_gem_gtt_pwrite_fast(struct drm_device *dev,
  676.                          struct drm_i915_gem_object *obj,
  677.                          struct drm_i915_gem_pwrite *args,
  678.                          struct drm_file *file)
  679. {
  680.         drm_i915_private_t *dev_priv = dev->dev_private;
  681.         ssize_t remain;
  682.         loff_t offset, page_base;
  683.         char __user *user_data;
  684.         int page_offset, page_length, ret;
  685.  
  686.         ret = i915_gem_object_pin(obj, 0, true, true);
  687.         if (ret)
  688.                 goto out;
  689.  
  690.         ret = i915_gem_object_set_to_gtt_domain(obj, true);
  691.         if (ret)
  692.                 goto out_unpin;
  693.  
  694.         ret = i915_gem_object_put_fence(obj);
  695.         if (ret)
  696.                 goto out_unpin;
  697.  
  698.         user_data = (char __user *) (uintptr_t) args->data_ptr;
  699.         remain = args->size;
  700.  
  701.         offset = obj->gtt_offset + args->offset;
  702.  
  703.         while (remain > 0) {
  704.                 /* Operation in this page
  705.                  *
  706.                  * page_base = page offset within aperture
  707.                  * page_offset = offset within page
  708.                  * page_length = bytes to copy for this page
  709.                  */
  710.                 page_base = offset & PAGE_MASK;
  711.                 page_offset = offset_in_page(offset);
  712.                 page_length = remain;
  713.                 if ((page_offset + remain) > PAGE_SIZE)
  714.                         page_length = PAGE_SIZE - page_offset;
  715.  
  716.                 /* If we get a fault while copying data, then (presumably) our
  717.                  * source page isn't available.  Return the error and we'll
  718.                  * retry in the slow path.
  719.                  */
  720.                 if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
  721.                                     page_offset, user_data, page_length)) {
  722.                         ret = -EFAULT;
  723.                         goto out_unpin;
  724.                 }
  725.  
  726.                 remain -= page_length;
  727.                 user_data += page_length;
  728.                 offset += page_length;
  729.         }
  730.  
  731. out_unpin:
  732.         i915_gem_object_unpin(obj);
  733. out:
  734.         return ret;
  735. }
  736.  
  737. /* Per-page copy function for the shmem pwrite fastpath.
  738.  * Flushes invalid cachelines before writing to the target if
  739.  * needs_clflush_before is set and flushes out any written cachelines after
  740.  * writing if needs_clflush is set. */
  741. static int
  742. shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
  743.                   char __user *user_data,
  744.                   bool page_do_bit17_swizzling,
  745.                   bool needs_clflush_before,
  746.                   bool needs_clflush_after)
  747. {
  748.         char *vaddr;
  749.         int ret;
  750.  
  751.         if (unlikely(page_do_bit17_swizzling))
  752.                 return -EINVAL;
  753.  
  754.         vaddr = kmap_atomic(page);
  755.         if (needs_clflush_before)
  756.                 drm_clflush_virt_range(vaddr + shmem_page_offset,
  757.                                        page_length);
  758.         ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,
  759.                                                 user_data,
  760.                                                 page_length);
  761.         if (needs_clflush_after)
  762.                 drm_clflush_virt_range(vaddr + shmem_page_offset,
  763.                                        page_length);
  764.         kunmap_atomic(vaddr);
  765.  
  766.         return ret ? -EFAULT : 0;
  767. }
  768.  
  769. /* Only difference to the fast-path function is that this can handle bit17
  770.  * and uses non-atomic copy and kmap functions. */
  771. static int
  772. shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
  773.                   char __user *user_data,
  774.                   bool page_do_bit17_swizzling,
  775.                   bool needs_clflush_before,
  776.                   bool needs_clflush_after)
  777. {
  778.         char *vaddr;
  779.         int ret;
  780.  
  781.         vaddr = kmap(page);
  782.         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
  783.                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  784.                                              page_length,
  785.                                              page_do_bit17_swizzling);
  786.         if (page_do_bit17_swizzling)
  787.                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
  788.                                                 user_data,
  789.                                                 page_length);
  790.         else
  791.                 ret = __copy_from_user(vaddr + shmem_page_offset,
  792.                                        user_data,
  793.                                        page_length);
  794.         if (needs_clflush_after)
  795.                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  796.                                              page_length,
  797.                                              page_do_bit17_swizzling);
  798.         kunmap(page);
  799.  
  800.         return ret ? -EFAULT : 0;
  801. }
  802.  
  803. static int
  804. i915_gem_shmem_pwrite(struct drm_device *dev,
  805.                       struct drm_i915_gem_object *obj,
  806.                       struct drm_i915_gem_pwrite *args,
  807.                       struct drm_file *file)
  808. {
  809.         ssize_t remain;
  810.         loff_t offset;
  811.         char __user *user_data;
  812.         int shmem_page_offset, page_length, ret = 0;
  813.         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
  814.         int hit_slowpath = 0;
  815.         int needs_clflush_after = 0;
  816.         int needs_clflush_before = 0;
  817.         int i;
  818.         struct scatterlist *sg;
  819.  
  820.         user_data = (char __user *) (uintptr_t) args->data_ptr;
  821.         remain = args->size;
  822.  
  823.         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
  824.  
  825.         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  826.                 /* If we're not in the cpu write domain, set ourself into the gtt
  827.                  * write domain and manually flush cachelines (if required). This
  828.                  * optimizes for the case when the gpu will use the data
  829.                  * right away and we therefore have to clflush anyway. */
  830.                 if (obj->cache_level == I915_CACHE_NONE)
  831.                         needs_clflush_after = 1;
  832.                 if (obj->gtt_space) {
  833.                         ret = i915_gem_object_set_to_gtt_domain(obj, true);
  834.                         if (ret)
  835.                                 return ret;
  836.                 }
  837.         }
  838.         /* Same trick applies for invalidate partially written cachelines before
  839.          * writing.  */
  840.         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
  841.             && obj->cache_level == I915_CACHE_NONE)
  842.                 needs_clflush_before = 1;
  843.  
  844.         ret = i915_gem_object_get_pages(obj);
  845.         if (ret)
  846.                 return ret;
  847.  
  848.         i915_gem_object_pin_pages(obj);
  849.  
  850.         offset = args->offset;
  851.         obj->dirty = 1;
  852.  
  853.         for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
  854.                 struct page *page;
  855.                 int partial_cacheline_write;
  856.  
  857.                 if (i < offset >> PAGE_SHIFT)
  858.                         continue;
  859.  
  860.                 if (remain <= 0)
  861.                         break;
  862.  
  863.                 /* Operation in this page
  864.                  *
  865.                  * shmem_page_offset = offset within page in shmem file
  866.                  * page_length = bytes to copy for this page
  867.                  */
  868.                 shmem_page_offset = offset_in_page(offset);
  869.  
  870.                 page_length = remain;
  871.                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
  872.                         page_length = PAGE_SIZE - shmem_page_offset;
  873.  
  874.                 /* If we don't overwrite a cacheline completely we need to be
  875.                  * careful to have up-to-date data by first clflushing. Don't
  876.                  * overcomplicate things and flush the entire patch. */
  877.                 partial_cacheline_write = needs_clflush_before &&
  878.                         ((shmem_page_offset | page_length)
  879.                                 & (boot_cpu_data.x86_clflush_size - 1));
  880.  
  881.                 page = sg_page(sg);
  882.                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
  883.                         (page_to_phys(page) & (1 << 17)) != 0;
  884.  
  885.                 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
  886.                                         user_data, page_do_bit17_swizzling,
  887.                                         partial_cacheline_write,
  888.                                         needs_clflush_after);
  889.                 if (ret == 0)
  890.                         goto next_page;
  891.  
  892.                 hit_slowpath = 1;
  893.                 mutex_unlock(&dev->struct_mutex);
  894.                 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
  895.                                         user_data, page_do_bit17_swizzling,
  896.                                         partial_cacheline_write,
  897.                                         needs_clflush_after);
  898.  
  899.                 mutex_lock(&dev->struct_mutex);
  900.  
  901. next_page:
  902.                 set_page_dirty(page);
  903.                 mark_page_accessed(page);
  904.  
  905.                 if (ret)
  906.                         goto out;
  907.  
  908.                 remain -= page_length;
  909.                 user_data += page_length;
  910.                 offset += page_length;
  911.         }
  912.  
  913. out:
  914.         i915_gem_object_unpin_pages(obj);
  915.  
  916.         if (hit_slowpath) {
  917.                 /* Fixup: Kill any reinstated backing storage pages */
  918.                 if (obj->madv == __I915_MADV_PURGED)
  919.                         i915_gem_object_truncate(obj);
  920.                 /* and flush dirty cachelines in case the object isn't in the cpu write
  921.                  * domain anymore. */
  922.                 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  923.                         i915_gem_clflush_object(obj);
  924.                         intel_gtt_chipset_flush();
  925.                 }
  926.         }
  927.  
  928.         if (needs_clflush_after)
  929.                 intel_gtt_chipset_flush();
  930.  
  931.         return ret;
  932. }
  933.  
  934. /**
  935.  * Writes data to the object referenced by handle.
  936.  *
  937.  * On error, the contents of the buffer that were to be modified are undefined.
  938.  */
  939. int
  940. i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
  941.                       struct drm_file *file)
  942. {
  943.         struct drm_i915_gem_pwrite *args = data;
  944.         struct drm_i915_gem_object *obj;
  945.         int ret;
  946.  
  947.         if (args->size == 0)
  948.                 return 0;
  949.  
  950.         if (!access_ok(VERIFY_READ,
  951.                        (char __user *)(uintptr_t)args->data_ptr,
  952.                        args->size))
  953.                 return -EFAULT;
  954.  
  955.         ret = fault_in_multipages_readable((char __user *)(uintptr_t)args->data_ptr,
  956.                                            args->size);
  957.         if (ret)
  958.                 return -EFAULT;
  959.  
  960.         ret = i915_mutex_lock_interruptible(dev);
  961.         if (ret)
  962.                 return ret;
  963.  
  964.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  965.         if (&obj->base == NULL) {
  966.                 ret = -ENOENT;
  967.                 goto unlock;
  968.         }
  969.  
  970.         /* Bounds check destination. */
  971.         if (args->offset > obj->base.size ||
  972.             args->size > obj->base.size - args->offset) {
  973.                 ret = -EINVAL;
  974.                 goto out;
  975.         }
  976.  
  977.         /* prime objects have no backing filp to GEM pread/pwrite
  978.          * pages from.
  979.          */
  980.         if (!obj->base.filp) {
  981.                 ret = -EINVAL;
  982.                 goto out;
  983.         }
  984.  
  985.         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
  986.  
  987.         ret = -EFAULT;
  988.         /* We can only do the GTT pwrite on untiled buffers, as otherwise
  989.          * it would end up going through the fenced access, and we'll get
  990.          * different detiling behavior between reading and writing.
  991.          * pread/pwrite currently are reading and writing from the CPU
  992.          * perspective, requiring manual detiling by the client.
  993.          */
  994.         if (obj->phys_obj) {
  995.                 ret = i915_gem_phys_pwrite(dev, obj, args, file);
  996.                 goto out;
  997.         }
  998.  
  999.         if (obj->cache_level == I915_CACHE_NONE &&
  1000.             obj->tiling_mode == I915_TILING_NONE &&
  1001.             obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  1002.                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
  1003.                 /* Note that the gtt paths might fail with non-page-backed user
  1004.                  * pointers (e.g. gtt mappings when moving data between
  1005.                  * textures). Fallback to the shmem path in that case. */
  1006.         }
  1007.  
  1008.         if (ret == -EFAULT || ret == -ENOSPC)
  1009.                 ret = i915_gem_shmem_pwrite(dev, obj, args, file);
  1010.  
  1011. out:
  1012.         drm_gem_object_unreference(&obj->base);
  1013. unlock:
  1014.         mutex_unlock(&dev->struct_mutex);
  1015.         return ret;
  1016. }
  1017.  
  1018. #endif
  1019.  
  1020. int
  1021. i915_gem_check_wedge(struct drm_i915_private *dev_priv,
  1022.                      bool interruptible)
  1023. {
  1024.         if (atomic_read(&dev_priv->mm.wedged)) {
  1025.                 struct completion *x = &dev_priv->error_completion;
  1026.                 bool recovery_complete;
  1027.                 unsigned long flags;
  1028.  
  1029.                 /* Give the error handler a chance to run. */
  1030.                 spin_lock_irqsave(&x->wait.lock, flags);
  1031.                 recovery_complete = x->done > 0;
  1032.                 spin_unlock_irqrestore(&x->wait.lock, flags);
  1033.  
  1034.                 /* Non-interruptible callers can't handle -EAGAIN, hence return
  1035.                  * -EIO unconditionally for these. */
  1036.                 if (!interruptible)
  1037.                         return -EIO;
  1038.  
  1039.                 /* Recovery complete, but still wedged means reset failure. */
  1040.                 if (recovery_complete)
  1041.                         return -EIO;
  1042.  
  1043.                 return -EAGAIN;
  1044.         }
  1045.  
  1046.         return 0;
  1047. }
  1048.  
  1049. /*
  1050.  * Compare seqno against outstanding lazy request. Emit a request if they are
  1051.  * equal.
  1052.  */
  1053. static int
  1054. i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
  1055. {
  1056.         int ret;
  1057.  
  1058.         BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
  1059.  
  1060.         ret = 0;
  1061.         if (seqno == ring->outstanding_lazy_request)
  1062.                 ret = i915_add_request(ring, NULL, NULL);
  1063.  
  1064.         return ret;
  1065. }
  1066.  
  1067. /**
  1068.  * __wait_seqno - wait until execution of seqno has finished
  1069.  * @ring: the ring expected to report seqno
  1070.  * @seqno: duh!
  1071.  * @interruptible: do an interruptible wait (normally yes)
  1072.  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
  1073.  *
  1074.  * Returns 0 if the seqno was found within the alloted time. Else returns the
  1075.  * errno with remaining time filled in timeout argument.
  1076.  */
  1077. static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
  1078.                         bool interruptible, struct timespec *timeout)
  1079. {
  1080.         drm_i915_private_t *dev_priv = ring->dev->dev_private;
  1081.         struct timespec before, now, wait_time={1,0};
  1082.         unsigned long timeout_jiffies;
  1083.         long end;
  1084.         bool wait_forever = true;
  1085.         int ret;
  1086.  
  1087.         if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
  1088.                 return 0;
  1089.  
  1090.         trace_i915_gem_request_wait_begin(ring, seqno);
  1091.  
  1092.         if (timeout != NULL) {
  1093.                 wait_time = *timeout;
  1094.                 wait_forever = false;
  1095.         }
  1096.  
  1097. //   timeout_jiffies = timespec_to_jiffies(&wait_time);
  1098.  
  1099.         if (WARN_ON(!ring->irq_get(ring)))
  1100.                 return -ENODEV;
  1101. #if 0
  1102.  
  1103.     /* Record current time in case interrupted by signal, or wedged * */
  1104.         getrawmonotonic(&before);
  1105.  
  1106. #define EXIT_COND \
  1107.         (i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \
  1108.         atomic_read(&dev_priv->mm.wedged))
  1109.         do {
  1110.                         end = wait_event_timeout(ring->irq_queue, EXIT_COND,
  1111.                                                  timeout_jiffies);
  1112.  
  1113.                 ret = i915_gem_check_wedge(dev_priv, interruptible);
  1114.                 if (ret)
  1115.                         end = ret;
  1116.         } while (end == 0 && wait_forever);
  1117.  
  1118.         getrawmonotonic(&now);
  1119.  
  1120.         ring->irq_put(ring);
  1121.         trace_i915_gem_request_wait_end(ring, seqno);
  1122. #undef EXIT_COND
  1123.  
  1124.         if (timeout) {
  1125. //       struct timespec sleep_time = timespec_sub(now, before);
  1126. //       *timeout = timespec_sub(*timeout, sleep_time);
  1127.         }
  1128.  
  1129.         switch (end) {
  1130.         case -EIO:
  1131.         case -EAGAIN: /* Wedged */
  1132.         case -ERESTARTSYS: /* Signal */
  1133.                 return (int)end;
  1134.         case 0: /* Timeout */
  1135. //              if (timeout)
  1136. //                      set_normalized_timespec(timeout, 0, 0);
  1137.                 return -ETIME;
  1138.         default: /* Completed */
  1139.                 WARN_ON(end < 0); /* We're not aware of other errors */
  1140.                 return 0;
  1141.         }
  1142. #endif
  1143.  
  1144. #define EXIT_COND \
  1145.     (i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \
  1146.     atomic_read(&dev_priv->mm.wedged))
  1147.     wait_event(ring->irq_queue, EXIT_COND);
  1148. #undef EXIT_COND
  1149.     ring->irq_put(ring);
  1150.  
  1151.     return 0;
  1152. }
  1153.  
  1154. /**
  1155.  * Waits for a sequence number to be signaled, and cleans up the
  1156.  * request and object lists appropriately for that event.
  1157.  */
  1158. int
  1159. i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
  1160. {
  1161.         struct drm_device *dev = ring->dev;
  1162.         struct drm_i915_private *dev_priv = dev->dev_private;
  1163.         bool interruptible = dev_priv->mm.interruptible;
  1164.         int ret;
  1165.  
  1166.         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
  1167.         BUG_ON(seqno == 0);
  1168.  
  1169.         ret = i915_gem_check_wedge(dev_priv, interruptible);
  1170.         if (ret)
  1171.                 return ret;
  1172.  
  1173.         ret = i915_gem_check_olr(ring, seqno);
  1174.         if (ret)
  1175.                 return ret;
  1176.  
  1177.         return __wait_seqno(ring, seqno, interruptible, NULL);
  1178. }
  1179.  
  1180. /**
  1181.  * Ensures that all rendering to the object has completed and the object is
  1182.  * safe to unbind from the GTT or access from the CPU.
  1183.  */
  1184. static __must_check int
  1185. i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
  1186.                                bool readonly)
  1187. {
  1188.         struct intel_ring_buffer *ring = obj->ring;
  1189.         u32 seqno;
  1190.         int ret;
  1191.  
  1192.         seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
  1193.         if (seqno == 0)
  1194.                 return 0;
  1195.  
  1196.         ret = i915_wait_seqno(ring, seqno);
  1197.        if (ret)
  1198.            return ret;
  1199.  
  1200.         i915_gem_retire_requests_ring(ring);
  1201.  
  1202.         /* Manually manage the write flush as we may have not yet
  1203.          * retired the buffer.
  1204.          */
  1205.         if (obj->last_write_seqno &&
  1206.             i915_seqno_passed(seqno, obj->last_write_seqno)) {
  1207.                 obj->last_write_seqno = 0;
  1208.                 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
  1209.         }
  1210.  
  1211.         return 0;
  1212. }
  1213.  
  1214.  
  1215.  
  1216.  
  1217.  
  1218.  
  1219.  
  1220.  
  1221.  
  1222.  
  1223.  
  1224.  
  1225.  
  1226.  
  1227.  
  1228.  
  1229.  
  1230.  
  1231.  
  1232.  
  1233.  
  1234.  
  1235.  
  1236.  
  1237.  
  1238.  
  1239.  
  1240.  
  1241.  
  1242.  
  1243.  
  1244.  
  1245.  
  1246.  
  1247.  
  1248.  
  1249.  
  1250.  
  1251.  
  1252.  
  1253.  
  1254.  
  1255.  
  1256. /**
  1257.  * i915_gem_release_mmap - remove physical page mappings
  1258.  * @obj: obj in question
  1259.  *
  1260.  * Preserve the reservation of the mmapping with the DRM core code, but
  1261.  * relinquish ownership of the pages back to the system.
  1262.  *
  1263.  * It is vital that we remove the page mapping if we have mapped a tiled
  1264.  * object through the GTT and then lose the fence register due to
  1265.  * resource pressure. Similarly if the object has been moved out of the
  1266.  * aperture, than pages mapped into userspace must be revoked. Removing the
  1267.  * mapping will then trigger a page fault on the next user access, allowing
  1268.  * fixup by i915_gem_fault().
  1269.  */
  1270. void
  1271. i915_gem_release_mmap(struct drm_i915_gem_object *obj)
  1272. {
  1273.         if (!obj->fault_mappable)
  1274.                 return;
  1275.  
  1276.         if (obj->base.dev->dev_mapping)
  1277. //              unmap_mapping_range(obj->base.dev->dev_mapping,
  1278. //                                  (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
  1279. //                                  obj->base.size, 1);
  1280.  
  1281.         obj->fault_mappable = false;
  1282. }
  1283.  
  1284. static uint32_t
  1285. i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
  1286. {
  1287.         uint32_t gtt_size;
  1288.  
  1289.         if (INTEL_INFO(dev)->gen >= 4 ||
  1290.             tiling_mode == I915_TILING_NONE)
  1291.                 return size;
  1292.  
  1293.         /* Previous chips need a power-of-two fence region when tiling */
  1294.         if (INTEL_INFO(dev)->gen == 3)
  1295.                 gtt_size = 1024*1024;
  1296.         else
  1297.                 gtt_size = 512*1024;
  1298.  
  1299.         while (gtt_size < size)
  1300.                 gtt_size <<= 1;
  1301.  
  1302.         return gtt_size;
  1303. }
  1304.  
  1305. /**
  1306.  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
  1307.  * @obj: object to check
  1308.  *
  1309.  * Return the required GTT alignment for an object, taking into account
  1310.  * potential fence register mapping.
  1311.  */
  1312. static uint32_t
  1313. i915_gem_get_gtt_alignment(struct drm_device *dev,
  1314.                            uint32_t size,
  1315.                            int tiling_mode)
  1316. {
  1317.         /*
  1318.          * Minimum alignment is 4k (GTT page size), but might be greater
  1319.          * if a fence register is needed for the object.
  1320.          */
  1321.         if (INTEL_INFO(dev)->gen >= 4 ||
  1322.             tiling_mode == I915_TILING_NONE)
  1323.                 return 4096;
  1324.  
  1325.         /*
  1326.          * Previous chips need to be aligned to the size of the smallest
  1327.          * fence register that can contain the object.
  1328.          */
  1329.         return i915_gem_get_gtt_size(dev, size, tiling_mode);
  1330. }
  1331.  
  1332. /**
  1333.  * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
  1334.  *                                       unfenced object
  1335.  * @dev: the device
  1336.  * @size: size of the object
  1337.  * @tiling_mode: tiling mode of the object
  1338.  *
  1339.  * Return the required GTT alignment for an object, only taking into account
  1340.  * unfenced tiled surface requirements.
  1341.  */
  1342. uint32_t
  1343. i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
  1344.                                     uint32_t size,
  1345.                                     int tiling_mode)
  1346. {
  1347.         /*
  1348.          * Minimum alignment is 4k (GTT page size) for sane hw.
  1349.          */
  1350.         if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
  1351.             tiling_mode == I915_TILING_NONE)
  1352.                 return 4096;
  1353.  
  1354.         /* Previous hardware however needs to be aligned to a power-of-two
  1355.          * tile height. The simplest method for determining this is to reuse
  1356.          * the power-of-tile object size.
  1357.          */
  1358.         return i915_gem_get_gtt_size(dev, size, tiling_mode);
  1359. }
  1360.  
  1361. /* Immediately discard the backing storage */
  1362. static void
  1363. i915_gem_object_truncate(struct drm_i915_gem_object *obj)
  1364. {
  1365. //      struct inode *inode;
  1366.  
  1367. //      i915_gem_object_free_mmap_offset(obj);
  1368.  
  1369. //      if (obj->base.filp == NULL)
  1370. //              return;
  1371.  
  1372.         /* Our goal here is to return as much of the memory as
  1373.          * is possible back to the system as we are called from OOM.
  1374.          * To do this we must instruct the shmfs to drop all of its
  1375.          * backing pages, *now*.
  1376.          */
  1377. //      inode = obj->base.filp->f_path.dentry->d_inode;
  1378. //      shmem_truncate_range(inode, 0, (loff_t)-1);
  1379.  
  1380.         obj->madv = __I915_MADV_PURGED;
  1381. }
  1382.  
  1383. static inline int
  1384. i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
  1385. {
  1386.         return obj->madv == I915_MADV_DONTNEED;
  1387. }
  1388.  
  1389. static void
  1390. i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
  1391. {
  1392.         int ret, i;
  1393.  
  1394.         BUG_ON(obj->madv == __I915_MADV_PURGED);
  1395.  
  1396.         ret = i915_gem_object_set_to_cpu_domain(obj, true);
  1397.         if (ret) {
  1398.                 /* In the event of a disaster, abandon all caches and
  1399.                  * hope for the best.
  1400.                  */
  1401.                 WARN_ON(ret != -EIO);
  1402.                 i915_gem_clflush_object(obj);
  1403.                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  1404.         }
  1405.  
  1406.         if (obj->madv == I915_MADV_DONTNEED)
  1407.                 obj->dirty = 0;
  1408.  
  1409.     for (i = 0; i < obj->pages.nents; i++)
  1410.         FreePage(obj->pages.page[i]);
  1411.  
  1412.     DRM_DEBUG_KMS("%s release %d pages\n", __FUNCTION__, obj->pages.nents);
  1413.         obj->dirty = 0;
  1414.         kfree(obj->pages.page);
  1415. }
  1416.  
  1417. static int
  1418. i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
  1419. {
  1420.         const struct drm_i915_gem_object_ops *ops = obj->ops;
  1421.  
  1422. //    printf("page %x pin count %d\n",
  1423. //           obj->pages.page, obj->pages_pin_count );
  1424.  
  1425.         if (obj->pages.page == NULL)
  1426.                 return 0;
  1427.  
  1428.         BUG_ON(obj->gtt_space);
  1429.  
  1430.         if (obj->pages_pin_count)
  1431.                 return -EBUSY;
  1432.  
  1433.         ops->put_pages(obj);
  1434.     obj->pages.page = NULL;
  1435.  
  1436.         list_del(&obj->gtt_list);
  1437.         if (i915_gem_object_is_purgeable(obj))
  1438.                 i915_gem_object_truncate(obj);
  1439.  
  1440.         return 0;
  1441. }
  1442.  
  1443.  
  1444.  
  1445.  
  1446.  
  1447.  
  1448.  
  1449.  
  1450. static int
  1451. i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
  1452. {
  1453.     dma_addr_t page;
  1454.         int page_count, i;
  1455.  
  1456.         /* Get the list of pages out of our struct file.  They'll be pinned
  1457.          * at this point until we release them.
  1458.          */
  1459.         page_count = obj->base.size / PAGE_SIZE;
  1460.     BUG_ON(obj->pages.page != NULL);
  1461.     obj->pages.page = malloc(page_count * sizeof(dma_addr_t));
  1462.     if (obj->pages.page == NULL)
  1463.                 return -ENOMEM;
  1464.  
  1465.         for (i = 0; i < page_count; i++) {
  1466.         page = AllocPage(); // oh-oh
  1467.         if ( page == 0 )
  1468.                         goto err_pages;
  1469.  
  1470.         obj->pages.page[i] = page;
  1471.     };
  1472.     DRM_DEBUG_KMS("%s alloc %d pages\n", __FUNCTION__, page_count);
  1473.     obj->pages.nents = page_count;
  1474.  
  1475.  
  1476. //   if (obj->tiling_mode != I915_TILING_NONE)
  1477. //       i915_gem_object_do_bit_17_swizzle(obj);
  1478.  
  1479.         return 0;
  1480.  
  1481. err_pages:
  1482.     while (i--)
  1483.         FreePage(obj->pages.page[i]);
  1484.  
  1485.     free(obj->pages.page);
  1486.     obj->pages.page  = NULL;
  1487.     obj->pages.nents = 0;
  1488.  
  1489.     return -ENOMEM;
  1490. }
  1491.  
  1492. /* Ensure that the associated pages are gathered from the backing storage
  1493.  * and pinned into our object. i915_gem_object_get_pages() may be called
  1494.  * multiple times before they are released by a single call to
  1495.  * i915_gem_object_put_pages() - once the pages are no longer referenced
  1496.  * either as a result of memory pressure (reaping pages under the shrinker)
  1497.  * or as the object is itself released.
  1498.  */
  1499. int
  1500. i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
  1501. {
  1502.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  1503.         const struct drm_i915_gem_object_ops *ops = obj->ops;
  1504.         int ret;
  1505.  
  1506.     if (obj->pages.page)
  1507.                 return 0;
  1508.  
  1509.         BUG_ON(obj->pages_pin_count);
  1510.  
  1511.         ret = ops->get_pages(obj);
  1512.         if (ret)
  1513.                 return ret;
  1514.  
  1515.         list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
  1516.         return 0;
  1517. }
  1518.  
  1519. void
  1520. i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
  1521.                                struct intel_ring_buffer *ring,
  1522.                                u32 seqno)
  1523. {
  1524.         struct drm_device *dev = obj->base.dev;
  1525.         struct drm_i915_private *dev_priv = dev->dev_private;
  1526.  
  1527.         BUG_ON(ring == NULL);
  1528.         obj->ring = ring;
  1529.  
  1530.         /* Add a reference if we're newly entering the active list. */
  1531.         if (!obj->active) {
  1532.                 drm_gem_object_reference(&obj->base);
  1533.                 obj->active = 1;
  1534.         }
  1535.  
  1536.         /* Move from whatever list we were on to the tail of execution. */
  1537.         list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
  1538.         list_move_tail(&obj->ring_list, &ring->active_list);
  1539.  
  1540.         obj->last_read_seqno = seqno;
  1541.  
  1542.         if (obj->fenced_gpu_access) {
  1543.                 obj->last_fenced_seqno = seqno;
  1544.  
  1545.                 /* Bump MRU to take account of the delayed flush */
  1546.                 if (obj->fence_reg != I915_FENCE_REG_NONE) {
  1547.                 struct drm_i915_fence_reg *reg;
  1548.  
  1549.                 reg = &dev_priv->fence_regs[obj->fence_reg];
  1550.                         list_move_tail(&reg->lru_list,
  1551.                                        &dev_priv->mm.fence_list);
  1552.                 }
  1553.         }
  1554. }
  1555.  
  1556. static void
  1557. i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
  1558. {
  1559.         struct drm_device *dev = obj->base.dev;
  1560.         struct drm_i915_private *dev_priv = dev->dev_private;
  1561.  
  1562.         BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
  1563.         BUG_ON(!obj->active);
  1564.  
  1565.         if (obj->pin_count) /* are we a framebuffer? */
  1566.                 intel_mark_fb_idle(obj);
  1567.  
  1568.                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
  1569.  
  1570.         list_del_init(&obj->ring_list);
  1571.         obj->ring = NULL;
  1572.  
  1573.         obj->last_read_seqno = 0;
  1574.         obj->last_write_seqno = 0;
  1575.         obj->base.write_domain = 0;
  1576.  
  1577.         obj->last_fenced_seqno = 0;
  1578.         obj->fenced_gpu_access = false;
  1579.  
  1580.         obj->active = 0;
  1581.         drm_gem_object_unreference(&obj->base);
  1582.  
  1583.         WARN_ON(i915_verify_lists(dev));
  1584. }
  1585.  
  1586. static u32
  1587. i915_gem_get_seqno(struct drm_device *dev)
  1588. {
  1589.         drm_i915_private_t *dev_priv = dev->dev_private;
  1590.         u32 seqno = dev_priv->next_seqno;
  1591.  
  1592.         /* reserve 0 for non-seqno */
  1593.         if (++dev_priv->next_seqno == 0)
  1594.                 dev_priv->next_seqno = 1;
  1595.  
  1596.         return seqno;
  1597. }
  1598.  
  1599. u32
  1600. i915_gem_next_request_seqno(struct intel_ring_buffer *ring)
  1601. {
  1602.         if (ring->outstanding_lazy_request == 0)
  1603.                 ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev);
  1604.  
  1605.         return ring->outstanding_lazy_request;
  1606. }
  1607.  
  1608. int
  1609. i915_add_request(struct intel_ring_buffer *ring,
  1610.                  struct drm_file *file,
  1611.                  u32 *out_seqno)
  1612. {
  1613.         drm_i915_private_t *dev_priv = ring->dev->dev_private;
  1614.         struct drm_i915_gem_request *request;
  1615.         u32 request_ring_position;
  1616.         u32 seqno;
  1617.         int was_empty;
  1618.         int ret;
  1619.  
  1620.         /*
  1621.          * Emit any outstanding flushes - execbuf can fail to emit the flush
  1622.          * after having emitted the batchbuffer command. Hence we need to fix
  1623.          * things up similar to emitting the lazy request. The difference here
  1624.          * is that the flush _must_ happen before the next request, no matter
  1625.          * what.
  1626.          */
  1627.         ret = intel_ring_flush_all_caches(ring);
  1628.         if (ret)
  1629.                 return ret;
  1630.  
  1631.         request = kmalloc(sizeof(*request), GFP_KERNEL);
  1632.         if (request == NULL)
  1633.                 return -ENOMEM;
  1634.  
  1635.         seqno = i915_gem_next_request_seqno(ring);
  1636.  
  1637.         /* Record the position of the start of the request so that
  1638.          * should we detect the updated seqno part-way through the
  1639.          * GPU processing the request, we never over-estimate the
  1640.          * position of the head.
  1641.          */
  1642.         request_ring_position = intel_ring_get_tail(ring);
  1643.  
  1644.         ret = ring->add_request(ring, &seqno);
  1645.         if (ret) {
  1646.                 kfree(request);
  1647.             return ret;
  1648.         }
  1649.  
  1650.         trace_i915_gem_request_add(ring, seqno);
  1651.  
  1652.         request->seqno = seqno;
  1653.         request->ring = ring;
  1654.         request->tail = request_ring_position;
  1655.     request->emitted_jiffies = GetTimerTicks();
  1656.         was_empty = list_empty(&ring->request_list);
  1657.         list_add_tail(&request->list, &ring->request_list);
  1658.         request->file_priv = NULL;
  1659.  
  1660.  
  1661.         ring->outstanding_lazy_request = 0;
  1662.  
  1663.         if (!dev_priv->mm.suspended) {
  1664.                 if (i915_enable_hangcheck) {
  1665. //                      mod_timer(&dev_priv->hangcheck_timer,
  1666. //                                jiffies +
  1667. //                                msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
  1668.                 }
  1669.                 if (was_empty) {
  1670.            queue_delayed_work(dev_priv->wq,
  1671.                       &dev_priv->mm.retire_work, HZ);
  1672.                         intel_mark_busy(dev_priv->dev);
  1673.                 }
  1674.         }
  1675.  
  1676.         if (out_seqno)
  1677.                 *out_seqno = seqno;
  1678.         return 0;
  1679. }
  1680.  
  1681.  
  1682.  
  1683.  
  1684. static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
  1685.                                       struct intel_ring_buffer *ring)
  1686. {
  1687.         while (!list_empty(&ring->request_list)) {
  1688.                 struct drm_i915_gem_request *request;
  1689.  
  1690.                 request = list_first_entry(&ring->request_list,
  1691.                                            struct drm_i915_gem_request,
  1692.                                            list);
  1693.  
  1694.                 list_del(&request->list);
  1695. //       i915_gem_request_remove_from_client(request);
  1696.                 kfree(request);
  1697.         }
  1698.  
  1699.         while (!list_empty(&ring->active_list)) {
  1700.                 struct drm_i915_gem_object *obj;
  1701.  
  1702.                 obj = list_first_entry(&ring->active_list,
  1703.                                        struct drm_i915_gem_object,
  1704.                                        ring_list);
  1705.  
  1706.                 i915_gem_object_move_to_inactive(obj);
  1707.         }
  1708. }
  1709.  
  1710. static void i915_gem_reset_fences(struct drm_device *dev)
  1711. {
  1712.         struct drm_i915_private *dev_priv = dev->dev_private;
  1713.         int i;
  1714.  
  1715.         for (i = 0; i < dev_priv->num_fence_regs; i++) {
  1716.                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
  1717.  
  1718.                 i915_gem_write_fence(dev, i, NULL);
  1719.  
  1720.                 if (reg->obj)
  1721.                         i915_gem_object_fence_lost(reg->obj);
  1722.  
  1723.                 reg->pin_count = 0;
  1724.                 reg->obj = NULL;
  1725.                 INIT_LIST_HEAD(&reg->lru_list);
  1726.         }
  1727.  
  1728.         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
  1729. }
  1730.  
  1731. void i915_gem_reset(struct drm_device *dev)
  1732. {
  1733.         struct drm_i915_private *dev_priv = dev->dev_private;
  1734.         struct drm_i915_gem_object *obj;
  1735.         struct intel_ring_buffer *ring;
  1736.         int i;
  1737.  
  1738.         for_each_ring(ring, dev_priv, i)
  1739.                 i915_gem_reset_ring_lists(dev_priv, ring);
  1740.  
  1741.         /* Move everything out of the GPU domains to ensure we do any
  1742.          * necessary invalidation upon reuse.
  1743.          */
  1744.         list_for_each_entry(obj,
  1745.                             &dev_priv->mm.inactive_list,
  1746.                             mm_list)
  1747.         {
  1748.                 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
  1749.         }
  1750.  
  1751.         /* The fence registers are invalidated so clear them out */
  1752.         i915_gem_reset_fences(dev);
  1753. }
  1754.  
  1755. /**
  1756.  * This function clears the request list as sequence numbers are passed.
  1757.  */
  1758. void
  1759. i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
  1760. {
  1761.         uint32_t seqno;
  1762.         int i;
  1763.  
  1764.         if (list_empty(&ring->request_list))
  1765.                 return;
  1766.  
  1767.         WARN_ON(i915_verify_lists(ring->dev));
  1768.  
  1769.         seqno = ring->get_seqno(ring, true);
  1770.  
  1771.         for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++)
  1772.                 if (seqno >= ring->sync_seqno[i])
  1773.                         ring->sync_seqno[i] = 0;
  1774.  
  1775.         while (!list_empty(&ring->request_list)) {
  1776.                 struct drm_i915_gem_request *request;
  1777.  
  1778.                 request = list_first_entry(&ring->request_list,
  1779.                                            struct drm_i915_gem_request,
  1780.                                            list);
  1781.  
  1782.                 if (!i915_seqno_passed(seqno, request->seqno))
  1783.                         break;
  1784.  
  1785.                 trace_i915_gem_request_retire(ring, request->seqno);
  1786.                 /* We know the GPU must have read the request to have
  1787.                  * sent us the seqno + interrupt, so use the position
  1788.                  * of tail of the request to update the last known position
  1789.                  * of the GPU head.
  1790.                  */
  1791.                 ring->last_retired_head = request->tail;
  1792.  
  1793.                 list_del(&request->list);
  1794.                 kfree(request);
  1795.         }
  1796.  
  1797.         /* Move any buffers on the active list that are no longer referenced
  1798.          * by the ringbuffer to the flushing/inactive lists as appropriate.
  1799.          */
  1800.         while (!list_empty(&ring->active_list)) {
  1801.                 struct drm_i915_gem_object *obj;
  1802.  
  1803.                 obj = list_first_entry(&ring->active_list,
  1804.                                       struct drm_i915_gem_object,
  1805.                                       ring_list);
  1806.  
  1807.                 if (!i915_seqno_passed(seqno, obj->last_read_seqno))
  1808.                         break;
  1809.  
  1810.                         i915_gem_object_move_to_inactive(obj);
  1811.         }
  1812.  
  1813.         if (unlikely(ring->trace_irq_seqno &&
  1814.                      i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
  1815.                 ring->irq_put(ring);
  1816.                 ring->trace_irq_seqno = 0;
  1817.         }
  1818.  
  1819.         WARN_ON(i915_verify_lists(ring->dev));
  1820. }
  1821.  
  1822. void
  1823. i915_gem_retire_requests(struct drm_device *dev)
  1824. {
  1825.         drm_i915_private_t *dev_priv = dev->dev_private;
  1826.         struct intel_ring_buffer *ring;
  1827.         int i;
  1828.  
  1829.         for_each_ring(ring, dev_priv, i)
  1830.                 i915_gem_retire_requests_ring(ring);
  1831. }
  1832.  
  1833. static void
  1834. i915_gem_retire_work_handler(struct work_struct *work)
  1835. {
  1836.         drm_i915_private_t *dev_priv;
  1837.         struct drm_device *dev;
  1838.         struct intel_ring_buffer *ring;
  1839.         bool idle;
  1840.         int i;
  1841.  
  1842.         dev_priv = container_of(work, drm_i915_private_t,
  1843.                                 mm.retire_work.work);
  1844.         dev = dev_priv->dev;
  1845.  
  1846.         /* Come back later if the device is busy... */
  1847.         if (!mutex_trylock(&dev->struct_mutex)) {
  1848.         queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
  1849.                 return;
  1850.         }
  1851.  
  1852.         i915_gem_retire_requests(dev);
  1853.  
  1854.         /* Send a periodic flush down the ring so we don't hold onto GEM
  1855.          * objects indefinitely.
  1856.          */
  1857.         idle = true;
  1858.         for_each_ring(ring, dev_priv, i) {
  1859.                 if (ring->gpu_caches_dirty)
  1860.                         i915_add_request(ring, NULL, NULL);
  1861.  
  1862.                 idle &= list_empty(&ring->request_list);
  1863.         }
  1864.  
  1865.    if (!dev_priv->mm.suspended && !idle)
  1866.        queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
  1867.         if (idle)
  1868.                 intel_mark_idle(dev);
  1869.  
  1870.         mutex_unlock(&dev->struct_mutex);
  1871. }
  1872.  
  1873. /**
  1874.  * Ensures that an object will eventually get non-busy by flushing any required
  1875.  * write domains, emitting any outstanding lazy request and retiring and
  1876.  * completed requests.
  1877.  */
  1878. static int
  1879. i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
  1880. {
  1881.         int ret;
  1882.  
  1883.         if (obj->active) {
  1884.                 ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
  1885.                 if (ret)
  1886.                         return ret;
  1887.  
  1888.                 i915_gem_retire_requests_ring(obj->ring);
  1889.         }
  1890.  
  1891.         return 0;
  1892. }
  1893.  
  1894.  
  1895.  
  1896.  
  1897.  
  1898.  
  1899.  
  1900.  
  1901.  
  1902.  
  1903. /**
  1904.  * i915_gem_object_sync - sync an object to a ring.
  1905.  *
  1906.  * @obj: object which may be in use on another ring.
  1907.  * @to: ring we wish to use the object on. May be NULL.
  1908.  *
  1909.  * This code is meant to abstract object synchronization with the GPU.
  1910.  * Calling with NULL implies synchronizing the object with the CPU
  1911.  * rather than a particular GPU ring.
  1912.  *
  1913.  * Returns 0 if successful, else propagates up the lower layer error.
  1914.  */
  1915. int
  1916. i915_gem_object_sync(struct drm_i915_gem_object *obj,
  1917.                      struct intel_ring_buffer *to)
  1918. {
  1919.         struct intel_ring_buffer *from = obj->ring;
  1920.         u32 seqno;
  1921.         int ret, idx;
  1922.  
  1923.         if (from == NULL || to == from)
  1924.                 return 0;
  1925.  
  1926.         if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
  1927.                 return i915_gem_object_wait_rendering(obj, false);
  1928.  
  1929.         idx = intel_ring_sync_index(from, to);
  1930.  
  1931.         seqno = obj->last_read_seqno;
  1932.         if (seqno <= from->sync_seqno[idx])
  1933.                 return 0;
  1934.  
  1935.         ret = i915_gem_check_olr(obj->ring, seqno);
  1936.         if (ret)
  1937.                 return ret;
  1938.  
  1939.         ret = to->sync_to(to, from, seqno);
  1940.         if (!ret)
  1941.                 from->sync_seqno[idx] = seqno;
  1942.  
  1943.         return ret;
  1944. }
  1945.  
  1946. static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
  1947. {
  1948.         u32 old_write_domain, old_read_domains;
  1949.  
  1950.         /* Act a barrier for all accesses through the GTT */
  1951.         mb();
  1952.  
  1953.         /* Force a pagefault for domain tracking on next user access */
  1954. //      i915_gem_release_mmap(obj);
  1955.  
  1956.         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
  1957.                 return;
  1958.  
  1959.         old_read_domains = obj->base.read_domains;
  1960.         old_write_domain = obj->base.write_domain;
  1961.  
  1962.         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
  1963.         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
  1964.  
  1965.         trace_i915_gem_object_change_domain(obj,
  1966.                                             old_read_domains,
  1967.                                             old_write_domain);
  1968. }
  1969.  
  1970. /**
  1971.  * Unbinds an object from the GTT aperture.
  1972.  */
  1973. int
  1974. i915_gem_object_unbind(struct drm_i915_gem_object *obj)
  1975. {
  1976.         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
  1977.         int ret = 0;
  1978.  
  1979.         if (obj->gtt_space == NULL)
  1980.                 return 0;
  1981.  
  1982.         if (obj->pin_count)
  1983.                 return -EBUSY;
  1984.  
  1985.     BUG_ON(obj->pages.page == NULL);
  1986.  
  1987.         ret = i915_gem_object_finish_gpu(obj);
  1988.         if (ret)
  1989.                 return ret;
  1990.         /* Continue on if we fail due to EIO, the GPU is hung so we
  1991.          * should be safe and we need to cleanup or else we might
  1992.          * cause memory corruption through use-after-free.
  1993.          */
  1994.  
  1995.         i915_gem_object_finish_gtt(obj);
  1996.  
  1997.         /* release the fence reg _after_ flushing */
  1998.         ret = i915_gem_object_put_fence(obj);
  1999.         if (ret)
  2000.                 return ret;
  2001.  
  2002.         trace_i915_gem_object_unbind(obj);
  2003.  
  2004.         if (obj->has_global_gtt_mapping)
  2005.         i915_gem_gtt_unbind_object(obj);
  2006.         if (obj->has_aliasing_ppgtt_mapping) {
  2007.                 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
  2008.                 obj->has_aliasing_ppgtt_mapping = 0;
  2009.         }
  2010.         i915_gem_gtt_finish_object(obj);
  2011.  
  2012.         list_del(&obj->mm_list);
  2013.         list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
  2014.         /* Avoid an unnecessary call to unbind on rebind. */
  2015.         obj->map_and_fenceable = true;
  2016.  
  2017.         drm_mm_put_block(obj->gtt_space);
  2018.         obj->gtt_space = NULL;
  2019.         obj->gtt_offset = 0;
  2020.  
  2021.         return 0;
  2022. }
  2023.  
  2024. static int i915_ring_idle(struct intel_ring_buffer *ring)
  2025. {
  2026.         if (list_empty(&ring->active_list))
  2027.                 return 0;
  2028.  
  2029.         return i915_wait_seqno(ring, i915_gem_next_request_seqno(ring));
  2030. }
  2031.  
  2032. int i915_gpu_idle(struct drm_device *dev)
  2033. {
  2034.         drm_i915_private_t *dev_priv = dev->dev_private;
  2035.         struct intel_ring_buffer *ring;
  2036.         int ret, i;
  2037.  
  2038.         /* Flush everything onto the inactive list. */
  2039.         for_each_ring(ring, dev_priv, i) {
  2040.                 ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
  2041.                 if (ret)
  2042.                         return ret;
  2043.  
  2044.                 ret = i915_ring_idle(ring);
  2045.                 if (ret)
  2046.                         return ret;
  2047.         }
  2048.  
  2049.         return 0;
  2050. }
  2051.  
  2052. static void sandybridge_write_fence_reg(struct drm_device *dev, int reg,
  2053.                                         struct drm_i915_gem_object *obj)
  2054. {
  2055.         drm_i915_private_t *dev_priv = dev->dev_private;
  2056.         uint64_t val;
  2057.  
  2058.         if (obj) {
  2059.                 u32 size = obj->gtt_space->size;
  2060.  
  2061.                 val = (uint64_t)((obj->gtt_offset + size - 4096) &
  2062.                                  0xfffff000) << 32;
  2063.                 val |= obj->gtt_offset & 0xfffff000;
  2064.                 val |= (uint64_t)((obj->stride / 128) - 1) <<
  2065.                         SANDYBRIDGE_FENCE_PITCH_SHIFT;
  2066.  
  2067.                 if (obj->tiling_mode == I915_TILING_Y)
  2068.                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
  2069.                 val |= I965_FENCE_REG_VALID;
  2070.         } else
  2071.                 val = 0;
  2072.  
  2073.         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val);
  2074.         POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8);
  2075. }
  2076.  
  2077. static void i965_write_fence_reg(struct drm_device *dev, int reg,
  2078.                                  struct drm_i915_gem_object *obj)
  2079. {
  2080.         drm_i915_private_t *dev_priv = dev->dev_private;
  2081.         uint64_t val;
  2082.  
  2083.         if (obj) {
  2084.                 u32 size = obj->gtt_space->size;
  2085.  
  2086.                 val = (uint64_t)((obj->gtt_offset + size - 4096) &
  2087.                                  0xfffff000) << 32;
  2088.                 val |= obj->gtt_offset & 0xfffff000;
  2089.                 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
  2090.                 if (obj->tiling_mode == I915_TILING_Y)
  2091.                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
  2092.                 val |= I965_FENCE_REG_VALID;
  2093.         } else
  2094.                 val = 0;
  2095.  
  2096.         I915_WRITE64(FENCE_REG_965_0 + reg * 8, val);
  2097.         POSTING_READ(FENCE_REG_965_0 + reg * 8);
  2098. }
  2099.  
  2100. static void i915_write_fence_reg(struct drm_device *dev, int reg,
  2101.                                  struct drm_i915_gem_object *obj)
  2102. {
  2103.         drm_i915_private_t *dev_priv = dev->dev_private;
  2104.         u32 val;
  2105.  
  2106.         if (obj) {
  2107.                 u32 size = obj->gtt_space->size;
  2108.                 int pitch_val;
  2109.                 int tile_width;
  2110.  
  2111.                 WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
  2112.                      (size & -size) != size ||
  2113.                      (obj->gtt_offset & (size - 1)),
  2114.                      "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
  2115.                      obj->gtt_offset, obj->map_and_fenceable, size);
  2116.  
  2117.                 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
  2118.                         tile_width = 128;
  2119.                 else
  2120.                         tile_width = 512;
  2121.  
  2122.                 /* Note: pitch better be a power of two tile widths */
  2123.                 pitch_val = obj->stride / tile_width;
  2124.                 pitch_val = ffs(pitch_val) - 1;
  2125.  
  2126.                 val = obj->gtt_offset;
  2127.                 if (obj->tiling_mode == I915_TILING_Y)
  2128.                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
  2129.                 val |= I915_FENCE_SIZE_BITS(size);
  2130.                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
  2131.                 val |= I830_FENCE_REG_VALID;
  2132.         } else
  2133.                 val = 0;
  2134.  
  2135.         if (reg < 8)
  2136.                 reg = FENCE_REG_830_0 + reg * 4;
  2137.         else
  2138.                 reg = FENCE_REG_945_8 + (reg - 8) * 4;
  2139.  
  2140.         I915_WRITE(reg, val);
  2141.         POSTING_READ(reg);
  2142. }
  2143.  
  2144. static void i830_write_fence_reg(struct drm_device *dev, int reg,
  2145.                                 struct drm_i915_gem_object *obj)
  2146. {
  2147.         drm_i915_private_t *dev_priv = dev->dev_private;
  2148.         uint32_t val;
  2149.  
  2150.         if (obj) {
  2151.                 u32 size = obj->gtt_space->size;
  2152.                 uint32_t pitch_val;
  2153.  
  2154.                 WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
  2155.                      (size & -size) != size ||
  2156.                      (obj->gtt_offset & (size - 1)),
  2157.                      "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
  2158.                      obj->gtt_offset, size);
  2159.  
  2160.                 pitch_val = obj->stride / 128;
  2161.                 pitch_val = ffs(pitch_val) - 1;
  2162.  
  2163.                 val = obj->gtt_offset;
  2164.                 if (obj->tiling_mode == I915_TILING_Y)
  2165.                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
  2166.                 val |= I830_FENCE_SIZE_BITS(size);
  2167.                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
  2168.                 val |= I830_FENCE_REG_VALID;
  2169.         } else
  2170.                 val = 0;
  2171.  
  2172.         I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
  2173.         POSTING_READ(FENCE_REG_830_0 + reg * 4);
  2174. }
  2175.  
  2176. static void i915_gem_write_fence(struct drm_device *dev, int reg,
  2177.                                  struct drm_i915_gem_object *obj)
  2178. {
  2179.         switch (INTEL_INFO(dev)->gen) {
  2180.         case 7:
  2181.         case 6: sandybridge_write_fence_reg(dev, reg, obj); break;
  2182.         case 5:
  2183.         case 4: i965_write_fence_reg(dev, reg, obj); break;
  2184.         case 3: i915_write_fence_reg(dev, reg, obj); break;
  2185.         case 2: i830_write_fence_reg(dev, reg, obj); break;
  2186.         default: break;
  2187.         }
  2188. }
  2189.  
  2190. static inline int fence_number(struct drm_i915_private *dev_priv,
  2191.                                struct drm_i915_fence_reg *fence)
  2192. {
  2193.         return fence - dev_priv->fence_regs;
  2194. }
  2195.  
  2196. static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
  2197.                                          struct drm_i915_fence_reg *fence,
  2198.                                          bool enable)
  2199. {
  2200.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  2201.         int reg = fence_number(dev_priv, fence);
  2202.  
  2203.         i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
  2204.  
  2205.         if (enable) {
  2206.                 obj->fence_reg = reg;
  2207.                 fence->obj = obj;
  2208.                 list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
  2209.         } else {
  2210.                 obj->fence_reg = I915_FENCE_REG_NONE;
  2211.                 fence->obj = NULL;
  2212.                 list_del_init(&fence->lru_list);
  2213.         }
  2214. }
  2215.  
  2216. static int
  2217. i915_gem_object_flush_fence(struct drm_i915_gem_object *obj)
  2218. {
  2219.         if (obj->last_fenced_seqno) {
  2220.                 int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
  2221.                         if (ret)
  2222.                                 return ret;
  2223.  
  2224.                 obj->last_fenced_seqno = 0;
  2225.         }
  2226.  
  2227.         /* Ensure that all CPU reads are completed before installing a fence
  2228.          * and all writes before removing the fence.
  2229.          */
  2230.         if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
  2231.                 mb();
  2232.  
  2233.         obj->fenced_gpu_access = false;
  2234.         return 0;
  2235. }
  2236.  
  2237. int
  2238. i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
  2239. {
  2240.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  2241.         int ret;
  2242.  
  2243.         ret = i915_gem_object_flush_fence(obj);
  2244.         if (ret)
  2245.                 return ret;
  2246.  
  2247.         if (obj->fence_reg == I915_FENCE_REG_NONE)
  2248.                 return 0;
  2249.  
  2250.         i915_gem_object_update_fence(obj,
  2251.                                      &dev_priv->fence_regs[obj->fence_reg],
  2252.                                      false);
  2253.         i915_gem_object_fence_lost(obj);
  2254.  
  2255.         return 0;
  2256. }
  2257.  
  2258. static struct drm_i915_fence_reg *
  2259. i915_find_fence_reg(struct drm_device *dev)
  2260. {
  2261.         struct drm_i915_private *dev_priv = dev->dev_private;
  2262.         struct drm_i915_fence_reg *reg, *avail;
  2263.         int i;
  2264.  
  2265.         /* First try to find a free reg */
  2266.         avail = NULL;
  2267.         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
  2268.                 reg = &dev_priv->fence_regs[i];
  2269.                 if (!reg->obj)
  2270.                         return reg;
  2271.  
  2272.                 if (!reg->pin_count)
  2273.                         avail = reg;
  2274.         }
  2275.  
  2276.         if (avail == NULL)
  2277.                 return NULL;
  2278.  
  2279.         /* None available, try to steal one or wait for a user to finish */
  2280.         list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
  2281.                 if (reg->pin_count)
  2282.                         continue;
  2283.  
  2284.                 return reg;
  2285.         }
  2286.  
  2287.         return NULL;
  2288. }
  2289.  
  2290. /**
  2291.  * i915_gem_object_get_fence - set up fencing for an object
  2292.  * @obj: object to map through a fence reg
  2293.  *
  2294.  * When mapping objects through the GTT, userspace wants to be able to write
  2295.  * to them without having to worry about swizzling if the object is tiled.
  2296.  * This function walks the fence regs looking for a free one for @obj,
  2297.  * stealing one if it can't find any.
  2298.  *
  2299.  * It then sets up the reg based on the object's properties: address, pitch
  2300.  * and tiling format.
  2301.  *
  2302.  * For an untiled surface, this removes any existing fence.
  2303.  */
  2304. int
  2305. i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
  2306. {
  2307.         struct drm_device *dev = obj->base.dev;
  2308.         struct drm_i915_private *dev_priv = dev->dev_private;
  2309.         bool enable = obj->tiling_mode != I915_TILING_NONE;
  2310.         struct drm_i915_fence_reg *reg;
  2311.         int ret;
  2312.  
  2313.         /* Have we updated the tiling parameters upon the object and so
  2314.          * will need to serialise the write to the associated fence register?
  2315.          */
  2316.         if (obj->fence_dirty) {
  2317.                 ret = i915_gem_object_flush_fence(obj);
  2318.                 if (ret)
  2319.                         return ret;
  2320.         }
  2321.  
  2322.         /* Just update our place in the LRU if our fence is getting reused. */
  2323.         if (obj->fence_reg != I915_FENCE_REG_NONE) {
  2324.                 reg = &dev_priv->fence_regs[obj->fence_reg];
  2325.                 if (!obj->fence_dirty) {
  2326.                         list_move_tail(&reg->lru_list,
  2327.                                        &dev_priv->mm.fence_list);
  2328.                         return 0;
  2329.                 }
  2330.         } else if (enable) {
  2331.                 reg = i915_find_fence_reg(dev);
  2332.                 if (reg == NULL)
  2333.                         return -EDEADLK;
  2334.  
  2335.                 if (reg->obj) {
  2336.                         struct drm_i915_gem_object *old = reg->obj;
  2337.  
  2338.                         ret = i915_gem_object_flush_fence(old);
  2339.                         if (ret)
  2340.                                 return ret;
  2341.  
  2342.                         i915_gem_object_fence_lost(old);
  2343.                 }
  2344.         } else
  2345.                 return 0;
  2346.  
  2347.         i915_gem_object_update_fence(obj, reg, enable);
  2348.         obj->fence_dirty = false;
  2349.  
  2350.         return 0;
  2351. }
  2352.  
  2353. static bool i915_gem_valid_gtt_space(struct drm_device *dev,
  2354.                                      struct drm_mm_node *gtt_space,
  2355.                                      unsigned long cache_level)
  2356. {
  2357.         struct drm_mm_node *other;
  2358.  
  2359.         /* On non-LLC machines we have to be careful when putting differing
  2360.          * types of snoopable memory together to avoid the prefetcher
  2361.          * crossing memory domains and dieing.
  2362.          */
  2363.         if (HAS_LLC(dev))
  2364.                 return true;
  2365.  
  2366.         if (gtt_space == NULL)
  2367.                 return true;
  2368.  
  2369.         if (list_empty(&gtt_space->node_list))
  2370.                 return true;
  2371.  
  2372.         other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
  2373.         if (other->allocated && !other->hole_follows && other->color != cache_level)
  2374.                 return false;
  2375.  
  2376.         other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
  2377.         if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
  2378.                 return false;
  2379.  
  2380.         return true;
  2381. }
  2382.  
  2383. static void i915_gem_verify_gtt(struct drm_device *dev)
  2384. {
  2385. #if WATCH_GTT
  2386.         struct drm_i915_private *dev_priv = dev->dev_private;
  2387.         struct drm_i915_gem_object *obj;
  2388.         int err = 0;
  2389.  
  2390.         list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
  2391.                 if (obj->gtt_space == NULL) {
  2392.                         printk(KERN_ERR "object found on GTT list with no space reserved\n");
  2393.                         err++;
  2394.                         continue;
  2395.                 }
  2396.  
  2397.                 if (obj->cache_level != obj->gtt_space->color) {
  2398.                         printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
  2399.                                obj->gtt_space->start,
  2400.                                obj->gtt_space->start + obj->gtt_space->size,
  2401.                                obj->cache_level,
  2402.                                obj->gtt_space->color);
  2403.                         err++;
  2404.                         continue;
  2405.                 }
  2406.  
  2407.                 if (!i915_gem_valid_gtt_space(dev,
  2408.                                               obj->gtt_space,
  2409.                                               obj->cache_level)) {
  2410.                         printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
  2411.                                obj->gtt_space->start,
  2412.                                obj->gtt_space->start + obj->gtt_space->size,
  2413.                                obj->cache_level);
  2414.                         err++;
  2415.                         continue;
  2416.                 }
  2417.         }
  2418.  
  2419.         WARN_ON(err);
  2420. #endif
  2421. }
  2422.  
  2423. /**
  2424.  * Finds free space in the GTT aperture and binds the object there.
  2425.  */
  2426. static int
  2427. i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  2428.                             unsigned alignment,
  2429.                             bool map_and_fenceable,
  2430.                             bool nonblocking)
  2431. {
  2432.         struct drm_device *dev = obj->base.dev;
  2433.         drm_i915_private_t *dev_priv = dev->dev_private;
  2434.         struct drm_mm_node *free_space;
  2435.         u32 size, fence_size, fence_alignment, unfenced_alignment;
  2436.         bool mappable, fenceable;
  2437.         int ret;
  2438.  
  2439.         if (obj->madv != I915_MADV_WILLNEED) {
  2440.                 DRM_ERROR("Attempting to bind a purgeable object\n");
  2441.                 return -EINVAL;
  2442.         }
  2443.  
  2444.         fence_size = i915_gem_get_gtt_size(dev,
  2445.                                            obj->base.size,
  2446.                                            obj->tiling_mode);
  2447.         fence_alignment = i915_gem_get_gtt_alignment(dev,
  2448.                                                      obj->base.size,
  2449.                                                      obj->tiling_mode);
  2450.         unfenced_alignment =
  2451.                 i915_gem_get_unfenced_gtt_alignment(dev,
  2452.                                                     obj->base.size,
  2453.                                                     obj->tiling_mode);
  2454.  
  2455.         if (alignment == 0)
  2456.                 alignment = map_and_fenceable ? fence_alignment :
  2457.                                                 unfenced_alignment;
  2458.         if (map_and_fenceable && alignment & (fence_alignment - 1)) {
  2459.                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
  2460.                 return -EINVAL;
  2461.         }
  2462.  
  2463.         size = map_and_fenceable ? fence_size : obj->base.size;
  2464.  
  2465.         /* If the object is bigger than the entire aperture, reject it early
  2466.          * before evicting everything in a vain attempt to find space.
  2467.          */
  2468.         if (obj->base.size >
  2469.             (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
  2470.                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
  2471.                 return -E2BIG;
  2472.         }
  2473.  
  2474.         ret = i915_gem_object_get_pages(obj);
  2475.         if (ret)
  2476.                 return ret;
  2477.  
  2478.  search_free:
  2479.         if (map_and_fenceable)
  2480.                 free_space =
  2481.                         drm_mm_search_free_in_range_color(&dev_priv->mm.gtt_space,
  2482.                                                           size, alignment, obj->cache_level,
  2483.                                                           0, dev_priv->mm.gtt_mappable_end,
  2484.                                                           false);
  2485.         else
  2486.                 free_space = drm_mm_search_free_color(&dev_priv->mm.gtt_space,
  2487.                                                       size, alignment, obj->cache_level,
  2488.                                                       false);
  2489.  
  2490.         if (free_space != NULL) {
  2491.                 if (map_and_fenceable)
  2492.                         obj->gtt_space =
  2493.                                 drm_mm_get_block_range_generic(free_space,
  2494.                                                                size, alignment, obj->cache_level,
  2495.                                                                0, dev_priv->mm.gtt_mappable_end,
  2496.                                                                false);
  2497.                 else
  2498.                         obj->gtt_space =
  2499.                                 drm_mm_get_block_generic(free_space,
  2500.                                                          size, alignment, obj->cache_level,
  2501.                                                          false);
  2502.         }
  2503.         if (obj->gtt_space == NULL) {
  2504.         ret = 1; //i915_gem_evict_something(dev, size, alignment,
  2505.                  //         map_and_fenceable);
  2506.                 if (ret)
  2507.                         return ret;
  2508.  
  2509.                 goto search_free;
  2510.         }
  2511.         if (WARN_ON(!i915_gem_valid_gtt_space(dev,
  2512.                                               obj->gtt_space,
  2513.                                               obj->cache_level))) {
  2514.                 drm_mm_put_block(obj->gtt_space);
  2515.                 obj->gtt_space = NULL;
  2516.                 return -EINVAL;
  2517.         }
  2518.  
  2519.  
  2520.         ret = i915_gem_gtt_prepare_object(obj);
  2521.         if (ret) {
  2522.                 drm_mm_put_block(obj->gtt_space);
  2523.                 obj->gtt_space = NULL;
  2524.                         return ret;
  2525.         }
  2526.  
  2527.         if (!dev_priv->mm.aliasing_ppgtt)
  2528.                 i915_gem_gtt_bind_object(obj, obj->cache_level);
  2529.  
  2530.         list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list);
  2531.         list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
  2532.  
  2533.         obj->gtt_offset = obj->gtt_space->start;
  2534.  
  2535.         fenceable =
  2536.                 obj->gtt_space->size == fence_size &&
  2537.                 (obj->gtt_space->start & (fence_alignment - 1)) == 0;
  2538.  
  2539.         mappable =
  2540.                 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
  2541.  
  2542.         obj->map_and_fenceable = mappable && fenceable;
  2543.  
  2544.         trace_i915_gem_object_bind(obj, map_and_fenceable);
  2545.         i915_gem_verify_gtt(dev);
  2546.         return 0;
  2547. }
  2548.  
  2549. void
  2550. i915_gem_clflush_object(struct drm_i915_gem_object *obj)
  2551. {
  2552.         /* If we don't have a page list set up, then we're not pinned
  2553.          * to GPU, and we can ignore the cache flush because it'll happen
  2554.          * again at bind time.
  2555.          */
  2556.     if (obj->pages.page == NULL)
  2557.                 return;
  2558.  
  2559.         /* If the GPU is snooping the contents of the CPU cache,
  2560.          * we do not need to manually clear the CPU cache lines.  However,
  2561.          * the caches are only snooped when the render cache is
  2562.          * flushed/invalidated.  As we always have to emit invalidations
  2563.          * and flushes when moving into and out of the RENDER domain, correct
  2564.          * snooping behaviour occurs naturally as the result of our domain
  2565.          * tracking.
  2566.          */
  2567.         if (obj->cache_level != I915_CACHE_NONE)
  2568.                 return;
  2569.  
  2570.      if(obj->mapped != NULL)
  2571.      {
  2572.         uint8_t *page_virtual;
  2573.         unsigned int i;
  2574.  
  2575.         page_virtual = obj->mapped;
  2576.         asm volatile("mfence");
  2577.         for (i = 0; i < obj->base.size; i += x86_clflush_size)
  2578.             clflush(page_virtual + i);
  2579.         asm volatile("mfence");
  2580.      }
  2581.      else
  2582.      {
  2583.         uint8_t *page_virtual;
  2584.         unsigned int i;
  2585.         page_virtual = AllocKernelSpace(obj->base.size);
  2586.         if(page_virtual != NULL)
  2587.         {
  2588.             dma_addr_t *src, *dst;
  2589.             u32 count;
  2590.  
  2591. #define page_tabs  0xFDC00000      /* really dirty hack */
  2592.  
  2593.             src =  obj->pages.page;
  2594.             dst =  &((dma_addr_t*)page_tabs)[(u32_t)page_virtual >> 12];
  2595.             count = obj->base.size/4096;
  2596.  
  2597.             while(count--)
  2598.             {
  2599.                 *dst++ = (0xFFFFF000 & *src++) | 0x001 ;
  2600.             };
  2601.  
  2602.             asm volatile("mfence");
  2603.             for (i = 0; i < obj->base.size; i += x86_clflush_size)
  2604.                 clflush(page_virtual + i);
  2605.             asm volatile("mfence");
  2606.             FreeKernelSpace(page_virtual);
  2607.         }
  2608.         else
  2609.         {
  2610.             asm volatile (
  2611.             "mfence         \n"
  2612.             "wbinvd         \n"                 /* this is really ugly  */
  2613.             "mfence");
  2614.         }
  2615.      }
  2616. }
  2617.  
  2618. /** Flushes the GTT write domain for the object if it's dirty. */
  2619. static void
  2620. i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
  2621. {
  2622.         uint32_t old_write_domain;
  2623.  
  2624.         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
  2625.                 return;
  2626.  
  2627.         /* No actual flushing is required for the GTT write domain.  Writes
  2628.          * to it immediately go to main memory as far as we know, so there's
  2629.          * no chipset flush.  It also doesn't land in render cache.
  2630.          *
  2631.          * However, we do have to enforce the order so that all writes through
  2632.          * the GTT land before any writes to the device, such as updates to
  2633.          * the GATT itself.
  2634.          */
  2635.         wmb();
  2636.  
  2637.         old_write_domain = obj->base.write_domain;
  2638.         obj->base.write_domain = 0;
  2639.  
  2640.         trace_i915_gem_object_change_domain(obj,
  2641.                                             obj->base.read_domains,
  2642.                                             old_write_domain);
  2643. }
  2644.  
  2645. /** Flushes the CPU write domain for the object if it's dirty. */
  2646. static void
  2647. i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
  2648. {
  2649.         uint32_t old_write_domain;
  2650.  
  2651.         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
  2652.                 return;
  2653.  
  2654.         i915_gem_clflush_object(obj);
  2655.         intel_gtt_chipset_flush();
  2656.         old_write_domain = obj->base.write_domain;
  2657.         obj->base.write_domain = 0;
  2658.  
  2659.         trace_i915_gem_object_change_domain(obj,
  2660.                                             obj->base.read_domains,
  2661.                                             old_write_domain);
  2662. }
  2663.  
  2664. /**
  2665.  * Moves a single object to the GTT read, and possibly write domain.
  2666.  *
  2667.  * This function returns when the move is complete, including waiting on
  2668.  * flushes to occur.
  2669.  */
  2670. int
  2671. i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
  2672. {
  2673.         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
  2674.         uint32_t old_write_domain, old_read_domains;
  2675.         int ret;
  2676.  
  2677.         /* Not valid to be called on unbound objects. */
  2678.         if (obj->gtt_space == NULL)
  2679.                 return -EINVAL;
  2680.  
  2681.         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
  2682.                 return 0;
  2683.  
  2684.         ret = i915_gem_object_wait_rendering(obj, !write);
  2685.                 if (ret)
  2686.                         return ret;
  2687.  
  2688.         i915_gem_object_flush_cpu_write_domain(obj);
  2689.  
  2690.         old_write_domain = obj->base.write_domain;
  2691.         old_read_domains = obj->base.read_domains;
  2692.  
  2693.         /* It should now be out of any other write domains, and we can update
  2694.          * the domain values for our changes.
  2695.          */
  2696.         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
  2697.         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
  2698.         if (write) {
  2699.                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
  2700.                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
  2701.                 obj->dirty = 1;
  2702.         }
  2703.  
  2704.         trace_i915_gem_object_change_domain(obj,
  2705.                                             old_read_domains,
  2706.                                             old_write_domain);
  2707.  
  2708.         /* And bump the LRU for this access */
  2709.         if (i915_gem_object_is_inactive(obj))
  2710.                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
  2711.  
  2712.         return 0;
  2713. }
  2714.  
  2715. int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
  2716.                                     enum i915_cache_level cache_level)
  2717. {
  2718.         struct drm_device *dev = obj->base.dev;
  2719.         drm_i915_private_t *dev_priv = dev->dev_private;
  2720.         int ret;
  2721.  
  2722.         if (obj->cache_level == cache_level)
  2723.                 return 0;
  2724.  
  2725.         if (obj->pin_count) {
  2726.                 DRM_DEBUG("can not change the cache level of pinned objects\n");
  2727.                 return -EBUSY;
  2728.         }
  2729.  
  2730.         if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) {
  2731.                 ret = i915_gem_object_unbind(obj);
  2732.                 if (ret)
  2733.                         return ret;
  2734.         }
  2735.  
  2736.         if (obj->gtt_space) {
  2737.                 ret = i915_gem_object_finish_gpu(obj);
  2738.                 if (ret)
  2739.                         return ret;
  2740.  
  2741.                 i915_gem_object_finish_gtt(obj);
  2742.  
  2743.                 /* Before SandyBridge, you could not use tiling or fence
  2744.                  * registers with snooped memory, so relinquish any fences
  2745.                  * currently pointing to our region in the aperture.
  2746.                  */
  2747.                 if (INTEL_INFO(dev)->gen < 6) {
  2748.                         ret = i915_gem_object_put_fence(obj);
  2749.                         if (ret)
  2750.                                 return ret;
  2751.                 }
  2752.  
  2753.                 if (obj->has_global_gtt_mapping)
  2754.                         i915_gem_gtt_bind_object(obj, cache_level);
  2755.                 if (obj->has_aliasing_ppgtt_mapping)
  2756.                         i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
  2757.                                                obj, cache_level);
  2758.  
  2759.                 obj->gtt_space->color = cache_level;
  2760.         }
  2761.  
  2762.         if (cache_level == I915_CACHE_NONE) {
  2763.                 u32 old_read_domains, old_write_domain;
  2764.  
  2765.                 /* If we're coming from LLC cached, then we haven't
  2766.                  * actually been tracking whether the data is in the
  2767.                  * CPU cache or not, since we only allow one bit set
  2768.                  * in obj->write_domain and have been skipping the clflushes.
  2769.                  * Just set it to the CPU cache for now.
  2770.                  */
  2771.                 WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
  2772.                 WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
  2773.  
  2774.                 old_read_domains = obj->base.read_domains;
  2775.                 old_write_domain = obj->base.write_domain;
  2776.  
  2777.                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  2778.                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  2779.  
  2780.                 trace_i915_gem_object_change_domain(obj,
  2781.                                                     old_read_domains,
  2782.                                                     old_write_domain);
  2783.     }
  2784.  
  2785.         obj->cache_level = cache_level;
  2786.         i915_gem_verify_gtt(dev);
  2787.         return 0;
  2788. }
  2789.  
  2790. /*
  2791.  * Prepare buffer for display plane (scanout, cursors, etc).
  2792.  * Can be called from an uninterruptible phase (modesetting) and allows
  2793.  * any flushes to be pipelined (for pageflips).
  2794.  */
  2795. int
  2796. i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
  2797.                                      u32 alignment,
  2798.                                      struct intel_ring_buffer *pipelined)
  2799. {
  2800.         u32 old_read_domains, old_write_domain;
  2801.         int ret;
  2802.  
  2803.         if (pipelined != obj->ring) {
  2804.                 ret = i915_gem_object_sync(obj, pipelined);
  2805.         if (ret)
  2806.                 return ret;
  2807.         }
  2808.  
  2809.         /* The display engine is not coherent with the LLC cache on gen6.  As
  2810.          * a result, we make sure that the pinning that is about to occur is
  2811.          * done with uncached PTEs. This is lowest common denominator for all
  2812.          * chipsets.
  2813.          *
  2814.          * However for gen6+, we could do better by using the GFDT bit instead
  2815.          * of uncaching, which would allow us to flush all the LLC-cached data
  2816.          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
  2817.          */
  2818.         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
  2819.         if (ret)
  2820.                 return ret;
  2821.  
  2822.         /* As the user may map the buffer once pinned in the display plane
  2823.          * (e.g. libkms for the bootup splash), we have to ensure that we
  2824.          * always use map_and_fenceable for all scanout buffers.
  2825.          */
  2826.         ret = i915_gem_object_pin(obj, alignment, true, false);
  2827.         if (ret)
  2828.                 return ret;
  2829.  
  2830.         i915_gem_object_flush_cpu_write_domain(obj);
  2831.  
  2832.         old_write_domain = obj->base.write_domain;
  2833.         old_read_domains = obj->base.read_domains;
  2834.  
  2835.         /* It should now be out of any other write domains, and we can update
  2836.          * the domain values for our changes.
  2837.          */
  2838.         obj->base.write_domain = 0;
  2839.         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
  2840.  
  2841.         trace_i915_gem_object_change_domain(obj,
  2842.                                             old_read_domains,
  2843.                                             old_write_domain);
  2844.  
  2845.         return 0;
  2846. }
  2847.  
  2848. int
  2849. i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
  2850. {
  2851.         int ret;
  2852.  
  2853.         if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
  2854.                 return 0;
  2855.  
  2856.         ret = i915_gem_object_wait_rendering(obj, false);
  2857.                 if (ret)
  2858.                         return ret;
  2859.  
  2860.         /* Ensure that we invalidate the GPU's caches and TLBs. */
  2861.         obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
  2862.         return 0;
  2863. }
  2864.  
  2865. /**
  2866.  * Moves a single object to the CPU read, and possibly write domain.
  2867.  *
  2868.  * This function returns when the move is complete, including waiting on
  2869.  * flushes to occur.
  2870.  */
  2871. int
  2872. i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
  2873. {
  2874.         uint32_t old_write_domain, old_read_domains;
  2875.         int ret;
  2876.  
  2877.         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
  2878.                 return 0;
  2879.  
  2880.         ret = i915_gem_object_wait_rendering(obj, !write);
  2881.         if (ret)
  2882.                 return ret;
  2883.  
  2884.         i915_gem_object_flush_gtt_write_domain(obj);
  2885.  
  2886.         old_write_domain = obj->base.write_domain;
  2887.         old_read_domains = obj->base.read_domains;
  2888.  
  2889.         /* Flush the CPU cache if it's still invalid. */
  2890.         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
  2891.                 i915_gem_clflush_object(obj);
  2892.  
  2893.                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
  2894.         }
  2895.  
  2896.         /* It should now be out of any other write domains, and we can update
  2897.          * the domain values for our changes.
  2898.          */
  2899.         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
  2900.  
  2901.         /* If we're writing through the CPU, then the GPU read domains will
  2902.          * need to be invalidated at next use.
  2903.          */
  2904.         if (write) {
  2905.                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  2906.                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  2907.         }
  2908.  
  2909.         trace_i915_gem_object_change_domain(obj,
  2910.                                             old_read_domains,
  2911.                                             old_write_domain);
  2912.  
  2913.         return 0;
  2914. }
  2915.  
  2916. #if 0
  2917. /* Throttle our rendering by waiting until the ring has completed our requests
  2918.  * emitted over 20 msec ago.
  2919.  *
  2920.  * Note that if we were to use the current jiffies each time around the loop,
  2921.  * we wouldn't escape the function with any frames outstanding if the time to
  2922.  * render a frame was over 20ms.
  2923.  *
  2924.  * This should get us reasonable parallelism between CPU and GPU but also
  2925.  * relatively low latency when blocking on a particular request to finish.
  2926.  */
  2927. static int
  2928. i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
  2929. {
  2930.         struct drm_i915_private *dev_priv = dev->dev_private;
  2931.         struct drm_i915_file_private *file_priv = file->driver_priv;
  2932.         unsigned long recent_enough = GetTimerTics() - msecs_to_jiffies(20);
  2933.         struct drm_i915_gem_request *request;
  2934.         struct intel_ring_buffer *ring = NULL;
  2935.         u32 seqno = 0;
  2936.         int ret;
  2937.  
  2938.         if (atomic_read(&dev_priv->mm.wedged))
  2939.                 return -EIO;
  2940.  
  2941.         spin_lock(&file_priv->mm.lock);
  2942.         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
  2943.                 if (time_after_eq(request->emitted_jiffies, recent_enough))
  2944.                         break;
  2945.  
  2946.                 ring = request->ring;
  2947.                 seqno = request->seqno;
  2948.         }
  2949.         spin_unlock(&file_priv->mm.lock);
  2950.  
  2951.         if (seqno == 0)
  2952.                 return 0;
  2953.  
  2954.         ret = __wait_seqno(ring, seqno, true, NULL);
  2955.         if (ret == 0)
  2956.                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
  2957.  
  2958.         return ret;
  2959. }
  2960. #endif
  2961.  
  2962. int
  2963. i915_gem_object_pin(struct drm_i915_gem_object *obj,
  2964.                     uint32_t alignment,
  2965.                     bool map_and_fenceable,
  2966.                     bool nonblocking)
  2967. {
  2968.         int ret;
  2969.  
  2970.         if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
  2971.                 return -EBUSY;
  2972.  
  2973. #if 0
  2974.         if (obj->gtt_space != NULL) {
  2975.                 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
  2976.                     (map_and_fenceable && !obj->map_and_fenceable)) {
  2977.                         WARN(obj->pin_count,
  2978.                              "bo is already pinned with incorrect alignment:"
  2979.                              " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
  2980.                              " obj->map_and_fenceable=%d\n",
  2981.                              obj->gtt_offset, alignment,
  2982.                              map_and_fenceable,
  2983.                              obj->map_and_fenceable);
  2984.                         ret = i915_gem_object_unbind(obj);
  2985.                         if (ret)
  2986.                                 return ret;
  2987.                 }
  2988.         }
  2989. #endif
  2990.  
  2991.         if (obj->gtt_space == NULL) {
  2992.                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
  2993.                                                   map_and_fenceable,
  2994.                                                   nonblocking);
  2995.                 if (ret)
  2996.                         return ret;
  2997.         }
  2998.  
  2999.         if (!obj->has_global_gtt_mapping && map_and_fenceable)
  3000.                 i915_gem_gtt_bind_object(obj, obj->cache_level);
  3001.  
  3002.         obj->pin_count++;
  3003.         obj->pin_mappable |= map_and_fenceable;
  3004.  
  3005.         return 0;
  3006. }
  3007.  
  3008. void
  3009. i915_gem_object_unpin(struct drm_i915_gem_object *obj)
  3010. {
  3011.         BUG_ON(obj->pin_count == 0);
  3012.         BUG_ON(obj->gtt_space == NULL);
  3013.  
  3014.         if (--obj->pin_count == 0)
  3015.                 obj->pin_mappable = false;
  3016. }
  3017.  
  3018. #if 0
  3019. int
  3020. i915_gem_pin_ioctl(struct drm_device *dev, void *data,
  3021.                    struct drm_file *file)
  3022. {
  3023.         struct drm_i915_gem_pin *args = data;
  3024.         struct drm_i915_gem_object *obj;
  3025.         int ret;
  3026.  
  3027.         ret = i915_mutex_lock_interruptible(dev);
  3028.         if (ret)
  3029.                 return ret;
  3030.  
  3031.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  3032.         if (&obj->base == NULL) {
  3033.                 ret = -ENOENT;
  3034.                 goto unlock;
  3035.         }
  3036.  
  3037.         if (obj->madv != I915_MADV_WILLNEED) {
  3038.                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
  3039.                 ret = -EINVAL;
  3040.                 goto out;
  3041.         }
  3042.  
  3043.         if (obj->pin_filp != NULL && obj->pin_filp != file) {
  3044.                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
  3045.                           args->handle);
  3046.                 ret = -EINVAL;
  3047.                 goto out;
  3048.         }
  3049.  
  3050.         obj->user_pin_count++;
  3051.         obj->pin_filp = file;
  3052.         if (obj->user_pin_count == 1) {
  3053.                 ret = i915_gem_object_pin(obj, args->alignment, true, false);
  3054.                 if (ret)
  3055.                         goto out;
  3056.         }
  3057.  
  3058.         /* XXX - flush the CPU caches for pinned objects
  3059.          * as the X server doesn't manage domains yet
  3060.          */
  3061.         i915_gem_object_flush_cpu_write_domain(obj);
  3062.         args->offset = obj->gtt_offset;
  3063. out:
  3064.         drm_gem_object_unreference(&obj->base);
  3065. unlock:
  3066.         mutex_unlock(&dev->struct_mutex);
  3067.         return ret;
  3068. }
  3069.  
  3070. int
  3071. i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
  3072.                      struct drm_file *file)
  3073. {
  3074.         struct drm_i915_gem_pin *args = data;
  3075.         struct drm_i915_gem_object *obj;
  3076.         int ret;
  3077.  
  3078.         ret = i915_mutex_lock_interruptible(dev);
  3079.         if (ret)
  3080.                 return ret;
  3081.  
  3082.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  3083.         if (&obj->base == NULL) {
  3084.                 ret = -ENOENT;
  3085.                 goto unlock;
  3086.         }
  3087.  
  3088.         if (obj->pin_filp != file) {
  3089.                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
  3090.                           args->handle);
  3091.                 ret = -EINVAL;
  3092.                 goto out;
  3093.         }
  3094.         obj->user_pin_count--;
  3095.         if (obj->user_pin_count == 0) {
  3096.                 obj->pin_filp = NULL;
  3097.                 i915_gem_object_unpin(obj);
  3098.         }
  3099.  
  3100. out:
  3101.         drm_gem_object_unreference(&obj->base);
  3102. unlock:
  3103.         mutex_unlock(&dev->struct_mutex);
  3104.         return ret;
  3105. }
  3106.  
  3107. int
  3108. i915_gem_busy_ioctl(struct drm_device *dev, void *data,
  3109.                     struct drm_file *file)
  3110. {
  3111.         struct drm_i915_gem_busy *args = data;
  3112.         struct drm_i915_gem_object *obj;
  3113.         int ret;
  3114.  
  3115.         ret = i915_mutex_lock_interruptible(dev);
  3116.         if (ret)
  3117.                 return ret;
  3118.  
  3119.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  3120.         if (&obj->base == NULL) {
  3121.                 ret = -ENOENT;
  3122.                 goto unlock;
  3123.         }
  3124.  
  3125.         /* Count all active objects as busy, even if they are currently not used
  3126.          * by the gpu. Users of this interface expect objects to eventually
  3127.          * become non-busy without any further actions, therefore emit any
  3128.          * necessary flushes here.
  3129.          */
  3130.         ret = i915_gem_object_flush_active(obj);
  3131.  
  3132.         args->busy = obj->active;
  3133.         if (obj->ring) {
  3134.                 BUILD_BUG_ON(I915_NUM_RINGS > 16);
  3135.                 args->busy |= intel_ring_flag(obj->ring) << 16;
  3136.         }
  3137.  
  3138.         drm_gem_object_unreference(&obj->base);
  3139. unlock:
  3140.         mutex_unlock(&dev->struct_mutex);
  3141.         return ret;
  3142. }
  3143.  
  3144. int
  3145. i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
  3146.                         struct drm_file *file_priv)
  3147. {
  3148.         return i915_gem_ring_throttle(dev, file_priv);
  3149. }
  3150.  
  3151. int
  3152. i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
  3153.                        struct drm_file *file_priv)
  3154. {
  3155.         struct drm_i915_gem_madvise *args = data;
  3156.         struct drm_i915_gem_object *obj;
  3157.         int ret;
  3158.  
  3159.         switch (args->madv) {
  3160.         case I915_MADV_DONTNEED:
  3161.         case I915_MADV_WILLNEED:
  3162.             break;
  3163.         default:
  3164.             return -EINVAL;
  3165.         }
  3166.  
  3167.         ret = i915_mutex_lock_interruptible(dev);
  3168.         if (ret)
  3169.                 return ret;
  3170.  
  3171.         obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
  3172.         if (&obj->base == NULL) {
  3173.                 ret = -ENOENT;
  3174.                 goto unlock;
  3175.         }
  3176.  
  3177.         if (obj->pin_count) {
  3178.                 ret = -EINVAL;
  3179.                 goto out;
  3180.         }
  3181.  
  3182.         if (obj->madv != __I915_MADV_PURGED)
  3183.                 obj->madv = args->madv;
  3184.  
  3185.         /* if the object is no longer attached, discard its backing storage */
  3186.         if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
  3187.                 i915_gem_object_truncate(obj);
  3188.  
  3189.         args->retained = obj->madv != __I915_MADV_PURGED;
  3190.  
  3191. out:
  3192.         drm_gem_object_unreference(&obj->base);
  3193. unlock:
  3194.         mutex_unlock(&dev->struct_mutex);
  3195.         return ret;
  3196. }
  3197. #endif
  3198.  
  3199. void i915_gem_object_init(struct drm_i915_gem_object *obj,
  3200.                           const struct drm_i915_gem_object_ops *ops)
  3201. {
  3202.         INIT_LIST_HEAD(&obj->mm_list);
  3203.         INIT_LIST_HEAD(&obj->gtt_list);
  3204.         INIT_LIST_HEAD(&obj->ring_list);
  3205.         INIT_LIST_HEAD(&obj->exec_list);
  3206.  
  3207.         obj->ops = ops;
  3208.  
  3209.         obj->fence_reg = I915_FENCE_REG_NONE;
  3210.         obj->madv = I915_MADV_WILLNEED;
  3211.         /* Avoid an unnecessary call to unbind on the first bind. */
  3212.         obj->map_and_fenceable = true;
  3213.  
  3214.         i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
  3215. }
  3216.  
  3217. static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
  3218.         .get_pages = i915_gem_object_get_pages_gtt,
  3219.         .put_pages = i915_gem_object_put_pages_gtt,
  3220. };
  3221.  
  3222. struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
  3223.                                                   size_t size)
  3224. {
  3225.         struct drm_i915_gem_object *obj;
  3226.         struct address_space *mapping;
  3227.         u32 mask;
  3228.  
  3229.         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
  3230.         if (obj == NULL)
  3231.                 return NULL;
  3232.  
  3233.         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
  3234.                 kfree(obj);
  3235.                 return NULL;
  3236.         }
  3237.  
  3238.  
  3239.         i915_gem_object_init(obj, &i915_gem_object_ops);
  3240.  
  3241.         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  3242.         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  3243.  
  3244.         if (HAS_LLC(dev)) {
  3245.                 /* On some devices, we can have the GPU use the LLC (the CPU
  3246.                  * cache) for about a 10% performance improvement
  3247.                  * compared to uncached.  Graphics requests other than
  3248.                  * display scanout are coherent with the CPU in
  3249.                  * accessing this cache.  This means in this mode we
  3250.                  * don't need to clflush on the CPU side, and on the
  3251.                  * GPU side we only need to flush internal caches to
  3252.                  * get data visible to the CPU.
  3253.                  *
  3254.                  * However, we maintain the display planes as UC, and so
  3255.                  * need to rebind when first used as such.
  3256.                  */
  3257.                 obj->cache_level = I915_CACHE_LLC;
  3258.         } else
  3259.                 obj->cache_level = I915_CACHE_NONE;
  3260.  
  3261.         return obj;
  3262. }
  3263.  
  3264. int i915_gem_init_object(struct drm_gem_object *obj)
  3265. {
  3266.         BUG();
  3267.  
  3268.         return 0;
  3269. }
  3270.  
  3271. void i915_gem_free_object(struct drm_gem_object *gem_obj)
  3272. {
  3273.         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
  3274.         struct drm_device *dev = obj->base.dev;
  3275.         drm_i915_private_t *dev_priv = dev->dev_private;
  3276.  
  3277.         trace_i915_gem_object_destroy(obj);
  3278.  
  3279. //   if (obj->phys_obj)
  3280. //       i915_gem_detach_phys_object(dev, obj);
  3281.  
  3282.         obj->pin_count = 0;
  3283.         if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
  3284.                 bool was_interruptible;
  3285.  
  3286.                 was_interruptible = dev_priv->mm.interruptible;
  3287.                 dev_priv->mm.interruptible = false;
  3288.  
  3289.                 WARN_ON(i915_gem_object_unbind(obj));
  3290.  
  3291.                 dev_priv->mm.interruptible = was_interruptible;
  3292.         }
  3293.  
  3294.         obj->pages_pin_count = 0;
  3295.         i915_gem_object_put_pages(obj);
  3296. //   i915_gem_object_free_mmap_offset(obj);
  3297.  
  3298.     BUG_ON(obj->pages.page);
  3299.  
  3300. //   if (obj->base.import_attach)
  3301. //       drm_prime_gem_destroy(&obj->base, NULL);
  3302.  
  3303.         drm_gem_object_release(&obj->base);
  3304.         i915_gem_info_remove_obj(dev_priv, obj->base.size);
  3305.  
  3306.         kfree(obj->bit_17);
  3307.         kfree(obj);
  3308. }
  3309.  
  3310. #if 0
  3311. int
  3312. i915_gem_idle(struct drm_device *dev)
  3313. {
  3314.         drm_i915_private_t *dev_priv = dev->dev_private;
  3315.         int ret;
  3316.  
  3317.         mutex_lock(&dev->struct_mutex);
  3318.  
  3319.         if (dev_priv->mm.suspended) {
  3320.                 mutex_unlock(&dev->struct_mutex);
  3321.                 return 0;
  3322.         }
  3323.  
  3324.         ret = i915_gpu_idle(dev);
  3325.         if (ret) {
  3326.                 mutex_unlock(&dev->struct_mutex);
  3327.                 return ret;
  3328.         }
  3329.         i915_gem_retire_requests(dev);
  3330.  
  3331.         i915_gem_reset_fences(dev);
  3332.  
  3333.         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
  3334.          * We need to replace this with a semaphore, or something.
  3335.          * And not confound mm.suspended!
  3336.          */
  3337.         dev_priv->mm.suspended = 1;
  3338.         del_timer_sync(&dev_priv->hangcheck_timer);
  3339.  
  3340.         i915_kernel_lost_context(dev);
  3341.         i915_gem_cleanup_ringbuffer(dev);
  3342.  
  3343.         mutex_unlock(&dev->struct_mutex);
  3344.  
  3345.         /* Cancel the retire work handler, which should be idle now. */
  3346. //   cancel_delayed_work_sync(&dev_priv->mm.retire_work);
  3347.  
  3348.         return 0;
  3349. }
  3350. #endif
  3351.  
  3352. void i915_gem_l3_remap(struct drm_device *dev)
  3353. {
  3354.         drm_i915_private_t *dev_priv = dev->dev_private;
  3355.         u32 misccpctl;
  3356.         int i;
  3357.  
  3358.         if (!IS_IVYBRIDGE(dev))
  3359.                 return;
  3360.  
  3361.         if (!dev_priv->mm.l3_remap_info)
  3362.                 return;
  3363.  
  3364.         misccpctl = I915_READ(GEN7_MISCCPCTL);
  3365.         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
  3366.         POSTING_READ(GEN7_MISCCPCTL);
  3367.  
  3368.         for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
  3369.                 u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
  3370.                 if (remap && remap != dev_priv->mm.l3_remap_info[i/4])
  3371.                         DRM_DEBUG("0x%x was already programmed to %x\n",
  3372.                                   GEN7_L3LOG_BASE + i, remap);
  3373.                 if (remap && !dev_priv->mm.l3_remap_info[i/4])
  3374.                         DRM_DEBUG_DRIVER("Clearing remapped register\n");
  3375.                 I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->mm.l3_remap_info[i/4]);
  3376.         }
  3377.  
  3378.         /* Make sure all the writes land before disabling dop clock gating */
  3379.         POSTING_READ(GEN7_L3LOG_BASE);
  3380.  
  3381.         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
  3382. }
  3383.  
  3384. void i915_gem_init_swizzling(struct drm_device *dev)
  3385. {
  3386.         drm_i915_private_t *dev_priv = dev->dev_private;
  3387.  
  3388.         if (INTEL_INFO(dev)->gen < 5 ||
  3389.             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
  3390.                 return;
  3391.  
  3392.         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
  3393.                                  DISP_TILE_SURFACE_SWIZZLING);
  3394.  
  3395.         if (IS_GEN5(dev))
  3396.                 return;
  3397.  
  3398.         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
  3399.         if (IS_GEN6(dev))
  3400.                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
  3401.         else
  3402.                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
  3403. }
  3404.  
  3405. void i915_gem_init_ppgtt(struct drm_device *dev)
  3406. {
  3407.         drm_i915_private_t *dev_priv = dev->dev_private;
  3408.         uint32_t pd_offset;
  3409.         struct intel_ring_buffer *ring;
  3410.         struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
  3411.         uint32_t __iomem *pd_addr;
  3412.         uint32_t pd_entry;
  3413.         int i;
  3414.  
  3415.         if (!dev_priv->mm.aliasing_ppgtt)
  3416.                 return;
  3417.  
  3418.  
  3419.         pd_addr = dev_priv->mm.gtt->gtt + ppgtt->pd_offset/sizeof(uint32_t);
  3420.         for (i = 0; i < ppgtt->num_pd_entries; i++) {
  3421.                 dma_addr_t pt_addr;
  3422.  
  3423.                 if (dev_priv->mm.gtt->needs_dmar)
  3424.                         pt_addr = ppgtt->pt_dma_addr[i];
  3425.                 else
  3426.             pt_addr = ppgtt->pt_pages[i];
  3427.  
  3428.                 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
  3429.                 pd_entry |= GEN6_PDE_VALID;
  3430.  
  3431.                 writel(pd_entry, pd_addr + i);
  3432.         }
  3433.         readl(pd_addr);
  3434.  
  3435.         pd_offset = ppgtt->pd_offset;
  3436.         pd_offset /= 64; /* in cachelines, */
  3437.         pd_offset <<= 16;
  3438.  
  3439.         if (INTEL_INFO(dev)->gen == 6) {
  3440.                 uint32_t ecochk, gab_ctl, ecobits;
  3441.  
  3442.                 ecobits = I915_READ(GAC_ECO_BITS);
  3443.                 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
  3444.  
  3445.                 gab_ctl = I915_READ(GAB_CTL);
  3446.                 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
  3447.  
  3448.                 ecochk = I915_READ(GAM_ECOCHK);
  3449.                 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT |
  3450.                                        ECOCHK_PPGTT_CACHE64B);
  3451.                 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
  3452.         } else if (INTEL_INFO(dev)->gen >= 7) {
  3453.                 I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B);
  3454.                 /* GFX_MODE is per-ring on gen7+ */
  3455.         }
  3456.  
  3457.         for_each_ring(ring, dev_priv, i) {
  3458.                 if (INTEL_INFO(dev)->gen >= 7)
  3459.                         I915_WRITE(RING_MODE_GEN7(ring),
  3460.                                    _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
  3461.  
  3462.                 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
  3463.                 I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
  3464.         }
  3465. }
  3466.  
  3467. static bool
  3468. intel_enable_blt(struct drm_device *dev)
  3469. {
  3470.         if (!HAS_BLT(dev))
  3471.                 return false;
  3472.  
  3473.         /* The blitter was dysfunctional on early prototypes */
  3474.         if (IS_GEN6(dev) && dev->pdev->revision < 8) {
  3475.                 DRM_INFO("BLT not supported on this pre-production hardware;"
  3476.                          " graphics performance will be degraded.\n");
  3477.                 return false;
  3478.         }
  3479.  
  3480.         return true;
  3481. }
  3482.  
  3483. int
  3484. i915_gem_init_hw(struct drm_device *dev)
  3485. {
  3486.         drm_i915_private_t *dev_priv = dev->dev_private;
  3487.         int ret;
  3488.  
  3489.         if (!intel_enable_gtt())
  3490.                 return -EIO;
  3491.  
  3492.         if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))
  3493.                 I915_WRITE(0x9008, I915_READ(0x9008) | 0xf0000);
  3494.  
  3495.         i915_gem_l3_remap(dev);
  3496.  
  3497.         i915_gem_init_swizzling(dev);
  3498.  
  3499.         ret = intel_init_render_ring_buffer(dev);
  3500.         if (ret)
  3501.                 return ret;
  3502.  
  3503.     if (HAS_BSD(dev)) {
  3504.                 ret = intel_init_bsd_ring_buffer(dev);
  3505.                 if (ret)
  3506.                         goto cleanup_render_ring;
  3507.         }
  3508.  
  3509.         if (intel_enable_blt(dev)) {
  3510.                 ret = intel_init_blt_ring_buffer(dev);
  3511.                 if (ret)
  3512.                         goto cleanup_bsd_ring;
  3513.         }
  3514.  
  3515.         dev_priv->next_seqno = 1;
  3516.  
  3517.         /*
  3518.          * XXX: There was some w/a described somewhere suggesting loading
  3519.          * contexts before PPGTT.
  3520.          */
  3521.         i915_gem_context_init(dev);
  3522.         i915_gem_init_ppgtt(dev);
  3523.  
  3524.         return 0;
  3525.  
  3526. cleanup_bsd_ring:
  3527.         intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
  3528. cleanup_render_ring:
  3529.         intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
  3530.         return ret;
  3531. }
  3532.  
  3533. static bool
  3534. intel_enable_ppgtt(struct drm_device *dev)
  3535. {
  3536.         if (i915_enable_ppgtt >= 0)
  3537.                 return i915_enable_ppgtt;
  3538.  
  3539. #ifdef CONFIG_INTEL_IOMMU
  3540.         /* Disable ppgtt on SNB if VT-d is on. */
  3541.         if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
  3542.                 return false;
  3543. #endif
  3544.  
  3545.         return true;
  3546. }
  3547.  
  3548. #define LFB_SIZE 0xC00000
  3549.  
  3550. int i915_gem_init(struct drm_device *dev)
  3551. {
  3552.         struct drm_i915_private *dev_priv = dev->dev_private;
  3553.         unsigned long gtt_size, mappable_size;
  3554.         int ret;
  3555.  
  3556.         gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT;
  3557.         mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
  3558.  
  3559.         mutex_lock(&dev->struct_mutex);
  3560.         if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
  3561.                 /* PPGTT pdes are stolen from global gtt ptes, so shrink the
  3562.                  * aperture accordingly when using aliasing ppgtt. */
  3563.                 gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
  3564.  
  3565.         i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE);
  3566.  
  3567.                 ret = i915_gem_init_aliasing_ppgtt(dev);
  3568.                 if (ret) {
  3569.                         mutex_unlock(&dev->struct_mutex);
  3570.                         return ret;
  3571.                 }
  3572.         } else {
  3573.                 /* Let GEM Manage all of the aperture.
  3574.                  *
  3575.                  * However, leave one page at the end still bound to the scratch
  3576.                  * page.  There are a number of places where the hardware
  3577.                  * apparently prefetches past the end of the object, and we've
  3578.                  * seen multiple hangs with the GPU head pointer stuck in a
  3579.                  * batchbuffer bound at the last page of the aperture.  One page
  3580.                  * should be enough to keep any prefetching inside of the
  3581.                  * aperture.
  3582.                  */
  3583.         i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE);
  3584.         }
  3585.  
  3586.         ret = i915_gem_init_hw(dev);
  3587.         mutex_unlock(&dev->struct_mutex);
  3588.         if (ret) {
  3589.                 i915_gem_cleanup_aliasing_ppgtt(dev);
  3590.                 return ret;
  3591.         }
  3592.  
  3593.     return 0;
  3594. }
  3595.  
  3596. void
  3597. i915_gem_cleanup_ringbuffer(struct drm_device *dev)
  3598. {
  3599.         drm_i915_private_t *dev_priv = dev->dev_private;
  3600.         struct intel_ring_buffer *ring;
  3601.         int i;
  3602.  
  3603.         for_each_ring(ring, dev_priv, i)
  3604.                 intel_cleanup_ring_buffer(ring);
  3605. }
  3606.  
  3607. #if 0
  3608.  
  3609. int
  3610. i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
  3611.                        struct drm_file *file_priv)
  3612. {
  3613.         drm_i915_private_t *dev_priv = dev->dev_private;
  3614.         int ret;
  3615.  
  3616.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  3617.                 return 0;
  3618.  
  3619.         if (atomic_read(&dev_priv->mm.wedged)) {
  3620.                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
  3621.                 atomic_set(&dev_priv->mm.wedged, 0);
  3622.         }
  3623.  
  3624.         mutex_lock(&dev->struct_mutex);
  3625.         dev_priv->mm.suspended = 0;
  3626.  
  3627.         ret = i915_gem_init_hw(dev);
  3628.         if (ret != 0) {
  3629.                 mutex_unlock(&dev->struct_mutex);
  3630.                 return ret;
  3631.         }
  3632.  
  3633.         BUG_ON(!list_empty(&dev_priv->mm.active_list));
  3634.         mutex_unlock(&dev->struct_mutex);
  3635.  
  3636.         ret = drm_irq_install(dev);
  3637.         if (ret)
  3638.                 goto cleanup_ringbuffer;
  3639.  
  3640.         return 0;
  3641.  
  3642. cleanup_ringbuffer:
  3643.         mutex_lock(&dev->struct_mutex);
  3644.         i915_gem_cleanup_ringbuffer(dev);
  3645.         dev_priv->mm.suspended = 1;
  3646.         mutex_unlock(&dev->struct_mutex);
  3647.  
  3648.         return ret;
  3649. }
  3650.  
  3651. int
  3652. i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
  3653.                        struct drm_file *file_priv)
  3654. {
  3655.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  3656.                 return 0;
  3657.  
  3658.         drm_irq_uninstall(dev);
  3659.         return i915_gem_idle(dev);
  3660. }
  3661.  
  3662. void
  3663. i915_gem_lastclose(struct drm_device *dev)
  3664. {
  3665.         int ret;
  3666.  
  3667.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  3668.                 return;
  3669.  
  3670.         ret = i915_gem_idle(dev);
  3671.         if (ret)
  3672.                 DRM_ERROR("failed to idle hardware: %d\n", ret);
  3673. }
  3674. #endif
  3675.  
  3676. static void
  3677. init_ring_lists(struct intel_ring_buffer *ring)
  3678. {
  3679.     INIT_LIST_HEAD(&ring->active_list);
  3680.     INIT_LIST_HEAD(&ring->request_list);
  3681. }
  3682.  
  3683. void
  3684. i915_gem_load(struct drm_device *dev)
  3685. {
  3686.     int i;
  3687.     drm_i915_private_t *dev_priv = dev->dev_private;
  3688.  
  3689.     INIT_LIST_HEAD(&dev_priv->mm.active_list);
  3690.     INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
  3691.         INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
  3692.         INIT_LIST_HEAD(&dev_priv->mm.bound_list);
  3693.     INIT_LIST_HEAD(&dev_priv->mm.fence_list);
  3694.     for (i = 0; i < I915_NUM_RINGS; i++)
  3695.         init_ring_lists(&dev_priv->ring[i]);
  3696.         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
  3697.         INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
  3698.         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
  3699.                           i915_gem_retire_work_handler);
  3700.  
  3701.     /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
  3702.     if (IS_GEN3(dev)) {
  3703.                 I915_WRITE(MI_ARB_STATE,
  3704.                            _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
  3705.     }
  3706.  
  3707.     dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
  3708.  
  3709.     if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
  3710.         dev_priv->num_fence_regs = 16;
  3711.     else
  3712.         dev_priv->num_fence_regs = 8;
  3713.  
  3714.     /* Initialize fence registers to zero */
  3715.         i915_gem_reset_fences(dev);
  3716.  
  3717.     i915_gem_detect_bit_6_swizzle(dev);
  3718.  
  3719.     dev_priv->mm.interruptible = true;
  3720.  
  3721. //    dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
  3722. //    dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
  3723. //    register_shrinker(&dev_priv->mm.inactive_shrinker);
  3724. }
  3725.  
  3726.  
  3727.