Subversion Repositories Kolibri OS

Rev

Rev 3255 | Rev 3263 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright © 2008 Intel Corporation
  3.  *
  4.  * Permission is hereby granted, free of charge, to any person obtaining a
  5.  * copy of this software and associated documentation files (the "Software"),
  6.  * to deal in the Software without restriction, including without limitation
  7.  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8.  * and/or sell copies of the Software, and to permit persons to whom the
  9.  * Software is furnished to do so, subject to the following conditions:
  10.  *
  11.  * The above copyright notice and this permission notice (including the next
  12.  * paragraph) shall be included in all copies or substantial portions of the
  13.  * Software.
  14.  *
  15.  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16.  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17.  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18.  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19.  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20.  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21.  * IN THE SOFTWARE.
  22.  *
  23.  * Authors:
  24.  *    Eric Anholt <eric@anholt.net>
  25.  *
  26.  */
  27.  
  28. #include <drm/drmP.h>
  29. #include <drm/i915_drm.h>
  30. #include "i915_drv.h"
  31. #include "i915_trace.h"
  32. #include "intel_drv.h"
  33. #include <linux/shmem_fs.h>
  34. #include <linux/slab.h>
  35. //#include <linux/swap.h>
  36. #include <linux/pci.h>
  37.  
  38. extern int x86_clflush_size;
  39.  
  40. #undef mb
  41. #undef rmb
  42. #undef wmb
  43. #define mb() asm volatile("mfence")
  44. #define rmb() asm volatile ("lfence")
  45. #define wmb() asm volatile ("sfence")
  46.  
  47. static inline void clflush(volatile void *__p)
  48. {
  49.     asm volatile("clflush %0" : "+m" (*(volatile char*)__p));
  50. }
  51.  
  52. #define MAX_ERRNO       4095
  53.  
  54. #define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
  55.  
  56.  
  57. #define I915_EXEC_CONSTANTS_MASK        (3<<6)
  58. #define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */
  59. #define I915_EXEC_CONSTANTS_ABSOLUTE    (1<<6)
  60. #define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */
  61.  
  62. static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  63. static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  64. static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  65.                                                     unsigned alignment,
  66.                                                     bool map_and_fenceable,
  67.                                                     bool nonblocking);
  68. static int i915_gem_phys_pwrite(struct drm_device *dev,
  69.                                 struct drm_i915_gem_object *obj,
  70.                                 struct drm_i915_gem_pwrite *args,
  71.                                 struct drm_file *file);
  72.  
  73. static void i915_gem_write_fence(struct drm_device *dev, int reg,
  74.                                  struct drm_i915_gem_object *obj);
  75. static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
  76.                                          struct drm_i915_fence_reg *fence,
  77.                                          bool enable);
  78.  
  79. static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
  80. static void i915_gem_shrink_all(struct drm_i915_private *dev_priv);
  81. static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
  82.  
  83. static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
  84. {
  85.         if (obj->tiling_mode)
  86.                 i915_gem_release_mmap(obj);
  87.  
  88.         /* As we do not have an associated fence register, we will force
  89.          * a tiling change if we ever need to acquire one.
  90.          */
  91.         obj->fence_dirty = false;
  92.         obj->fence_reg = I915_FENCE_REG_NONE;
  93. }
  94.  
  95. /* some bookkeeping */
  96. static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  97.                                   size_t size)
  98. {
  99.         dev_priv->mm.object_count++;
  100.         dev_priv->mm.object_memory += size;
  101. }
  102.  
  103. static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  104.                                      size_t size)
  105. {
  106.         dev_priv->mm.object_count--;
  107.         dev_priv->mm.object_memory -= size;
  108. }
  109.  
  110. static int
  111. i915_gem_wait_for_error(struct drm_device *dev)
  112. {
  113.         struct drm_i915_private *dev_priv = dev->dev_private;
  114.         struct completion *x = &dev_priv->error_completion;
  115.         unsigned long flags;
  116.         int ret;
  117.  
  118.         if (!atomic_read(&dev_priv->mm.wedged))
  119.                 return 0;
  120. #if 0
  121.         /*
  122.          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
  123.          * userspace. If it takes that long something really bad is going on and
  124.          * we should simply try to bail out and fail as gracefully as possible.
  125.          */
  126.         ret = wait_for_completion_interruptible_timeout(x, 10*HZ);
  127.         if (ret == 0) {
  128.                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
  129.                 return -EIO;
  130.         } else if (ret < 0) {
  131.                 return ret;
  132.         }
  133.  
  134.         if (atomic_read(&dev_priv->mm.wedged)) {
  135.                 /* GPU is hung, bump the completion count to account for
  136.                  * the token we just consumed so that we never hit zero and
  137.                  * end up waiting upon a subsequent completion event that
  138.                  * will never happen.
  139.                  */
  140.                 spin_lock_irqsave(&x->wait.lock, flags);
  141.                 x->done++;
  142.                 spin_unlock_irqrestore(&x->wait.lock, flags);
  143.         }
  144. #endif
  145.  
  146.         return 0;
  147. }
  148.  
  149. int i915_mutex_lock_interruptible(struct drm_device *dev)
  150. {
  151.         int ret;
  152.  
  153.         ret = i915_gem_wait_for_error(dev);
  154.         if (ret)
  155.                 return ret;
  156.  
  157.     mutex_lock(&dev->struct_mutex);
  158.  
  159.         WARN_ON(i915_verify_lists(dev));
  160.         return 0;
  161. }
  162.  
  163. static inline bool
  164. i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
  165. {
  166.         return obj->gtt_space && !obj->active;
  167. }
  168.  
  169.  
  170. #if 0
  171.  
  172. int
  173. i915_gem_init_ioctl(struct drm_device *dev, void *data,
  174.                     struct drm_file *file)
  175. {
  176.         struct drm_i915_gem_init *args = data;
  177.  
  178.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  179.                 return -ENODEV;
  180.  
  181.         if (args->gtt_start >= args->gtt_end ||
  182.             (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
  183.                 return -EINVAL;
  184.  
  185.         /* GEM with user mode setting was never supported on ilk and later. */
  186.         if (INTEL_INFO(dev)->gen >= 5)
  187.                 return -ENODEV;
  188.  
  189.         mutex_lock(&dev->struct_mutex);
  190.         i915_gem_init_global_gtt(dev, args->gtt_start,
  191.                                  args->gtt_end, args->gtt_end);
  192.         mutex_unlock(&dev->struct_mutex);
  193.  
  194.         return 0;
  195. }
  196. #endif
  197.  
  198. int
  199. i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  200.                             struct drm_file *file)
  201. {
  202.         struct drm_i915_private *dev_priv = dev->dev_private;
  203.         struct drm_i915_gem_get_aperture *args = data;
  204.         struct drm_i915_gem_object *obj;
  205.         size_t pinned;
  206.  
  207.         pinned = 0;
  208.         mutex_lock(&dev->struct_mutex);
  209.         list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
  210.                 if (obj->pin_count)
  211.                 pinned += obj->gtt_space->size;
  212.         mutex_unlock(&dev->struct_mutex);
  213.  
  214.         args->aper_size = dev_priv->mm.gtt_total;
  215.         args->aper_available_size = args->aper_size - pinned;
  216.  
  217.         return 0;
  218. }
  219.  
  220. static int
  221. i915_gem_create(struct drm_file *file,
  222.                 struct drm_device *dev,
  223.                 uint64_t size,
  224.                 uint32_t *handle_p)
  225. {
  226.         struct drm_i915_gem_object *obj;
  227.         int ret;
  228.         u32 handle;
  229.  
  230.         size = roundup(size, PAGE_SIZE);
  231.         if (size == 0)
  232.                 return -EINVAL;
  233.  
  234.         /* Allocate the new object */
  235.         obj = i915_gem_alloc_object(dev, size);
  236.         if (obj == NULL)
  237.                 return -ENOMEM;
  238.  
  239.         ret = drm_gem_handle_create(file, &obj->base, &handle);
  240.         if (ret) {
  241.                 drm_gem_object_release(&obj->base);
  242.                 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
  243.                 kfree(obj);
  244.                 return ret;
  245.         }
  246.  
  247.         /* drop reference from allocate - handle holds it now */
  248.         drm_gem_object_unreference(&obj->base);
  249.         trace_i915_gem_object_create(obj);
  250.  
  251.         *handle_p = handle;
  252.         return 0;
  253. }
  254.  
  255. int
  256. i915_gem_dumb_create(struct drm_file *file,
  257.                      struct drm_device *dev,
  258.                      struct drm_mode_create_dumb *args)
  259. {
  260.         /* have to work out size/pitch and return them */
  261.         args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
  262.         args->size = args->pitch * args->height;
  263.         return i915_gem_create(file, dev,
  264.                                args->size, &args->handle);
  265. }
  266.  
  267. int i915_gem_dumb_destroy(struct drm_file *file,
  268.                           struct drm_device *dev,
  269.                           uint32_t handle)
  270. {
  271.         return drm_gem_handle_delete(file, handle);
  272. }
  273.  
  274. /**
  275.  * Creates a new mm object and returns a handle to it.
  276.  */
  277. int
  278. i915_gem_create_ioctl(struct drm_device *dev, void *data,
  279.                       struct drm_file *file)
  280. {
  281.         struct drm_i915_gem_create *args = data;
  282.  
  283.         return i915_gem_create(file, dev,
  284.                                args->size, &args->handle);
  285. }
  286.  
  287. static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
  288. {
  289.         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
  290.  
  291.         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
  292.                 obj->tiling_mode != I915_TILING_NONE;
  293. }
  294. #if 0
  295.  
  296. static inline int
  297. __copy_to_user_swizzled(char __user *cpu_vaddr,
  298.                         const char *gpu_vaddr, int gpu_offset,
  299.                 int length)
  300. {
  301.         int ret, cpu_offset = 0;
  302.  
  303.         while (length > 0) {
  304.                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
  305.                 int this_length = min(cacheline_end - gpu_offset, length);
  306.                 int swizzled_gpu_offset = gpu_offset ^ 64;
  307.  
  308.                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
  309.                                      gpu_vaddr + swizzled_gpu_offset,
  310.                                      this_length);
  311.                 if (ret)
  312.                         return ret + length;
  313.  
  314.                 cpu_offset += this_length;
  315.                 gpu_offset += this_length;
  316.                 length -= this_length;
  317.         }
  318.  
  319.         return 0;
  320. }
  321.  
  322. static inline int
  323. __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
  324.                           const char __user *cpu_vaddr,
  325.                           int length)
  326. {
  327.         int ret, cpu_offset = 0;
  328.  
  329.         while (length > 0) {
  330.                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
  331.                 int this_length = min(cacheline_end - gpu_offset, length);
  332.                 int swizzled_gpu_offset = gpu_offset ^ 64;
  333.  
  334.                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
  335.                                cpu_vaddr + cpu_offset,
  336.                                this_length);
  337.                 if (ret)
  338.                         return ret + length;
  339.  
  340.                 cpu_offset += this_length;
  341.                 gpu_offset += this_length;
  342.                 length -= this_length;
  343.         }
  344.  
  345.         return 0;
  346. }
  347.  
  348. /* Per-page copy function for the shmem pread fastpath.
  349.  * Flushes invalid cachelines before reading the target if
  350.  * needs_clflush is set. */
  351. static int
  352. shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
  353.                  char __user *user_data,
  354.                  bool page_do_bit17_swizzling, bool needs_clflush)
  355. {
  356.                 char *vaddr;
  357.                 int ret;
  358.  
  359.         if (unlikely(page_do_bit17_swizzling))
  360.                 return -EINVAL;
  361.  
  362.                 vaddr = kmap_atomic(page);
  363.         if (needs_clflush)
  364.                 drm_clflush_virt_range(vaddr + shmem_page_offset,
  365.                                        page_length);
  366.                 ret = __copy_to_user_inatomic(user_data,
  367.                                       vaddr + shmem_page_offset,
  368.                                               page_length);
  369.                 kunmap_atomic(vaddr);
  370.  
  371.         return ret ? -EFAULT : 0;
  372. }
  373.  
  374. static void
  375. shmem_clflush_swizzled_range(char *addr, unsigned long length,
  376.                              bool swizzled)
  377. {
  378.         if (unlikely(swizzled)) {
  379.                 unsigned long start = (unsigned long) addr;
  380.                 unsigned long end = (unsigned long) addr + length;
  381.  
  382.                 /* For swizzling simply ensure that we always flush both
  383.                  * channels. Lame, but simple and it works. Swizzled
  384.                  * pwrite/pread is far from a hotpath - current userspace
  385.                  * doesn't use it at all. */
  386.                 start = round_down(start, 128);
  387.                 end = round_up(end, 128);
  388.  
  389.                 drm_clflush_virt_range((void *)start, end - start);
  390.         } else {
  391.                 drm_clflush_virt_range(addr, length);
  392.         }
  393.  
  394. }
  395.  
  396. /* Only difference to the fast-path function is that this can handle bit17
  397.  * and uses non-atomic copy and kmap functions. */
  398. static int
  399. shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
  400.                  char __user *user_data,
  401.                  bool page_do_bit17_swizzling, bool needs_clflush)
  402. {
  403.         char *vaddr;
  404.         int ret;
  405.  
  406.         vaddr = kmap(page);
  407.         if (needs_clflush)
  408.                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  409.                                              page_length,
  410.                                              page_do_bit17_swizzling);
  411.  
  412.         if (page_do_bit17_swizzling)
  413.                 ret = __copy_to_user_swizzled(user_data,
  414.                                               vaddr, shmem_page_offset,
  415.                                               page_length);
  416.         else
  417.                 ret = __copy_to_user(user_data,
  418.                                      vaddr + shmem_page_offset,
  419.                                      page_length);
  420.         kunmap(page);
  421.  
  422.         return ret ? - EFAULT : 0;
  423. }
  424.  
  425. static int
  426. i915_gem_shmem_pread(struct drm_device *dev,
  427.                           struct drm_i915_gem_object *obj,
  428.                           struct drm_i915_gem_pread *args,
  429.                           struct drm_file *file)
  430. {
  431.         char __user *user_data;
  432.         ssize_t remain;
  433.         loff_t offset;
  434.         int shmem_page_offset, page_length, ret = 0;
  435.         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
  436.         int hit_slowpath = 0;
  437.         int prefaulted = 0;
  438.         int needs_clflush = 0;
  439.         struct scatterlist *sg;
  440.         int i;
  441.  
  442.         user_data = (char __user *) (uintptr_t) args->data_ptr;
  443.         remain = args->size;
  444.  
  445.         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
  446.  
  447.         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
  448.                 /* If we're not in the cpu read domain, set ourself into the gtt
  449.                  * read domain and manually flush cachelines (if required). This
  450.                  * optimizes for the case when the gpu will dirty the data
  451.                  * anyway again before the next pread happens. */
  452.                 if (obj->cache_level == I915_CACHE_NONE)
  453.                         needs_clflush = 1;
  454.                 if (obj->gtt_space) {
  455.                         ret = i915_gem_object_set_to_gtt_domain(obj, false);
  456.                         if (ret)
  457.                                 return ret;
  458.                 }
  459.         }
  460.  
  461.         ret = i915_gem_object_get_pages(obj);
  462.         if (ret)
  463.                 return ret;
  464.  
  465.         i915_gem_object_pin_pages(obj);
  466.  
  467.         offset = args->offset;
  468.  
  469.         for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
  470.                 struct page *page;
  471.  
  472.                 if (i < offset >> PAGE_SHIFT)
  473.                         continue;
  474.  
  475.                 if (remain <= 0)
  476.                         break;
  477.  
  478.                 /* Operation in this page
  479.                  *
  480.                  * shmem_page_offset = offset within page in shmem file
  481.                  * page_length = bytes to copy for this page
  482.                  */
  483.                 shmem_page_offset = offset_in_page(offset);
  484.                 page_length = remain;
  485.                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
  486.                         page_length = PAGE_SIZE - shmem_page_offset;
  487.  
  488.                 page = sg_page(sg);
  489.                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
  490.                         (page_to_phys(page) & (1 << 17)) != 0;
  491.  
  492.                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
  493.                                        user_data, page_do_bit17_swizzling,
  494.                                        needs_clflush);
  495.                 if (ret == 0)
  496.                         goto next_page;
  497.  
  498.                 hit_slowpath = 1;
  499.                 mutex_unlock(&dev->struct_mutex);
  500.  
  501.                 if (!prefaulted) {
  502.                         ret = fault_in_multipages_writeable(user_data, remain);
  503.                         /* Userspace is tricking us, but we've already clobbered
  504.                          * its pages with the prefault and promised to write the
  505.                          * data up to the first fault. Hence ignore any errors
  506.                          * and just continue. */
  507.                         (void)ret;
  508.                         prefaulted = 1;
  509.                 }
  510.  
  511.                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
  512.                                        user_data, page_do_bit17_swizzling,
  513.                                        needs_clflush);
  514.  
  515.                 mutex_lock(&dev->struct_mutex);
  516.  
  517. next_page:
  518.                 mark_page_accessed(page);
  519.  
  520.                 if (ret)
  521.                         goto out;
  522.  
  523.                 remain -= page_length;
  524.                 user_data += page_length;
  525.                 offset += page_length;
  526.         }
  527.  
  528. out:
  529.         i915_gem_object_unpin_pages(obj);
  530.  
  531.         if (hit_slowpath) {
  532.                 /* Fixup: Kill any reinstated backing storage pages */
  533.                 if (obj->madv == __I915_MADV_PURGED)
  534.                         i915_gem_object_truncate(obj);
  535.         }
  536.  
  537.         return ret;
  538. }
  539.  
  540. /**
  541.  * Reads data from the object referenced by handle.
  542.  *
  543.  * On error, the contents of *data are undefined.
  544.  */
  545. int
  546. i915_gem_pread_ioctl(struct drm_device *dev, void *data,
  547.                      struct drm_file *file)
  548. {
  549.         struct drm_i915_gem_pread *args = data;
  550.         struct drm_i915_gem_object *obj;
  551.         int ret = 0;
  552.  
  553.         if (args->size == 0)
  554.                 return 0;
  555.  
  556.         if (!access_ok(VERIFY_WRITE,
  557.                        (char __user *)(uintptr_t)args->data_ptr,
  558.                        args->size))
  559.                 return -EFAULT;
  560.  
  561.         ret = i915_mutex_lock_interruptible(dev);
  562.         if (ret)
  563.                 return ret;
  564.  
  565.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  566.         if (&obj->base == NULL) {
  567.                 ret = -ENOENT;
  568.                 goto unlock;
  569.         }
  570.  
  571.         /* Bounds check source.  */
  572.         if (args->offset > obj->base.size ||
  573.             args->size > obj->base.size - args->offset) {
  574.                 ret = -EINVAL;
  575.                 goto out;
  576.         }
  577.  
  578.         /* prime objects have no backing filp to GEM pread/pwrite
  579.          * pages from.
  580.          */
  581.         if (!obj->base.filp) {
  582.                 ret = -EINVAL;
  583.                 goto out;
  584.         }
  585.  
  586.         trace_i915_gem_object_pread(obj, args->offset, args->size);
  587.  
  588.         ret = i915_gem_shmem_pread(dev, obj, args, file);
  589.  
  590. out:
  591.         drm_gem_object_unreference(&obj->base);
  592. unlock:
  593.         mutex_unlock(&dev->struct_mutex);
  594.         return ret;
  595. }
  596.  
  597. /* This is the fast write path which cannot handle
  598.  * page faults in the source data
  599.  */
  600.  
  601. static inline int
  602. fast_user_write(struct io_mapping *mapping,
  603.                 loff_t page_base, int page_offset,
  604.                 char __user *user_data,
  605.                 int length)
  606. {
  607.         void __iomem *vaddr_atomic;
  608.         void *vaddr;
  609.         unsigned long unwritten;
  610.  
  611.         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
  612.         /* We can use the cpu mem copy function because this is X86. */
  613.         vaddr = (void __force*)vaddr_atomic + page_offset;
  614.         unwritten = __copy_from_user_inatomic_nocache(vaddr,
  615.                                                       user_data, length);
  616.         io_mapping_unmap_atomic(vaddr_atomic);
  617.         return unwritten;
  618. }
  619. #endif
  620.  
  621. #define offset_in_page(p)       ((unsigned long)(p) & ~PAGE_MASK)
  622. /**
  623.  * This is the fast pwrite path, where we copy the data directly from the
  624.  * user into the GTT, uncached.
  625.  */
  626. static int
  627. i915_gem_gtt_pwrite_fast(struct drm_device *dev,
  628.                          struct drm_i915_gem_object *obj,
  629.                          struct drm_i915_gem_pwrite *args,
  630.                          struct drm_file *file)
  631. {
  632.         drm_i915_private_t *dev_priv = dev->dev_private;
  633.         ssize_t remain;
  634.         loff_t offset, page_base;
  635.         char __user *user_data;
  636.         int page_offset, page_length, ret;
  637.     char *vaddr;
  638.  
  639.         ret = i915_gem_object_pin(obj, 0, true, true);
  640.         if (ret)
  641.                 goto out;
  642.  
  643.         ret = i915_gem_object_set_to_gtt_domain(obj, true);
  644.         if (ret)
  645.                 goto out_unpin;
  646.  
  647.         ret = i915_gem_object_put_fence(obj);
  648.         if (ret)
  649.                 goto out_unpin;
  650.  
  651.     vaddr = AllocKernelSpace(4096);
  652.     if(vaddr == NULL)
  653.     {
  654.         ret = -ENOSPC;
  655.         goto out_unpin;
  656.     };
  657.  
  658.         user_data = (char __user *) (uintptr_t) args->data_ptr;
  659.         remain = args->size;
  660.  
  661.         offset = obj->gtt_offset + args->offset;
  662.  
  663.         while (remain > 0) {
  664.                 /* Operation in this page
  665.                  *
  666.                  * page_base = page offset within aperture
  667.                  * page_offset = offset within page
  668.                  * page_length = bytes to copy for this page
  669.                  */
  670.                 page_base = offset & PAGE_MASK;
  671.                 page_offset = offset_in_page(offset);
  672.                 page_length = remain;
  673.                 if ((page_offset + remain) > PAGE_SIZE)
  674.                         page_length = PAGE_SIZE - page_offset;
  675.  
  676.         MapPage(vaddr, page_base, PG_SW|PG_NOCACHE);
  677.  
  678.         memcpy(vaddr+page_offset, user_data, page_length);
  679.  
  680.                 remain -= page_length;
  681.                 user_data += page_length;
  682.                 offset += page_length;
  683.         }
  684.  
  685.     FreeKernelSpace(vaddr);
  686.  
  687. out_unpin:
  688.         i915_gem_object_unpin(obj);
  689. out:
  690.     printf("% s ret = %d\n", __FUNCTION__, ret);
  691.  
  692.         return ret;
  693. }
  694.  
  695. /* Per-page copy function for the shmem pwrite fastpath.
  696.  * Flushes invalid cachelines before writing to the target if
  697.  * needs_clflush_before is set and flushes out any written cachelines after
  698.  * writing if needs_clflush is set. */
  699. static int
  700. shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
  701.                   char __user *user_data,
  702.                   bool page_do_bit17_swizzling,
  703.                   bool needs_clflush_before,
  704.                   bool needs_clflush_after)
  705. {
  706.         char *vaddr;
  707.         int ret = 0;
  708.  
  709.         if (unlikely(page_do_bit17_swizzling))
  710.                 return -EINVAL;
  711.  
  712.         vaddr = (char *)MapIoMem((addr_t)page, 4096, PG_SW);
  713.         if (needs_clflush_before)
  714.                 drm_clflush_virt_range(vaddr + shmem_page_offset,
  715.                                        page_length);
  716.         memcpy(vaddr + shmem_page_offset,
  717.                                                 user_data,
  718.                                                 page_length);
  719.         if (needs_clflush_after)
  720.                 drm_clflush_virt_range(vaddr + shmem_page_offset,
  721.                                        page_length);
  722.         FreeKernelSpace(vaddr);
  723.  
  724.         return ret ? -EFAULT : 0;
  725. }
  726. #if 0
  727.  
  728. /* Only difference to the fast-path function is that this can handle bit17
  729.  * and uses non-atomic copy and kmap functions. */
  730. static int
  731. shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
  732.                   char __user *user_data,
  733.                   bool page_do_bit17_swizzling,
  734.                   bool needs_clflush_before,
  735.                   bool needs_clflush_after)
  736. {
  737.         char *vaddr;
  738.         int ret;
  739.  
  740.         vaddr = kmap(page);
  741.         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
  742.                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  743.                                              page_length,
  744.                                              page_do_bit17_swizzling);
  745.         if (page_do_bit17_swizzling)
  746.                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
  747.                                                 user_data,
  748.                                                 page_length);
  749.         else
  750.                 ret = __copy_from_user(vaddr + shmem_page_offset,
  751.                                        user_data,
  752.                                        page_length);
  753.         if (needs_clflush_after)
  754.                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  755.                                              page_length,
  756.                                              page_do_bit17_swizzling);
  757.         kunmap(page);
  758.  
  759.         return ret ? -EFAULT : 0;
  760. }
  761. #endif
  762.  
  763.  
  764. static int
  765. i915_gem_shmem_pwrite(struct drm_device *dev,
  766.                       struct drm_i915_gem_object *obj,
  767.                       struct drm_i915_gem_pwrite *args,
  768.                       struct drm_file *file)
  769. {
  770.         ssize_t remain;
  771.         loff_t offset;
  772.         char __user *user_data;
  773.         int shmem_page_offset, page_length, ret = 0;
  774.         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
  775.         int hit_slowpath = 0;
  776.         int needs_clflush_after = 0;
  777.         int needs_clflush_before = 0;
  778.         int i;
  779.         struct scatterlist *sg;
  780.  
  781.         user_data = (char __user *) (uintptr_t) args->data_ptr;
  782.         remain = args->size;
  783.  
  784.         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
  785.  
  786.         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  787.                 /* If we're not in the cpu write domain, set ourself into the gtt
  788.                  * write domain and manually flush cachelines (if required). This
  789.                  * optimizes for the case when the gpu will use the data
  790.                  * right away and we therefore have to clflush anyway. */
  791.                 if (obj->cache_level == I915_CACHE_NONE)
  792.                         needs_clflush_after = 1;
  793.                 if (obj->gtt_space) {
  794.                         ret = i915_gem_object_set_to_gtt_domain(obj, true);
  795.                         if (ret)
  796.                                 return ret;
  797.                 }
  798.         }
  799.         /* Same trick applies for invalidate partially written cachelines before
  800.          * writing.  */
  801.         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
  802.             && obj->cache_level == I915_CACHE_NONE)
  803.                 needs_clflush_before = 1;
  804.  
  805.         ret = i915_gem_object_get_pages(obj);
  806.         if (ret)
  807.                 return ret;
  808.  
  809.         i915_gem_object_pin_pages(obj);
  810.  
  811.         offset = args->offset;
  812.         obj->dirty = 1;
  813.  
  814.         for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
  815.                 struct page *page;
  816.                 int partial_cacheline_write;
  817.  
  818.                 if (i < offset >> PAGE_SHIFT)
  819.                         continue;
  820.  
  821.                 if (remain <= 0)
  822.                         break;
  823.  
  824.                 /* Operation in this page
  825.                  *
  826.                  * shmem_page_offset = offset within page in shmem file
  827.                  * page_length = bytes to copy for this page
  828.                  */
  829.                 shmem_page_offset = offset_in_page(offset);
  830.  
  831.                 page_length = remain;
  832.                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
  833.                         page_length = PAGE_SIZE - shmem_page_offset;
  834.  
  835.                 /* If we don't overwrite a cacheline completely we need to be
  836.                  * careful to have up-to-date data by first clflushing. Don't
  837.                  * overcomplicate things and flush the entire patch. */
  838.                 partial_cacheline_write = needs_clflush_before &&
  839.                         ((shmem_page_offset | page_length)
  840.                                 & (x86_clflush_size - 1));
  841.  
  842.                 page = sg_page(sg);
  843.                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
  844.                         (page_to_phys(page) & (1 << 17)) != 0;
  845.  
  846.                 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
  847.                                         user_data, page_do_bit17_swizzling,
  848.                                         partial_cacheline_write,
  849.                                         needs_clflush_after);
  850.                 if (ret == 0)
  851.                         goto next_page;
  852.  
  853.                 hit_slowpath = 1;
  854.                 mutex_unlock(&dev->struct_mutex);
  855.                 dbgprintf("%s need shmem_pwrite_slow\n",__FUNCTION__);
  856.  
  857. //              ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
  858. //                                      user_data, page_do_bit17_swizzling,
  859. //                                      partial_cacheline_write,
  860. //                                      needs_clflush_after);
  861.  
  862.                 mutex_lock(&dev->struct_mutex);
  863.  
  864. next_page:
  865.  
  866.                 if (ret)
  867.                         goto out;
  868.  
  869.                 remain -= page_length;
  870.                 user_data += page_length;
  871.                 offset += page_length;
  872.         }
  873.  
  874. out:
  875.         i915_gem_object_unpin_pages(obj);
  876.  
  877.         if (hit_slowpath) {
  878.                 /* Fixup: Kill any reinstated backing storage pages */
  879.                 if (obj->madv == __I915_MADV_PURGED)
  880.                         i915_gem_object_truncate(obj);
  881.                 /* and flush dirty cachelines in case the object isn't in the cpu write
  882.                  * domain anymore. */
  883.                 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  884.                         i915_gem_clflush_object(obj);
  885.                         i915_gem_chipset_flush(dev);
  886.                 }
  887.         }
  888.  
  889.         if (needs_clflush_after)
  890.                 i915_gem_chipset_flush(dev);
  891.  
  892.         return ret;
  893. }
  894.  
  895. /**
  896.  * Writes data to the object referenced by handle.
  897.  *
  898.  * On error, the contents of the buffer that were to be modified are undefined.
  899.  */
  900. int
  901. i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
  902.                       struct drm_file *file)
  903. {
  904.         struct drm_i915_gem_pwrite *args = data;
  905.         struct drm_i915_gem_object *obj;
  906.         int ret;
  907.  
  908.         if (args->size == 0)
  909.                 return 0;
  910.  
  911.         ret = i915_mutex_lock_interruptible(dev);
  912.         if (ret)
  913.                 return ret;
  914.  
  915.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  916.         if (&obj->base == NULL) {
  917.                 ret = -ENOENT;
  918.                 goto unlock;
  919.         }
  920.  
  921.         /* Bounds check destination. */
  922.         if (args->offset > obj->base.size ||
  923.             args->size > obj->base.size - args->offset) {
  924.                 ret = -EINVAL;
  925.                 goto out;
  926.         }
  927.  
  928.         /* prime objects have no backing filp to GEM pread/pwrite
  929.          * pages from.
  930.          */
  931.         if (!obj->base.filp) {
  932.                 ret = -EINVAL;
  933.                 goto out;
  934.         }
  935.  
  936.         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
  937.  
  938.         ret = -EFAULT;
  939.         /* We can only do the GTT pwrite on untiled buffers, as otherwise
  940.          * it would end up going through the fenced access, and we'll get
  941.          * different detiling behavior between reading and writing.
  942.          * pread/pwrite currently are reading and writing from the CPU
  943.          * perspective, requiring manual detiling by the client.
  944.          */
  945. //   if (obj->phys_obj) {
  946. //       ret = i915_gem_phys_pwrite(dev, obj, args, file);
  947. //       goto out;
  948. //   }
  949.  
  950.         if (obj->cache_level == I915_CACHE_NONE &&
  951.             obj->tiling_mode == I915_TILING_NONE &&
  952.             obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  953.                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
  954.                 /* Note that the gtt paths might fail with non-page-backed user
  955.                  * pointers (e.g. gtt mappings when moving data between
  956.                  * textures). Fallback to the shmem path in that case. */
  957.         }
  958.  
  959.         if (ret == -EFAULT || ret == -ENOSPC)
  960.        ret = i915_gem_shmem_pwrite(dev, obj, args, file);
  961.  
  962. out:
  963.         drm_gem_object_unreference(&obj->base);
  964. unlock:
  965.         mutex_unlock(&dev->struct_mutex);
  966.         return ret;
  967. }
  968.  
  969. int
  970. i915_gem_check_wedge(struct drm_i915_private *dev_priv,
  971.                      bool interruptible)
  972. {
  973.         if (atomic_read(&dev_priv->mm.wedged)) {
  974.                 struct completion *x = &dev_priv->error_completion;
  975.                 bool recovery_complete;
  976.                 unsigned long flags;
  977.  
  978.                 /* Give the error handler a chance to run. */
  979.                 spin_lock_irqsave(&x->wait.lock, flags);
  980.                 recovery_complete = x->done > 0;
  981.                 spin_unlock_irqrestore(&x->wait.lock, flags);
  982.  
  983.                 /* Non-interruptible callers can't handle -EAGAIN, hence return
  984.                  * -EIO unconditionally for these. */
  985.                 if (!interruptible)
  986.                         return -EIO;
  987.  
  988.                 /* Recovery complete, but still wedged means reset failure. */
  989.                 if (recovery_complete)
  990.                         return -EIO;
  991.  
  992.                 return -EAGAIN;
  993.         }
  994.  
  995.         return 0;
  996. }
  997.  
  998. /*
  999.  * Compare seqno against outstanding lazy request. Emit a request if they are
  1000.  * equal.
  1001.  */
  1002. static int
  1003. i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
  1004. {
  1005.         int ret;
  1006.  
  1007.         BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
  1008.  
  1009.         ret = 0;
  1010.         if (seqno == ring->outstanding_lazy_request)
  1011.                 ret = i915_add_request(ring, NULL, NULL);
  1012.  
  1013.         return ret;
  1014. }
  1015.  
  1016. /**
  1017.  * __wait_seqno - wait until execution of seqno has finished
  1018.  * @ring: the ring expected to report seqno
  1019.  * @seqno: duh!
  1020.  * @interruptible: do an interruptible wait (normally yes)
  1021.  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
  1022.  *
  1023.  * Returns 0 if the seqno was found within the alloted time. Else returns the
  1024.  * errno with remaining time filled in timeout argument.
  1025.  */
  1026. static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
  1027.                         bool interruptible, struct timespec *timeout)
  1028. {
  1029.         drm_i915_private_t *dev_priv = ring->dev->dev_private;
  1030.         struct timespec before, now, wait_time={1,0};
  1031.         unsigned long timeout_jiffies;
  1032.         long end;
  1033.         bool wait_forever = true;
  1034.         int ret;
  1035.  
  1036.         if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
  1037.                 return 0;
  1038.  
  1039.         trace_i915_gem_request_wait_begin(ring, seqno);
  1040.  
  1041.         if (timeout != NULL) {
  1042.                 wait_time = *timeout;
  1043.                 wait_forever = false;
  1044.         }
  1045.  
  1046. //   timeout_jiffies = timespec_to_jiffies(&wait_time);
  1047.  
  1048.         if (WARN_ON(!ring->irq_get(ring)))
  1049.                 return -ENODEV;
  1050. #if 0
  1051.  
  1052.     /* Record current time in case interrupted by signal, or wedged * */
  1053.         getrawmonotonic(&before);
  1054.  
  1055. #define EXIT_COND \
  1056.         (i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \
  1057.         atomic_read(&dev_priv->mm.wedged))
  1058.         do {
  1059.                         end = wait_event_timeout(ring->irq_queue, EXIT_COND,
  1060.                                                  timeout_jiffies);
  1061.  
  1062.                 ret = i915_gem_check_wedge(dev_priv, interruptible);
  1063.                 if (ret)
  1064.                         end = ret;
  1065.         } while (end == 0 && wait_forever);
  1066.  
  1067.         getrawmonotonic(&now);
  1068.  
  1069.         ring->irq_put(ring);
  1070.         trace_i915_gem_request_wait_end(ring, seqno);
  1071. #undef EXIT_COND
  1072.  
  1073.         if (timeout) {
  1074. //       struct timespec sleep_time = timespec_sub(now, before);
  1075. //       *timeout = timespec_sub(*timeout, sleep_time);
  1076.         }
  1077.  
  1078.         switch (end) {
  1079.         case -EIO:
  1080.         case -EAGAIN: /* Wedged */
  1081.         case -ERESTARTSYS: /* Signal */
  1082.                 return (int)end;
  1083.         case 0: /* Timeout */
  1084. //              if (timeout)
  1085. //                      set_normalized_timespec(timeout, 0, 0);
  1086.                 return -ETIME;
  1087.         default: /* Completed */
  1088.                 WARN_ON(end < 0); /* We're not aware of other errors */
  1089.                 return 0;
  1090.         }
  1091.  
  1092. #endif
  1093.  
  1094. #define EXIT_COND \
  1095.     (i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \
  1096.     atomic_read(&dev_priv->mm.wedged))
  1097.     wait_event(ring->irq_queue, EXIT_COND);
  1098. #undef EXIT_COND
  1099.     ring->irq_put(ring);
  1100.  
  1101.     return 0;
  1102. }
  1103.  
  1104. /**
  1105.  * Waits for a sequence number to be signaled, and cleans up the
  1106.  * request and object lists appropriately for that event.
  1107.  */
  1108. int
  1109. i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
  1110. {
  1111.         struct drm_device *dev = ring->dev;
  1112.         struct drm_i915_private *dev_priv = dev->dev_private;
  1113.         bool interruptible = dev_priv->mm.interruptible;
  1114.         int ret;
  1115.  
  1116.         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
  1117.         BUG_ON(seqno == 0);
  1118.  
  1119.         ret = i915_gem_check_wedge(dev_priv, interruptible);
  1120.         if (ret)
  1121.                 return ret;
  1122.  
  1123.         ret = i915_gem_check_olr(ring, seqno);
  1124.         if (ret)
  1125.                 return ret;
  1126.  
  1127.         return __wait_seqno(ring, seqno, interruptible, NULL);
  1128. }
  1129.  
  1130. /**
  1131.  * Ensures that all rendering to the object has completed and the object is
  1132.  * safe to unbind from the GTT or access from the CPU.
  1133.  */
  1134. static __must_check int
  1135. i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
  1136.                                bool readonly)
  1137. {
  1138.         struct intel_ring_buffer *ring = obj->ring;
  1139.         u32 seqno;
  1140.         int ret;
  1141.  
  1142.         seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
  1143.         if (seqno == 0)
  1144.                 return 0;
  1145.  
  1146.         ret = i915_wait_seqno(ring, seqno);
  1147.        if (ret)
  1148.            return ret;
  1149.  
  1150.         i915_gem_retire_requests_ring(ring);
  1151.  
  1152.         /* Manually manage the write flush as we may have not yet
  1153.          * retired the buffer.
  1154.          */
  1155.         if (obj->last_write_seqno &&
  1156.             i915_seqno_passed(seqno, obj->last_write_seqno)) {
  1157.                 obj->last_write_seqno = 0;
  1158.                 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
  1159.         }
  1160.  
  1161.         return 0;
  1162. }
  1163.  
  1164. /* A nonblocking variant of the above wait. This is a highly dangerous routine
  1165.  * as the object state may change during this call.
  1166.  */
  1167. static __must_check int
  1168. i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
  1169.                                             bool readonly)
  1170. {
  1171.         struct drm_device *dev = obj->base.dev;
  1172.         struct drm_i915_private *dev_priv = dev->dev_private;
  1173.         struct intel_ring_buffer *ring = obj->ring;
  1174.         u32 seqno;
  1175.         int ret;
  1176.  
  1177.         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
  1178.         BUG_ON(!dev_priv->mm.interruptible);
  1179.  
  1180.         seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
  1181.         if (seqno == 0)
  1182.                 return 0;
  1183.  
  1184.         ret = i915_gem_check_wedge(dev_priv, true);
  1185.         if (ret)
  1186.                 return ret;
  1187.  
  1188.         ret = i915_gem_check_olr(ring, seqno);
  1189.         if (ret)
  1190.                 return ret;
  1191.  
  1192.         mutex_unlock(&dev->struct_mutex);
  1193.         ret = __wait_seqno(ring, seqno, true, NULL);
  1194.         mutex_lock(&dev->struct_mutex);
  1195.  
  1196.         i915_gem_retire_requests_ring(ring);
  1197.  
  1198.         /* Manually manage the write flush as we may have not yet
  1199.          * retired the buffer.
  1200.          */
  1201.         if (obj->last_write_seqno &&
  1202.             i915_seqno_passed(seqno, obj->last_write_seqno)) {
  1203.                 obj->last_write_seqno = 0;
  1204.                 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
  1205.         }
  1206.  
  1207.         return ret;
  1208. }
  1209.  
  1210. /**
  1211.  * Called when user space prepares to use an object with the CPU, either
  1212.  * through the mmap ioctl's mapping or a GTT mapping.
  1213.  */
  1214. int
  1215. i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
  1216.                           struct drm_file *file)
  1217. {
  1218.         struct drm_i915_gem_set_domain *args = data;
  1219.         struct drm_i915_gem_object *obj;
  1220.         uint32_t read_domains = args->read_domains;
  1221.         uint32_t write_domain = args->write_domain;
  1222.         int ret;
  1223.  
  1224.         /* Only handle setting domains to types used by the CPU. */
  1225.         if (write_domain & I915_GEM_GPU_DOMAINS)
  1226.                 return -EINVAL;
  1227.  
  1228.         if (read_domains & I915_GEM_GPU_DOMAINS)
  1229.                 return -EINVAL;
  1230.  
  1231.         /* Having something in the write domain implies it's in the read
  1232.          * domain, and only that read domain.  Enforce that in the request.
  1233.          */
  1234.         if (write_domain != 0 && read_domains != write_domain)
  1235.                 return -EINVAL;
  1236.  
  1237.         ret = i915_mutex_lock_interruptible(dev);
  1238.         if (ret)
  1239.                 return ret;
  1240.  
  1241.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  1242.         if (&obj->base == NULL) {
  1243.                 ret = -ENOENT;
  1244.                 goto unlock;
  1245.         }
  1246.  
  1247.         /* Try to flush the object off the GPU without holding the lock.
  1248.          * We will repeat the flush holding the lock in the normal manner
  1249.          * to catch cases where we are gazumped.
  1250.          */
  1251.         ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
  1252.         if (ret)
  1253.                 goto unref;
  1254.  
  1255.         if (read_domains & I915_GEM_DOMAIN_GTT) {
  1256.                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
  1257.  
  1258.                 /* Silently promote "you're not bound, there was nothing to do"
  1259.                  * to success, since the client was just asking us to
  1260.                  * make sure everything was done.
  1261.                  */
  1262.                 if (ret == -EINVAL)
  1263.                         ret = 0;
  1264.         } else {
  1265.                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
  1266.         }
  1267.  
  1268. unref:
  1269.         drm_gem_object_unreference(&obj->base);
  1270. unlock:
  1271.         mutex_unlock(&dev->struct_mutex);
  1272.         return ret;
  1273. }
  1274.  
  1275.  
  1276.  
  1277.  
  1278.  
  1279.  
  1280. /**
  1281.  * Maps the contents of an object, returning the address it is mapped
  1282.  * into.
  1283.  *
  1284.  * While the mapping holds a reference on the contents of the object, it doesn't
  1285.  * imply a ref on the object itself.
  1286.  */
  1287. int
  1288. i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
  1289.                     struct drm_file *file)
  1290. {
  1291.         struct drm_i915_gem_mmap *args = data;
  1292.         struct drm_gem_object *obj;
  1293.         unsigned long addr = 0;
  1294.  
  1295.         obj = drm_gem_object_lookup(dev, file, args->handle);
  1296.         if (obj == NULL)
  1297.                 return -ENOENT;
  1298.        
  1299.         dbgprintf("%s offset %lld size %lld not supported\n",
  1300.                                 args->offset, args->size);  
  1301.         /* prime objects have no backing filp to GEM mmap
  1302.          * pages from.
  1303.          */
  1304.         if (!obj->filp) {
  1305.                 drm_gem_object_unreference_unlocked(obj);
  1306.                 return -EINVAL;
  1307.         }
  1308.  
  1309. //      addr = vm_mmap(obj->filp, 0, args->size,
  1310. //                     PROT_READ | PROT_WRITE, MAP_SHARED,
  1311. //                     args->offset);
  1312.         drm_gem_object_unreference_unlocked(obj);
  1313. //      if (IS_ERR((void *)addr))
  1314. //              return addr;
  1315.  
  1316.         args->addr_ptr = (uint64_t) addr;
  1317.         return -EINVAL;
  1318.  
  1319. //      return 0;
  1320. }
  1321.  
  1322.  
  1323.  
  1324.  
  1325.  
  1326.  
  1327.  
  1328.  
  1329.  
  1330.  
  1331.  
  1332.  
  1333.  
  1334. /**
  1335.  * i915_gem_release_mmap - remove physical page mappings
  1336.  * @obj: obj in question
  1337.  *
  1338.  * Preserve the reservation of the mmapping with the DRM core code, but
  1339.  * relinquish ownership of the pages back to the system.
  1340.  *
  1341.  * It is vital that we remove the page mapping if we have mapped a tiled
  1342.  * object through the GTT and then lose the fence register due to
  1343.  * resource pressure. Similarly if the object has been moved out of the
  1344.  * aperture, than pages mapped into userspace must be revoked. Removing the
  1345.  * mapping will then trigger a page fault on the next user access, allowing
  1346.  * fixup by i915_gem_fault().
  1347.  */
  1348. void
  1349. i915_gem_release_mmap(struct drm_i915_gem_object *obj)
  1350. {
  1351.         if (!obj->fault_mappable)
  1352.                 return;
  1353.  
  1354.         if (obj->base.dev->dev_mapping)
  1355. //              unmap_mapping_range(obj->base.dev->dev_mapping,
  1356. //                                  (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
  1357. //                                  obj->base.size, 1);
  1358.  
  1359.         obj->fault_mappable = false;
  1360. }
  1361.  
  1362. static uint32_t
  1363. i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
  1364. {
  1365.         uint32_t gtt_size;
  1366.  
  1367.         if (INTEL_INFO(dev)->gen >= 4 ||
  1368.             tiling_mode == I915_TILING_NONE)
  1369.                 return size;
  1370.  
  1371.         /* Previous chips need a power-of-two fence region when tiling */
  1372.         if (INTEL_INFO(dev)->gen == 3)
  1373.                 gtt_size = 1024*1024;
  1374.         else
  1375.                 gtt_size = 512*1024;
  1376.  
  1377.         while (gtt_size < size)
  1378.                 gtt_size <<= 1;
  1379.  
  1380.         return gtt_size;
  1381. }
  1382.  
  1383. /**
  1384.  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
  1385.  * @obj: object to check
  1386.  *
  1387.  * Return the required GTT alignment for an object, taking into account
  1388.  * potential fence register mapping.
  1389.  */
  1390. static uint32_t
  1391. i915_gem_get_gtt_alignment(struct drm_device *dev,
  1392.                            uint32_t size,
  1393.                            int tiling_mode)
  1394. {
  1395.         /*
  1396.          * Minimum alignment is 4k (GTT page size), but might be greater
  1397.          * if a fence register is needed for the object.
  1398.          */
  1399.         if (INTEL_INFO(dev)->gen >= 4 ||
  1400.             tiling_mode == I915_TILING_NONE)
  1401.                 return 4096;
  1402.  
  1403.         /*
  1404.          * Previous chips need to be aligned to the size of the smallest
  1405.          * fence register that can contain the object.
  1406.          */
  1407.         return i915_gem_get_gtt_size(dev, size, tiling_mode);
  1408. }
  1409.  
  1410. /**
  1411.  * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
  1412.  *                                       unfenced object
  1413.  * @dev: the device
  1414.  * @size: size of the object
  1415.  * @tiling_mode: tiling mode of the object
  1416.  *
  1417.  * Return the required GTT alignment for an object, only taking into account
  1418.  * unfenced tiled surface requirements.
  1419.  */
  1420. uint32_t
  1421. i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
  1422.                                     uint32_t size,
  1423.                                     int tiling_mode)
  1424. {
  1425.         /*
  1426.          * Minimum alignment is 4k (GTT page size) for sane hw.
  1427.          */
  1428.         if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
  1429.             tiling_mode == I915_TILING_NONE)
  1430.                 return 4096;
  1431.  
  1432.         /* Previous hardware however needs to be aligned to a power-of-two
  1433.          * tile height. The simplest method for determining this is to reuse
  1434.          * the power-of-tile object size.
  1435.          */
  1436.         return i915_gem_get_gtt_size(dev, size, tiling_mode);
  1437. }
  1438.  
  1439. /* Immediately discard the backing storage */
  1440. static void
  1441. i915_gem_object_truncate(struct drm_i915_gem_object *obj)
  1442. {
  1443. //      struct inode *inode;
  1444.  
  1445. //      i915_gem_object_free_mmap_offset(obj);
  1446.  
  1447. //      if (obj->base.filp == NULL)
  1448. //              return;
  1449.  
  1450.         /* Our goal here is to return as much of the memory as
  1451.          * is possible back to the system as we are called from OOM.
  1452.          * To do this we must instruct the shmfs to drop all of its
  1453.          * backing pages, *now*.
  1454.          */
  1455. //      inode = obj->base.filp->f_path.dentry->d_inode;
  1456. //      shmem_truncate_range(inode, 0, (loff_t)-1);
  1457.  
  1458.         obj->madv = __I915_MADV_PURGED;
  1459. }
  1460.  
  1461. static inline int
  1462. i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
  1463. {
  1464.         return obj->madv == I915_MADV_DONTNEED;
  1465. }
  1466.  
  1467. static void
  1468. i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
  1469. {
  1470.         int page_count = obj->base.size / PAGE_SIZE;
  1471.         struct scatterlist *sg;
  1472.         int ret, i;
  1473.  
  1474.         BUG_ON(obj->madv == __I915_MADV_PURGED);
  1475.  
  1476.         ret = i915_gem_object_set_to_cpu_domain(obj, true);
  1477.         if (ret) {
  1478.                 /* In the event of a disaster, abandon all caches and
  1479.                  * hope for the best.
  1480.                  */
  1481.                 WARN_ON(ret != -EIO);
  1482.                 i915_gem_clflush_object(obj);
  1483.                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  1484.         }
  1485.  
  1486.         if (obj->madv == I915_MADV_DONTNEED)
  1487.                 obj->dirty = 0;
  1488.  
  1489.         for_each_sg(obj->pages->sgl, sg, page_count, i) {
  1490.                 struct page *page = sg_page(sg);
  1491.  
  1492.  
  1493.  
  1494.                 page_cache_release(page);
  1495.         }
  1496.     //DRM_DEBUG_KMS("%s release %d pages\n", __FUNCTION__, page_count);
  1497.         obj->dirty = 0;
  1498.  
  1499.         sg_free_table(obj->pages);
  1500.         kfree(obj->pages);
  1501. }
  1502.  
  1503. static int
  1504. i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
  1505. {
  1506.         const struct drm_i915_gem_object_ops *ops = obj->ops;
  1507.  
  1508.         if (obj->pages == NULL)
  1509.                 return 0;
  1510.  
  1511.         BUG_ON(obj->gtt_space);
  1512.  
  1513.         if (obj->pages_pin_count)
  1514.                 return -EBUSY;
  1515.  
  1516.         /* ->put_pages might need to allocate memory for the bit17 swizzle
  1517.          * array, hence protect them from being reaped by removing them from gtt
  1518.          * lists early. */
  1519.         list_del(&obj->gtt_list);
  1520.  
  1521.         ops->put_pages(obj);
  1522.         obj->pages = NULL;
  1523.  
  1524.         if (i915_gem_object_is_purgeable(obj))
  1525.                 i915_gem_object_truncate(obj);
  1526.  
  1527.         return 0;
  1528. }
  1529.  
  1530.  
  1531.  
  1532.  
  1533.  
  1534.  
  1535.  
  1536.  
  1537. static int
  1538. i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
  1539. {
  1540.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  1541.     int page_count, i;
  1542.         struct address_space *mapping;
  1543.         struct sg_table *st;
  1544.         struct scatterlist *sg;
  1545.         struct page *page;
  1546.         gfp_t gfp;
  1547.  
  1548.         /* Assert that the object is not currently in any GPU domain. As it
  1549.          * wasn't in the GTT, there shouldn't be any way it could have been in
  1550.          * a GPU cache
  1551.          */
  1552.         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
  1553.         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
  1554.  
  1555.         st = kmalloc(sizeof(*st), GFP_KERNEL);
  1556.         if (st == NULL)
  1557.                 return -ENOMEM;
  1558.  
  1559.         page_count = obj->base.size / PAGE_SIZE;
  1560.         if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
  1561.                 sg_free_table(st);
  1562.                 kfree(st);
  1563.                 return -ENOMEM;
  1564.         }
  1565.  
  1566.         /* Get the list of pages out of our struct file.  They'll be pinned
  1567.          * at this point until we release them.
  1568.          *
  1569.          * Fail silently without starting the shrinker
  1570.          */
  1571.         for_each_sg(st->sgl, sg, page_count, i) {
  1572.                 page = shmem_read_mapping_page_gfp(obj->base.filp, i, gfp);
  1573.                 if (IS_ERR(page)) {
  1574.             dbgprintf("%s invalid page %p\n", __FUNCTION__, page);
  1575.                         goto err_pages;
  1576.  
  1577.                 }
  1578.                 sg_set_page(sg, page, PAGE_SIZE, 0);
  1579.         }
  1580.  
  1581.         obj->pages = st;
  1582.  
  1583.     DRM_DEBUG_KMS("%s alloc %d pages\n", __FUNCTION__, page_count);
  1584.  
  1585.         return 0;
  1586.  
  1587. err_pages:
  1588.         for_each_sg(st->sgl, sg, i, page_count)
  1589.                 page_cache_release(sg_page(sg));
  1590.         sg_free_table(st);
  1591.         kfree(st);
  1592.         return PTR_ERR(page);
  1593. }
  1594.  
  1595. /* Ensure that the associated pages are gathered from the backing storage
  1596.  * and pinned into our object. i915_gem_object_get_pages() may be called
  1597.  * multiple times before they are released by a single call to
  1598.  * i915_gem_object_put_pages() - once the pages are no longer referenced
  1599.  * either as a result of memory pressure (reaping pages under the shrinker)
  1600.  * or as the object is itself released.
  1601.  */
  1602. int
  1603. i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
  1604. {
  1605.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  1606.         const struct drm_i915_gem_object_ops *ops = obj->ops;
  1607.         int ret;
  1608.  
  1609.         if (obj->pages)
  1610.                 return 0;
  1611.  
  1612.         BUG_ON(obj->pages_pin_count);
  1613.  
  1614.         ret = ops->get_pages(obj);
  1615.         if (ret)
  1616.                 return ret;
  1617.  
  1618.         list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
  1619.     return 0;
  1620. }
  1621.  
  1622. void
  1623. i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
  1624.                                struct intel_ring_buffer *ring)
  1625. {
  1626.         struct drm_device *dev = obj->base.dev;
  1627.         struct drm_i915_private *dev_priv = dev->dev_private;
  1628.         u32 seqno = intel_ring_get_seqno(ring);
  1629.  
  1630.         BUG_ON(ring == NULL);
  1631.         obj->ring = ring;
  1632.  
  1633.         /* Add a reference if we're newly entering the active list. */
  1634.         if (!obj->active) {
  1635.                 drm_gem_object_reference(&obj->base);
  1636.                 obj->active = 1;
  1637.         }
  1638.  
  1639.         /* Move from whatever list we were on to the tail of execution. */
  1640.         list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
  1641.         list_move_tail(&obj->ring_list, &ring->active_list);
  1642.  
  1643.         obj->last_read_seqno = seqno;
  1644.  
  1645.         if (obj->fenced_gpu_access) {
  1646.                 obj->last_fenced_seqno = seqno;
  1647.  
  1648.                 /* Bump MRU to take account of the delayed flush */
  1649.                 if (obj->fence_reg != I915_FENCE_REG_NONE) {
  1650.                 struct drm_i915_fence_reg *reg;
  1651.  
  1652.                 reg = &dev_priv->fence_regs[obj->fence_reg];
  1653.                         list_move_tail(&reg->lru_list,
  1654.                                        &dev_priv->mm.fence_list);
  1655.                 }
  1656.         }
  1657. }
  1658.  
  1659. static void
  1660. i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
  1661. {
  1662.         struct drm_device *dev = obj->base.dev;
  1663.         struct drm_i915_private *dev_priv = dev->dev_private;
  1664.  
  1665.         BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
  1666.         BUG_ON(!obj->active);
  1667.  
  1668.         if (obj->pin_count) /* are we a framebuffer? */
  1669.                 intel_mark_fb_idle(obj);
  1670.  
  1671.                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
  1672.  
  1673.         list_del_init(&obj->ring_list);
  1674.         obj->ring = NULL;
  1675.  
  1676.         obj->last_read_seqno = 0;
  1677.         obj->last_write_seqno = 0;
  1678.         obj->base.write_domain = 0;
  1679.  
  1680.         obj->last_fenced_seqno = 0;
  1681.         obj->fenced_gpu_access = false;
  1682.  
  1683.         obj->active = 0;
  1684.         drm_gem_object_unreference(&obj->base);
  1685.  
  1686.         WARN_ON(i915_verify_lists(dev));
  1687. }
  1688.  
  1689. static int
  1690. i915_gem_handle_seqno_wrap(struct drm_device *dev)
  1691. {
  1692.         struct drm_i915_private *dev_priv = dev->dev_private;
  1693.         struct intel_ring_buffer *ring;
  1694.         int ret, i, j;
  1695.  
  1696.         /* The hardware uses various monotonic 32-bit counters, if we
  1697.          * detect that they will wraparound we need to idle the GPU
  1698.          * and reset those counters.
  1699.          */
  1700.         ret = 0;
  1701.         for_each_ring(ring, dev_priv, i) {
  1702.                 for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
  1703.                         ret |= ring->sync_seqno[j] != 0;
  1704.         }
  1705.         if (ret == 0)
  1706.                 return ret;
  1707.  
  1708.         ret = i915_gpu_idle(dev);
  1709.         if (ret)
  1710.                 return ret;
  1711.  
  1712.         i915_gem_retire_requests(dev);
  1713.         for_each_ring(ring, dev_priv, i) {
  1714.                 for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
  1715.                         ring->sync_seqno[j] = 0;
  1716.         }
  1717.  
  1718.         return 0;
  1719. }
  1720.  
  1721. int
  1722. i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
  1723. {
  1724.         struct drm_i915_private *dev_priv = dev->dev_private;
  1725.  
  1726.         /* reserve 0 for non-seqno */
  1727.         if (dev_priv->next_seqno == 0) {
  1728.                 int ret = i915_gem_handle_seqno_wrap(dev);
  1729.                 if (ret)
  1730.                         return ret;
  1731.  
  1732.                 dev_priv->next_seqno = 1;
  1733.         }
  1734.  
  1735.         *seqno = dev_priv->next_seqno++;
  1736.         return 0;
  1737. }
  1738.  
  1739. int
  1740. i915_add_request(struct intel_ring_buffer *ring,
  1741.                  struct drm_file *file,
  1742.                  u32 *out_seqno)
  1743. {
  1744.         drm_i915_private_t *dev_priv = ring->dev->dev_private;
  1745.         struct drm_i915_gem_request *request;
  1746.         u32 request_ring_position;
  1747.         int was_empty;
  1748.         int ret;
  1749.  
  1750.         /*
  1751.          * Emit any outstanding flushes - execbuf can fail to emit the flush
  1752.          * after having emitted the batchbuffer command. Hence we need to fix
  1753.          * things up similar to emitting the lazy request. The difference here
  1754.          * is that the flush _must_ happen before the next request, no matter
  1755.          * what.
  1756.          */
  1757.         ret = intel_ring_flush_all_caches(ring);
  1758.         if (ret)
  1759.                 return ret;
  1760.  
  1761.         request = kmalloc(sizeof(*request), GFP_KERNEL);
  1762.         if (request == NULL)
  1763.                 return -ENOMEM;
  1764.  
  1765.  
  1766.         /* Record the position of the start of the request so that
  1767.          * should we detect the updated seqno part-way through the
  1768.          * GPU processing the request, we never over-estimate the
  1769.          * position of the head.
  1770.          */
  1771.         request_ring_position = intel_ring_get_tail(ring);
  1772.  
  1773.         ret = ring->add_request(ring);
  1774.         if (ret) {
  1775.                 kfree(request);
  1776.             return ret;
  1777.         }
  1778.  
  1779.         request->seqno = intel_ring_get_seqno(ring);
  1780.         request->ring = ring;
  1781.         request->tail = request_ring_position;
  1782.     request->emitted_jiffies = GetTimerTicks();
  1783.         was_empty = list_empty(&ring->request_list);
  1784.         list_add_tail(&request->list, &ring->request_list);
  1785.         request->file_priv = NULL;
  1786.  
  1787.  
  1788.         ring->outstanding_lazy_request = 0;
  1789.  
  1790.         if (!dev_priv->mm.suspended) {
  1791.                 if (i915_enable_hangcheck) {
  1792. //                      mod_timer(&dev_priv->hangcheck_timer,
  1793. //                                jiffies +
  1794. //                                msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
  1795.                 }
  1796.                 if (was_empty) {
  1797.            queue_delayed_work(dev_priv->wq,
  1798.                       &dev_priv->mm.retire_work, HZ);
  1799.                         intel_mark_busy(dev_priv->dev);
  1800.                 }
  1801.         }
  1802.  
  1803.         if (out_seqno)
  1804.                 *out_seqno = request->seqno;
  1805.         return 0;
  1806. }
  1807.  
  1808.  
  1809.  
  1810.  
  1811. static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
  1812.                                       struct intel_ring_buffer *ring)
  1813. {
  1814.         while (!list_empty(&ring->request_list)) {
  1815.                 struct drm_i915_gem_request *request;
  1816.  
  1817.                 request = list_first_entry(&ring->request_list,
  1818.                                            struct drm_i915_gem_request,
  1819.                                            list);
  1820.  
  1821.                 list_del(&request->list);
  1822. //       i915_gem_request_remove_from_client(request);
  1823.                 kfree(request);
  1824.         }
  1825.  
  1826.         while (!list_empty(&ring->active_list)) {
  1827.                 struct drm_i915_gem_object *obj;
  1828.  
  1829.                 obj = list_first_entry(&ring->active_list,
  1830.                                        struct drm_i915_gem_object,
  1831.                                        ring_list);
  1832.  
  1833.                 i915_gem_object_move_to_inactive(obj);
  1834.         }
  1835. }
  1836.  
  1837. static void i915_gem_reset_fences(struct drm_device *dev)
  1838. {
  1839.         struct drm_i915_private *dev_priv = dev->dev_private;
  1840.         int i;
  1841.  
  1842.         for (i = 0; i < dev_priv->num_fence_regs; i++) {
  1843.                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
  1844.  
  1845.                 i915_gem_write_fence(dev, i, NULL);
  1846.  
  1847.                 if (reg->obj)
  1848.                         i915_gem_object_fence_lost(reg->obj);
  1849.  
  1850.                 reg->pin_count = 0;
  1851.                 reg->obj = NULL;
  1852.                 INIT_LIST_HEAD(&reg->lru_list);
  1853.         }
  1854.  
  1855.         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
  1856. }
  1857.  
  1858. void i915_gem_reset(struct drm_device *dev)
  1859. {
  1860.         struct drm_i915_private *dev_priv = dev->dev_private;
  1861.         struct drm_i915_gem_object *obj;
  1862.         struct intel_ring_buffer *ring;
  1863.         int i;
  1864.  
  1865.         for_each_ring(ring, dev_priv, i)
  1866.                 i915_gem_reset_ring_lists(dev_priv, ring);
  1867.  
  1868.         /* Move everything out of the GPU domains to ensure we do any
  1869.          * necessary invalidation upon reuse.
  1870.          */
  1871.         list_for_each_entry(obj,
  1872.                             &dev_priv->mm.inactive_list,
  1873.                             mm_list)
  1874.         {
  1875.                 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
  1876.         }
  1877.  
  1878.         /* The fence registers are invalidated so clear them out */
  1879.         i915_gem_reset_fences(dev);
  1880. }
  1881.  
  1882. /**
  1883.  * This function clears the request list as sequence numbers are passed.
  1884.  */
  1885. void
  1886. i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
  1887. {
  1888.         uint32_t seqno;
  1889.  
  1890.         if (list_empty(&ring->request_list))
  1891.                 return;
  1892.  
  1893.         WARN_ON(i915_verify_lists(ring->dev));
  1894.  
  1895.         seqno = ring->get_seqno(ring, true);
  1896.  
  1897.         while (!list_empty(&ring->request_list)) {
  1898.                 struct drm_i915_gem_request *request;
  1899.  
  1900.                 request = list_first_entry(&ring->request_list,
  1901.                                            struct drm_i915_gem_request,
  1902.                                            list);
  1903.  
  1904.                 if (!i915_seqno_passed(seqno, request->seqno))
  1905.                         break;
  1906.  
  1907.                 trace_i915_gem_request_retire(ring, request->seqno);
  1908.                 /* We know the GPU must have read the request to have
  1909.                  * sent us the seqno + interrupt, so use the position
  1910.                  * of tail of the request to update the last known position
  1911.                  * of the GPU head.
  1912.                  */
  1913.                 ring->last_retired_head = request->tail;
  1914.  
  1915.                 list_del(&request->list);
  1916.                 kfree(request);
  1917.         }
  1918.  
  1919.         /* Move any buffers on the active list that are no longer referenced
  1920.          * by the ringbuffer to the flushing/inactive lists as appropriate.
  1921.          */
  1922.         while (!list_empty(&ring->active_list)) {
  1923.                 struct drm_i915_gem_object *obj;
  1924.  
  1925.                 obj = list_first_entry(&ring->active_list,
  1926.                                       struct drm_i915_gem_object,
  1927.                                       ring_list);
  1928.  
  1929.                 if (!i915_seqno_passed(seqno, obj->last_read_seqno))
  1930.                         break;
  1931.  
  1932.                         i915_gem_object_move_to_inactive(obj);
  1933.         }
  1934.  
  1935.         if (unlikely(ring->trace_irq_seqno &&
  1936.                      i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
  1937.                 ring->irq_put(ring);
  1938.                 ring->trace_irq_seqno = 0;
  1939.         }
  1940.  
  1941.         WARN_ON(i915_verify_lists(ring->dev));
  1942. }
  1943.  
  1944. void
  1945. i915_gem_retire_requests(struct drm_device *dev)
  1946. {
  1947.         drm_i915_private_t *dev_priv = dev->dev_private;
  1948.         struct intel_ring_buffer *ring;
  1949.         int i;
  1950.  
  1951.         for_each_ring(ring, dev_priv, i)
  1952.                 i915_gem_retire_requests_ring(ring);
  1953. }
  1954.  
  1955. static void
  1956. i915_gem_retire_work_handler(struct work_struct *work)
  1957. {
  1958.         drm_i915_private_t *dev_priv;
  1959.         struct drm_device *dev;
  1960.         struct intel_ring_buffer *ring;
  1961.         bool idle;
  1962.         int i;
  1963.  
  1964.         dev_priv = container_of(work, drm_i915_private_t,
  1965.                                 mm.retire_work.work);
  1966.         dev = dev_priv->dev;
  1967.  
  1968.         /* Come back later if the device is busy... */
  1969.         if (!mutex_trylock(&dev->struct_mutex)) {
  1970.         queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
  1971.         return;
  1972.         }
  1973.  
  1974.         i915_gem_retire_requests(dev);
  1975.  
  1976.         /* Send a periodic flush down the ring so we don't hold onto GEM
  1977.          * objects indefinitely.
  1978.          */
  1979.         idle = true;
  1980.         for_each_ring(ring, dev_priv, i) {
  1981.                 if (ring->gpu_caches_dirty)
  1982.                         i915_add_request(ring, NULL, NULL);
  1983.  
  1984.                 idle &= list_empty(&ring->request_list);
  1985.         }
  1986.  
  1987.    if (!dev_priv->mm.suspended && !idle)
  1988.        queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
  1989.         if (idle)
  1990.                 intel_mark_idle(dev);
  1991.  
  1992.         mutex_unlock(&dev->struct_mutex);
  1993. }
  1994.  
  1995. /**
  1996.  * Ensures that an object will eventually get non-busy by flushing any required
  1997.  * write domains, emitting any outstanding lazy request and retiring and
  1998.  * completed requests.
  1999.  */
  2000. static int
  2001. i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
  2002. {
  2003.         int ret;
  2004.  
  2005.         if (obj->active) {
  2006.                 ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
  2007.                 if (ret)
  2008.                         return ret;
  2009.  
  2010.                 i915_gem_retire_requests_ring(obj->ring);
  2011.         }
  2012.  
  2013.         return 0;
  2014. }
  2015.  
  2016. /**
  2017.  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
  2018.  * @DRM_IOCTL_ARGS: standard ioctl arguments
  2019.  *
  2020.  * Returns 0 if successful, else an error is returned with the remaining time in
  2021.  * the timeout parameter.
  2022.  *  -ETIME: object is still busy after timeout
  2023.  *  -ERESTARTSYS: signal interrupted the wait
  2024.  *  -ENONENT: object doesn't exist
  2025.  * Also possible, but rare:
  2026.  *  -EAGAIN: GPU wedged
  2027.  *  -ENOMEM: damn
  2028.  *  -ENODEV: Internal IRQ fail
  2029.  *  -E?: The add request failed
  2030.  *
  2031.  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
  2032.  * non-zero timeout parameter the wait ioctl will wait for the given number of
  2033.  * nanoseconds on an object becoming unbusy. Since the wait itself does so
  2034.  * without holding struct_mutex the object may become re-busied before this
  2035.  * function completes. A similar but shorter * race condition exists in the busy
  2036.  * ioctl
  2037.  */
  2038.  
  2039.  
  2040.  
  2041.  
  2042.  
  2043.  
  2044.  
  2045.  
  2046.  
  2047.  
  2048.  
  2049.  
  2050. /**
  2051.  * i915_gem_object_sync - sync an object to a ring.
  2052.  *
  2053.  * @obj: object which may be in use on another ring.
  2054.  * @to: ring we wish to use the object on. May be NULL.
  2055.  *
  2056.  * This code is meant to abstract object synchronization with the GPU.
  2057.  * Calling with NULL implies synchronizing the object with the CPU
  2058.  * rather than a particular GPU ring.
  2059.  *
  2060.  * Returns 0 if successful, else propagates up the lower layer error.
  2061.  */
  2062. int
  2063. i915_gem_object_sync(struct drm_i915_gem_object *obj,
  2064.                      struct intel_ring_buffer *to)
  2065. {
  2066.         struct intel_ring_buffer *from = obj->ring;
  2067.         u32 seqno;
  2068.         int ret, idx;
  2069.  
  2070.         if (from == NULL || to == from)
  2071.                 return 0;
  2072.  
  2073.         if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
  2074.                 return i915_gem_object_wait_rendering(obj, false);
  2075.  
  2076.         idx = intel_ring_sync_index(from, to);
  2077.  
  2078.         seqno = obj->last_read_seqno;
  2079.         if (seqno <= from->sync_seqno[idx])
  2080.                 return 0;
  2081.  
  2082.         ret = i915_gem_check_olr(obj->ring, seqno);
  2083.         if (ret)
  2084.                 return ret;
  2085.  
  2086.         ret = to->sync_to(to, from, seqno);
  2087.         if (!ret)
  2088.                 /* We use last_read_seqno because sync_to()
  2089.                  * might have just caused seqno wrap under
  2090.                  * the radar.
  2091.                  */
  2092.                 from->sync_seqno[idx] = obj->last_read_seqno;
  2093.  
  2094.         return ret;
  2095. }
  2096.  
  2097. static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
  2098. {
  2099.         u32 old_write_domain, old_read_domains;
  2100.  
  2101.         /* Act a barrier for all accesses through the GTT */
  2102.         mb();
  2103.  
  2104.         /* Force a pagefault for domain tracking on next user access */
  2105. //      i915_gem_release_mmap(obj);
  2106.  
  2107.         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
  2108.                 return;
  2109.  
  2110.         old_read_domains = obj->base.read_domains;
  2111.         old_write_domain = obj->base.write_domain;
  2112.  
  2113.         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
  2114.         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
  2115.  
  2116.         trace_i915_gem_object_change_domain(obj,
  2117.                                             old_read_domains,
  2118.                                             old_write_domain);
  2119. }
  2120.  
  2121. /**
  2122.  * Unbinds an object from the GTT aperture.
  2123.  */
  2124. int
  2125. i915_gem_object_unbind(struct drm_i915_gem_object *obj)
  2126. {
  2127.         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
  2128.         int ret = 0;
  2129.  
  2130.         if (obj->gtt_space == NULL)
  2131.                 return 0;
  2132.  
  2133.         if (obj->pin_count)
  2134.                 return -EBUSY;
  2135.  
  2136.         BUG_ON(obj->pages == NULL);
  2137.  
  2138.         ret = i915_gem_object_finish_gpu(obj);
  2139.         if (ret)
  2140.                 return ret;
  2141.         /* Continue on if we fail due to EIO, the GPU is hung so we
  2142.          * should be safe and we need to cleanup or else we might
  2143.          * cause memory corruption through use-after-free.
  2144.          */
  2145.  
  2146.         i915_gem_object_finish_gtt(obj);
  2147.  
  2148.         /* release the fence reg _after_ flushing */
  2149.         ret = i915_gem_object_put_fence(obj);
  2150.         if (ret)
  2151.                 return ret;
  2152.  
  2153.         trace_i915_gem_object_unbind(obj);
  2154.  
  2155.         if (obj->has_global_gtt_mapping)
  2156.         i915_gem_gtt_unbind_object(obj);
  2157.         if (obj->has_aliasing_ppgtt_mapping) {
  2158.                 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
  2159.                 obj->has_aliasing_ppgtt_mapping = 0;
  2160.         }
  2161.         i915_gem_gtt_finish_object(obj);
  2162.  
  2163.         list_del(&obj->mm_list);
  2164.         list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
  2165.         /* Avoid an unnecessary call to unbind on rebind. */
  2166.         obj->map_and_fenceable = true;
  2167.  
  2168.         drm_mm_put_block(obj->gtt_space);
  2169.         obj->gtt_space = NULL;
  2170.         obj->gtt_offset = 0;
  2171.  
  2172.         return 0;
  2173. }
  2174.  
  2175. int i915_gpu_idle(struct drm_device *dev)
  2176. {
  2177.         drm_i915_private_t *dev_priv = dev->dev_private;
  2178.         struct intel_ring_buffer *ring;
  2179.         int ret, i;
  2180.  
  2181.         /* Flush everything onto the inactive list. */
  2182.         for_each_ring(ring, dev_priv, i) {
  2183.                 ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
  2184.                 if (ret)
  2185.                         return ret;
  2186.  
  2187.                 ret = intel_ring_idle(ring);
  2188.                 if (ret)
  2189.                         return ret;
  2190.         }
  2191.  
  2192.         return 0;
  2193. }
  2194.  
  2195. static void sandybridge_write_fence_reg(struct drm_device *dev, int reg,
  2196.                                         struct drm_i915_gem_object *obj)
  2197. {
  2198.         drm_i915_private_t *dev_priv = dev->dev_private;
  2199.         uint64_t val;
  2200.  
  2201.         if (obj) {
  2202.                 u32 size = obj->gtt_space->size;
  2203.  
  2204.                 val = (uint64_t)((obj->gtt_offset + size - 4096) &
  2205.                                  0xfffff000) << 32;
  2206.                 val |= obj->gtt_offset & 0xfffff000;
  2207.                 val |= (uint64_t)((obj->stride / 128) - 1) <<
  2208.                         SANDYBRIDGE_FENCE_PITCH_SHIFT;
  2209.  
  2210.                 if (obj->tiling_mode == I915_TILING_Y)
  2211.                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
  2212.                 val |= I965_FENCE_REG_VALID;
  2213.         } else
  2214.                 val = 0;
  2215.  
  2216.         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val);
  2217.         POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8);
  2218. }
  2219.  
  2220. static void i965_write_fence_reg(struct drm_device *dev, int reg,
  2221.                                  struct drm_i915_gem_object *obj)
  2222. {
  2223.         drm_i915_private_t *dev_priv = dev->dev_private;
  2224.         uint64_t val;
  2225.  
  2226.         if (obj) {
  2227.                 u32 size = obj->gtt_space->size;
  2228.  
  2229.                 val = (uint64_t)((obj->gtt_offset + size - 4096) &
  2230.                                  0xfffff000) << 32;
  2231.                 val |= obj->gtt_offset & 0xfffff000;
  2232.                 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
  2233.                 if (obj->tiling_mode == I915_TILING_Y)
  2234.                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
  2235.                 val |= I965_FENCE_REG_VALID;
  2236.         } else
  2237.                 val = 0;
  2238.  
  2239.         I915_WRITE64(FENCE_REG_965_0 + reg * 8, val);
  2240.         POSTING_READ(FENCE_REG_965_0 + reg * 8);
  2241. }
  2242.  
  2243. static void i915_write_fence_reg(struct drm_device *dev, int reg,
  2244.                                  struct drm_i915_gem_object *obj)
  2245. {
  2246.         drm_i915_private_t *dev_priv = dev->dev_private;
  2247.         u32 val;
  2248.  
  2249.         if (obj) {
  2250.                 u32 size = obj->gtt_space->size;
  2251.                 int pitch_val;
  2252.                 int tile_width;
  2253.  
  2254.                 WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
  2255.                      (size & -size) != size ||
  2256.                      (obj->gtt_offset & (size - 1)),
  2257.                      "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
  2258.                      obj->gtt_offset, obj->map_and_fenceable, size);
  2259.  
  2260.                 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
  2261.                         tile_width = 128;
  2262.                 else
  2263.                         tile_width = 512;
  2264.  
  2265.                 /* Note: pitch better be a power of two tile widths */
  2266.                 pitch_val = obj->stride / tile_width;
  2267.                 pitch_val = ffs(pitch_val) - 1;
  2268.  
  2269.                 val = obj->gtt_offset;
  2270.                 if (obj->tiling_mode == I915_TILING_Y)
  2271.                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
  2272.                 val |= I915_FENCE_SIZE_BITS(size);
  2273.                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
  2274.                 val |= I830_FENCE_REG_VALID;
  2275.         } else
  2276.                 val = 0;
  2277.  
  2278.         if (reg < 8)
  2279.                 reg = FENCE_REG_830_0 + reg * 4;
  2280.         else
  2281.                 reg = FENCE_REG_945_8 + (reg - 8) * 4;
  2282.  
  2283.         I915_WRITE(reg, val);
  2284.         POSTING_READ(reg);
  2285. }
  2286.  
  2287. static void i830_write_fence_reg(struct drm_device *dev, int reg,
  2288.                                 struct drm_i915_gem_object *obj)
  2289. {
  2290.         drm_i915_private_t *dev_priv = dev->dev_private;
  2291.         uint32_t val;
  2292.  
  2293.         if (obj) {
  2294.                 u32 size = obj->gtt_space->size;
  2295.                 uint32_t pitch_val;
  2296.  
  2297.                 WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
  2298.                      (size & -size) != size ||
  2299.                      (obj->gtt_offset & (size - 1)),
  2300.                      "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
  2301.                      obj->gtt_offset, size);
  2302.  
  2303.                 pitch_val = obj->stride / 128;
  2304.                 pitch_val = ffs(pitch_val) - 1;
  2305.  
  2306.                 val = obj->gtt_offset;
  2307.                 if (obj->tiling_mode == I915_TILING_Y)
  2308.                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
  2309.                 val |= I830_FENCE_SIZE_BITS(size);
  2310.                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
  2311.                 val |= I830_FENCE_REG_VALID;
  2312.         } else
  2313.                 val = 0;
  2314.  
  2315.         I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
  2316.         POSTING_READ(FENCE_REG_830_0 + reg * 4);
  2317. }
  2318.  
  2319. static void i915_gem_write_fence(struct drm_device *dev, int reg,
  2320.                                  struct drm_i915_gem_object *obj)
  2321. {
  2322.         switch (INTEL_INFO(dev)->gen) {
  2323.         case 7:
  2324.         case 6: sandybridge_write_fence_reg(dev, reg, obj); break;
  2325.         case 5:
  2326.         case 4: i965_write_fence_reg(dev, reg, obj); break;
  2327.         case 3: i915_write_fence_reg(dev, reg, obj); break;
  2328.         case 2: i830_write_fence_reg(dev, reg, obj); break;
  2329.         default: break;
  2330.         }
  2331. }
  2332.  
  2333. static inline int fence_number(struct drm_i915_private *dev_priv,
  2334.                                struct drm_i915_fence_reg *fence)
  2335. {
  2336.         return fence - dev_priv->fence_regs;
  2337. }
  2338.  
  2339. static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
  2340.                                          struct drm_i915_fence_reg *fence,
  2341.                                          bool enable)
  2342. {
  2343.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  2344.         int reg = fence_number(dev_priv, fence);
  2345.  
  2346.         i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
  2347.  
  2348.         if (enable) {
  2349.                 obj->fence_reg = reg;
  2350.                 fence->obj = obj;
  2351.                 list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
  2352.         } else {
  2353.                 obj->fence_reg = I915_FENCE_REG_NONE;
  2354.                 fence->obj = NULL;
  2355.                 list_del_init(&fence->lru_list);
  2356.         }
  2357. }
  2358.  
  2359. static int
  2360. i915_gem_object_flush_fence(struct drm_i915_gem_object *obj)
  2361. {
  2362.         if (obj->last_fenced_seqno) {
  2363.                 int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
  2364.                         if (ret)
  2365.                                 return ret;
  2366.  
  2367.                 obj->last_fenced_seqno = 0;
  2368.         }
  2369.  
  2370.         /* Ensure that all CPU reads are completed before installing a fence
  2371.          * and all writes before removing the fence.
  2372.          */
  2373.         if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
  2374.                 mb();
  2375.  
  2376.         obj->fenced_gpu_access = false;
  2377.         return 0;
  2378. }
  2379.  
  2380. int
  2381. i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
  2382. {
  2383.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  2384.         int ret;
  2385.  
  2386.         ret = i915_gem_object_flush_fence(obj);
  2387.         if (ret)
  2388.                 return ret;
  2389.  
  2390.         if (obj->fence_reg == I915_FENCE_REG_NONE)
  2391.                 return 0;
  2392.  
  2393.         i915_gem_object_update_fence(obj,
  2394.                                      &dev_priv->fence_regs[obj->fence_reg],
  2395.                                      false);
  2396.         i915_gem_object_fence_lost(obj);
  2397.  
  2398.         return 0;
  2399. }
  2400.  
  2401. static struct drm_i915_fence_reg *
  2402. i915_find_fence_reg(struct drm_device *dev)
  2403. {
  2404.         struct drm_i915_private *dev_priv = dev->dev_private;
  2405.         struct drm_i915_fence_reg *reg, *avail;
  2406.         int i;
  2407.  
  2408.         /* First try to find a free reg */
  2409.         avail = NULL;
  2410.         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
  2411.                 reg = &dev_priv->fence_regs[i];
  2412.                 if (!reg->obj)
  2413.                         return reg;
  2414.  
  2415.                 if (!reg->pin_count)
  2416.                         avail = reg;
  2417.         }
  2418.  
  2419.         if (avail == NULL)
  2420.                 return NULL;
  2421.  
  2422.         /* None available, try to steal one or wait for a user to finish */
  2423.         list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
  2424.                 if (reg->pin_count)
  2425.                         continue;
  2426.  
  2427.                 return reg;
  2428.         }
  2429.  
  2430.         return NULL;
  2431. }
  2432.  
  2433. /**
  2434.  * i915_gem_object_get_fence - set up fencing for an object
  2435.  * @obj: object to map through a fence reg
  2436.  *
  2437.  * When mapping objects through the GTT, userspace wants to be able to write
  2438.  * to them without having to worry about swizzling if the object is tiled.
  2439.  * This function walks the fence regs looking for a free one for @obj,
  2440.  * stealing one if it can't find any.
  2441.  *
  2442.  * It then sets up the reg based on the object's properties: address, pitch
  2443.  * and tiling format.
  2444.  *
  2445.  * For an untiled surface, this removes any existing fence.
  2446.  */
  2447. int
  2448. i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
  2449. {
  2450.         struct drm_device *dev = obj->base.dev;
  2451.         struct drm_i915_private *dev_priv = dev->dev_private;
  2452.         bool enable = obj->tiling_mode != I915_TILING_NONE;
  2453.         struct drm_i915_fence_reg *reg;
  2454.         int ret;
  2455.  
  2456.         /* Have we updated the tiling parameters upon the object and so
  2457.          * will need to serialise the write to the associated fence register?
  2458.          */
  2459.         if (obj->fence_dirty) {
  2460.                 ret = i915_gem_object_flush_fence(obj);
  2461.                 if (ret)
  2462.                         return ret;
  2463.         }
  2464.  
  2465.         /* Just update our place in the LRU if our fence is getting reused. */
  2466.         if (obj->fence_reg != I915_FENCE_REG_NONE) {
  2467.                 reg = &dev_priv->fence_regs[obj->fence_reg];
  2468.                 if (!obj->fence_dirty) {
  2469.                         list_move_tail(&reg->lru_list,
  2470.                                        &dev_priv->mm.fence_list);
  2471.                         return 0;
  2472.                 }
  2473.         } else if (enable) {
  2474.                 reg = i915_find_fence_reg(dev);
  2475.                 if (reg == NULL)
  2476.                         return -EDEADLK;
  2477.  
  2478.                 if (reg->obj) {
  2479.                         struct drm_i915_gem_object *old = reg->obj;
  2480.  
  2481.                         ret = i915_gem_object_flush_fence(old);
  2482.                         if (ret)
  2483.                                 return ret;
  2484.  
  2485.                         i915_gem_object_fence_lost(old);
  2486.                 }
  2487.         } else
  2488.                 return 0;
  2489.  
  2490.         i915_gem_object_update_fence(obj, reg, enable);
  2491.         obj->fence_dirty = false;
  2492.  
  2493.         return 0;
  2494. }
  2495.  
  2496. static bool i915_gem_valid_gtt_space(struct drm_device *dev,
  2497.                                      struct drm_mm_node *gtt_space,
  2498.                                      unsigned long cache_level)
  2499. {
  2500.         struct drm_mm_node *other;
  2501.  
  2502.         /* On non-LLC machines we have to be careful when putting differing
  2503.          * types of snoopable memory together to avoid the prefetcher
  2504.          * crossing memory domains and dieing.
  2505.          */
  2506.         if (HAS_LLC(dev))
  2507.                 return true;
  2508.  
  2509.         if (gtt_space == NULL)
  2510.                 return true;
  2511.  
  2512.         if (list_empty(&gtt_space->node_list))
  2513.                 return true;
  2514.  
  2515.         other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
  2516.         if (other->allocated && !other->hole_follows && other->color != cache_level)
  2517.                 return false;
  2518.  
  2519.         other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
  2520.         if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
  2521.                 return false;
  2522.  
  2523.         return true;
  2524. }
  2525.  
  2526. static void i915_gem_verify_gtt(struct drm_device *dev)
  2527. {
  2528. #if WATCH_GTT
  2529.         struct drm_i915_private *dev_priv = dev->dev_private;
  2530.         struct drm_i915_gem_object *obj;
  2531.         int err = 0;
  2532.  
  2533.         list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
  2534.                 if (obj->gtt_space == NULL) {
  2535.                         printk(KERN_ERR "object found on GTT list with no space reserved\n");
  2536.                         err++;
  2537.                         continue;
  2538.                 }
  2539.  
  2540.                 if (obj->cache_level != obj->gtt_space->color) {
  2541.                         printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
  2542.                                obj->gtt_space->start,
  2543.                                obj->gtt_space->start + obj->gtt_space->size,
  2544.                                obj->cache_level,
  2545.                                obj->gtt_space->color);
  2546.                         err++;
  2547.                         continue;
  2548.                 }
  2549.  
  2550.                 if (!i915_gem_valid_gtt_space(dev,
  2551.                                               obj->gtt_space,
  2552.                                               obj->cache_level)) {
  2553.                         printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
  2554.                                obj->gtt_space->start,
  2555.                                obj->gtt_space->start + obj->gtt_space->size,
  2556.                                obj->cache_level);
  2557.                         err++;
  2558.                         continue;
  2559.                 }
  2560.         }
  2561.  
  2562.         WARN_ON(err);
  2563. #endif
  2564. }
  2565.  
  2566. /**
  2567.  * Finds free space in the GTT aperture and binds the object there.
  2568.  */
  2569. static int
  2570. i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  2571.                             unsigned alignment,
  2572.                             bool map_and_fenceable,
  2573.                             bool nonblocking)
  2574. {
  2575.         struct drm_device *dev = obj->base.dev;
  2576.         drm_i915_private_t *dev_priv = dev->dev_private;
  2577.         struct drm_mm_node *node;
  2578.         u32 size, fence_size, fence_alignment, unfenced_alignment;
  2579.         bool mappable, fenceable;
  2580.         int ret;
  2581.  
  2582.         if (obj->madv != I915_MADV_WILLNEED) {
  2583.         DRM_ERROR("Attempting to bind a purgeable object\n");
  2584.                 return -EINVAL;
  2585.         }
  2586.  
  2587.         fence_size = i915_gem_get_gtt_size(dev,
  2588.                                            obj->base.size,
  2589.                                            obj->tiling_mode);
  2590.         fence_alignment = i915_gem_get_gtt_alignment(dev,
  2591.                                                      obj->base.size,
  2592.                                                      obj->tiling_mode);
  2593.         unfenced_alignment =
  2594.                 i915_gem_get_unfenced_gtt_alignment(dev,
  2595.                                                     obj->base.size,
  2596.                                                     obj->tiling_mode);
  2597.  
  2598.         if (alignment == 0)
  2599.                 alignment = map_and_fenceable ? fence_alignment :
  2600.                                                 unfenced_alignment;
  2601.         if (map_and_fenceable && alignment & (fence_alignment - 1)) {
  2602.                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
  2603.                 return -EINVAL;
  2604.         }
  2605.  
  2606.         size = map_and_fenceable ? fence_size : obj->base.size;
  2607.  
  2608.         /* If the object is bigger than the entire aperture, reject it early
  2609.          * before evicting everything in a vain attempt to find space.
  2610.          */
  2611.         if (obj->base.size >
  2612.             (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
  2613.                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
  2614.                 return -E2BIG;
  2615.         }
  2616.  
  2617.         ret = i915_gem_object_get_pages(obj);
  2618.         if (ret)
  2619.                 return ret;
  2620.  
  2621.         i915_gem_object_pin_pages(obj);
  2622.  
  2623.         node = kzalloc(sizeof(*node), GFP_KERNEL);
  2624.         if (node == NULL) {
  2625.                 i915_gem_object_unpin_pages(obj);
  2626.                 return -ENOMEM;
  2627.         }
  2628.  
  2629.  search_free:
  2630.         if (map_and_fenceable)
  2631.                 ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space, node,
  2632.                                                           size, alignment, obj->cache_level,
  2633.                                                           0, dev_priv->mm.gtt_mappable_end);
  2634.         else
  2635.                 ret = drm_mm_insert_node_generic(&dev_priv->mm.gtt_space, node,
  2636.                                                  size, alignment, obj->cache_level);
  2637.         if (ret) {
  2638.  
  2639.                 i915_gem_object_unpin_pages(obj);
  2640.                 kfree(node);
  2641.                         return ret;
  2642.         }
  2643.         if (WARN_ON(!i915_gem_valid_gtt_space(dev, node, obj->cache_level))) {
  2644.                 i915_gem_object_unpin_pages(obj);
  2645.                 drm_mm_put_block(node);
  2646.                 return -EINVAL;
  2647.         }
  2648.  
  2649.         ret = i915_gem_gtt_prepare_object(obj);
  2650.         if (ret) {
  2651.                 i915_gem_object_unpin_pages(obj);
  2652.                 drm_mm_put_block(node);
  2653.                         return ret;
  2654.         }
  2655.  
  2656.         list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list);
  2657.         list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
  2658.  
  2659.         obj->gtt_space = node;
  2660.         obj->gtt_offset = node->start;
  2661.  
  2662.         fenceable =
  2663.                 node->size == fence_size &&
  2664.                 (node->start & (fence_alignment - 1)) == 0;
  2665.  
  2666.         mappable =
  2667.                 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
  2668.  
  2669.         obj->map_and_fenceable = mappable && fenceable;
  2670.  
  2671.         i915_gem_object_unpin_pages(obj);
  2672.         trace_i915_gem_object_bind(obj, map_and_fenceable);
  2673.         i915_gem_verify_gtt(dev);
  2674.         return 0;
  2675. }
  2676.  
  2677. void
  2678. i915_gem_clflush_object(struct drm_i915_gem_object *obj)
  2679. {
  2680.         /* If we don't have a page list set up, then we're not pinned
  2681.          * to GPU, and we can ignore the cache flush because it'll happen
  2682.          * again at bind time.
  2683.          */
  2684.         if (obj->pages == NULL)
  2685.                 return;
  2686.  
  2687.         /* If the GPU is snooping the contents of the CPU cache,
  2688.          * we do not need to manually clear the CPU cache lines.  However,
  2689.          * the caches are only snooped when the render cache is
  2690.          * flushed/invalidated.  As we always have to emit invalidations
  2691.          * and flushes when moving into and out of the RENDER domain, correct
  2692.          * snooping behaviour occurs naturally as the result of our domain
  2693.          * tracking.
  2694.          */
  2695.         if (obj->cache_level != I915_CACHE_NONE)
  2696.                 return;
  2697. #if 0
  2698.      if(obj->mapped != NULL)
  2699.      {
  2700.         uint8_t *page_virtual;
  2701.         unsigned int i;
  2702.  
  2703.         page_virtual = obj->mapped;
  2704.         asm volatile("mfence");
  2705.         for (i = 0; i < obj->base.size; i += x86_clflush_size)
  2706.             clflush(page_virtual + i);
  2707.         asm volatile("mfence");
  2708.      }
  2709.      else
  2710.      {
  2711.         uint8_t *page_virtual;
  2712.         unsigned int i;
  2713.         page_virtual = AllocKernelSpace(obj->base.size);
  2714.         if(page_virtual != NULL)
  2715.         {
  2716.             dma_addr_t *src, *dst;
  2717.             u32 count;
  2718.  
  2719. #define page_tabs  0xFDC00000      /* really dirty hack */
  2720.  
  2721.             src =  obj->pages.page;
  2722.             dst =  &((dma_addr_t*)page_tabs)[(u32_t)page_virtual >> 12];
  2723.             count = obj->base.size/4096;
  2724.  
  2725.             while(count--)
  2726.             {
  2727.                 *dst++ = (0xFFFFF000 & *src++) | 0x001 ;
  2728.             };
  2729.  
  2730.             asm volatile("mfence");
  2731.             for (i = 0; i < obj->base.size; i += x86_clflush_size)
  2732.                 clflush(page_virtual + i);
  2733.             asm volatile("mfence");
  2734.             FreeKernelSpace(page_virtual);
  2735.         }
  2736.         else
  2737.         {
  2738.             asm volatile (
  2739.             "mfence         \n"
  2740.             "wbinvd         \n"                 /* this is really ugly  */
  2741.             "mfence");
  2742.         }
  2743.      }
  2744. #endif
  2745.  
  2746. }
  2747.  
  2748. /** Flushes the GTT write domain for the object if it's dirty. */
  2749. static void
  2750. i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
  2751. {
  2752.         uint32_t old_write_domain;
  2753.  
  2754.         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
  2755.                 return;
  2756.  
  2757.         /* No actual flushing is required for the GTT write domain.  Writes
  2758.          * to it immediately go to main memory as far as we know, so there's
  2759.          * no chipset flush.  It also doesn't land in render cache.
  2760.          *
  2761.          * However, we do have to enforce the order so that all writes through
  2762.          * the GTT land before any writes to the device, such as updates to
  2763.          * the GATT itself.
  2764.          */
  2765.         wmb();
  2766.  
  2767.         old_write_domain = obj->base.write_domain;
  2768.         obj->base.write_domain = 0;
  2769.  
  2770.         trace_i915_gem_object_change_domain(obj,
  2771.                                             obj->base.read_domains,
  2772.                                             old_write_domain);
  2773. }
  2774.  
  2775. /** Flushes the CPU write domain for the object if it's dirty. */
  2776. static void
  2777. i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
  2778. {
  2779.         uint32_t old_write_domain;
  2780.  
  2781.         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
  2782.                 return;
  2783.  
  2784.         i915_gem_clflush_object(obj);
  2785.         i915_gem_chipset_flush(obj->base.dev);
  2786.         old_write_domain = obj->base.write_domain;
  2787.         obj->base.write_domain = 0;
  2788.  
  2789.         trace_i915_gem_object_change_domain(obj,
  2790.                                             obj->base.read_domains,
  2791.                                             old_write_domain);
  2792. }
  2793.  
  2794. /**
  2795.  * Moves a single object to the GTT read, and possibly write domain.
  2796.  *
  2797.  * This function returns when the move is complete, including waiting on
  2798.  * flushes to occur.
  2799.  */
  2800. int
  2801. i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
  2802. {
  2803.         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
  2804.         uint32_t old_write_domain, old_read_domains;
  2805.         int ret;
  2806.  
  2807.         /* Not valid to be called on unbound objects. */
  2808.         if (obj->gtt_space == NULL)
  2809.                 return -EINVAL;
  2810.  
  2811.         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
  2812.                 return 0;
  2813.  
  2814.         ret = i915_gem_object_wait_rendering(obj, !write);
  2815.                 if (ret)
  2816.                         return ret;
  2817.  
  2818.         i915_gem_object_flush_cpu_write_domain(obj);
  2819.  
  2820.         old_write_domain = obj->base.write_domain;
  2821.         old_read_domains = obj->base.read_domains;
  2822.  
  2823.         /* It should now be out of any other write domains, and we can update
  2824.          * the domain values for our changes.
  2825.          */
  2826.         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
  2827.         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
  2828.         if (write) {
  2829.                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
  2830.                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
  2831.                 obj->dirty = 1;
  2832.         }
  2833.  
  2834.         trace_i915_gem_object_change_domain(obj,
  2835.                                             old_read_domains,
  2836.                                             old_write_domain);
  2837.  
  2838.         /* And bump the LRU for this access */
  2839.         if (i915_gem_object_is_inactive(obj))
  2840.                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
  2841.  
  2842.         return 0;
  2843. }
  2844.  
  2845. int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
  2846.                                     enum i915_cache_level cache_level)
  2847. {
  2848.         struct drm_device *dev = obj->base.dev;
  2849.         drm_i915_private_t *dev_priv = dev->dev_private;
  2850.         int ret;
  2851.  
  2852.         if (obj->cache_level == cache_level)
  2853.                 return 0;
  2854.  
  2855.         if (obj->pin_count) {
  2856.                 DRM_DEBUG("can not change the cache level of pinned objects\n");
  2857.                 return -EBUSY;
  2858.         }
  2859.  
  2860.         if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) {
  2861.                 ret = i915_gem_object_unbind(obj);
  2862.                 if (ret)
  2863.                         return ret;
  2864.         }
  2865.  
  2866.         if (obj->gtt_space) {
  2867.                 ret = i915_gem_object_finish_gpu(obj);
  2868.                 if (ret)
  2869.                         return ret;
  2870.  
  2871.                 i915_gem_object_finish_gtt(obj);
  2872.  
  2873.                 /* Before SandyBridge, you could not use tiling or fence
  2874.                  * registers with snooped memory, so relinquish any fences
  2875.                  * currently pointing to our region in the aperture.
  2876.                  */
  2877.                 if (INTEL_INFO(dev)->gen < 6) {
  2878.                         ret = i915_gem_object_put_fence(obj);
  2879.                         if (ret)
  2880.                                 return ret;
  2881.                 }
  2882.  
  2883.                 if (obj->has_global_gtt_mapping)
  2884.                         i915_gem_gtt_bind_object(obj, cache_level);
  2885.                 if (obj->has_aliasing_ppgtt_mapping)
  2886.                         i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
  2887.                                                obj, cache_level);
  2888.  
  2889.                 obj->gtt_space->color = cache_level;
  2890.         }
  2891.  
  2892.         if (cache_level == I915_CACHE_NONE) {
  2893.                 u32 old_read_domains, old_write_domain;
  2894.  
  2895.                 /* If we're coming from LLC cached, then we haven't
  2896.                  * actually been tracking whether the data is in the
  2897.                  * CPU cache or not, since we only allow one bit set
  2898.                  * in obj->write_domain and have been skipping the clflushes.
  2899.                  * Just set it to the CPU cache for now.
  2900.                  */
  2901.                 WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
  2902.                 WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
  2903.  
  2904.                 old_read_domains = obj->base.read_domains;
  2905.                 old_write_domain = obj->base.write_domain;
  2906.  
  2907.                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  2908.                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  2909.  
  2910.                 trace_i915_gem_object_change_domain(obj,
  2911.                                                     old_read_domains,
  2912.                                                     old_write_domain);
  2913.     }
  2914.  
  2915.         obj->cache_level = cache_level;
  2916.         i915_gem_verify_gtt(dev);
  2917.         return 0;
  2918. }
  2919.  
  2920. int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
  2921.                                struct drm_file *file)
  2922. {
  2923.         struct drm_i915_gem_caching *args = data;
  2924.         struct drm_i915_gem_object *obj;
  2925.         int ret;
  2926.  
  2927.         ret = i915_mutex_lock_interruptible(dev);
  2928.         if (ret)
  2929.                 return ret;
  2930.  
  2931.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  2932.         if (&obj->base == NULL) {
  2933.                 ret = -ENOENT;
  2934.                 goto unlock;
  2935.         }
  2936.  
  2937.         args->caching = obj->cache_level != I915_CACHE_NONE;
  2938.  
  2939.         drm_gem_object_unreference(&obj->base);
  2940. unlock:
  2941.         mutex_unlock(&dev->struct_mutex);
  2942.         return ret;
  2943. }
  2944.  
  2945. int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
  2946.                                struct drm_file *file)
  2947. {
  2948.         struct drm_i915_gem_caching *args = data;
  2949.         struct drm_i915_gem_object *obj;
  2950.         enum i915_cache_level level;
  2951.         int ret;
  2952.  
  2953.         switch (args->caching) {
  2954.         case I915_CACHING_NONE:
  2955.                 level = I915_CACHE_NONE;
  2956.                 break;
  2957.         case I915_CACHING_CACHED:
  2958.                 level = I915_CACHE_LLC;
  2959.                 break;
  2960.         default:
  2961.                 return -EINVAL;
  2962.         }
  2963.  
  2964.         ret = i915_mutex_lock_interruptible(dev);
  2965.         if (ret)
  2966.                 return ret;
  2967.  
  2968.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  2969.         if (&obj->base == NULL) {
  2970.                 ret = -ENOENT;
  2971.                 goto unlock;
  2972.         }
  2973.  
  2974.         ret = i915_gem_object_set_cache_level(obj, level);
  2975.  
  2976.         drm_gem_object_unreference(&obj->base);
  2977. unlock:
  2978.         mutex_unlock(&dev->struct_mutex);
  2979.         return ret;
  2980. }
  2981.  
  2982. /*
  2983.  * Prepare buffer for display plane (scanout, cursors, etc).
  2984.  * Can be called from an uninterruptible phase (modesetting) and allows
  2985.  * any flushes to be pipelined (for pageflips).
  2986.  */
  2987. int
  2988. i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
  2989.                                      u32 alignment,
  2990.                                      struct intel_ring_buffer *pipelined)
  2991. {
  2992.         u32 old_read_domains, old_write_domain;
  2993.         int ret;
  2994.  
  2995.         if (pipelined != obj->ring) {
  2996.                 ret = i915_gem_object_sync(obj, pipelined);
  2997.         if (ret)
  2998.                 return ret;
  2999.         }
  3000.  
  3001.         /* The display engine is not coherent with the LLC cache on gen6.  As
  3002.          * a result, we make sure that the pinning that is about to occur is
  3003.          * done with uncached PTEs. This is lowest common denominator for all
  3004.          * chipsets.
  3005.          *
  3006.          * However for gen6+, we could do better by using the GFDT bit instead
  3007.          * of uncaching, which would allow us to flush all the LLC-cached data
  3008.          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
  3009.          */
  3010.         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
  3011.         if (ret)
  3012.                 return ret;
  3013.  
  3014.         /* As the user may map the buffer once pinned in the display plane
  3015.          * (e.g. libkms for the bootup splash), we have to ensure that we
  3016.          * always use map_and_fenceable for all scanout buffers.
  3017.          */
  3018.         ret = i915_gem_object_pin(obj, alignment, true, false);
  3019.         if (ret)
  3020.                 return ret;
  3021.  
  3022.         i915_gem_object_flush_cpu_write_domain(obj);
  3023.  
  3024.         old_write_domain = obj->base.write_domain;
  3025.         old_read_domains = obj->base.read_domains;
  3026.  
  3027.         /* It should now be out of any other write domains, and we can update
  3028.          * the domain values for our changes.
  3029.          */
  3030.         obj->base.write_domain = 0;
  3031.         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
  3032.  
  3033.         trace_i915_gem_object_change_domain(obj,
  3034.                                             old_read_domains,
  3035.                                             old_write_domain);
  3036.  
  3037.         return 0;
  3038. }
  3039.  
  3040. int
  3041. i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
  3042. {
  3043.         int ret;
  3044.  
  3045.         if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
  3046.                 return 0;
  3047.  
  3048.         ret = i915_gem_object_wait_rendering(obj, false);
  3049.     if (ret)
  3050.         return ret;
  3051.  
  3052.         /* Ensure that we invalidate the GPU's caches and TLBs. */
  3053.         obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
  3054.         return 0;
  3055. }
  3056.  
  3057. /**
  3058.  * Moves a single object to the CPU read, and possibly write domain.
  3059.  *
  3060.  * This function returns when the move is complete, including waiting on
  3061.  * flushes to occur.
  3062.  */
  3063. int
  3064. i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
  3065. {
  3066.         uint32_t old_write_domain, old_read_domains;
  3067.         int ret;
  3068.  
  3069.         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
  3070.                 return 0;
  3071.  
  3072.         ret = i915_gem_object_wait_rendering(obj, !write);
  3073.         if (ret)
  3074.                 return ret;
  3075.  
  3076.         i915_gem_object_flush_gtt_write_domain(obj);
  3077.  
  3078.         old_write_domain = obj->base.write_domain;
  3079.         old_read_domains = obj->base.read_domains;
  3080.  
  3081.         /* Flush the CPU cache if it's still invalid. */
  3082.         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
  3083.                 i915_gem_clflush_object(obj);
  3084.  
  3085.                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
  3086.         }
  3087.  
  3088.         /* It should now be out of any other write domains, and we can update
  3089.          * the domain values for our changes.
  3090.          */
  3091.         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
  3092.  
  3093.         /* If we're writing through the CPU, then the GPU read domains will
  3094.          * need to be invalidated at next use.
  3095.          */
  3096.         if (write) {
  3097.                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  3098.                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  3099.         }
  3100.  
  3101.         trace_i915_gem_object_change_domain(obj,
  3102.                                             old_read_domains,
  3103.                                             old_write_domain);
  3104.  
  3105.         return 0;
  3106. }
  3107.  
  3108. #if 0
  3109. /* Throttle our rendering by waiting until the ring has completed our requests
  3110.  * emitted over 20 msec ago.
  3111.  *
  3112.  * Note that if we were to use the current jiffies each time around the loop,
  3113.  * we wouldn't escape the function with any frames outstanding if the time to
  3114.  * render a frame was over 20ms.
  3115.  *
  3116.  * This should get us reasonable parallelism between CPU and GPU but also
  3117.  * relatively low latency when blocking on a particular request to finish.
  3118.  */
  3119. static int
  3120. i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
  3121. {
  3122.         struct drm_i915_private *dev_priv = dev->dev_private;
  3123.         struct drm_i915_file_private *file_priv = file->driver_priv;
  3124.         unsigned long recent_enough = GetTimerTics() - msecs_to_jiffies(20);
  3125.         struct drm_i915_gem_request *request;
  3126.         struct intel_ring_buffer *ring = NULL;
  3127.         u32 seqno = 0;
  3128.         int ret;
  3129.  
  3130.         if (atomic_read(&dev_priv->mm.wedged))
  3131.                 return -EIO;
  3132.  
  3133.         spin_lock(&file_priv->mm.lock);
  3134.         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
  3135.                 if (time_after_eq(request->emitted_jiffies, recent_enough))
  3136.                         break;
  3137.  
  3138.                 ring = request->ring;
  3139.                 seqno = request->seqno;
  3140.         }
  3141.         spin_unlock(&file_priv->mm.lock);
  3142.  
  3143.         if (seqno == 0)
  3144.                 return 0;
  3145.  
  3146.         ret = __wait_seqno(ring, seqno, true, NULL);
  3147.         if (ret == 0)
  3148.                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
  3149.  
  3150.         return ret;
  3151. }
  3152. #endif
  3153.  
  3154. int
  3155. i915_gem_object_pin(struct drm_i915_gem_object *obj,
  3156.                     uint32_t alignment,
  3157.                     bool map_and_fenceable,
  3158.                     bool nonblocking)
  3159. {
  3160.         int ret;
  3161.  
  3162.         if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
  3163.                 return -EBUSY;
  3164.  
  3165. #if 0
  3166.         if (obj->gtt_space != NULL) {
  3167.                 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
  3168.                     (map_and_fenceable && !obj->map_and_fenceable)) {
  3169.                         WARN(obj->pin_count,
  3170.                              "bo is already pinned with incorrect alignment:"
  3171.                              " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
  3172.                              " obj->map_and_fenceable=%d\n",
  3173.                              obj->gtt_offset, alignment,
  3174.                              map_and_fenceable,
  3175.                              obj->map_and_fenceable);
  3176.                         ret = i915_gem_object_unbind(obj);
  3177.                         if (ret)
  3178.                                 return ret;
  3179.                 }
  3180.         }
  3181. #endif
  3182.  
  3183.         if (obj->gtt_space == NULL) {
  3184.                 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  3185.  
  3186.                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
  3187.                                                   map_and_fenceable,
  3188.                                                   nonblocking);
  3189.                 if (ret)
  3190.                         return ret;
  3191.  
  3192.                 if (!dev_priv->mm.aliasing_ppgtt)
  3193.                         i915_gem_gtt_bind_object(obj, obj->cache_level);
  3194.         }
  3195.  
  3196.         if (!obj->has_global_gtt_mapping && map_and_fenceable)
  3197.                 i915_gem_gtt_bind_object(obj, obj->cache_level);
  3198.  
  3199.         obj->pin_count++;
  3200.         obj->pin_mappable |= map_and_fenceable;
  3201.  
  3202.         return 0;
  3203. }
  3204.  
  3205. void
  3206. i915_gem_object_unpin(struct drm_i915_gem_object *obj)
  3207. {
  3208.         BUG_ON(obj->pin_count == 0);
  3209.         BUG_ON(obj->gtt_space == NULL);
  3210.  
  3211.         if (--obj->pin_count == 0)
  3212.                 obj->pin_mappable = false;
  3213. }
  3214.  
  3215. int
  3216. i915_gem_pin_ioctl(struct drm_device *dev, void *data,
  3217.                    struct drm_file *file)
  3218. {
  3219.         struct drm_i915_gem_pin *args = data;
  3220.         struct drm_i915_gem_object *obj;
  3221.         int ret;
  3222.  
  3223.         ret = i915_mutex_lock_interruptible(dev);
  3224.         if (ret)
  3225.                 return ret;
  3226.  
  3227.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  3228.         if (&obj->base == NULL) {
  3229.                 ret = -ENOENT;
  3230.                 goto unlock;
  3231.         }
  3232.  
  3233.         if (obj->madv != I915_MADV_WILLNEED) {
  3234.                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
  3235.                 ret = -EINVAL;
  3236.                 goto out;
  3237.         }
  3238.  
  3239.         if (obj->pin_filp != NULL && obj->pin_filp != file) {
  3240.                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
  3241.                           args->handle);
  3242.                 ret = -EINVAL;
  3243.                 goto out;
  3244.         }
  3245.  
  3246.         if (obj->user_pin_count == 0) {
  3247.                 ret = i915_gem_object_pin(obj, args->alignment, true, false);
  3248.                 if (ret)
  3249.                         goto out;
  3250.         }
  3251.  
  3252.         obj->user_pin_count++;
  3253.         obj->pin_filp = file;
  3254.  
  3255.         /* XXX - flush the CPU caches for pinned objects
  3256.          * as the X server doesn't manage domains yet
  3257.          */
  3258.         i915_gem_object_flush_cpu_write_domain(obj);
  3259.         args->offset = obj->gtt_offset;
  3260. out:
  3261.         drm_gem_object_unreference(&obj->base);
  3262. unlock:
  3263.         mutex_unlock(&dev->struct_mutex);
  3264.         return ret;
  3265. }
  3266.  
  3267. #if 0
  3268.  
  3269. int
  3270. i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
  3271.                      struct drm_file *file)
  3272. {
  3273.         struct drm_i915_gem_pin *args = data;
  3274.         struct drm_i915_gem_object *obj;
  3275.         int ret;
  3276.  
  3277.         ret = i915_mutex_lock_interruptible(dev);
  3278.         if (ret)
  3279.                 return ret;
  3280.  
  3281.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  3282.         if (&obj->base == NULL) {
  3283.                 ret = -ENOENT;
  3284.                 goto unlock;
  3285.         }
  3286.  
  3287.         if (obj->pin_filp != file) {
  3288.                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
  3289.                           args->handle);
  3290.                 ret = -EINVAL;
  3291.                 goto out;
  3292.         }
  3293.         obj->user_pin_count--;
  3294.         if (obj->user_pin_count == 0) {
  3295.                 obj->pin_filp = NULL;
  3296.                 i915_gem_object_unpin(obj);
  3297.         }
  3298.  
  3299. out:
  3300.         drm_gem_object_unreference(&obj->base);
  3301. unlock:
  3302.         mutex_unlock(&dev->struct_mutex);
  3303.         return ret;
  3304. }
  3305.  
  3306. #endif
  3307.  
  3308. int
  3309. i915_gem_busy_ioctl(struct drm_device *dev, void *data,
  3310.                     struct drm_file *file)
  3311. {
  3312.         struct drm_i915_gem_busy *args = data;
  3313.         struct drm_i915_gem_object *obj;
  3314.         int ret;
  3315.  
  3316.         ret = i915_mutex_lock_interruptible(dev);
  3317.         if (ret)
  3318.                 return ret;
  3319.  
  3320.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  3321.         if (&obj->base == NULL) {
  3322.                 ret = -ENOENT;
  3323.                 goto unlock;
  3324.         }
  3325.  
  3326.         /* Count all active objects as busy, even if they are currently not used
  3327.          * by the gpu. Users of this interface expect objects to eventually
  3328.          * become non-busy without any further actions, therefore emit any
  3329.          * necessary flushes here.
  3330.          */
  3331.         ret = i915_gem_object_flush_active(obj);
  3332.  
  3333.         args->busy = obj->active;
  3334.         if (obj->ring) {
  3335.                 BUILD_BUG_ON(I915_NUM_RINGS > 16);
  3336.                 args->busy |= intel_ring_flag(obj->ring) << 16;
  3337.         }
  3338.  
  3339.         drm_gem_object_unreference(&obj->base);
  3340. unlock:
  3341.         mutex_unlock(&dev->struct_mutex);
  3342.         return ret;
  3343. }
  3344.  
  3345. #if 0
  3346. int
  3347. i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
  3348.                         struct drm_file *file_priv)
  3349. {
  3350.         return i915_gem_ring_throttle(dev, file_priv);
  3351. }
  3352.  
  3353. int
  3354. i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
  3355.                        struct drm_file *file_priv)
  3356. {
  3357.         struct drm_i915_gem_madvise *args = data;
  3358.         struct drm_i915_gem_object *obj;
  3359.         int ret;
  3360.  
  3361.         switch (args->madv) {
  3362.         case I915_MADV_DONTNEED:
  3363.         case I915_MADV_WILLNEED:
  3364.             break;
  3365.         default:
  3366.             return -EINVAL;
  3367.         }
  3368.  
  3369.         ret = i915_mutex_lock_interruptible(dev);
  3370.         if (ret)
  3371.                 return ret;
  3372.  
  3373.         obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
  3374.         if (&obj->base == NULL) {
  3375.                 ret = -ENOENT;
  3376.                 goto unlock;
  3377.         }
  3378.  
  3379.         if (obj->pin_count) {
  3380.                 ret = -EINVAL;
  3381.                 goto out;
  3382.         }
  3383.  
  3384.         if (obj->madv != __I915_MADV_PURGED)
  3385.                 obj->madv = args->madv;
  3386.  
  3387.         /* if the object is no longer attached, discard its backing storage */
  3388.         if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
  3389.                 i915_gem_object_truncate(obj);
  3390.  
  3391.         args->retained = obj->madv != __I915_MADV_PURGED;
  3392.  
  3393. out:
  3394.         drm_gem_object_unreference(&obj->base);
  3395. unlock:
  3396.         mutex_unlock(&dev->struct_mutex);
  3397.         return ret;
  3398. }
  3399. #endif
  3400.  
  3401. void i915_gem_object_init(struct drm_i915_gem_object *obj,
  3402.                           const struct drm_i915_gem_object_ops *ops)
  3403. {
  3404.         INIT_LIST_HEAD(&obj->mm_list);
  3405.         INIT_LIST_HEAD(&obj->gtt_list);
  3406.         INIT_LIST_HEAD(&obj->ring_list);
  3407.         INIT_LIST_HEAD(&obj->exec_list);
  3408.  
  3409.         obj->ops = ops;
  3410.  
  3411.         obj->fence_reg = I915_FENCE_REG_NONE;
  3412.         obj->madv = I915_MADV_WILLNEED;
  3413.         /* Avoid an unnecessary call to unbind on the first bind. */
  3414.         obj->map_and_fenceable = true;
  3415.  
  3416.         i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
  3417. }
  3418.  
  3419. static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
  3420.         .get_pages = i915_gem_object_get_pages_gtt,
  3421.         .put_pages = i915_gem_object_put_pages_gtt,
  3422. };
  3423.  
  3424. struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
  3425.                                                   size_t size)
  3426. {
  3427.         struct drm_i915_gem_object *obj;
  3428.         struct address_space *mapping;
  3429.         u32 mask;
  3430.  
  3431.         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
  3432.         if (obj == NULL)
  3433.                 return NULL;
  3434.  
  3435.         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
  3436.                 kfree(obj);
  3437.                 return NULL;
  3438.         }
  3439.  
  3440.  
  3441.         i915_gem_object_init(obj, &i915_gem_object_ops);
  3442.  
  3443.         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  3444.         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  3445.  
  3446.         if (HAS_LLC(dev)) {
  3447.                 /* On some devices, we can have the GPU use the LLC (the CPU
  3448.                  * cache) for about a 10% performance improvement
  3449.                  * compared to uncached.  Graphics requests other than
  3450.                  * display scanout are coherent with the CPU in
  3451.                  * accessing this cache.  This means in this mode we
  3452.                  * don't need to clflush on the CPU side, and on the
  3453.                  * GPU side we only need to flush internal caches to
  3454.                  * get data visible to the CPU.
  3455.                  *
  3456.                  * However, we maintain the display planes as UC, and so
  3457.                  * need to rebind when first used as such.
  3458.                  */
  3459.                 obj->cache_level = I915_CACHE_LLC;
  3460.         } else
  3461.                 obj->cache_level = I915_CACHE_NONE;
  3462.  
  3463.         return obj;
  3464. }
  3465.  
  3466. int i915_gem_init_object(struct drm_gem_object *obj)
  3467. {
  3468.         BUG();
  3469.  
  3470.         return 0;
  3471. }
  3472.  
  3473. void i915_gem_free_object(struct drm_gem_object *gem_obj)
  3474. {
  3475.         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
  3476.         struct drm_device *dev = obj->base.dev;
  3477.         drm_i915_private_t *dev_priv = dev->dev_private;
  3478.  
  3479.         trace_i915_gem_object_destroy(obj);
  3480.  
  3481. //   if (obj->phys_obj)
  3482. //       i915_gem_detach_phys_object(dev, obj);
  3483.  
  3484.         obj->pin_count = 0;
  3485.         if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
  3486.                 bool was_interruptible;
  3487.  
  3488.                 was_interruptible = dev_priv->mm.interruptible;
  3489.                 dev_priv->mm.interruptible = false;
  3490.  
  3491.                 WARN_ON(i915_gem_object_unbind(obj));
  3492.  
  3493.                 dev_priv->mm.interruptible = was_interruptible;
  3494.         }
  3495.  
  3496.         obj->pages_pin_count = 0;
  3497.         i915_gem_object_put_pages(obj);
  3498. //   i915_gem_object_free_mmap_offset(obj);
  3499.  
  3500.         BUG_ON(obj->pages);
  3501.  
  3502. //   if (obj->base.import_attach)
  3503. //       drm_prime_gem_destroy(&obj->base, NULL);
  3504.  
  3505.         drm_gem_object_release(&obj->base);
  3506.         i915_gem_info_remove_obj(dev_priv, obj->base.size);
  3507.  
  3508.         kfree(obj->bit_17);
  3509.         kfree(obj);
  3510. }
  3511.  
  3512. #if 0
  3513. int
  3514. i915_gem_idle(struct drm_device *dev)
  3515. {
  3516.         drm_i915_private_t *dev_priv = dev->dev_private;
  3517.         int ret;
  3518.  
  3519.         mutex_lock(&dev->struct_mutex);
  3520.  
  3521.         if (dev_priv->mm.suspended) {
  3522.                 mutex_unlock(&dev->struct_mutex);
  3523.                 return 0;
  3524.         }
  3525.  
  3526.         ret = i915_gpu_idle(dev);
  3527.         if (ret) {
  3528.                 mutex_unlock(&dev->struct_mutex);
  3529.                 return ret;
  3530.         }
  3531.         i915_gem_retire_requests(dev);
  3532.  
  3533.         i915_gem_reset_fences(dev);
  3534.  
  3535.         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
  3536.          * We need to replace this with a semaphore, or something.
  3537.          * And not confound mm.suspended!
  3538.          */
  3539.         dev_priv->mm.suspended = 1;
  3540.         del_timer_sync(&dev_priv->hangcheck_timer);
  3541.  
  3542.         i915_kernel_lost_context(dev);
  3543.         i915_gem_cleanup_ringbuffer(dev);
  3544.  
  3545.         mutex_unlock(&dev->struct_mutex);
  3546.  
  3547.         /* Cancel the retire work handler, which should be idle now. */
  3548. //   cancel_delayed_work_sync(&dev_priv->mm.retire_work);
  3549.  
  3550.         return 0;
  3551. }
  3552. #endif
  3553.  
  3554. void i915_gem_l3_remap(struct drm_device *dev)
  3555. {
  3556.         drm_i915_private_t *dev_priv = dev->dev_private;
  3557.         u32 misccpctl;
  3558.         int i;
  3559.  
  3560.         if (!IS_IVYBRIDGE(dev))
  3561.                 return;
  3562.  
  3563.         if (!dev_priv->l3_parity.remap_info)
  3564.                 return;
  3565.  
  3566.         misccpctl = I915_READ(GEN7_MISCCPCTL);
  3567.         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
  3568.         POSTING_READ(GEN7_MISCCPCTL);
  3569.  
  3570.         for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
  3571.                 u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
  3572.                 if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
  3573.                         DRM_DEBUG("0x%x was already programmed to %x\n",
  3574.                                   GEN7_L3LOG_BASE + i, remap);
  3575.                 if (remap && !dev_priv->l3_parity.remap_info[i/4])
  3576.                         DRM_DEBUG_DRIVER("Clearing remapped register\n");
  3577.                 I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
  3578.         }
  3579.  
  3580.         /* Make sure all the writes land before disabling dop clock gating */
  3581.         POSTING_READ(GEN7_L3LOG_BASE);
  3582.  
  3583.         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
  3584. }
  3585.  
  3586. void i915_gem_init_swizzling(struct drm_device *dev)
  3587. {
  3588.         drm_i915_private_t *dev_priv = dev->dev_private;
  3589.  
  3590.         if (INTEL_INFO(dev)->gen < 5 ||
  3591.             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
  3592.                 return;
  3593.  
  3594.         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
  3595.                                  DISP_TILE_SURFACE_SWIZZLING);
  3596.  
  3597.         if (IS_GEN5(dev))
  3598.                 return;
  3599.  
  3600.         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
  3601.         if (IS_GEN6(dev))
  3602.                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
  3603.         else
  3604.                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
  3605. }
  3606.  
  3607. static bool
  3608. intel_enable_blt(struct drm_device *dev)
  3609. {
  3610.         if (!HAS_BLT(dev))
  3611.                 return false;
  3612.  
  3613.         /* The blitter was dysfunctional on early prototypes */
  3614.         if (IS_GEN6(dev) && dev->pdev->revision < 8) {
  3615.                 DRM_INFO("BLT not supported on this pre-production hardware;"
  3616.                          " graphics performance will be degraded.\n");
  3617.                 return false;
  3618.         }
  3619.  
  3620.         return true;
  3621. }
  3622.  
  3623. int
  3624. i915_gem_init_hw(struct drm_device *dev)
  3625. {
  3626.         drm_i915_private_t *dev_priv = dev->dev_private;
  3627.         int ret;
  3628.  
  3629.         if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
  3630.                 return -EIO;
  3631.  
  3632.         if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))
  3633.                 I915_WRITE(0x9008, I915_READ(0x9008) | 0xf0000);
  3634.  
  3635.         i915_gem_l3_remap(dev);
  3636.  
  3637.         i915_gem_init_swizzling(dev);
  3638.  
  3639.         ret = intel_init_render_ring_buffer(dev);
  3640.         if (ret)
  3641.                 return ret;
  3642.  
  3643.     if (HAS_BSD(dev)) {
  3644.                 ret = intel_init_bsd_ring_buffer(dev);
  3645.                 if (ret)
  3646.                         goto cleanup_render_ring;
  3647.         }
  3648.  
  3649.         if (intel_enable_blt(dev)) {
  3650.                 ret = intel_init_blt_ring_buffer(dev);
  3651.                 if (ret)
  3652.                         goto cleanup_bsd_ring;
  3653.         }
  3654.  
  3655.         dev_priv->next_seqno = 1;
  3656.  
  3657.         /*
  3658.          * XXX: There was some w/a described somewhere suggesting loading
  3659.          * contexts before PPGTT.
  3660.          */
  3661.         i915_gem_context_init(dev);
  3662.         i915_gem_init_ppgtt(dev);
  3663.  
  3664.         return 0;
  3665.  
  3666. cleanup_bsd_ring:
  3667.         intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
  3668. cleanup_render_ring:
  3669.         intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
  3670.         return ret;
  3671. }
  3672.  
  3673. static bool
  3674. intel_enable_ppgtt(struct drm_device *dev)
  3675. {
  3676.         if (i915_enable_ppgtt >= 0)
  3677.                 return i915_enable_ppgtt;
  3678.  
  3679. #ifdef CONFIG_INTEL_IOMMU
  3680.         /* Disable ppgtt on SNB if VT-d is on. */
  3681.         if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
  3682.                 return false;
  3683. #endif
  3684.  
  3685.         return true;
  3686. }
  3687.  
  3688. #define LFB_SIZE 0xC00000
  3689.  
  3690. int i915_gem_init(struct drm_device *dev)
  3691. {
  3692.         struct drm_i915_private *dev_priv = dev->dev_private;
  3693.         unsigned long gtt_size, mappable_size;
  3694.         int ret;
  3695.  
  3696.         gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT;
  3697.         mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
  3698.  
  3699.         mutex_lock(&dev->struct_mutex);
  3700.         if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
  3701.                 /* PPGTT pdes are stolen from global gtt ptes, so shrink the
  3702.                  * aperture accordingly when using aliasing ppgtt. */
  3703.                 gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
  3704.  
  3705.         i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE);
  3706.  
  3707.                 ret = i915_gem_init_aliasing_ppgtt(dev);
  3708.                 if (ret) {
  3709.                         mutex_unlock(&dev->struct_mutex);
  3710.                         return ret;
  3711.                 }
  3712.         } else {
  3713.                 /* Let GEM Manage all of the aperture.
  3714.                  *
  3715.                  * However, leave one page at the end still bound to the scratch
  3716.                  * page.  There are a number of places where the hardware
  3717.                  * apparently prefetches past the end of the object, and we've
  3718.                  * seen multiple hangs with the GPU head pointer stuck in a
  3719.                  * batchbuffer bound at the last page of the aperture.  One page
  3720.                  * should be enough to keep any prefetching inside of the
  3721.                  * aperture.
  3722.                  */
  3723.         i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE);
  3724.         }
  3725.  
  3726.         ret = i915_gem_init_hw(dev);
  3727.         mutex_unlock(&dev->struct_mutex);
  3728.         if (ret) {
  3729.                 i915_gem_cleanup_aliasing_ppgtt(dev);
  3730.                 return ret;
  3731.         }
  3732.  
  3733.     return 0;
  3734. }
  3735.  
  3736. void
  3737. i915_gem_cleanup_ringbuffer(struct drm_device *dev)
  3738. {
  3739.         drm_i915_private_t *dev_priv = dev->dev_private;
  3740.         struct intel_ring_buffer *ring;
  3741.         int i;
  3742.  
  3743.         for_each_ring(ring, dev_priv, i)
  3744.                 intel_cleanup_ring_buffer(ring);
  3745. }
  3746.  
  3747. #if 0
  3748.  
  3749. int
  3750. i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
  3751.                        struct drm_file *file_priv)
  3752. {
  3753.         drm_i915_private_t *dev_priv = dev->dev_private;
  3754.         int ret;
  3755.  
  3756.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  3757.                 return 0;
  3758.  
  3759.         if (atomic_read(&dev_priv->mm.wedged)) {
  3760.                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
  3761.                 atomic_set(&dev_priv->mm.wedged, 0);
  3762.         }
  3763.  
  3764.         mutex_lock(&dev->struct_mutex);
  3765.         dev_priv->mm.suspended = 0;
  3766.  
  3767.         ret = i915_gem_init_hw(dev);
  3768.         if (ret != 0) {
  3769.                 mutex_unlock(&dev->struct_mutex);
  3770.                 return ret;
  3771.         }
  3772.  
  3773.         BUG_ON(!list_empty(&dev_priv->mm.active_list));
  3774.         mutex_unlock(&dev->struct_mutex);
  3775.  
  3776.         ret = drm_irq_install(dev);
  3777.         if (ret)
  3778.                 goto cleanup_ringbuffer;
  3779.  
  3780.         return 0;
  3781.  
  3782. cleanup_ringbuffer:
  3783.         mutex_lock(&dev->struct_mutex);
  3784.         i915_gem_cleanup_ringbuffer(dev);
  3785.         dev_priv->mm.suspended = 1;
  3786.         mutex_unlock(&dev->struct_mutex);
  3787.  
  3788.         return ret;
  3789. }
  3790.  
  3791. int
  3792. i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
  3793.                        struct drm_file *file_priv)
  3794. {
  3795.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  3796.                 return 0;
  3797.  
  3798.         drm_irq_uninstall(dev);
  3799.         return i915_gem_idle(dev);
  3800. }
  3801.  
  3802. void
  3803. i915_gem_lastclose(struct drm_device *dev)
  3804. {
  3805.         int ret;
  3806.  
  3807.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  3808.                 return;
  3809.  
  3810.         ret = i915_gem_idle(dev);
  3811.         if (ret)
  3812.                 DRM_ERROR("failed to idle hardware: %d\n", ret);
  3813. }
  3814. #endif
  3815.  
  3816. static void
  3817. init_ring_lists(struct intel_ring_buffer *ring)
  3818. {
  3819.     INIT_LIST_HEAD(&ring->active_list);
  3820.     INIT_LIST_HEAD(&ring->request_list);
  3821. }
  3822.  
  3823. void
  3824. i915_gem_load(struct drm_device *dev)
  3825. {
  3826.     int i;
  3827.     drm_i915_private_t *dev_priv = dev->dev_private;
  3828.  
  3829.     INIT_LIST_HEAD(&dev_priv->mm.active_list);
  3830.     INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
  3831.         INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
  3832.         INIT_LIST_HEAD(&dev_priv->mm.bound_list);
  3833.     INIT_LIST_HEAD(&dev_priv->mm.fence_list);
  3834.     for (i = 0; i < I915_NUM_RINGS; i++)
  3835.         init_ring_lists(&dev_priv->ring[i]);
  3836.         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
  3837.         INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
  3838.         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
  3839.                           i915_gem_retire_work_handler);
  3840.  
  3841.     /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
  3842.     if (IS_GEN3(dev)) {
  3843.                 I915_WRITE(MI_ARB_STATE,
  3844.                            _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
  3845.     }
  3846.  
  3847.     dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
  3848.  
  3849.     if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
  3850.         dev_priv->num_fence_regs = 16;
  3851.     else
  3852.         dev_priv->num_fence_regs = 8;
  3853.  
  3854.     /* Initialize fence registers to zero */
  3855.         i915_gem_reset_fences(dev);
  3856.  
  3857.     i915_gem_detect_bit_6_swizzle(dev);
  3858.  
  3859.     dev_priv->mm.interruptible = true;
  3860.  
  3861. //    dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
  3862. //    dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
  3863. //    register_shrinker(&dev_priv->mm.inactive_shrinker);
  3864. }
  3865.  
  3866.  
  3867.