Subversion Repositories Kolibri OS

Rev

Rev 3260 | Rev 3266 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright © 2008 Intel Corporation
  3.  *
  4.  * Permission is hereby granted, free of charge, to any person obtaining a
  5.  * copy of this software and associated documentation files (the "Software"),
  6.  * to deal in the Software without restriction, including without limitation
  7.  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8.  * and/or sell copies of the Software, and to permit persons to whom the
  9.  * Software is furnished to do so, subject to the following conditions:
  10.  *
  11.  * The above copyright notice and this permission notice (including the next
  12.  * paragraph) shall be included in all copies or substantial portions of the
  13.  * Software.
  14.  *
  15.  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16.  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17.  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18.  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19.  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20.  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21.  * IN THE SOFTWARE.
  22.  *
  23.  * Authors:
  24.  *    Eric Anholt <eric@anholt.net>
  25.  *
  26.  */
  27.  
  28. #include <drm/drmP.h>
  29. #include <drm/i915_drm.h>
  30. #include "i915_drv.h"
  31. #include "i915_trace.h"
  32. #include "intel_drv.h"
  33. #include <linux/shmem_fs.h>
  34. #include <linux/slab.h>
  35. //#include <linux/swap.h>
  36. #include <linux/pci.h>
  37.  
  38. extern int x86_clflush_size;
  39.  
  40. #define PROT_READ       0x1             /* page can be read */
  41. #define PROT_WRITE      0x2             /* page can be written */
  42. #define MAP_SHARED      0x01            /* Share changes */
  43.  
  44. #undef mb
  45. #undef rmb
  46. #undef wmb
  47. #define mb() asm volatile("mfence")
  48. #define rmb() asm volatile ("lfence")
  49. #define wmb() asm volatile ("sfence")
  50.  
  51. unsigned long vm_mmap(struct file *file, unsigned long addr,
  52.          unsigned long len, unsigned long prot,
  53.          unsigned long flag, unsigned long offset);
  54.  
  55. static inline void clflush(volatile void *__p)
  56. {
  57.     asm volatile("clflush %0" : "+m" (*(volatile char*)__p));
  58. }
  59.  
  60. #define MAX_ERRNO       4095
  61.  
  62. #define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
  63.  
  64.  
  65. #define I915_EXEC_CONSTANTS_MASK        (3<<6)
  66. #define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */
  67. #define I915_EXEC_CONSTANTS_ABSOLUTE    (1<<6)
  68. #define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */
  69.  
  70. static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  71. static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  72. static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  73.                                                     unsigned alignment,
  74.                                                     bool map_and_fenceable,
  75.                                                     bool nonblocking);
  76. static int i915_gem_phys_pwrite(struct drm_device *dev,
  77.                                 struct drm_i915_gem_object *obj,
  78.                                 struct drm_i915_gem_pwrite *args,
  79.                                 struct drm_file *file);
  80.  
  81. static void i915_gem_write_fence(struct drm_device *dev, int reg,
  82.                                  struct drm_i915_gem_object *obj);
  83. static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
  84.                                          struct drm_i915_fence_reg *fence,
  85.                                          bool enable);
  86.  
  87. static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
  88. static void i915_gem_shrink_all(struct drm_i915_private *dev_priv);
  89. static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
  90.  
  91. static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
  92. {
  93.         if (obj->tiling_mode)
  94.                 i915_gem_release_mmap(obj);
  95.  
  96.         /* As we do not have an associated fence register, we will force
  97.          * a tiling change if we ever need to acquire one.
  98.          */
  99.         obj->fence_dirty = false;
  100.         obj->fence_reg = I915_FENCE_REG_NONE;
  101. }
  102.  
  103. /* some bookkeeping */
  104. static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  105.                                   size_t size)
  106. {
  107.         dev_priv->mm.object_count++;
  108.         dev_priv->mm.object_memory += size;
  109. }
  110.  
  111. static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  112.                                      size_t size)
  113. {
  114.         dev_priv->mm.object_count--;
  115.         dev_priv->mm.object_memory -= size;
  116. }
  117.  
  118. static int
  119. i915_gem_wait_for_error(struct drm_device *dev)
  120. {
  121.         struct drm_i915_private *dev_priv = dev->dev_private;
  122.         struct completion *x = &dev_priv->error_completion;
  123.         unsigned long flags;
  124.         int ret;
  125.  
  126.         if (!atomic_read(&dev_priv->mm.wedged))
  127.                 return 0;
  128. #if 0
  129.         /*
  130.          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
  131.          * userspace. If it takes that long something really bad is going on and
  132.          * we should simply try to bail out and fail as gracefully as possible.
  133.          */
  134.         ret = wait_for_completion_interruptible_timeout(x, 10*HZ);
  135.         if (ret == 0) {
  136.                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
  137.                 return -EIO;
  138.         } else if (ret < 0) {
  139.                 return ret;
  140.         }
  141.  
  142.         if (atomic_read(&dev_priv->mm.wedged)) {
  143.                 /* GPU is hung, bump the completion count to account for
  144.                  * the token we just consumed so that we never hit zero and
  145.                  * end up waiting upon a subsequent completion event that
  146.                  * will never happen.
  147.                  */
  148.                 spin_lock_irqsave(&x->wait.lock, flags);
  149.                 x->done++;
  150.                 spin_unlock_irqrestore(&x->wait.lock, flags);
  151.         }
  152. #endif
  153.  
  154.         return 0;
  155. }
  156.  
  157. int i915_mutex_lock_interruptible(struct drm_device *dev)
  158. {
  159.         int ret;
  160.  
  161.         ret = i915_gem_wait_for_error(dev);
  162.         if (ret)
  163.                 return ret;
  164.  
  165.     mutex_lock(&dev->struct_mutex);
  166.  
  167.         WARN_ON(i915_verify_lists(dev));
  168.         return 0;
  169. }
  170.  
  171. static inline bool
  172. i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
  173. {
  174.         return obj->gtt_space && !obj->active;
  175. }
  176.  
  177.  
  178. #if 0
  179.  
  180. int
  181. i915_gem_init_ioctl(struct drm_device *dev, void *data,
  182.                     struct drm_file *file)
  183. {
  184.         struct drm_i915_gem_init *args = data;
  185.  
  186.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  187.                 return -ENODEV;
  188.  
  189.         if (args->gtt_start >= args->gtt_end ||
  190.             (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
  191.                 return -EINVAL;
  192.  
  193.         /* GEM with user mode setting was never supported on ilk and later. */
  194.         if (INTEL_INFO(dev)->gen >= 5)
  195.                 return -ENODEV;
  196.  
  197.         mutex_lock(&dev->struct_mutex);
  198.         i915_gem_init_global_gtt(dev, args->gtt_start,
  199.                                  args->gtt_end, args->gtt_end);
  200.         mutex_unlock(&dev->struct_mutex);
  201.  
  202.         return 0;
  203. }
  204. #endif
  205.  
  206. int
  207. i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  208.                             struct drm_file *file)
  209. {
  210.         struct drm_i915_private *dev_priv = dev->dev_private;
  211.         struct drm_i915_gem_get_aperture *args = data;
  212.         struct drm_i915_gem_object *obj;
  213.         size_t pinned;
  214.  
  215.         pinned = 0;
  216.         mutex_lock(&dev->struct_mutex);
  217.         list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
  218.                 if (obj->pin_count)
  219.                 pinned += obj->gtt_space->size;
  220.         mutex_unlock(&dev->struct_mutex);
  221.  
  222.         args->aper_size = dev_priv->mm.gtt_total;
  223.         args->aper_available_size = args->aper_size - pinned;
  224.  
  225.         return 0;
  226. }
  227.  
  228. static int
  229. i915_gem_create(struct drm_file *file,
  230.                 struct drm_device *dev,
  231.                 uint64_t size,
  232.                 uint32_t *handle_p)
  233. {
  234.         struct drm_i915_gem_object *obj;
  235.         int ret;
  236.         u32 handle;
  237.  
  238.         size = roundup(size, PAGE_SIZE);
  239.         if (size == 0)
  240.                 return -EINVAL;
  241.  
  242.         /* Allocate the new object */
  243.         obj = i915_gem_alloc_object(dev, size);
  244.         if (obj == NULL)
  245.                 return -ENOMEM;
  246.  
  247.         ret = drm_gem_handle_create(file, &obj->base, &handle);
  248.         if (ret) {
  249.                 drm_gem_object_release(&obj->base);
  250.                 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
  251.                 kfree(obj);
  252.                 return ret;
  253.         }
  254.  
  255.         /* drop reference from allocate - handle holds it now */
  256.         drm_gem_object_unreference(&obj->base);
  257.         trace_i915_gem_object_create(obj);
  258.  
  259.         *handle_p = handle;
  260.         return 0;
  261. }
  262.  
  263. int
  264. i915_gem_dumb_create(struct drm_file *file,
  265.                      struct drm_device *dev,
  266.                      struct drm_mode_create_dumb *args)
  267. {
  268.         /* have to work out size/pitch and return them */
  269.         args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
  270.         args->size = args->pitch * args->height;
  271.         return i915_gem_create(file, dev,
  272.                                args->size, &args->handle);
  273. }
  274.  
  275. int i915_gem_dumb_destroy(struct drm_file *file,
  276.                           struct drm_device *dev,
  277.                           uint32_t handle)
  278. {
  279.         return drm_gem_handle_delete(file, handle);
  280. }
  281.  
  282. /**
  283.  * Creates a new mm object and returns a handle to it.
  284.  */
  285. int
  286. i915_gem_create_ioctl(struct drm_device *dev, void *data,
  287.                       struct drm_file *file)
  288. {
  289.         struct drm_i915_gem_create *args = data;
  290.  
  291.         return i915_gem_create(file, dev,
  292.                                args->size, &args->handle);
  293. }
  294.  
  295. static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
  296. {
  297.         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
  298.  
  299.         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
  300.                 obj->tiling_mode != I915_TILING_NONE;
  301. }
  302. #if 0
  303.  
  304. static inline int
  305. __copy_to_user_swizzled(char __user *cpu_vaddr,
  306.                         const char *gpu_vaddr, int gpu_offset,
  307.                 int length)
  308. {
  309.         int ret, cpu_offset = 0;
  310.  
  311.         while (length > 0) {
  312.                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
  313.                 int this_length = min(cacheline_end - gpu_offset, length);
  314.                 int swizzled_gpu_offset = gpu_offset ^ 64;
  315.  
  316.                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
  317.                                      gpu_vaddr + swizzled_gpu_offset,
  318.                                      this_length);
  319.                 if (ret)
  320.                         return ret + length;
  321.  
  322.                 cpu_offset += this_length;
  323.                 gpu_offset += this_length;
  324.                 length -= this_length;
  325.         }
  326.  
  327.         return 0;
  328. }
  329.  
  330. static inline int
  331. __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
  332.                           const char __user *cpu_vaddr,
  333.                           int length)
  334. {
  335.         int ret, cpu_offset = 0;
  336.  
  337.         while (length > 0) {
  338.                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
  339.                 int this_length = min(cacheline_end - gpu_offset, length);
  340.                 int swizzled_gpu_offset = gpu_offset ^ 64;
  341.  
  342.                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
  343.                                cpu_vaddr + cpu_offset,
  344.                                this_length);
  345.                 if (ret)
  346.                         return ret + length;
  347.  
  348.                 cpu_offset += this_length;
  349.                 gpu_offset += this_length;
  350.                 length -= this_length;
  351.         }
  352.  
  353.         return 0;
  354. }
  355.  
  356. /* Per-page copy function for the shmem pread fastpath.
  357.  * Flushes invalid cachelines before reading the target if
  358.  * needs_clflush is set. */
  359. static int
  360. shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
  361.                  char __user *user_data,
  362.                  bool page_do_bit17_swizzling, bool needs_clflush)
  363. {
  364.                 char *vaddr;
  365.                 int ret;
  366.  
  367.         if (unlikely(page_do_bit17_swizzling))
  368.                 return -EINVAL;
  369.  
  370.                 vaddr = kmap_atomic(page);
  371.         if (needs_clflush)
  372.                 drm_clflush_virt_range(vaddr + shmem_page_offset,
  373.                                        page_length);
  374.                 ret = __copy_to_user_inatomic(user_data,
  375.                                       vaddr + shmem_page_offset,
  376.                                               page_length);
  377.                 kunmap_atomic(vaddr);
  378.  
  379.         return ret ? -EFAULT : 0;
  380. }
  381.  
  382. static void
  383. shmem_clflush_swizzled_range(char *addr, unsigned long length,
  384.                              bool swizzled)
  385. {
  386.         if (unlikely(swizzled)) {
  387.                 unsigned long start = (unsigned long) addr;
  388.                 unsigned long end = (unsigned long) addr + length;
  389.  
  390.                 /* For swizzling simply ensure that we always flush both
  391.                  * channels. Lame, but simple and it works. Swizzled
  392.                  * pwrite/pread is far from a hotpath - current userspace
  393.                  * doesn't use it at all. */
  394.                 start = round_down(start, 128);
  395.                 end = round_up(end, 128);
  396.  
  397.                 drm_clflush_virt_range((void *)start, end - start);
  398.         } else {
  399.                 drm_clflush_virt_range(addr, length);
  400.         }
  401.  
  402. }
  403.  
  404. /* Only difference to the fast-path function is that this can handle bit17
  405.  * and uses non-atomic copy and kmap functions. */
  406. static int
  407. shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
  408.                  char __user *user_data,
  409.                  bool page_do_bit17_swizzling, bool needs_clflush)
  410. {
  411.         char *vaddr;
  412.         int ret;
  413.  
  414.         vaddr = kmap(page);
  415.         if (needs_clflush)
  416.                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  417.                                              page_length,
  418.                                              page_do_bit17_swizzling);
  419.  
  420.         if (page_do_bit17_swizzling)
  421.                 ret = __copy_to_user_swizzled(user_data,
  422.                                               vaddr, shmem_page_offset,
  423.                                               page_length);
  424.         else
  425.                 ret = __copy_to_user(user_data,
  426.                                      vaddr + shmem_page_offset,
  427.                                      page_length);
  428.         kunmap(page);
  429.  
  430.         return ret ? - EFAULT : 0;
  431. }
  432.  
  433. static int
  434. i915_gem_shmem_pread(struct drm_device *dev,
  435.                           struct drm_i915_gem_object *obj,
  436.                           struct drm_i915_gem_pread *args,
  437.                           struct drm_file *file)
  438. {
  439.         char __user *user_data;
  440.         ssize_t remain;
  441.         loff_t offset;
  442.         int shmem_page_offset, page_length, ret = 0;
  443.         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
  444.         int hit_slowpath = 0;
  445.         int prefaulted = 0;
  446.         int needs_clflush = 0;
  447.         struct scatterlist *sg;
  448.         int i;
  449.  
  450.         user_data = (char __user *) (uintptr_t) args->data_ptr;
  451.         remain = args->size;
  452.  
  453.         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
  454.  
  455.         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
  456.                 /* If we're not in the cpu read domain, set ourself into the gtt
  457.                  * read domain and manually flush cachelines (if required). This
  458.                  * optimizes for the case when the gpu will dirty the data
  459.                  * anyway again before the next pread happens. */
  460.                 if (obj->cache_level == I915_CACHE_NONE)
  461.                         needs_clflush = 1;
  462.                 if (obj->gtt_space) {
  463.                         ret = i915_gem_object_set_to_gtt_domain(obj, false);
  464.                         if (ret)
  465.                                 return ret;
  466.                 }
  467.         }
  468.  
  469.         ret = i915_gem_object_get_pages(obj);
  470.         if (ret)
  471.                 return ret;
  472.  
  473.         i915_gem_object_pin_pages(obj);
  474.  
  475.         offset = args->offset;
  476.  
  477.         for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
  478.                 struct page *page;
  479.  
  480.                 if (i < offset >> PAGE_SHIFT)
  481.                         continue;
  482.  
  483.                 if (remain <= 0)
  484.                         break;
  485.  
  486.                 /* Operation in this page
  487.                  *
  488.                  * shmem_page_offset = offset within page in shmem file
  489.                  * page_length = bytes to copy for this page
  490.                  */
  491.                 shmem_page_offset = offset_in_page(offset);
  492.                 page_length = remain;
  493.                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
  494.                         page_length = PAGE_SIZE - shmem_page_offset;
  495.  
  496.                 page = sg_page(sg);
  497.                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
  498.                         (page_to_phys(page) & (1 << 17)) != 0;
  499.  
  500.                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
  501.                                        user_data, page_do_bit17_swizzling,
  502.                                        needs_clflush);
  503.                 if (ret == 0)
  504.                         goto next_page;
  505.  
  506.                 hit_slowpath = 1;
  507.                 mutex_unlock(&dev->struct_mutex);
  508.  
  509.                 if (!prefaulted) {
  510.                         ret = fault_in_multipages_writeable(user_data, remain);
  511.                         /* Userspace is tricking us, but we've already clobbered
  512.                          * its pages with the prefault and promised to write the
  513.                          * data up to the first fault. Hence ignore any errors
  514.                          * and just continue. */
  515.                         (void)ret;
  516.                         prefaulted = 1;
  517.                 }
  518.  
  519.                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
  520.                                        user_data, page_do_bit17_swizzling,
  521.                                        needs_clflush);
  522.  
  523.                 mutex_lock(&dev->struct_mutex);
  524.  
  525. next_page:
  526.                 mark_page_accessed(page);
  527.  
  528.                 if (ret)
  529.                         goto out;
  530.  
  531.                 remain -= page_length;
  532.                 user_data += page_length;
  533.                 offset += page_length;
  534.         }
  535.  
  536. out:
  537.         i915_gem_object_unpin_pages(obj);
  538.  
  539.         if (hit_slowpath) {
  540.                 /* Fixup: Kill any reinstated backing storage pages */
  541.                 if (obj->madv == __I915_MADV_PURGED)
  542.                         i915_gem_object_truncate(obj);
  543.         }
  544.  
  545.         return ret;
  546. }
  547.  
  548. /**
  549.  * Reads data from the object referenced by handle.
  550.  *
  551.  * On error, the contents of *data are undefined.
  552.  */
  553. int
  554. i915_gem_pread_ioctl(struct drm_device *dev, void *data,
  555.                      struct drm_file *file)
  556. {
  557.         struct drm_i915_gem_pread *args = data;
  558.         struct drm_i915_gem_object *obj;
  559.         int ret = 0;
  560.  
  561.         if (args->size == 0)
  562.                 return 0;
  563.  
  564.         if (!access_ok(VERIFY_WRITE,
  565.                        (char __user *)(uintptr_t)args->data_ptr,
  566.                        args->size))
  567.                 return -EFAULT;
  568.  
  569.         ret = i915_mutex_lock_interruptible(dev);
  570.         if (ret)
  571.                 return ret;
  572.  
  573.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  574.         if (&obj->base == NULL) {
  575.                 ret = -ENOENT;
  576.                 goto unlock;
  577.         }
  578.  
  579.         /* Bounds check source.  */
  580.         if (args->offset > obj->base.size ||
  581.             args->size > obj->base.size - args->offset) {
  582.                 ret = -EINVAL;
  583.                 goto out;
  584.         }
  585.  
  586.         /* prime objects have no backing filp to GEM pread/pwrite
  587.          * pages from.
  588.          */
  589.         if (!obj->base.filp) {
  590.                 ret = -EINVAL;
  591.                 goto out;
  592.         }
  593.  
  594.         trace_i915_gem_object_pread(obj, args->offset, args->size);
  595.  
  596.         ret = i915_gem_shmem_pread(dev, obj, args, file);
  597.  
  598. out:
  599.         drm_gem_object_unreference(&obj->base);
  600. unlock:
  601.         mutex_unlock(&dev->struct_mutex);
  602.         return ret;
  603. }
  604.  
  605. /* This is the fast write path which cannot handle
  606.  * page faults in the source data
  607.  */
  608.  
  609. static inline int
  610. fast_user_write(struct io_mapping *mapping,
  611.                 loff_t page_base, int page_offset,
  612.                 char __user *user_data,
  613.                 int length)
  614. {
  615.         void __iomem *vaddr_atomic;
  616.         void *vaddr;
  617.         unsigned long unwritten;
  618.  
  619.         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
  620.         /* We can use the cpu mem copy function because this is X86. */
  621.         vaddr = (void __force*)vaddr_atomic + page_offset;
  622.         unwritten = __copy_from_user_inatomic_nocache(vaddr,
  623.                                                       user_data, length);
  624.         io_mapping_unmap_atomic(vaddr_atomic);
  625.         return unwritten;
  626. }
  627. #endif
  628.  
  629. #define offset_in_page(p)       ((unsigned long)(p) & ~PAGE_MASK)
  630. /**
  631.  * This is the fast pwrite path, where we copy the data directly from the
  632.  * user into the GTT, uncached.
  633.  */
  634. static int
  635. i915_gem_gtt_pwrite_fast(struct drm_device *dev,
  636.                          struct drm_i915_gem_object *obj,
  637.                          struct drm_i915_gem_pwrite *args,
  638.                          struct drm_file *file)
  639. {
  640.         drm_i915_private_t *dev_priv = dev->dev_private;
  641.         ssize_t remain;
  642.         loff_t offset, page_base;
  643.         char __user *user_data;
  644.         int page_offset, page_length, ret;
  645.     char *vaddr;
  646.  
  647.         ret = i915_gem_object_pin(obj, 0, true, true);
  648.         if (ret)
  649.                 goto out;
  650.  
  651.         ret = i915_gem_object_set_to_gtt_domain(obj, true);
  652.         if (ret)
  653.                 goto out_unpin;
  654.  
  655.         ret = i915_gem_object_put_fence(obj);
  656.         if (ret)
  657.                 goto out_unpin;
  658.  
  659.     vaddr = AllocKernelSpace(4096);
  660.     if(vaddr == NULL)
  661.     {
  662.         ret = -ENOSPC;
  663.         goto out_unpin;
  664.     };
  665.  
  666.         user_data = (char __user *) (uintptr_t) args->data_ptr;
  667.         remain = args->size;
  668.  
  669.         offset = obj->gtt_offset + args->offset;
  670.  
  671.         while (remain > 0) {
  672.                 /* Operation in this page
  673.                  *
  674.                  * page_base = page offset within aperture
  675.                  * page_offset = offset within page
  676.                  * page_length = bytes to copy for this page
  677.                  */
  678.                 page_base = offset & PAGE_MASK;
  679.                 page_offset = offset_in_page(offset);
  680.                 page_length = remain;
  681.                 if ((page_offset + remain) > PAGE_SIZE)
  682.                         page_length = PAGE_SIZE - page_offset;
  683.  
  684.         MapPage(vaddr, page_base, PG_SW|PG_NOCACHE);
  685.  
  686.         memcpy(vaddr+page_offset, user_data, page_length);
  687.  
  688.                 remain -= page_length;
  689.                 user_data += page_length;
  690.                 offset += page_length;
  691.         }
  692.  
  693.     FreeKernelSpace(vaddr);
  694.  
  695. out_unpin:
  696.         i915_gem_object_unpin(obj);
  697. out:
  698.     printf("% s ret = %d\n", __FUNCTION__, ret);
  699.  
  700.         return ret;
  701. }
  702.  
  703. /* Per-page copy function for the shmem pwrite fastpath.
  704.  * Flushes invalid cachelines before writing to the target if
  705.  * needs_clflush_before is set and flushes out any written cachelines after
  706.  * writing if needs_clflush is set. */
  707. static int
  708. shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
  709.                   char __user *user_data,
  710.                   bool page_do_bit17_swizzling,
  711.                   bool needs_clflush_before,
  712.                   bool needs_clflush_after)
  713. {
  714.         char *vaddr;
  715.         int ret = 0;
  716.  
  717.         if (unlikely(page_do_bit17_swizzling))
  718.                 return -EINVAL;
  719.  
  720.         vaddr = (char *)MapIoMem((addr_t)page, 4096, PG_SW);
  721.         if (needs_clflush_before)
  722.                 drm_clflush_virt_range(vaddr + shmem_page_offset,
  723.                                        page_length);
  724.         memcpy(vaddr + shmem_page_offset,
  725.                                                 user_data,
  726.                                                 page_length);
  727.         if (needs_clflush_after)
  728.                 drm_clflush_virt_range(vaddr + shmem_page_offset,
  729.                                        page_length);
  730.         FreeKernelSpace(vaddr);
  731.  
  732.         return ret ? -EFAULT : 0;
  733. }
  734. #if 0
  735.  
  736. /* Only difference to the fast-path function is that this can handle bit17
  737.  * and uses non-atomic copy and kmap functions. */
  738. static int
  739. shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
  740.                   char __user *user_data,
  741.                   bool page_do_bit17_swizzling,
  742.                   bool needs_clflush_before,
  743.                   bool needs_clflush_after)
  744. {
  745.         char *vaddr;
  746.         int ret;
  747.  
  748.         vaddr = kmap(page);
  749.         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
  750.                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  751.                                              page_length,
  752.                                              page_do_bit17_swizzling);
  753.         if (page_do_bit17_swizzling)
  754.                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
  755.                                                 user_data,
  756.                                                 page_length);
  757.         else
  758.                 ret = __copy_from_user(vaddr + shmem_page_offset,
  759.                                        user_data,
  760.                                        page_length);
  761.         if (needs_clflush_after)
  762.                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  763.                                              page_length,
  764.                                              page_do_bit17_swizzling);
  765.         kunmap(page);
  766.  
  767.         return ret ? -EFAULT : 0;
  768. }
  769. #endif
  770.  
  771.  
  772. static int
  773. i915_gem_shmem_pwrite(struct drm_device *dev,
  774.                       struct drm_i915_gem_object *obj,
  775.                       struct drm_i915_gem_pwrite *args,
  776.                       struct drm_file *file)
  777. {
  778.         ssize_t remain;
  779.         loff_t offset;
  780.         char __user *user_data;
  781.         int shmem_page_offset, page_length, ret = 0;
  782.         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
  783.         int hit_slowpath = 0;
  784.         int needs_clflush_after = 0;
  785.         int needs_clflush_before = 0;
  786.         int i;
  787.         struct scatterlist *sg;
  788.  
  789.         user_data = (char __user *) (uintptr_t) args->data_ptr;
  790.         remain = args->size;
  791.  
  792.         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
  793.  
  794.         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  795.                 /* If we're not in the cpu write domain, set ourself into the gtt
  796.                  * write domain and manually flush cachelines (if required). This
  797.                  * optimizes for the case when the gpu will use the data
  798.                  * right away and we therefore have to clflush anyway. */
  799.                 if (obj->cache_level == I915_CACHE_NONE)
  800.                         needs_clflush_after = 1;
  801.                 if (obj->gtt_space) {
  802.                         ret = i915_gem_object_set_to_gtt_domain(obj, true);
  803.                         if (ret)
  804.                                 return ret;
  805.                 }
  806.         }
  807.         /* Same trick applies for invalidate partially written cachelines before
  808.          * writing.  */
  809.         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
  810.             && obj->cache_level == I915_CACHE_NONE)
  811.                 needs_clflush_before = 1;
  812.  
  813.         ret = i915_gem_object_get_pages(obj);
  814.         if (ret)
  815.                 return ret;
  816.  
  817.         i915_gem_object_pin_pages(obj);
  818.  
  819.         offset = args->offset;
  820.         obj->dirty = 1;
  821.  
  822.         for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
  823.                 struct page *page;
  824.                 int partial_cacheline_write;
  825.  
  826.                 if (i < offset >> PAGE_SHIFT)
  827.                         continue;
  828.  
  829.                 if (remain <= 0)
  830.                         break;
  831.  
  832.                 /* Operation in this page
  833.                  *
  834.                  * shmem_page_offset = offset within page in shmem file
  835.                  * page_length = bytes to copy for this page
  836.                  */
  837.                 shmem_page_offset = offset_in_page(offset);
  838.  
  839.                 page_length = remain;
  840.                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
  841.                         page_length = PAGE_SIZE - shmem_page_offset;
  842.  
  843.                 /* If we don't overwrite a cacheline completely we need to be
  844.                  * careful to have up-to-date data by first clflushing. Don't
  845.                  * overcomplicate things and flush the entire patch. */
  846.                 partial_cacheline_write = needs_clflush_before &&
  847.                         ((shmem_page_offset | page_length)
  848.                                 & (x86_clflush_size - 1));
  849.  
  850.                 page = sg_page(sg);
  851.                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
  852.                         (page_to_phys(page) & (1 << 17)) != 0;
  853.  
  854.                 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
  855.                                         user_data, page_do_bit17_swizzling,
  856.                                         partial_cacheline_write,
  857.                                         needs_clflush_after);
  858.                 if (ret == 0)
  859.                         goto next_page;
  860.  
  861.                 hit_slowpath = 1;
  862.                 mutex_unlock(&dev->struct_mutex);
  863.                 dbgprintf("%s need shmem_pwrite_slow\n",__FUNCTION__);
  864.  
  865. //              ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
  866. //                                      user_data, page_do_bit17_swizzling,
  867. //                                      partial_cacheline_write,
  868. //                                      needs_clflush_after);
  869.  
  870.                 mutex_lock(&dev->struct_mutex);
  871.  
  872. next_page:
  873.  
  874.                 if (ret)
  875.                         goto out;
  876.  
  877.                 remain -= page_length;
  878.                 user_data += page_length;
  879.                 offset += page_length;
  880.         }
  881.  
  882. out:
  883.         i915_gem_object_unpin_pages(obj);
  884.  
  885.         if (hit_slowpath) {
  886.                 /* Fixup: Kill any reinstated backing storage pages */
  887.                 if (obj->madv == __I915_MADV_PURGED)
  888.                         i915_gem_object_truncate(obj);
  889.                 /* and flush dirty cachelines in case the object isn't in the cpu write
  890.                  * domain anymore. */
  891.                 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  892.                         i915_gem_clflush_object(obj);
  893.                         i915_gem_chipset_flush(dev);
  894.                 }
  895.         }
  896.  
  897.         if (needs_clflush_after)
  898.                 i915_gem_chipset_flush(dev);
  899.  
  900.         return ret;
  901. }
  902.  
  903. /**
  904.  * Writes data to the object referenced by handle.
  905.  *
  906.  * On error, the contents of the buffer that were to be modified are undefined.
  907.  */
  908. int
  909. i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
  910.                       struct drm_file *file)
  911. {
  912.         struct drm_i915_gem_pwrite *args = data;
  913.         struct drm_i915_gem_object *obj;
  914.         int ret;
  915.  
  916.         if (args->size == 0)
  917.                 return 0;
  918.  
  919.         ret = i915_mutex_lock_interruptible(dev);
  920.         if (ret)
  921.                 return ret;
  922.  
  923.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  924.         if (&obj->base == NULL) {
  925.                 ret = -ENOENT;
  926.                 goto unlock;
  927.         }
  928.  
  929.         /* Bounds check destination. */
  930.         if (args->offset > obj->base.size ||
  931.             args->size > obj->base.size - args->offset) {
  932.                 ret = -EINVAL;
  933.                 goto out;
  934.         }
  935.  
  936.         /* prime objects have no backing filp to GEM pread/pwrite
  937.          * pages from.
  938.          */
  939.         if (!obj->base.filp) {
  940.                 ret = -EINVAL;
  941.                 goto out;
  942.         }
  943.  
  944.         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
  945.  
  946.         ret = -EFAULT;
  947.         /* We can only do the GTT pwrite on untiled buffers, as otherwise
  948.          * it would end up going through the fenced access, and we'll get
  949.          * different detiling behavior between reading and writing.
  950.          * pread/pwrite currently are reading and writing from the CPU
  951.          * perspective, requiring manual detiling by the client.
  952.          */
  953. //   if (obj->phys_obj) {
  954. //       ret = i915_gem_phys_pwrite(dev, obj, args, file);
  955. //       goto out;
  956. //   }
  957.  
  958.         if (obj->cache_level == I915_CACHE_NONE &&
  959.             obj->tiling_mode == I915_TILING_NONE &&
  960.             obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  961.                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
  962.                 /* Note that the gtt paths might fail with non-page-backed user
  963.                  * pointers (e.g. gtt mappings when moving data between
  964.                  * textures). Fallback to the shmem path in that case. */
  965.         }
  966.  
  967.         if (ret == -EFAULT || ret == -ENOSPC)
  968.        ret = i915_gem_shmem_pwrite(dev, obj, args, file);
  969.  
  970. out:
  971.         drm_gem_object_unreference(&obj->base);
  972. unlock:
  973.         mutex_unlock(&dev->struct_mutex);
  974.         return ret;
  975. }
  976.  
  977. int
  978. i915_gem_check_wedge(struct drm_i915_private *dev_priv,
  979.                      bool interruptible)
  980. {
  981.         if (atomic_read(&dev_priv->mm.wedged)) {
  982.                 struct completion *x = &dev_priv->error_completion;
  983.                 bool recovery_complete;
  984.                 unsigned long flags;
  985.  
  986.                 /* Give the error handler a chance to run. */
  987.                 spin_lock_irqsave(&x->wait.lock, flags);
  988.                 recovery_complete = x->done > 0;
  989.                 spin_unlock_irqrestore(&x->wait.lock, flags);
  990.  
  991.                 /* Non-interruptible callers can't handle -EAGAIN, hence return
  992.                  * -EIO unconditionally for these. */
  993.                 if (!interruptible)
  994.                         return -EIO;
  995.  
  996.                 /* Recovery complete, but still wedged means reset failure. */
  997.                 if (recovery_complete)
  998.                         return -EIO;
  999.  
  1000.                 return -EAGAIN;
  1001.         }
  1002.  
  1003.         return 0;
  1004. }
  1005.  
  1006. /*
  1007.  * Compare seqno against outstanding lazy request. Emit a request if they are
  1008.  * equal.
  1009.  */
  1010. static int
  1011. i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
  1012. {
  1013.         int ret;
  1014.  
  1015.         BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
  1016.  
  1017.         ret = 0;
  1018.         if (seqno == ring->outstanding_lazy_request)
  1019.                 ret = i915_add_request(ring, NULL, NULL);
  1020.  
  1021.         return ret;
  1022. }
  1023.  
  1024. /**
  1025.  * __wait_seqno - wait until execution of seqno has finished
  1026.  * @ring: the ring expected to report seqno
  1027.  * @seqno: duh!
  1028.  * @interruptible: do an interruptible wait (normally yes)
  1029.  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
  1030.  *
  1031.  * Returns 0 if the seqno was found within the alloted time. Else returns the
  1032.  * errno with remaining time filled in timeout argument.
  1033.  */
  1034. static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
  1035.                         bool interruptible, struct timespec *timeout)
  1036. {
  1037.         drm_i915_private_t *dev_priv = ring->dev->dev_private;
  1038.         struct timespec before, now, wait_time={1,0};
  1039.         unsigned long timeout_jiffies;
  1040.         long end;
  1041.         bool wait_forever = true;
  1042.         int ret;
  1043.  
  1044.         if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
  1045.                 return 0;
  1046.  
  1047.         trace_i915_gem_request_wait_begin(ring, seqno);
  1048.  
  1049.         if (timeout != NULL) {
  1050.                 wait_time = *timeout;
  1051.                 wait_forever = false;
  1052.         }
  1053.  
  1054. //   timeout_jiffies = timespec_to_jiffies(&wait_time);
  1055.  
  1056.         if (WARN_ON(!ring->irq_get(ring)))
  1057.                 return -ENODEV;
  1058. #if 0
  1059.  
  1060.     /* Record current time in case interrupted by signal, or wedged * */
  1061.         getrawmonotonic(&before);
  1062.  
  1063. #define EXIT_COND \
  1064.         (i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \
  1065.         atomic_read(&dev_priv->mm.wedged))
  1066.         do {
  1067.                         end = wait_event_timeout(ring->irq_queue, EXIT_COND,
  1068.                                                  timeout_jiffies);
  1069.  
  1070.                 ret = i915_gem_check_wedge(dev_priv, interruptible);
  1071.                 if (ret)
  1072.                         end = ret;
  1073.         } while (end == 0 && wait_forever);
  1074.  
  1075.         getrawmonotonic(&now);
  1076.  
  1077.         ring->irq_put(ring);
  1078.         trace_i915_gem_request_wait_end(ring, seqno);
  1079. #undef EXIT_COND
  1080.  
  1081.         if (timeout) {
  1082. //       struct timespec sleep_time = timespec_sub(now, before);
  1083. //       *timeout = timespec_sub(*timeout, sleep_time);
  1084.         }
  1085.  
  1086.         switch (end) {
  1087.         case -EIO:
  1088.         case -EAGAIN: /* Wedged */
  1089.         case -ERESTARTSYS: /* Signal */
  1090.                 return (int)end;
  1091.         case 0: /* Timeout */
  1092. //              if (timeout)
  1093. //                      set_normalized_timespec(timeout, 0, 0);
  1094.                 return -ETIME;
  1095.         default: /* Completed */
  1096.                 WARN_ON(end < 0); /* We're not aware of other errors */
  1097.                 return 0;
  1098.         }
  1099.  
  1100. #endif
  1101.  
  1102. #define EXIT_COND \
  1103.     (i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \
  1104.     atomic_read(&dev_priv->mm.wedged))
  1105.     wait_event(ring->irq_queue, EXIT_COND);
  1106. #undef EXIT_COND
  1107.     ring->irq_put(ring);
  1108.  
  1109.     return 0;
  1110. }
  1111.  
  1112. /**
  1113.  * Waits for a sequence number to be signaled, and cleans up the
  1114.  * request and object lists appropriately for that event.
  1115.  */
  1116. int
  1117. i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
  1118. {
  1119.         struct drm_device *dev = ring->dev;
  1120.         struct drm_i915_private *dev_priv = dev->dev_private;
  1121.         bool interruptible = dev_priv->mm.interruptible;
  1122.         int ret;
  1123.  
  1124.         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
  1125.         BUG_ON(seqno == 0);
  1126.  
  1127.         ret = i915_gem_check_wedge(dev_priv, interruptible);
  1128.         if (ret)
  1129.                 return ret;
  1130.  
  1131.         ret = i915_gem_check_olr(ring, seqno);
  1132.         if (ret)
  1133.                 return ret;
  1134.  
  1135.         return __wait_seqno(ring, seqno, interruptible, NULL);
  1136. }
  1137.  
  1138. /**
  1139.  * Ensures that all rendering to the object has completed and the object is
  1140.  * safe to unbind from the GTT or access from the CPU.
  1141.  */
  1142. static __must_check int
  1143. i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
  1144.                                bool readonly)
  1145. {
  1146.         struct intel_ring_buffer *ring = obj->ring;
  1147.         u32 seqno;
  1148.         int ret;
  1149.  
  1150.         seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
  1151.         if (seqno == 0)
  1152.                 return 0;
  1153.  
  1154.         ret = i915_wait_seqno(ring, seqno);
  1155.        if (ret)
  1156.            return ret;
  1157.  
  1158.         i915_gem_retire_requests_ring(ring);
  1159.  
  1160.         /* Manually manage the write flush as we may have not yet
  1161.          * retired the buffer.
  1162.          */
  1163.         if (obj->last_write_seqno &&
  1164.             i915_seqno_passed(seqno, obj->last_write_seqno)) {
  1165.                 obj->last_write_seqno = 0;
  1166.                 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
  1167.         }
  1168.  
  1169.         return 0;
  1170. }
  1171.  
  1172. /* A nonblocking variant of the above wait. This is a highly dangerous routine
  1173.  * as the object state may change during this call.
  1174.  */
  1175. static __must_check int
  1176. i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
  1177.                                             bool readonly)
  1178. {
  1179.         struct drm_device *dev = obj->base.dev;
  1180.         struct drm_i915_private *dev_priv = dev->dev_private;
  1181.         struct intel_ring_buffer *ring = obj->ring;
  1182.         u32 seqno;
  1183.         int ret;
  1184.  
  1185.         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
  1186.         BUG_ON(!dev_priv->mm.interruptible);
  1187.  
  1188.         seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
  1189.         if (seqno == 0)
  1190.                 return 0;
  1191.  
  1192.         ret = i915_gem_check_wedge(dev_priv, true);
  1193.         if (ret)
  1194.                 return ret;
  1195.  
  1196.         ret = i915_gem_check_olr(ring, seqno);
  1197.         if (ret)
  1198.                 return ret;
  1199.  
  1200.         mutex_unlock(&dev->struct_mutex);
  1201.         ret = __wait_seqno(ring, seqno, true, NULL);
  1202.         mutex_lock(&dev->struct_mutex);
  1203.  
  1204.         i915_gem_retire_requests_ring(ring);
  1205.  
  1206.         /* Manually manage the write flush as we may have not yet
  1207.          * retired the buffer.
  1208.          */
  1209.         if (obj->last_write_seqno &&
  1210.             i915_seqno_passed(seqno, obj->last_write_seqno)) {
  1211.                 obj->last_write_seqno = 0;
  1212.                 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
  1213.         }
  1214.  
  1215.         return ret;
  1216. }
  1217.  
  1218. /**
  1219.  * Called when user space prepares to use an object with the CPU, either
  1220.  * through the mmap ioctl's mapping or a GTT mapping.
  1221.  */
  1222. int
  1223. i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
  1224.                           struct drm_file *file)
  1225. {
  1226.         struct drm_i915_gem_set_domain *args = data;
  1227.         struct drm_i915_gem_object *obj;
  1228.         uint32_t read_domains = args->read_domains;
  1229.         uint32_t write_domain = args->write_domain;
  1230.         int ret;
  1231.  
  1232.         /* Only handle setting domains to types used by the CPU. */
  1233.         if (write_domain & I915_GEM_GPU_DOMAINS)
  1234.                 return -EINVAL;
  1235.  
  1236.         if (read_domains & I915_GEM_GPU_DOMAINS)
  1237.                 return -EINVAL;
  1238.  
  1239.         /* Having something in the write domain implies it's in the read
  1240.          * domain, and only that read domain.  Enforce that in the request.
  1241.          */
  1242.         if (write_domain != 0 && read_domains != write_domain)
  1243.                 return -EINVAL;
  1244.  
  1245.         ret = i915_mutex_lock_interruptible(dev);
  1246.         if (ret)
  1247.                 return ret;
  1248.  
  1249.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  1250.         if (&obj->base == NULL) {
  1251.                 ret = -ENOENT;
  1252.                 goto unlock;
  1253.         }
  1254.  
  1255.         /* Try to flush the object off the GPU without holding the lock.
  1256.          * We will repeat the flush holding the lock in the normal manner
  1257.          * to catch cases where we are gazumped.
  1258.          */
  1259.         ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
  1260.         if (ret)
  1261.                 goto unref;
  1262.  
  1263.         if (read_domains & I915_GEM_DOMAIN_GTT) {
  1264.                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
  1265.  
  1266.                 /* Silently promote "you're not bound, there was nothing to do"
  1267.                  * to success, since the client was just asking us to
  1268.                  * make sure everything was done.
  1269.                  */
  1270.                 if (ret == -EINVAL)
  1271.                         ret = 0;
  1272.         } else {
  1273.                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
  1274.         }
  1275.  
  1276. unref:
  1277.         drm_gem_object_unreference(&obj->base);
  1278. unlock:
  1279.         mutex_unlock(&dev->struct_mutex);
  1280.         return ret;
  1281. }
  1282.  
  1283.  
  1284.  
  1285.  
  1286.  
  1287.  
  1288. /**
  1289.  * Maps the contents of an object, returning the address it is mapped
  1290.  * into.
  1291.  *
  1292.  * While the mapping holds a reference on the contents of the object, it doesn't
  1293.  * imply a ref on the object itself.
  1294.  */
  1295. int
  1296. i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
  1297.                     struct drm_file *file)
  1298. {
  1299.         struct drm_i915_gem_mmap *args = data;
  1300.         struct drm_gem_object *obj;
  1301.         unsigned long addr = 0;
  1302.  
  1303.         obj = drm_gem_object_lookup(dev, file, args->handle);
  1304.         if (obj == NULL)
  1305.                 return -ENOENT;
  1306.        
  1307.     dbgprintf("%s offset %lld size %lld\n",
  1308.                 __FUNCTION__, args->offset, args->size);
  1309.         /* prime objects have no backing filp to GEM mmap
  1310.          * pages from.
  1311.          */
  1312.         if (!obj->filp) {
  1313.                 drm_gem_object_unreference_unlocked(obj);
  1314.                 return -EINVAL;
  1315.         }
  1316.  
  1317.     addr = vm_mmap(obj->filp, 0, args->size,
  1318.               PROT_READ | PROT_WRITE, MAP_SHARED,
  1319.               args->offset);
  1320.         drm_gem_object_unreference_unlocked(obj);
  1321.     if (IS_ERR((void *)addr))
  1322.         return addr;
  1323.  
  1324.         args->addr_ptr = (uint64_t) addr;
  1325.  
  1326.     return 0;
  1327. }
  1328.  
  1329.  
  1330.  
  1331.  
  1332.  
  1333.  
  1334.  
  1335.  
  1336.  
  1337.  
  1338.  
  1339.  
  1340.  
  1341. /**
  1342.  * i915_gem_release_mmap - remove physical page mappings
  1343.  * @obj: obj in question
  1344.  *
  1345.  * Preserve the reservation of the mmapping with the DRM core code, but
  1346.  * relinquish ownership of the pages back to the system.
  1347.  *
  1348.  * It is vital that we remove the page mapping if we have mapped a tiled
  1349.  * object through the GTT and then lose the fence register due to
  1350.  * resource pressure. Similarly if the object has been moved out of the
  1351.  * aperture, than pages mapped into userspace must be revoked. Removing the
  1352.  * mapping will then trigger a page fault on the next user access, allowing
  1353.  * fixup by i915_gem_fault().
  1354.  */
  1355. void
  1356. i915_gem_release_mmap(struct drm_i915_gem_object *obj)
  1357. {
  1358.         if (!obj->fault_mappable)
  1359.                 return;
  1360.  
  1361.         if (obj->base.dev->dev_mapping)
  1362. //              unmap_mapping_range(obj->base.dev->dev_mapping,
  1363. //                                  (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
  1364. //                                  obj->base.size, 1);
  1365.  
  1366.         obj->fault_mappable = false;
  1367. }
  1368.  
  1369. static uint32_t
  1370. i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
  1371. {
  1372.         uint32_t gtt_size;
  1373.  
  1374.         if (INTEL_INFO(dev)->gen >= 4 ||
  1375.             tiling_mode == I915_TILING_NONE)
  1376.                 return size;
  1377.  
  1378.         /* Previous chips need a power-of-two fence region when tiling */
  1379.         if (INTEL_INFO(dev)->gen == 3)
  1380.                 gtt_size = 1024*1024;
  1381.         else
  1382.                 gtt_size = 512*1024;
  1383.  
  1384.         while (gtt_size < size)
  1385.                 gtt_size <<= 1;
  1386.  
  1387.         return gtt_size;
  1388. }
  1389.  
  1390. /**
  1391.  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
  1392.  * @obj: object to check
  1393.  *
  1394.  * Return the required GTT alignment for an object, taking into account
  1395.  * potential fence register mapping.
  1396.  */
  1397. static uint32_t
  1398. i915_gem_get_gtt_alignment(struct drm_device *dev,
  1399.                            uint32_t size,
  1400.                            int tiling_mode)
  1401. {
  1402.         /*
  1403.          * Minimum alignment is 4k (GTT page size), but might be greater
  1404.          * if a fence register is needed for the object.
  1405.          */
  1406.         if (INTEL_INFO(dev)->gen >= 4 ||
  1407.             tiling_mode == I915_TILING_NONE)
  1408.                 return 4096;
  1409.  
  1410.         /*
  1411.          * Previous chips need to be aligned to the size of the smallest
  1412.          * fence register that can contain the object.
  1413.          */
  1414.         return i915_gem_get_gtt_size(dev, size, tiling_mode);
  1415. }
  1416.  
  1417. /**
  1418.  * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
  1419.  *                                       unfenced object
  1420.  * @dev: the device
  1421.  * @size: size of the object
  1422.  * @tiling_mode: tiling mode of the object
  1423.  *
  1424.  * Return the required GTT alignment for an object, only taking into account
  1425.  * unfenced tiled surface requirements.
  1426.  */
  1427. uint32_t
  1428. i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
  1429.                                     uint32_t size,
  1430.                                     int tiling_mode)
  1431. {
  1432.         /*
  1433.          * Minimum alignment is 4k (GTT page size) for sane hw.
  1434.          */
  1435.         if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
  1436.             tiling_mode == I915_TILING_NONE)
  1437.                 return 4096;
  1438.  
  1439.         /* Previous hardware however needs to be aligned to a power-of-two
  1440.          * tile height. The simplest method for determining this is to reuse
  1441.          * the power-of-tile object size.
  1442.          */
  1443.         return i915_gem_get_gtt_size(dev, size, tiling_mode);
  1444. }
  1445.  
  1446. /* Immediately discard the backing storage */
  1447. static void
  1448. i915_gem_object_truncate(struct drm_i915_gem_object *obj)
  1449. {
  1450. //      struct inode *inode;
  1451.  
  1452. //      i915_gem_object_free_mmap_offset(obj);
  1453.  
  1454.         if (obj->base.filp == NULL)
  1455.                 return;
  1456.  
  1457.         /* Our goal here is to return as much of the memory as
  1458.          * is possible back to the system as we are called from OOM.
  1459.          * To do this we must instruct the shmfs to drop all of its
  1460.          * backing pages, *now*.
  1461.          */
  1462. //      inode = obj->base.filp->f_path.dentry->d_inode;
  1463. //      shmem_truncate_range(inode, 0, (loff_t)-1);
  1464.  
  1465.         obj->madv = __I915_MADV_PURGED;
  1466. }
  1467.  
  1468. static inline int
  1469. i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
  1470. {
  1471.         return obj->madv == I915_MADV_DONTNEED;
  1472. }
  1473.  
  1474. static void
  1475. i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
  1476. {
  1477.         int page_count = obj->base.size / PAGE_SIZE;
  1478.         struct scatterlist *sg;
  1479.         int ret, i;
  1480.  
  1481.         BUG_ON(obj->madv == __I915_MADV_PURGED);
  1482.  
  1483.         ret = i915_gem_object_set_to_cpu_domain(obj, true);
  1484.         if (ret) {
  1485.                 /* In the event of a disaster, abandon all caches and
  1486.                  * hope for the best.
  1487.                  */
  1488.                 WARN_ON(ret != -EIO);
  1489.                 i915_gem_clflush_object(obj);
  1490.                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  1491.         }
  1492.  
  1493.         if (obj->madv == I915_MADV_DONTNEED)
  1494.                 obj->dirty = 0;
  1495.  
  1496.         for_each_sg(obj->pages->sgl, sg, page_count, i) {
  1497.                 struct page *page = sg_page(sg);
  1498.  
  1499.  
  1500.  
  1501. //       page_cache_release(page);
  1502.         }
  1503.     //DRM_DEBUG_KMS("%s release %d pages\n", __FUNCTION__, page_count);
  1504.         obj->dirty = 0;
  1505.  
  1506.         sg_free_table(obj->pages);
  1507.         kfree(obj->pages);
  1508. }
  1509.  
  1510. static int
  1511. i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
  1512. {
  1513.         const struct drm_i915_gem_object_ops *ops = obj->ops;
  1514.  
  1515.         if (obj->pages == NULL)
  1516.                 return 0;
  1517.  
  1518.         BUG_ON(obj->gtt_space);
  1519.  
  1520.         if (obj->pages_pin_count)
  1521.                 return -EBUSY;
  1522.  
  1523.         /* ->put_pages might need to allocate memory for the bit17 swizzle
  1524.          * array, hence protect them from being reaped by removing them from gtt
  1525.          * lists early. */
  1526.         list_del(&obj->gtt_list);
  1527.  
  1528.         ops->put_pages(obj);
  1529.         obj->pages = NULL;
  1530.  
  1531.         if (i915_gem_object_is_purgeable(obj))
  1532.                 i915_gem_object_truncate(obj);
  1533.  
  1534.         return 0;
  1535. }
  1536.  
  1537.  
  1538.  
  1539.  
  1540.  
  1541.  
  1542.  
  1543.  
  1544. static int
  1545. i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
  1546. {
  1547.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  1548.     int page_count, i;
  1549.         struct address_space *mapping;
  1550.         struct sg_table *st;
  1551.         struct scatterlist *sg;
  1552.         struct page *page;
  1553.         gfp_t gfp;
  1554.  
  1555.         /* Assert that the object is not currently in any GPU domain. As it
  1556.          * wasn't in the GTT, there shouldn't be any way it could have been in
  1557.          * a GPU cache
  1558.          */
  1559.         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
  1560.         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
  1561.  
  1562.         st = kmalloc(sizeof(*st), GFP_KERNEL);
  1563.         if (st == NULL)
  1564.                 return -ENOMEM;
  1565.  
  1566.         page_count = obj->base.size / PAGE_SIZE;
  1567.         if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
  1568.                 sg_free_table(st);
  1569.                 kfree(st);
  1570.                 return -ENOMEM;
  1571.         }
  1572.  
  1573.         /* Get the list of pages out of our struct file.  They'll be pinned
  1574.          * at this point until we release them.
  1575.          *
  1576.          * Fail silently without starting the shrinker
  1577.          */
  1578.         for_each_sg(st->sgl, sg, page_count, i) {
  1579.                 page = shmem_read_mapping_page_gfp(obj->base.filp, i, gfp);
  1580.                 if (IS_ERR(page)) {
  1581.             dbgprintf("%s invalid page %p\n", __FUNCTION__, page);
  1582.                         goto err_pages;
  1583.  
  1584.                 }
  1585.                 sg_set_page(sg, page, PAGE_SIZE, 0);
  1586.         }
  1587.  
  1588.         obj->pages = st;
  1589.  
  1590.     DRM_DEBUG_KMS("%s alloc %d pages\n", __FUNCTION__, page_count);
  1591.  
  1592.         return 0;
  1593.  
  1594. err_pages:
  1595.         for_each_sg(st->sgl, sg, i, page_count)
  1596.                 page_cache_release(sg_page(sg));
  1597.         sg_free_table(st);
  1598.         kfree(st);
  1599.         return PTR_ERR(page);
  1600. }
  1601.  
  1602. /* Ensure that the associated pages are gathered from the backing storage
  1603.  * and pinned into our object. i915_gem_object_get_pages() may be called
  1604.  * multiple times before they are released by a single call to
  1605.  * i915_gem_object_put_pages() - once the pages are no longer referenced
  1606.  * either as a result of memory pressure (reaping pages under the shrinker)
  1607.  * or as the object is itself released.
  1608.  */
  1609. int
  1610. i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
  1611. {
  1612.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  1613.         const struct drm_i915_gem_object_ops *ops = obj->ops;
  1614.         int ret;
  1615.  
  1616.         if (obj->pages)
  1617.                 return 0;
  1618.  
  1619.         BUG_ON(obj->pages_pin_count);
  1620.  
  1621.         ret = ops->get_pages(obj);
  1622.         if (ret)
  1623.                 return ret;
  1624.  
  1625.         list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
  1626.     return 0;
  1627. }
  1628.  
  1629. void
  1630. i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
  1631.                                struct intel_ring_buffer *ring)
  1632. {
  1633.         struct drm_device *dev = obj->base.dev;
  1634.         struct drm_i915_private *dev_priv = dev->dev_private;
  1635.         u32 seqno = intel_ring_get_seqno(ring);
  1636.  
  1637.         BUG_ON(ring == NULL);
  1638.         obj->ring = ring;
  1639.  
  1640.         /* Add a reference if we're newly entering the active list. */
  1641.         if (!obj->active) {
  1642.                 drm_gem_object_reference(&obj->base);
  1643.                 obj->active = 1;
  1644.         }
  1645.  
  1646.         /* Move from whatever list we were on to the tail of execution. */
  1647.         list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
  1648.         list_move_tail(&obj->ring_list, &ring->active_list);
  1649.  
  1650.         obj->last_read_seqno = seqno;
  1651.  
  1652.         if (obj->fenced_gpu_access) {
  1653.                 obj->last_fenced_seqno = seqno;
  1654.  
  1655.                 /* Bump MRU to take account of the delayed flush */
  1656.                 if (obj->fence_reg != I915_FENCE_REG_NONE) {
  1657.                 struct drm_i915_fence_reg *reg;
  1658.  
  1659.                 reg = &dev_priv->fence_regs[obj->fence_reg];
  1660.                         list_move_tail(&reg->lru_list,
  1661.                                        &dev_priv->mm.fence_list);
  1662.                 }
  1663.         }
  1664. }
  1665.  
  1666. static void
  1667. i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
  1668. {
  1669.         struct drm_device *dev = obj->base.dev;
  1670.         struct drm_i915_private *dev_priv = dev->dev_private;
  1671.  
  1672.         BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
  1673.         BUG_ON(!obj->active);
  1674.  
  1675.         if (obj->pin_count) /* are we a framebuffer? */
  1676.                 intel_mark_fb_idle(obj);
  1677.  
  1678.                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
  1679.  
  1680.         list_del_init(&obj->ring_list);
  1681.         obj->ring = NULL;
  1682.  
  1683.         obj->last_read_seqno = 0;
  1684.         obj->last_write_seqno = 0;
  1685.         obj->base.write_domain = 0;
  1686.  
  1687.         obj->last_fenced_seqno = 0;
  1688.         obj->fenced_gpu_access = false;
  1689.  
  1690.         obj->active = 0;
  1691.         drm_gem_object_unreference(&obj->base);
  1692.  
  1693.         WARN_ON(i915_verify_lists(dev));
  1694. }
  1695.  
  1696. static int
  1697. i915_gem_handle_seqno_wrap(struct drm_device *dev)
  1698. {
  1699.         struct drm_i915_private *dev_priv = dev->dev_private;
  1700.         struct intel_ring_buffer *ring;
  1701.         int ret, i, j;
  1702.  
  1703.         /* The hardware uses various monotonic 32-bit counters, if we
  1704.          * detect that they will wraparound we need to idle the GPU
  1705.          * and reset those counters.
  1706.          */
  1707.         ret = 0;
  1708.         for_each_ring(ring, dev_priv, i) {
  1709.                 for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
  1710.                         ret |= ring->sync_seqno[j] != 0;
  1711.         }
  1712.         if (ret == 0)
  1713.                 return ret;
  1714.  
  1715.         ret = i915_gpu_idle(dev);
  1716.         if (ret)
  1717.                 return ret;
  1718.  
  1719.         i915_gem_retire_requests(dev);
  1720.         for_each_ring(ring, dev_priv, i) {
  1721.                 for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
  1722.                         ring->sync_seqno[j] = 0;
  1723.         }
  1724.  
  1725.         return 0;
  1726. }
  1727.  
  1728. int
  1729. i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
  1730. {
  1731.         struct drm_i915_private *dev_priv = dev->dev_private;
  1732.  
  1733.         /* reserve 0 for non-seqno */
  1734.         if (dev_priv->next_seqno == 0) {
  1735.                 int ret = i915_gem_handle_seqno_wrap(dev);
  1736.                 if (ret)
  1737.                         return ret;
  1738.  
  1739.                 dev_priv->next_seqno = 1;
  1740.         }
  1741.  
  1742.         *seqno = dev_priv->next_seqno++;
  1743.         return 0;
  1744. }
  1745.  
  1746. int
  1747. i915_add_request(struct intel_ring_buffer *ring,
  1748.                  struct drm_file *file,
  1749.                  u32 *out_seqno)
  1750. {
  1751.         drm_i915_private_t *dev_priv = ring->dev->dev_private;
  1752.         struct drm_i915_gem_request *request;
  1753.         u32 request_ring_position;
  1754.         int was_empty;
  1755.         int ret;
  1756.  
  1757.         /*
  1758.          * Emit any outstanding flushes - execbuf can fail to emit the flush
  1759.          * after having emitted the batchbuffer command. Hence we need to fix
  1760.          * things up similar to emitting the lazy request. The difference here
  1761.          * is that the flush _must_ happen before the next request, no matter
  1762.          * what.
  1763.          */
  1764.         ret = intel_ring_flush_all_caches(ring);
  1765.         if (ret)
  1766.                 return ret;
  1767.  
  1768.         request = kmalloc(sizeof(*request), GFP_KERNEL);
  1769.         if (request == NULL)
  1770.                 return -ENOMEM;
  1771.  
  1772.  
  1773.         /* Record the position of the start of the request so that
  1774.          * should we detect the updated seqno part-way through the
  1775.          * GPU processing the request, we never over-estimate the
  1776.          * position of the head.
  1777.          */
  1778.         request_ring_position = intel_ring_get_tail(ring);
  1779.  
  1780.         ret = ring->add_request(ring);
  1781.         if (ret) {
  1782.                 kfree(request);
  1783.             return ret;
  1784.         }
  1785.  
  1786.         request->seqno = intel_ring_get_seqno(ring);
  1787.         request->ring = ring;
  1788.         request->tail = request_ring_position;
  1789.     request->emitted_jiffies = GetTimerTicks();
  1790.         was_empty = list_empty(&ring->request_list);
  1791.         list_add_tail(&request->list, &ring->request_list);
  1792.         request->file_priv = NULL;
  1793.  
  1794.         if (file) {
  1795.                 struct drm_i915_file_private *file_priv = file->driver_priv;
  1796.  
  1797.                 spin_lock(&file_priv->mm.lock);
  1798.                 request->file_priv = file_priv;
  1799.                 list_add_tail(&request->client_list,
  1800.                               &file_priv->mm.request_list);
  1801.                 spin_unlock(&file_priv->mm.lock);
  1802.         }
  1803.  
  1804.         trace_i915_gem_request_add(ring, request->seqno);
  1805.         ring->outstanding_lazy_request = 0;
  1806.  
  1807.         if (!dev_priv->mm.suspended) {
  1808.                 if (i915_enable_hangcheck) {
  1809. //                      mod_timer(&dev_priv->hangcheck_timer,
  1810. //                                jiffies +
  1811. //                                msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
  1812.                 }
  1813.                 if (was_empty) {
  1814.            queue_delayed_work(dev_priv->wq,
  1815.                       &dev_priv->mm.retire_work, HZ);
  1816.                         intel_mark_busy(dev_priv->dev);
  1817.                 }
  1818.         }
  1819.  
  1820.         if (out_seqno)
  1821.                 *out_seqno = request->seqno;
  1822.         return 0;
  1823. }
  1824.  
  1825. static inline void
  1826. i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
  1827. {
  1828.         struct drm_i915_file_private *file_priv = request->file_priv;
  1829.  
  1830.         if (!file_priv)
  1831.                 return;
  1832.  
  1833.         spin_lock(&file_priv->mm.lock);
  1834.         if (request->file_priv) {
  1835.                 list_del(&request->client_list);
  1836.                 request->file_priv = NULL;
  1837.         }
  1838.         spin_unlock(&file_priv->mm.lock);
  1839. }
  1840.  
  1841. static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
  1842.                                       struct intel_ring_buffer *ring)
  1843. {
  1844.         while (!list_empty(&ring->request_list)) {
  1845.                 struct drm_i915_gem_request *request;
  1846.  
  1847.                 request = list_first_entry(&ring->request_list,
  1848.                                            struct drm_i915_gem_request,
  1849.                                            list);
  1850.  
  1851.                 list_del(&request->list);
  1852.                 i915_gem_request_remove_from_client(request);
  1853.                 kfree(request);
  1854.         }
  1855.  
  1856.         while (!list_empty(&ring->active_list)) {
  1857.                 struct drm_i915_gem_object *obj;
  1858.  
  1859.                 obj = list_first_entry(&ring->active_list,
  1860.                                        struct drm_i915_gem_object,
  1861.                                        ring_list);
  1862.  
  1863.                 i915_gem_object_move_to_inactive(obj);
  1864.         }
  1865. }
  1866.  
  1867. static void i915_gem_reset_fences(struct drm_device *dev)
  1868. {
  1869.         struct drm_i915_private *dev_priv = dev->dev_private;
  1870.         int i;
  1871.  
  1872.         for (i = 0; i < dev_priv->num_fence_regs; i++) {
  1873.                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
  1874.  
  1875.                 i915_gem_write_fence(dev, i, NULL);
  1876.  
  1877.                 if (reg->obj)
  1878.                         i915_gem_object_fence_lost(reg->obj);
  1879.  
  1880.                 reg->pin_count = 0;
  1881.                 reg->obj = NULL;
  1882.                 INIT_LIST_HEAD(&reg->lru_list);
  1883.         }
  1884.  
  1885.         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
  1886. }
  1887.  
  1888. void i915_gem_reset(struct drm_device *dev)
  1889. {
  1890.         struct drm_i915_private *dev_priv = dev->dev_private;
  1891.         struct drm_i915_gem_object *obj;
  1892.         struct intel_ring_buffer *ring;
  1893.         int i;
  1894.  
  1895.         for_each_ring(ring, dev_priv, i)
  1896.                 i915_gem_reset_ring_lists(dev_priv, ring);
  1897.  
  1898.         /* Move everything out of the GPU domains to ensure we do any
  1899.          * necessary invalidation upon reuse.
  1900.          */
  1901.         list_for_each_entry(obj,
  1902.                             &dev_priv->mm.inactive_list,
  1903.                             mm_list)
  1904.         {
  1905.                 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
  1906.         }
  1907.  
  1908.         /* The fence registers are invalidated so clear them out */
  1909.         i915_gem_reset_fences(dev);
  1910. }
  1911.  
  1912. /**
  1913.  * This function clears the request list as sequence numbers are passed.
  1914.  */
  1915. void
  1916. i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
  1917. {
  1918.         uint32_t seqno;
  1919.  
  1920.     ENTER();
  1921.  
  1922.         if (list_empty(&ring->request_list))
  1923.                 return;
  1924.  
  1925.         WARN_ON(i915_verify_lists(ring->dev));
  1926.  
  1927.         seqno = ring->get_seqno(ring, true);
  1928.  
  1929.         while (!list_empty(&ring->request_list)) {
  1930.                 struct drm_i915_gem_request *request;
  1931.  
  1932.                 request = list_first_entry(&ring->request_list,
  1933.                                            struct drm_i915_gem_request,
  1934.                                            list);
  1935.  
  1936.                 if (!i915_seqno_passed(seqno, request->seqno))
  1937.                         break;
  1938.  
  1939.                 trace_i915_gem_request_retire(ring, request->seqno);
  1940.                 /* We know the GPU must have read the request to have
  1941.                  * sent us the seqno + interrupt, so use the position
  1942.                  * of tail of the request to update the last known position
  1943.                  * of the GPU head.
  1944.                  */
  1945.                 ring->last_retired_head = request->tail;
  1946.  
  1947.                 list_del(&request->list);
  1948.                 i915_gem_request_remove_from_client(request);
  1949.                 kfree(request);
  1950.         }
  1951.  
  1952.         /* Move any buffers on the active list that are no longer referenced
  1953.          * by the ringbuffer to the flushing/inactive lists as appropriate.
  1954.          */
  1955.         while (!list_empty(&ring->active_list)) {
  1956.                 struct drm_i915_gem_object *obj;
  1957.  
  1958.                 obj = list_first_entry(&ring->active_list,
  1959.                                       struct drm_i915_gem_object,
  1960.                                       ring_list);
  1961.  
  1962.                 if (!i915_seqno_passed(seqno, obj->last_read_seqno))
  1963.                         break;
  1964.  
  1965.                         i915_gem_object_move_to_inactive(obj);
  1966.         }
  1967.  
  1968.         if (unlikely(ring->trace_irq_seqno &&
  1969.                      i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
  1970.                 ring->irq_put(ring);
  1971.                 ring->trace_irq_seqno = 0;
  1972.         }
  1973.  
  1974.         WARN_ON(i915_verify_lists(ring->dev));
  1975.     LEAVE();
  1976. }
  1977.  
  1978. void
  1979. i915_gem_retire_requests(struct drm_device *dev)
  1980. {
  1981.         drm_i915_private_t *dev_priv = dev->dev_private;
  1982.         struct intel_ring_buffer *ring;
  1983.         int i;
  1984.  
  1985.         for_each_ring(ring, dev_priv, i)
  1986.                 i915_gem_retire_requests_ring(ring);
  1987. }
  1988.  
  1989. static void
  1990. i915_gem_retire_work_handler(struct work_struct *work)
  1991. {
  1992.         drm_i915_private_t *dev_priv;
  1993.         struct drm_device *dev;
  1994.         struct intel_ring_buffer *ring;
  1995.         bool idle;
  1996.         int i;
  1997.  
  1998.     ENTER();
  1999.  
  2000.         dev_priv = container_of(work, drm_i915_private_t,
  2001.                                 mm.retire_work.work);
  2002.         dev = dev_priv->dev;
  2003.  
  2004.         /* Come back later if the device is busy... */
  2005.         if (!mutex_trylock(&dev->struct_mutex)) {
  2006.         queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
  2007.         return;
  2008.         }
  2009.  
  2010.         i915_gem_retire_requests(dev);
  2011.  
  2012.         /* Send a periodic flush down the ring so we don't hold onto GEM
  2013.          * objects indefinitely.
  2014.          */
  2015.         idle = true;
  2016.         for_each_ring(ring, dev_priv, i) {
  2017.                 if (ring->gpu_caches_dirty)
  2018.                         i915_add_request(ring, NULL, NULL);
  2019.  
  2020.                 idle &= list_empty(&ring->request_list);
  2021.         }
  2022.  
  2023.    if (!dev_priv->mm.suspended && !idle)
  2024.        queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
  2025.         if (idle)
  2026.                 intel_mark_idle(dev);
  2027.  
  2028.         mutex_unlock(&dev->struct_mutex);
  2029.  
  2030.     LEAVE();
  2031. }
  2032.  
  2033. /**
  2034.  * Ensures that an object will eventually get non-busy by flushing any required
  2035.  * write domains, emitting any outstanding lazy request and retiring and
  2036.  * completed requests.
  2037.  */
  2038. static int
  2039. i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
  2040. {
  2041.         int ret;
  2042.  
  2043.         if (obj->active) {
  2044.                 ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
  2045.                 if (ret)
  2046.                         return ret;
  2047.  
  2048.                 i915_gem_retire_requests_ring(obj->ring);
  2049.         }
  2050.  
  2051.         return 0;
  2052. }
  2053.  
  2054. /**
  2055.  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
  2056.  * @DRM_IOCTL_ARGS: standard ioctl arguments
  2057.  *
  2058.  * Returns 0 if successful, else an error is returned with the remaining time in
  2059.  * the timeout parameter.
  2060.  *  -ETIME: object is still busy after timeout
  2061.  *  -ERESTARTSYS: signal interrupted the wait
  2062.  *  -ENONENT: object doesn't exist
  2063.  * Also possible, but rare:
  2064.  *  -EAGAIN: GPU wedged
  2065.  *  -ENOMEM: damn
  2066.  *  -ENODEV: Internal IRQ fail
  2067.  *  -E?: The add request failed
  2068.  *
  2069.  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
  2070.  * non-zero timeout parameter the wait ioctl will wait for the given number of
  2071.  * nanoseconds on an object becoming unbusy. Since the wait itself does so
  2072.  * without holding struct_mutex the object may become re-busied before this
  2073.  * function completes. A similar but shorter * race condition exists in the busy
  2074.  * ioctl
  2075.  */
  2076.  
  2077.  
  2078.  
  2079.  
  2080.  
  2081.  
  2082.  
  2083.  
  2084.  
  2085.  
  2086.  
  2087.  
  2088. /**
  2089.  * i915_gem_object_sync - sync an object to a ring.
  2090.  *
  2091.  * @obj: object which may be in use on another ring.
  2092.  * @to: ring we wish to use the object on. May be NULL.
  2093.  *
  2094.  * This code is meant to abstract object synchronization with the GPU.
  2095.  * Calling with NULL implies synchronizing the object with the CPU
  2096.  * rather than a particular GPU ring.
  2097.  *
  2098.  * Returns 0 if successful, else propagates up the lower layer error.
  2099.  */
  2100. int
  2101. i915_gem_object_sync(struct drm_i915_gem_object *obj,
  2102.                      struct intel_ring_buffer *to)
  2103. {
  2104.         struct intel_ring_buffer *from = obj->ring;
  2105.         u32 seqno;
  2106.         int ret, idx;
  2107.  
  2108.         if (from == NULL || to == from)
  2109.                 return 0;
  2110.  
  2111.         if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
  2112.                 return i915_gem_object_wait_rendering(obj, false);
  2113.  
  2114.         idx = intel_ring_sync_index(from, to);
  2115.  
  2116.         seqno = obj->last_read_seqno;
  2117.         if (seqno <= from->sync_seqno[idx])
  2118.                 return 0;
  2119.  
  2120.         ret = i915_gem_check_olr(obj->ring, seqno);
  2121.         if (ret)
  2122.                 return ret;
  2123.  
  2124.         ret = to->sync_to(to, from, seqno);
  2125.         if (!ret)
  2126.                 /* We use last_read_seqno because sync_to()
  2127.                  * might have just caused seqno wrap under
  2128.                  * the radar.
  2129.                  */
  2130.                 from->sync_seqno[idx] = obj->last_read_seqno;
  2131.  
  2132.         return ret;
  2133. }
  2134.  
  2135. static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
  2136. {
  2137.         u32 old_write_domain, old_read_domains;
  2138.  
  2139.         /* Act a barrier for all accesses through the GTT */
  2140.         mb();
  2141.  
  2142.         /* Force a pagefault for domain tracking on next user access */
  2143. //      i915_gem_release_mmap(obj);
  2144.  
  2145.         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
  2146.                 return;
  2147.  
  2148.         old_read_domains = obj->base.read_domains;
  2149.         old_write_domain = obj->base.write_domain;
  2150.  
  2151.         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
  2152.         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
  2153.  
  2154.         trace_i915_gem_object_change_domain(obj,
  2155.                                             old_read_domains,
  2156.                                             old_write_domain);
  2157. }
  2158.  
  2159. /**
  2160.  * Unbinds an object from the GTT aperture.
  2161.  */
  2162. int
  2163. i915_gem_object_unbind(struct drm_i915_gem_object *obj)
  2164. {
  2165.         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
  2166.         int ret = 0;
  2167.  
  2168.     if(obj == get_fb_obj())
  2169.         return 0;
  2170.  
  2171.         if (obj->gtt_space == NULL)
  2172.                 return 0;
  2173.  
  2174.         if (obj->pin_count)
  2175.                 return -EBUSY;
  2176.  
  2177.         BUG_ON(obj->pages == NULL);
  2178.  
  2179.         ret = i915_gem_object_finish_gpu(obj);
  2180.         if (ret)
  2181.                 return ret;
  2182.         /* Continue on if we fail due to EIO, the GPU is hung so we
  2183.          * should be safe and we need to cleanup or else we might
  2184.          * cause memory corruption through use-after-free.
  2185.          */
  2186.  
  2187.         i915_gem_object_finish_gtt(obj);
  2188.  
  2189.         /* release the fence reg _after_ flushing */
  2190.         ret = i915_gem_object_put_fence(obj);
  2191.         if (ret)
  2192.                 return ret;
  2193.  
  2194.         trace_i915_gem_object_unbind(obj);
  2195.  
  2196.         if (obj->has_global_gtt_mapping)
  2197.         i915_gem_gtt_unbind_object(obj);
  2198.         if (obj->has_aliasing_ppgtt_mapping) {
  2199.                 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
  2200.                 obj->has_aliasing_ppgtt_mapping = 0;
  2201.         }
  2202.         i915_gem_gtt_finish_object(obj);
  2203.  
  2204.         list_del(&obj->mm_list);
  2205.         list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
  2206.         /* Avoid an unnecessary call to unbind on rebind. */
  2207.         obj->map_and_fenceable = true;
  2208.  
  2209.         drm_mm_put_block(obj->gtt_space);
  2210.         obj->gtt_space = NULL;
  2211.         obj->gtt_offset = 0;
  2212.  
  2213.         return 0;
  2214. }
  2215.  
  2216. int i915_gpu_idle(struct drm_device *dev)
  2217. {
  2218.         drm_i915_private_t *dev_priv = dev->dev_private;
  2219.         struct intel_ring_buffer *ring;
  2220.         int ret, i;
  2221.  
  2222.         /* Flush everything onto the inactive list. */
  2223.         for_each_ring(ring, dev_priv, i) {
  2224.                 ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
  2225.                 if (ret)
  2226.                         return ret;
  2227.  
  2228.                 ret = intel_ring_idle(ring);
  2229.                 if (ret)
  2230.                         return ret;
  2231.         }
  2232.  
  2233.         return 0;
  2234. }
  2235.  
  2236. static void sandybridge_write_fence_reg(struct drm_device *dev, int reg,
  2237.                                         struct drm_i915_gem_object *obj)
  2238. {
  2239.         drm_i915_private_t *dev_priv = dev->dev_private;
  2240.         uint64_t val;
  2241.  
  2242.         if (obj) {
  2243.                 u32 size = obj->gtt_space->size;
  2244.  
  2245.                 val = (uint64_t)((obj->gtt_offset + size - 4096) &
  2246.                                  0xfffff000) << 32;
  2247.                 val |= obj->gtt_offset & 0xfffff000;
  2248.                 val |= (uint64_t)((obj->stride / 128) - 1) <<
  2249.                         SANDYBRIDGE_FENCE_PITCH_SHIFT;
  2250.  
  2251.                 if (obj->tiling_mode == I915_TILING_Y)
  2252.                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
  2253.                 val |= I965_FENCE_REG_VALID;
  2254.         } else
  2255.                 val = 0;
  2256.  
  2257.         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val);
  2258.         POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8);
  2259. }
  2260.  
  2261. static void i965_write_fence_reg(struct drm_device *dev, int reg,
  2262.                                  struct drm_i915_gem_object *obj)
  2263. {
  2264.         drm_i915_private_t *dev_priv = dev->dev_private;
  2265.         uint64_t val;
  2266.  
  2267.         if (obj) {
  2268.                 u32 size = obj->gtt_space->size;
  2269.  
  2270.                 val = (uint64_t)((obj->gtt_offset + size - 4096) &
  2271.                                  0xfffff000) << 32;
  2272.                 val |= obj->gtt_offset & 0xfffff000;
  2273.                 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
  2274.                 if (obj->tiling_mode == I915_TILING_Y)
  2275.                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
  2276.                 val |= I965_FENCE_REG_VALID;
  2277.         } else
  2278.                 val = 0;
  2279.  
  2280.         I915_WRITE64(FENCE_REG_965_0 + reg * 8, val);
  2281.         POSTING_READ(FENCE_REG_965_0 + reg * 8);
  2282. }
  2283.  
  2284. static void i915_write_fence_reg(struct drm_device *dev, int reg,
  2285.                                  struct drm_i915_gem_object *obj)
  2286. {
  2287.         drm_i915_private_t *dev_priv = dev->dev_private;
  2288.         u32 val;
  2289.  
  2290.         if (obj) {
  2291.                 u32 size = obj->gtt_space->size;
  2292.                 int pitch_val;
  2293.                 int tile_width;
  2294.  
  2295.                 WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
  2296.                      (size & -size) != size ||
  2297.                      (obj->gtt_offset & (size - 1)),
  2298.                      "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
  2299.                      obj->gtt_offset, obj->map_and_fenceable, size);
  2300.  
  2301.                 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
  2302.                         tile_width = 128;
  2303.                 else
  2304.                         tile_width = 512;
  2305.  
  2306.                 /* Note: pitch better be a power of two tile widths */
  2307.                 pitch_val = obj->stride / tile_width;
  2308.                 pitch_val = ffs(pitch_val) - 1;
  2309.  
  2310.                 val = obj->gtt_offset;
  2311.                 if (obj->tiling_mode == I915_TILING_Y)
  2312.                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
  2313.                 val |= I915_FENCE_SIZE_BITS(size);
  2314.                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
  2315.                 val |= I830_FENCE_REG_VALID;
  2316.         } else
  2317.                 val = 0;
  2318.  
  2319.         if (reg < 8)
  2320.                 reg = FENCE_REG_830_0 + reg * 4;
  2321.         else
  2322.                 reg = FENCE_REG_945_8 + (reg - 8) * 4;
  2323.  
  2324.         I915_WRITE(reg, val);
  2325.         POSTING_READ(reg);
  2326. }
  2327.  
  2328. static void i830_write_fence_reg(struct drm_device *dev, int reg,
  2329.                                 struct drm_i915_gem_object *obj)
  2330. {
  2331.         drm_i915_private_t *dev_priv = dev->dev_private;
  2332.         uint32_t val;
  2333.  
  2334.         if (obj) {
  2335.                 u32 size = obj->gtt_space->size;
  2336.                 uint32_t pitch_val;
  2337.  
  2338.                 WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
  2339.                      (size & -size) != size ||
  2340.                      (obj->gtt_offset & (size - 1)),
  2341.                      "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
  2342.                      obj->gtt_offset, size);
  2343.  
  2344.                 pitch_val = obj->stride / 128;
  2345.                 pitch_val = ffs(pitch_val) - 1;
  2346.  
  2347.                 val = obj->gtt_offset;
  2348.                 if (obj->tiling_mode == I915_TILING_Y)
  2349.                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
  2350.                 val |= I830_FENCE_SIZE_BITS(size);
  2351.                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
  2352.                 val |= I830_FENCE_REG_VALID;
  2353.         } else
  2354.                 val = 0;
  2355.  
  2356.         I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
  2357.         POSTING_READ(FENCE_REG_830_0 + reg * 4);
  2358. }
  2359.  
  2360. static void i915_gem_write_fence(struct drm_device *dev, int reg,
  2361.                                  struct drm_i915_gem_object *obj)
  2362. {
  2363.         switch (INTEL_INFO(dev)->gen) {
  2364.         case 7:
  2365.         case 6: sandybridge_write_fence_reg(dev, reg, obj); break;
  2366.         case 5:
  2367.         case 4: i965_write_fence_reg(dev, reg, obj); break;
  2368.         case 3: i915_write_fence_reg(dev, reg, obj); break;
  2369.         case 2: i830_write_fence_reg(dev, reg, obj); break;
  2370.         default: break;
  2371.         }
  2372. }
  2373.  
  2374. static inline int fence_number(struct drm_i915_private *dev_priv,
  2375.                                struct drm_i915_fence_reg *fence)
  2376. {
  2377.         return fence - dev_priv->fence_regs;
  2378. }
  2379.  
  2380. static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
  2381.                                          struct drm_i915_fence_reg *fence,
  2382.                                          bool enable)
  2383. {
  2384.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  2385.         int reg = fence_number(dev_priv, fence);
  2386.  
  2387.         i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
  2388.  
  2389.         if (enable) {
  2390.                 obj->fence_reg = reg;
  2391.                 fence->obj = obj;
  2392.                 list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
  2393.         } else {
  2394.                 obj->fence_reg = I915_FENCE_REG_NONE;
  2395.                 fence->obj = NULL;
  2396.                 list_del_init(&fence->lru_list);
  2397.         }
  2398. }
  2399.  
  2400. static int
  2401. i915_gem_object_flush_fence(struct drm_i915_gem_object *obj)
  2402. {
  2403.         if (obj->last_fenced_seqno) {
  2404.                 int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
  2405.                         if (ret)
  2406.                                 return ret;
  2407.  
  2408.                 obj->last_fenced_seqno = 0;
  2409.         }
  2410.  
  2411.         /* Ensure that all CPU reads are completed before installing a fence
  2412.          * and all writes before removing the fence.
  2413.          */
  2414.         if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
  2415.                 mb();
  2416.  
  2417.         obj->fenced_gpu_access = false;
  2418.         return 0;
  2419. }
  2420.  
  2421. int
  2422. i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
  2423. {
  2424.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  2425.         int ret;
  2426.  
  2427.         ret = i915_gem_object_flush_fence(obj);
  2428.         if (ret)
  2429.                 return ret;
  2430.  
  2431.         if (obj->fence_reg == I915_FENCE_REG_NONE)
  2432.                 return 0;
  2433.  
  2434.         i915_gem_object_update_fence(obj,
  2435.                                      &dev_priv->fence_regs[obj->fence_reg],
  2436.                                      false);
  2437.         i915_gem_object_fence_lost(obj);
  2438.  
  2439.         return 0;
  2440. }
  2441.  
  2442. static struct drm_i915_fence_reg *
  2443. i915_find_fence_reg(struct drm_device *dev)
  2444. {
  2445.         struct drm_i915_private *dev_priv = dev->dev_private;
  2446.         struct drm_i915_fence_reg *reg, *avail;
  2447.         int i;
  2448.  
  2449.         /* First try to find a free reg */
  2450.         avail = NULL;
  2451.         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
  2452.                 reg = &dev_priv->fence_regs[i];
  2453.                 if (!reg->obj)
  2454.                         return reg;
  2455.  
  2456.                 if (!reg->pin_count)
  2457.                         avail = reg;
  2458.         }
  2459.  
  2460.         if (avail == NULL)
  2461.                 return NULL;
  2462.  
  2463.         /* None available, try to steal one or wait for a user to finish */
  2464.         list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
  2465.                 if (reg->pin_count)
  2466.                         continue;
  2467.  
  2468.                 return reg;
  2469.         }
  2470.  
  2471.         return NULL;
  2472. }
  2473.  
  2474. /**
  2475.  * i915_gem_object_get_fence - set up fencing for an object
  2476.  * @obj: object to map through a fence reg
  2477.  *
  2478.  * When mapping objects through the GTT, userspace wants to be able to write
  2479.  * to them without having to worry about swizzling if the object is tiled.
  2480.  * This function walks the fence regs looking for a free one for @obj,
  2481.  * stealing one if it can't find any.
  2482.  *
  2483.  * It then sets up the reg based on the object's properties: address, pitch
  2484.  * and tiling format.
  2485.  *
  2486.  * For an untiled surface, this removes any existing fence.
  2487.  */
  2488. int
  2489. i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
  2490. {
  2491.         struct drm_device *dev = obj->base.dev;
  2492.         struct drm_i915_private *dev_priv = dev->dev_private;
  2493.         bool enable = obj->tiling_mode != I915_TILING_NONE;
  2494.         struct drm_i915_fence_reg *reg;
  2495.         int ret;
  2496.  
  2497.         /* Have we updated the tiling parameters upon the object and so
  2498.          * will need to serialise the write to the associated fence register?
  2499.          */
  2500.         if (obj->fence_dirty) {
  2501.                 ret = i915_gem_object_flush_fence(obj);
  2502.                 if (ret)
  2503.                         return ret;
  2504.         }
  2505.  
  2506.         /* Just update our place in the LRU if our fence is getting reused. */
  2507.         if (obj->fence_reg != I915_FENCE_REG_NONE) {
  2508.                 reg = &dev_priv->fence_regs[obj->fence_reg];
  2509.                 if (!obj->fence_dirty) {
  2510.                         list_move_tail(&reg->lru_list,
  2511.                                        &dev_priv->mm.fence_list);
  2512.                         return 0;
  2513.                 }
  2514.         } else if (enable) {
  2515.                 reg = i915_find_fence_reg(dev);
  2516.                 if (reg == NULL)
  2517.                         return -EDEADLK;
  2518.  
  2519.                 if (reg->obj) {
  2520.                         struct drm_i915_gem_object *old = reg->obj;
  2521.  
  2522.                         ret = i915_gem_object_flush_fence(old);
  2523.                         if (ret)
  2524.                                 return ret;
  2525.  
  2526.                         i915_gem_object_fence_lost(old);
  2527.                 }
  2528.         } else
  2529.                 return 0;
  2530.  
  2531.         i915_gem_object_update_fence(obj, reg, enable);
  2532.         obj->fence_dirty = false;
  2533.  
  2534.         return 0;
  2535. }
  2536.  
  2537. static bool i915_gem_valid_gtt_space(struct drm_device *dev,
  2538.                                      struct drm_mm_node *gtt_space,
  2539.                                      unsigned long cache_level)
  2540. {
  2541.         struct drm_mm_node *other;
  2542.  
  2543.         /* On non-LLC machines we have to be careful when putting differing
  2544.          * types of snoopable memory together to avoid the prefetcher
  2545.          * crossing memory domains and dieing.
  2546.          */
  2547.         if (HAS_LLC(dev))
  2548.                 return true;
  2549.  
  2550.         if (gtt_space == NULL)
  2551.                 return true;
  2552.  
  2553.         if (list_empty(&gtt_space->node_list))
  2554.                 return true;
  2555.  
  2556.         other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
  2557.         if (other->allocated && !other->hole_follows && other->color != cache_level)
  2558.                 return false;
  2559.  
  2560.         other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
  2561.         if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
  2562.                 return false;
  2563.  
  2564.         return true;
  2565. }
  2566.  
  2567. static void i915_gem_verify_gtt(struct drm_device *dev)
  2568. {
  2569. #if WATCH_GTT
  2570.         struct drm_i915_private *dev_priv = dev->dev_private;
  2571.         struct drm_i915_gem_object *obj;
  2572.         int err = 0;
  2573.  
  2574.         list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
  2575.                 if (obj->gtt_space == NULL) {
  2576.                         printk(KERN_ERR "object found on GTT list with no space reserved\n");
  2577.                         err++;
  2578.                         continue;
  2579.                 }
  2580.  
  2581.                 if (obj->cache_level != obj->gtt_space->color) {
  2582.                         printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
  2583.                                obj->gtt_space->start,
  2584.                                obj->gtt_space->start + obj->gtt_space->size,
  2585.                                obj->cache_level,
  2586.                                obj->gtt_space->color);
  2587.                         err++;
  2588.                         continue;
  2589.                 }
  2590.  
  2591.                 if (!i915_gem_valid_gtt_space(dev,
  2592.                                               obj->gtt_space,
  2593.                                               obj->cache_level)) {
  2594.                         printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
  2595.                                obj->gtt_space->start,
  2596.                                obj->gtt_space->start + obj->gtt_space->size,
  2597.                                obj->cache_level);
  2598.                         err++;
  2599.                         continue;
  2600.                 }
  2601.         }
  2602.  
  2603.         WARN_ON(err);
  2604. #endif
  2605. }
  2606.  
  2607. /**
  2608.  * Finds free space in the GTT aperture and binds the object there.
  2609.  */
  2610. static int
  2611. i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  2612.                             unsigned alignment,
  2613.                             bool map_and_fenceable,
  2614.                             bool nonblocking)
  2615. {
  2616.         struct drm_device *dev = obj->base.dev;
  2617.         drm_i915_private_t *dev_priv = dev->dev_private;
  2618.         struct drm_mm_node *node;
  2619.         u32 size, fence_size, fence_alignment, unfenced_alignment;
  2620.         bool mappable, fenceable;
  2621.         int ret;
  2622.  
  2623.         if (obj->madv != I915_MADV_WILLNEED) {
  2624.         DRM_ERROR("Attempting to bind a purgeable object\n");
  2625.                 return -EINVAL;
  2626.         }
  2627.  
  2628.         fence_size = i915_gem_get_gtt_size(dev,
  2629.                                            obj->base.size,
  2630.                                            obj->tiling_mode);
  2631.         fence_alignment = i915_gem_get_gtt_alignment(dev,
  2632.                                                      obj->base.size,
  2633.                                                      obj->tiling_mode);
  2634.         unfenced_alignment =
  2635.                 i915_gem_get_unfenced_gtt_alignment(dev,
  2636.                                                     obj->base.size,
  2637.                                                     obj->tiling_mode);
  2638.  
  2639.         if (alignment == 0)
  2640.                 alignment = map_and_fenceable ? fence_alignment :
  2641.                                                 unfenced_alignment;
  2642.         if (map_and_fenceable && alignment & (fence_alignment - 1)) {
  2643.                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
  2644.                 return -EINVAL;
  2645.         }
  2646.  
  2647.         size = map_and_fenceable ? fence_size : obj->base.size;
  2648.  
  2649.         /* If the object is bigger than the entire aperture, reject it early
  2650.          * before evicting everything in a vain attempt to find space.
  2651.          */
  2652.         if (obj->base.size >
  2653.             (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
  2654.                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
  2655.                 return -E2BIG;
  2656.         }
  2657.  
  2658.         ret = i915_gem_object_get_pages(obj);
  2659.         if (ret)
  2660.                 return ret;
  2661.  
  2662.         i915_gem_object_pin_pages(obj);
  2663.  
  2664.         node = kzalloc(sizeof(*node), GFP_KERNEL);
  2665.         if (node == NULL) {
  2666.                 i915_gem_object_unpin_pages(obj);
  2667.                 return -ENOMEM;
  2668.         }
  2669.  
  2670.  search_free:
  2671.         if (map_and_fenceable)
  2672.                 ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space, node,
  2673.                                                           size, alignment, obj->cache_level,
  2674.                                                           0, dev_priv->mm.gtt_mappable_end);
  2675.         else
  2676.                 ret = drm_mm_insert_node_generic(&dev_priv->mm.gtt_space, node,
  2677.                                                  size, alignment, obj->cache_level);
  2678.         if (ret) {
  2679.  
  2680.                 i915_gem_object_unpin_pages(obj);
  2681.                 kfree(node);
  2682.                         return ret;
  2683.         }
  2684.         if (WARN_ON(!i915_gem_valid_gtt_space(dev, node, obj->cache_level))) {
  2685.                 i915_gem_object_unpin_pages(obj);
  2686.                 drm_mm_put_block(node);
  2687.                 return -EINVAL;
  2688.         }
  2689.  
  2690.         ret = i915_gem_gtt_prepare_object(obj);
  2691.         if (ret) {
  2692.                 i915_gem_object_unpin_pages(obj);
  2693.                 drm_mm_put_block(node);
  2694.                         return ret;
  2695.         }
  2696.  
  2697.         list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list);
  2698.         list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
  2699.  
  2700.         obj->gtt_space = node;
  2701.         obj->gtt_offset = node->start;
  2702.  
  2703.         fenceable =
  2704.                 node->size == fence_size &&
  2705.                 (node->start & (fence_alignment - 1)) == 0;
  2706.  
  2707.         mappable =
  2708.                 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
  2709.  
  2710.         obj->map_and_fenceable = mappable && fenceable;
  2711.  
  2712.         i915_gem_object_unpin_pages(obj);
  2713.         trace_i915_gem_object_bind(obj, map_and_fenceable);
  2714.         i915_gem_verify_gtt(dev);
  2715.         return 0;
  2716. }
  2717.  
  2718. void
  2719. i915_gem_clflush_object(struct drm_i915_gem_object *obj)
  2720. {
  2721.         /* If we don't have a page list set up, then we're not pinned
  2722.          * to GPU, and we can ignore the cache flush because it'll happen
  2723.          * again at bind time.
  2724.          */
  2725.         if (obj->pages == NULL)
  2726.                 return;
  2727.  
  2728.         /* If the GPU is snooping the contents of the CPU cache,
  2729.          * we do not need to manually clear the CPU cache lines.  However,
  2730.          * the caches are only snooped when the render cache is
  2731.          * flushed/invalidated.  As we always have to emit invalidations
  2732.          * and flushes when moving into and out of the RENDER domain, correct
  2733.          * snooping behaviour occurs naturally as the result of our domain
  2734.          * tracking.
  2735.          */
  2736.         if (obj->cache_level != I915_CACHE_NONE)
  2737.                 return;
  2738. #if 0
  2739.      if(obj->mapped != NULL)
  2740.      {
  2741.         uint8_t *page_virtual;
  2742.         unsigned int i;
  2743.  
  2744.         page_virtual = obj->mapped;
  2745.         asm volatile("mfence");
  2746.         for (i = 0; i < obj->base.size; i += x86_clflush_size)
  2747.             clflush(page_virtual + i);
  2748.         asm volatile("mfence");
  2749.      }
  2750.      else
  2751.      {
  2752.         uint8_t *page_virtual;
  2753.         unsigned int i;
  2754.         page_virtual = AllocKernelSpace(obj->base.size);
  2755.         if(page_virtual != NULL)
  2756.         {
  2757.             dma_addr_t *src, *dst;
  2758.             u32 count;
  2759.  
  2760. #define page_tabs  0xFDC00000      /* really dirty hack */
  2761.  
  2762.             src =  obj->pages.page;
  2763.             dst =  &((dma_addr_t*)page_tabs)[(u32_t)page_virtual >> 12];
  2764.             count = obj->base.size/4096;
  2765.  
  2766.             while(count--)
  2767.             {
  2768.                 *dst++ = (0xFFFFF000 & *src++) | 0x001 ;
  2769.             };
  2770.  
  2771.             asm volatile("mfence");
  2772.             for (i = 0; i < obj->base.size; i += x86_clflush_size)
  2773.                 clflush(page_virtual + i);
  2774.             asm volatile("mfence");
  2775.             FreeKernelSpace(page_virtual);
  2776.         }
  2777.         else
  2778.         {
  2779.             asm volatile (
  2780.             "mfence         \n"
  2781.             "wbinvd         \n"                 /* this is really ugly  */
  2782.             "mfence");
  2783.         }
  2784.      }
  2785. #endif
  2786.  
  2787. }
  2788.  
  2789. /** Flushes the GTT write domain for the object if it's dirty. */
  2790. static void
  2791. i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
  2792. {
  2793.         uint32_t old_write_domain;
  2794.  
  2795.         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
  2796.                 return;
  2797.  
  2798.         /* No actual flushing is required for the GTT write domain.  Writes
  2799.          * to it immediately go to main memory as far as we know, so there's
  2800.          * no chipset flush.  It also doesn't land in render cache.
  2801.          *
  2802.          * However, we do have to enforce the order so that all writes through
  2803.          * the GTT land before any writes to the device, such as updates to
  2804.          * the GATT itself.
  2805.          */
  2806.         wmb();
  2807.  
  2808.         old_write_domain = obj->base.write_domain;
  2809.         obj->base.write_domain = 0;
  2810.  
  2811.         trace_i915_gem_object_change_domain(obj,
  2812.                                             obj->base.read_domains,
  2813.                                             old_write_domain);
  2814. }
  2815.  
  2816. /** Flushes the CPU write domain for the object if it's dirty. */
  2817. static void
  2818. i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
  2819. {
  2820.         uint32_t old_write_domain;
  2821.  
  2822.         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
  2823.                 return;
  2824.  
  2825.         i915_gem_clflush_object(obj);
  2826.         i915_gem_chipset_flush(obj->base.dev);
  2827.         old_write_domain = obj->base.write_domain;
  2828.         obj->base.write_domain = 0;
  2829.  
  2830.         trace_i915_gem_object_change_domain(obj,
  2831.                                             obj->base.read_domains,
  2832.                                             old_write_domain);
  2833. }
  2834.  
  2835. /**
  2836.  * Moves a single object to the GTT read, and possibly write domain.
  2837.  *
  2838.  * This function returns when the move is complete, including waiting on
  2839.  * flushes to occur.
  2840.  */
  2841. int
  2842. i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
  2843. {
  2844.         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
  2845.         uint32_t old_write_domain, old_read_domains;
  2846.         int ret;
  2847.  
  2848.         /* Not valid to be called on unbound objects. */
  2849.         if (obj->gtt_space == NULL)
  2850.                 return -EINVAL;
  2851.  
  2852.         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
  2853.                 return 0;
  2854.  
  2855.         ret = i915_gem_object_wait_rendering(obj, !write);
  2856.                 if (ret)
  2857.                         return ret;
  2858.  
  2859.         i915_gem_object_flush_cpu_write_domain(obj);
  2860.  
  2861.         old_write_domain = obj->base.write_domain;
  2862.         old_read_domains = obj->base.read_domains;
  2863.  
  2864.         /* It should now be out of any other write domains, and we can update
  2865.          * the domain values for our changes.
  2866.          */
  2867.         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
  2868.         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
  2869.         if (write) {
  2870.                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
  2871.                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
  2872.                 obj->dirty = 1;
  2873.         }
  2874.  
  2875.         trace_i915_gem_object_change_domain(obj,
  2876.                                             old_read_domains,
  2877.                                             old_write_domain);
  2878.  
  2879.         /* And bump the LRU for this access */
  2880.         if (i915_gem_object_is_inactive(obj))
  2881.                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
  2882.  
  2883.         return 0;
  2884. }
  2885.  
  2886. int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
  2887.                                     enum i915_cache_level cache_level)
  2888. {
  2889.         struct drm_device *dev = obj->base.dev;
  2890.         drm_i915_private_t *dev_priv = dev->dev_private;
  2891.         int ret;
  2892.  
  2893.         if (obj->cache_level == cache_level)
  2894.                 return 0;
  2895.  
  2896.         if (obj->pin_count) {
  2897.                 DRM_DEBUG("can not change the cache level of pinned objects\n");
  2898.                 return -EBUSY;
  2899.         }
  2900.  
  2901.         if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) {
  2902.                 ret = i915_gem_object_unbind(obj);
  2903.                 if (ret)
  2904.                         return ret;
  2905.         }
  2906.  
  2907.         if (obj->gtt_space) {
  2908.                 ret = i915_gem_object_finish_gpu(obj);
  2909.                 if (ret)
  2910.                         return ret;
  2911.  
  2912.                 i915_gem_object_finish_gtt(obj);
  2913.  
  2914.                 /* Before SandyBridge, you could not use tiling or fence
  2915.                  * registers with snooped memory, so relinquish any fences
  2916.                  * currently pointing to our region in the aperture.
  2917.                  */
  2918.                 if (INTEL_INFO(dev)->gen < 6) {
  2919.                         ret = i915_gem_object_put_fence(obj);
  2920.                         if (ret)
  2921.                                 return ret;
  2922.                 }
  2923.  
  2924.                 if (obj->has_global_gtt_mapping)
  2925.                         i915_gem_gtt_bind_object(obj, cache_level);
  2926.                 if (obj->has_aliasing_ppgtt_mapping)
  2927.                         i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
  2928.                                                obj, cache_level);
  2929.  
  2930.                 obj->gtt_space->color = cache_level;
  2931.         }
  2932.  
  2933.         if (cache_level == I915_CACHE_NONE) {
  2934.                 u32 old_read_domains, old_write_domain;
  2935.  
  2936.                 /* If we're coming from LLC cached, then we haven't
  2937.                  * actually been tracking whether the data is in the
  2938.                  * CPU cache or not, since we only allow one bit set
  2939.                  * in obj->write_domain and have been skipping the clflushes.
  2940.                  * Just set it to the CPU cache for now.
  2941.                  */
  2942.                 WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
  2943.                 WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
  2944.  
  2945.                 old_read_domains = obj->base.read_domains;
  2946.                 old_write_domain = obj->base.write_domain;
  2947.  
  2948.                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  2949.                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  2950.  
  2951.                 trace_i915_gem_object_change_domain(obj,
  2952.                                                     old_read_domains,
  2953.                                                     old_write_domain);
  2954.     }
  2955.  
  2956.         obj->cache_level = cache_level;
  2957.         i915_gem_verify_gtt(dev);
  2958.         return 0;
  2959. }
  2960.  
  2961. int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
  2962.                                struct drm_file *file)
  2963. {
  2964.         struct drm_i915_gem_caching *args = data;
  2965.         struct drm_i915_gem_object *obj;
  2966.         int ret;
  2967.  
  2968.         ret = i915_mutex_lock_interruptible(dev);
  2969.         if (ret)
  2970.                 return ret;
  2971.  
  2972.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  2973.         if (&obj->base == NULL) {
  2974.                 ret = -ENOENT;
  2975.                 goto unlock;
  2976.         }
  2977.  
  2978.         args->caching = obj->cache_level != I915_CACHE_NONE;
  2979.  
  2980.         drm_gem_object_unreference(&obj->base);
  2981. unlock:
  2982.         mutex_unlock(&dev->struct_mutex);
  2983.         return ret;
  2984. }
  2985.  
  2986. int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
  2987.                                struct drm_file *file)
  2988. {
  2989.         struct drm_i915_gem_caching *args = data;
  2990.         struct drm_i915_gem_object *obj;
  2991.         enum i915_cache_level level;
  2992.         int ret;
  2993.  
  2994.         switch (args->caching) {
  2995.         case I915_CACHING_NONE:
  2996.                 level = I915_CACHE_NONE;
  2997.                 break;
  2998.         case I915_CACHING_CACHED:
  2999.                 level = I915_CACHE_LLC;
  3000.                 break;
  3001.         default:
  3002.                 return -EINVAL;
  3003.         }
  3004.  
  3005.         ret = i915_mutex_lock_interruptible(dev);
  3006.         if (ret)
  3007.                 return ret;
  3008.  
  3009.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  3010.         if (&obj->base == NULL) {
  3011.                 ret = -ENOENT;
  3012.                 goto unlock;
  3013.         }
  3014.  
  3015.         ret = i915_gem_object_set_cache_level(obj, level);
  3016.  
  3017.         drm_gem_object_unreference(&obj->base);
  3018. unlock:
  3019.         mutex_unlock(&dev->struct_mutex);
  3020.         return ret;
  3021. }
  3022.  
  3023. /*
  3024.  * Prepare buffer for display plane (scanout, cursors, etc).
  3025.  * Can be called from an uninterruptible phase (modesetting) and allows
  3026.  * any flushes to be pipelined (for pageflips).
  3027.  */
  3028. int
  3029. i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
  3030.                                      u32 alignment,
  3031.                                      struct intel_ring_buffer *pipelined)
  3032. {
  3033.         u32 old_read_domains, old_write_domain;
  3034.         int ret;
  3035.  
  3036.         if (pipelined != obj->ring) {
  3037.                 ret = i915_gem_object_sync(obj, pipelined);
  3038.         if (ret)
  3039.                 return ret;
  3040.         }
  3041.  
  3042.         /* The display engine is not coherent with the LLC cache on gen6.  As
  3043.          * a result, we make sure that the pinning that is about to occur is
  3044.          * done with uncached PTEs. This is lowest common denominator for all
  3045.          * chipsets.
  3046.          *
  3047.          * However for gen6+, we could do better by using the GFDT bit instead
  3048.          * of uncaching, which would allow us to flush all the LLC-cached data
  3049.          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
  3050.          */
  3051.         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
  3052.         if (ret)
  3053.                 return ret;
  3054.  
  3055.         /* As the user may map the buffer once pinned in the display plane
  3056.          * (e.g. libkms for the bootup splash), we have to ensure that we
  3057.          * always use map_and_fenceable for all scanout buffers.
  3058.          */
  3059.         ret = i915_gem_object_pin(obj, alignment, true, false);
  3060.         if (ret)
  3061.                 return ret;
  3062.  
  3063.         i915_gem_object_flush_cpu_write_domain(obj);
  3064.  
  3065.         old_write_domain = obj->base.write_domain;
  3066.         old_read_domains = obj->base.read_domains;
  3067.  
  3068.         /* It should now be out of any other write domains, and we can update
  3069.          * the domain values for our changes.
  3070.          */
  3071.         obj->base.write_domain = 0;
  3072.         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
  3073.  
  3074.         trace_i915_gem_object_change_domain(obj,
  3075.                                             old_read_domains,
  3076.                                             old_write_domain);
  3077.  
  3078.         return 0;
  3079. }
  3080.  
  3081. int
  3082. i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
  3083. {
  3084.         int ret;
  3085.  
  3086.         if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
  3087.                 return 0;
  3088.  
  3089.         ret = i915_gem_object_wait_rendering(obj, false);
  3090.     if (ret)
  3091.         return ret;
  3092.  
  3093.         /* Ensure that we invalidate the GPU's caches and TLBs. */
  3094.         obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
  3095.         return 0;
  3096. }
  3097.  
  3098. /**
  3099.  * Moves a single object to the CPU read, and possibly write domain.
  3100.  *
  3101.  * This function returns when the move is complete, including waiting on
  3102.  * flushes to occur.
  3103.  */
  3104. int
  3105. i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
  3106. {
  3107.         uint32_t old_write_domain, old_read_domains;
  3108.         int ret;
  3109.  
  3110.         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
  3111.                 return 0;
  3112.  
  3113.         ret = i915_gem_object_wait_rendering(obj, !write);
  3114.         if (ret)
  3115.                 return ret;
  3116.  
  3117.         i915_gem_object_flush_gtt_write_domain(obj);
  3118.  
  3119.         old_write_domain = obj->base.write_domain;
  3120.         old_read_domains = obj->base.read_domains;
  3121.  
  3122.         /* Flush the CPU cache if it's still invalid. */
  3123.         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
  3124.                 i915_gem_clflush_object(obj);
  3125.  
  3126.                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
  3127.         }
  3128.  
  3129.         /* It should now be out of any other write domains, and we can update
  3130.          * the domain values for our changes.
  3131.          */
  3132.         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
  3133.  
  3134.         /* If we're writing through the CPU, then the GPU read domains will
  3135.          * need to be invalidated at next use.
  3136.          */
  3137.         if (write) {
  3138.                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  3139.                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  3140.         }
  3141.  
  3142.         trace_i915_gem_object_change_domain(obj,
  3143.                                             old_read_domains,
  3144.                                             old_write_domain);
  3145.  
  3146.         return 0;
  3147. }
  3148.  
  3149. /* Throttle our rendering by waiting until the ring has completed our requests
  3150.  * emitted over 20 msec ago.
  3151.  *
  3152.  * Note that if we were to use the current jiffies each time around the loop,
  3153.  * we wouldn't escape the function with any frames outstanding if the time to
  3154.  * render a frame was over 20ms.
  3155.  *
  3156.  * This should get us reasonable parallelism between CPU and GPU but also
  3157.  * relatively low latency when blocking on a particular request to finish.
  3158.  */
  3159. static int
  3160. i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
  3161. {
  3162.         struct drm_i915_private *dev_priv = dev->dev_private;
  3163.         struct drm_i915_file_private *file_priv = file->driver_priv;
  3164.         unsigned long recent_enough = GetTimerTicks() - msecs_to_jiffies(20);
  3165.         struct drm_i915_gem_request *request;
  3166.         struct intel_ring_buffer *ring = NULL;
  3167.         u32 seqno = 0;
  3168.         int ret;
  3169.  
  3170.         if (atomic_read(&dev_priv->mm.wedged))
  3171.                 return -EIO;
  3172.  
  3173.         spin_lock(&file_priv->mm.lock);
  3174.         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
  3175.                 if (time_after_eq(request->emitted_jiffies, recent_enough))
  3176.                         break;
  3177.  
  3178.                 ring = request->ring;
  3179.                 seqno = request->seqno;
  3180.         }
  3181.         spin_unlock(&file_priv->mm.lock);
  3182.  
  3183.         if (seqno == 0)
  3184.                 return 0;
  3185.  
  3186.         ret = __wait_seqno(ring, seqno, true, NULL);
  3187.         if (ret == 0)
  3188.                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
  3189.  
  3190.         return ret;
  3191. }
  3192.  
  3193. int
  3194. i915_gem_object_pin(struct drm_i915_gem_object *obj,
  3195.                     uint32_t alignment,
  3196.                     bool map_and_fenceable,
  3197.                     bool nonblocking)
  3198. {
  3199.         int ret;
  3200.  
  3201.         if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
  3202.                 return -EBUSY;
  3203.  
  3204.         if (obj->gtt_space != NULL) {
  3205.                 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
  3206.                     (map_and_fenceable && !obj->map_and_fenceable)) {
  3207.                         WARN(obj->pin_count,
  3208.                              "bo is already pinned with incorrect alignment:"
  3209.                              " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
  3210.                              " obj->map_and_fenceable=%d\n",
  3211.                              obj->gtt_offset, alignment,
  3212.                              map_and_fenceable,
  3213.                              obj->map_and_fenceable);
  3214.                         ret = i915_gem_object_unbind(obj);
  3215.                         if (ret)
  3216.                                 return ret;
  3217.                 }
  3218.         }
  3219.  
  3220.         if (obj->gtt_space == NULL) {
  3221.                 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  3222.  
  3223.                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
  3224.                                                   map_and_fenceable,
  3225.                                                   nonblocking);
  3226.                 if (ret)
  3227.                         return ret;
  3228.  
  3229.                 if (!dev_priv->mm.aliasing_ppgtt)
  3230.                         i915_gem_gtt_bind_object(obj, obj->cache_level);
  3231.         }
  3232.  
  3233.         if (!obj->has_global_gtt_mapping && map_and_fenceable)
  3234.                 i915_gem_gtt_bind_object(obj, obj->cache_level);
  3235.  
  3236.         obj->pin_count++;
  3237.         obj->pin_mappable |= map_and_fenceable;
  3238.  
  3239.         return 0;
  3240. }
  3241.  
  3242. void
  3243. i915_gem_object_unpin(struct drm_i915_gem_object *obj)
  3244. {
  3245.         BUG_ON(obj->pin_count == 0);
  3246.         BUG_ON(obj->gtt_space == NULL);
  3247.  
  3248.         if (--obj->pin_count == 0)
  3249.                 obj->pin_mappable = false;
  3250. }
  3251.  
  3252. int
  3253. i915_gem_pin_ioctl(struct drm_device *dev, void *data,
  3254.                    struct drm_file *file)
  3255. {
  3256.         struct drm_i915_gem_pin *args = data;
  3257.         struct drm_i915_gem_object *obj;
  3258.         int ret;
  3259.  
  3260.         ret = i915_mutex_lock_interruptible(dev);
  3261.         if (ret)
  3262.                 return ret;
  3263.  
  3264.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  3265.         if (&obj->base == NULL) {
  3266.                 ret = -ENOENT;
  3267.                 goto unlock;
  3268.         }
  3269.  
  3270.         if (obj->madv != I915_MADV_WILLNEED) {
  3271.                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
  3272.                 ret = -EINVAL;
  3273.                 goto out;
  3274.         }
  3275.  
  3276.         if (obj->pin_filp != NULL && obj->pin_filp != file) {
  3277.                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
  3278.                           args->handle);
  3279.                 ret = -EINVAL;
  3280.                 goto out;
  3281.         }
  3282.  
  3283.         if (obj->user_pin_count == 0) {
  3284.                 ret = i915_gem_object_pin(obj, args->alignment, true, false);
  3285.                 if (ret)
  3286.                         goto out;
  3287.         }
  3288.  
  3289.         obj->user_pin_count++;
  3290.         obj->pin_filp = file;
  3291.  
  3292.         /* XXX - flush the CPU caches for pinned objects
  3293.          * as the X server doesn't manage domains yet
  3294.          */
  3295.         i915_gem_object_flush_cpu_write_domain(obj);
  3296.         args->offset = obj->gtt_offset;
  3297. out:
  3298.         drm_gem_object_unreference(&obj->base);
  3299. unlock:
  3300.         mutex_unlock(&dev->struct_mutex);
  3301.         return ret;
  3302. }
  3303.  
  3304. #if 0
  3305.  
  3306. int
  3307. i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
  3308.                      struct drm_file *file)
  3309. {
  3310.         struct drm_i915_gem_pin *args = data;
  3311.         struct drm_i915_gem_object *obj;
  3312.         int ret;
  3313.  
  3314.         ret = i915_mutex_lock_interruptible(dev);
  3315.         if (ret)
  3316.                 return ret;
  3317.  
  3318.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  3319.         if (&obj->base == NULL) {
  3320.                 ret = -ENOENT;
  3321.                 goto unlock;
  3322.         }
  3323.  
  3324.         if (obj->pin_filp != file) {
  3325.                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
  3326.                           args->handle);
  3327.                 ret = -EINVAL;
  3328.                 goto out;
  3329.         }
  3330.         obj->user_pin_count--;
  3331.         if (obj->user_pin_count == 0) {
  3332.                 obj->pin_filp = NULL;
  3333.                 i915_gem_object_unpin(obj);
  3334.         }
  3335.  
  3336. out:
  3337.         drm_gem_object_unreference(&obj->base);
  3338. unlock:
  3339.         mutex_unlock(&dev->struct_mutex);
  3340.         return ret;
  3341. }
  3342.  
  3343. #endif
  3344.  
  3345. int
  3346. i915_gem_busy_ioctl(struct drm_device *dev, void *data,
  3347.                     struct drm_file *file)
  3348. {
  3349.         struct drm_i915_gem_busy *args = data;
  3350.         struct drm_i915_gem_object *obj;
  3351.         int ret;
  3352.  
  3353.         ret = i915_mutex_lock_interruptible(dev);
  3354.         if (ret)
  3355.                 return ret;
  3356.  
  3357.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  3358.         if (&obj->base == NULL) {
  3359.                 ret = -ENOENT;
  3360.                 goto unlock;
  3361.         }
  3362.  
  3363.         /* Count all active objects as busy, even if they are currently not used
  3364.          * by the gpu. Users of this interface expect objects to eventually
  3365.          * become non-busy without any further actions, therefore emit any
  3366.          * necessary flushes here.
  3367.          */
  3368.         ret = i915_gem_object_flush_active(obj);
  3369.  
  3370.         args->busy = obj->active;
  3371.         if (obj->ring) {
  3372.                 BUILD_BUG_ON(I915_NUM_RINGS > 16);
  3373.                 args->busy |= intel_ring_flag(obj->ring) << 16;
  3374.         }
  3375.  
  3376.         drm_gem_object_unreference(&obj->base);
  3377. unlock:
  3378.         mutex_unlock(&dev->struct_mutex);
  3379.         return ret;
  3380. }
  3381.  
  3382. int
  3383. i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
  3384.                         struct drm_file *file_priv)
  3385. {
  3386.         return i915_gem_ring_throttle(dev, file_priv);
  3387. }
  3388.  
  3389. #if 0
  3390.  
  3391. int
  3392. i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
  3393.                        struct drm_file *file_priv)
  3394. {
  3395.         struct drm_i915_gem_madvise *args = data;
  3396.         struct drm_i915_gem_object *obj;
  3397.         int ret;
  3398.  
  3399.         switch (args->madv) {
  3400.         case I915_MADV_DONTNEED:
  3401.         case I915_MADV_WILLNEED:
  3402.             break;
  3403.         default:
  3404.             return -EINVAL;
  3405.         }
  3406.  
  3407.         ret = i915_mutex_lock_interruptible(dev);
  3408.         if (ret)
  3409.                 return ret;
  3410.  
  3411.         obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
  3412.         if (&obj->base == NULL) {
  3413.                 ret = -ENOENT;
  3414.                 goto unlock;
  3415.         }
  3416.  
  3417.         if (obj->pin_count) {
  3418.                 ret = -EINVAL;
  3419.                 goto out;
  3420.         }
  3421.  
  3422.         if (obj->madv != __I915_MADV_PURGED)
  3423.                 obj->madv = args->madv;
  3424.  
  3425.         /* if the object is no longer attached, discard its backing storage */
  3426.         if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
  3427.                 i915_gem_object_truncate(obj);
  3428.  
  3429.         args->retained = obj->madv != __I915_MADV_PURGED;
  3430.  
  3431. out:
  3432.         drm_gem_object_unreference(&obj->base);
  3433. unlock:
  3434.         mutex_unlock(&dev->struct_mutex);
  3435.         return ret;
  3436. }
  3437. #endif
  3438.  
  3439. void i915_gem_object_init(struct drm_i915_gem_object *obj,
  3440.                           const struct drm_i915_gem_object_ops *ops)
  3441. {
  3442.         INIT_LIST_HEAD(&obj->mm_list);
  3443.         INIT_LIST_HEAD(&obj->gtt_list);
  3444.         INIT_LIST_HEAD(&obj->ring_list);
  3445.         INIT_LIST_HEAD(&obj->exec_list);
  3446.  
  3447.         obj->ops = ops;
  3448.  
  3449.         obj->fence_reg = I915_FENCE_REG_NONE;
  3450.         obj->madv = I915_MADV_WILLNEED;
  3451.         /* Avoid an unnecessary call to unbind on the first bind. */
  3452.         obj->map_and_fenceable = true;
  3453.  
  3454.         i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
  3455. }
  3456.  
  3457. static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
  3458.         .get_pages = i915_gem_object_get_pages_gtt,
  3459.         .put_pages = i915_gem_object_put_pages_gtt,
  3460. };
  3461.  
  3462. struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
  3463.                                                   size_t size)
  3464. {
  3465.         struct drm_i915_gem_object *obj;
  3466.         struct address_space *mapping;
  3467.         u32 mask;
  3468.  
  3469.         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
  3470.         if (obj == NULL)
  3471.                 return NULL;
  3472.  
  3473.         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
  3474.                 kfree(obj);
  3475.                 return NULL;
  3476.         }
  3477.  
  3478.  
  3479.         i915_gem_object_init(obj, &i915_gem_object_ops);
  3480.  
  3481.         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  3482.         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  3483.  
  3484.         if (HAS_LLC(dev)) {
  3485.                 /* On some devices, we can have the GPU use the LLC (the CPU
  3486.                  * cache) for about a 10% performance improvement
  3487.                  * compared to uncached.  Graphics requests other than
  3488.                  * display scanout are coherent with the CPU in
  3489.                  * accessing this cache.  This means in this mode we
  3490.                  * don't need to clflush on the CPU side, and on the
  3491.                  * GPU side we only need to flush internal caches to
  3492.                  * get data visible to the CPU.
  3493.                  *
  3494.                  * However, we maintain the display planes as UC, and so
  3495.                  * need to rebind when first used as such.
  3496.                  */
  3497.                 obj->cache_level = I915_CACHE_LLC;
  3498.         } else
  3499.                 obj->cache_level = I915_CACHE_NONE;
  3500.  
  3501.         return obj;
  3502. }
  3503.  
  3504. int i915_gem_init_object(struct drm_gem_object *obj)
  3505. {
  3506.         BUG();
  3507.  
  3508.         return 0;
  3509. }
  3510.  
  3511. void i915_gem_free_object(struct drm_gem_object *gem_obj)
  3512. {
  3513.         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
  3514.         struct drm_device *dev = obj->base.dev;
  3515.         drm_i915_private_t *dev_priv = dev->dev_private;
  3516.  
  3517.         trace_i915_gem_object_destroy(obj);
  3518.  
  3519. //   if (obj->phys_obj)
  3520. //       i915_gem_detach_phys_object(dev, obj);
  3521.  
  3522.         obj->pin_count = 0;
  3523.         if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
  3524.                 bool was_interruptible;
  3525.  
  3526.                 was_interruptible = dev_priv->mm.interruptible;
  3527.                 dev_priv->mm.interruptible = false;
  3528.  
  3529.                 WARN_ON(i915_gem_object_unbind(obj));
  3530.  
  3531.                 dev_priv->mm.interruptible = was_interruptible;
  3532.         }
  3533.  
  3534.         obj->pages_pin_count = 0;
  3535.         i915_gem_object_put_pages(obj);
  3536. //   i915_gem_object_free_mmap_offset(obj);
  3537.  
  3538.         BUG_ON(obj->pages);
  3539.  
  3540. //   if (obj->base.import_attach)
  3541. //       drm_prime_gem_destroy(&obj->base, NULL);
  3542.  
  3543.         drm_gem_object_release(&obj->base);
  3544.         i915_gem_info_remove_obj(dev_priv, obj->base.size);
  3545.  
  3546.         kfree(obj->bit_17);
  3547.         kfree(obj);
  3548. }
  3549.  
  3550. #if 0
  3551. int
  3552. i915_gem_idle(struct drm_device *dev)
  3553. {
  3554.         drm_i915_private_t *dev_priv = dev->dev_private;
  3555.         int ret;
  3556.  
  3557.         mutex_lock(&dev->struct_mutex);
  3558.  
  3559.         if (dev_priv->mm.suspended) {
  3560.                 mutex_unlock(&dev->struct_mutex);
  3561.                 return 0;
  3562.         }
  3563.  
  3564.         ret = i915_gpu_idle(dev);
  3565.         if (ret) {
  3566.                 mutex_unlock(&dev->struct_mutex);
  3567.                 return ret;
  3568.         }
  3569.         i915_gem_retire_requests(dev);
  3570.  
  3571.         i915_gem_reset_fences(dev);
  3572.  
  3573.         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
  3574.          * We need to replace this with a semaphore, or something.
  3575.          * And not confound mm.suspended!
  3576.          */
  3577.         dev_priv->mm.suspended = 1;
  3578.         del_timer_sync(&dev_priv->hangcheck_timer);
  3579.  
  3580.         i915_kernel_lost_context(dev);
  3581.         i915_gem_cleanup_ringbuffer(dev);
  3582.  
  3583.         mutex_unlock(&dev->struct_mutex);
  3584.  
  3585.         /* Cancel the retire work handler, which should be idle now. */
  3586.         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
  3587.  
  3588.         return 0;
  3589. }
  3590. #endif
  3591.  
  3592. void i915_gem_l3_remap(struct drm_device *dev)
  3593. {
  3594.         drm_i915_private_t *dev_priv = dev->dev_private;
  3595.         u32 misccpctl;
  3596.         int i;
  3597.  
  3598.         if (!IS_IVYBRIDGE(dev))
  3599.                 return;
  3600.  
  3601.         if (!dev_priv->l3_parity.remap_info)
  3602.                 return;
  3603.  
  3604.         misccpctl = I915_READ(GEN7_MISCCPCTL);
  3605.         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
  3606.         POSTING_READ(GEN7_MISCCPCTL);
  3607.  
  3608.         for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
  3609.                 u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
  3610.                 if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
  3611.                         DRM_DEBUG("0x%x was already programmed to %x\n",
  3612.                                   GEN7_L3LOG_BASE + i, remap);
  3613.                 if (remap && !dev_priv->l3_parity.remap_info[i/4])
  3614.                         DRM_DEBUG_DRIVER("Clearing remapped register\n");
  3615.                 I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
  3616.         }
  3617.  
  3618.         /* Make sure all the writes land before disabling dop clock gating */
  3619.         POSTING_READ(GEN7_L3LOG_BASE);
  3620.  
  3621.         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
  3622. }
  3623.  
  3624. void i915_gem_init_swizzling(struct drm_device *dev)
  3625. {
  3626.         drm_i915_private_t *dev_priv = dev->dev_private;
  3627.  
  3628.         if (INTEL_INFO(dev)->gen < 5 ||
  3629.             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
  3630.                 return;
  3631.  
  3632.         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
  3633.                                  DISP_TILE_SURFACE_SWIZZLING);
  3634.  
  3635.         if (IS_GEN5(dev))
  3636.                 return;
  3637.  
  3638.         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
  3639.         if (IS_GEN6(dev))
  3640.                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
  3641.         else
  3642.                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
  3643. }
  3644.  
  3645. static bool
  3646. intel_enable_blt(struct drm_device *dev)
  3647. {
  3648.         if (!HAS_BLT(dev))
  3649.                 return false;
  3650.  
  3651.         /* The blitter was dysfunctional on early prototypes */
  3652.         if (IS_GEN6(dev) && dev->pdev->revision < 8) {
  3653.                 DRM_INFO("BLT not supported on this pre-production hardware;"
  3654.                          " graphics performance will be degraded.\n");
  3655.                 return false;
  3656.         }
  3657.  
  3658.         return true;
  3659. }
  3660.  
  3661. int
  3662. i915_gem_init_hw(struct drm_device *dev)
  3663. {
  3664.         drm_i915_private_t *dev_priv = dev->dev_private;
  3665.         int ret;
  3666.  
  3667.         if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
  3668.                 return -EIO;
  3669.  
  3670.         if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))
  3671.                 I915_WRITE(0x9008, I915_READ(0x9008) | 0xf0000);
  3672.  
  3673.         i915_gem_l3_remap(dev);
  3674.  
  3675.         i915_gem_init_swizzling(dev);
  3676.  
  3677.         ret = intel_init_render_ring_buffer(dev);
  3678.         if (ret)
  3679.                 return ret;
  3680.  
  3681.     if (HAS_BSD(dev)) {
  3682.                 ret = intel_init_bsd_ring_buffer(dev);
  3683.                 if (ret)
  3684.                         goto cleanup_render_ring;
  3685.         }
  3686.  
  3687.         if (intel_enable_blt(dev)) {
  3688.                 ret = intel_init_blt_ring_buffer(dev);
  3689.                 if (ret)
  3690.                         goto cleanup_bsd_ring;
  3691.         }
  3692.  
  3693.         dev_priv->next_seqno = 1;
  3694.  
  3695.         /*
  3696.          * XXX: There was some w/a described somewhere suggesting loading
  3697.          * contexts before PPGTT.
  3698.          */
  3699.         i915_gem_context_init(dev);
  3700.         i915_gem_init_ppgtt(dev);
  3701.  
  3702.         return 0;
  3703.  
  3704. cleanup_bsd_ring:
  3705.         intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
  3706. cleanup_render_ring:
  3707.         intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
  3708.         return ret;
  3709. }
  3710.  
  3711. static bool
  3712. intel_enable_ppgtt(struct drm_device *dev)
  3713. {
  3714.         if (i915_enable_ppgtt >= 0)
  3715.                 return i915_enable_ppgtt;
  3716.  
  3717. #ifdef CONFIG_INTEL_IOMMU
  3718.         /* Disable ppgtt on SNB if VT-d is on. */
  3719.         if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
  3720.                 return false;
  3721. #endif
  3722.  
  3723.         return true;
  3724. }
  3725.  
  3726. #define LFB_SIZE 0xC00000
  3727.  
  3728. int i915_gem_init(struct drm_device *dev)
  3729. {
  3730.         struct drm_i915_private *dev_priv = dev->dev_private;
  3731.         unsigned long gtt_size, mappable_size;
  3732.         int ret;
  3733.  
  3734.         gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT;
  3735.         mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
  3736.  
  3737.         mutex_lock(&dev->struct_mutex);
  3738.         if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
  3739.                 /* PPGTT pdes are stolen from global gtt ptes, so shrink the
  3740.                  * aperture accordingly when using aliasing ppgtt. */
  3741.                 gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
  3742.  
  3743.         i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE);
  3744.  
  3745.                 ret = i915_gem_init_aliasing_ppgtt(dev);
  3746.                 if (ret) {
  3747.                         mutex_unlock(&dev->struct_mutex);
  3748.                         return ret;
  3749.                 }
  3750.         } else {
  3751.                 /* Let GEM Manage all of the aperture.
  3752.                  *
  3753.                  * However, leave one page at the end still bound to the scratch
  3754.                  * page.  There are a number of places where the hardware
  3755.                  * apparently prefetches past the end of the object, and we've
  3756.                  * seen multiple hangs with the GPU head pointer stuck in a
  3757.                  * batchbuffer bound at the last page of the aperture.  One page
  3758.                  * should be enough to keep any prefetching inside of the
  3759.                  * aperture.
  3760.                  */
  3761.         i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE);
  3762.         }
  3763.  
  3764.         ret = i915_gem_init_hw(dev);
  3765.         mutex_unlock(&dev->struct_mutex);
  3766.         if (ret) {
  3767.                 i915_gem_cleanup_aliasing_ppgtt(dev);
  3768.                 return ret;
  3769.         }
  3770.  
  3771.     return 0;
  3772. }
  3773.  
  3774. void
  3775. i915_gem_cleanup_ringbuffer(struct drm_device *dev)
  3776. {
  3777.         drm_i915_private_t *dev_priv = dev->dev_private;
  3778.         struct intel_ring_buffer *ring;
  3779.         int i;
  3780.  
  3781.         for_each_ring(ring, dev_priv, i)
  3782.                 intel_cleanup_ring_buffer(ring);
  3783. }
  3784.  
  3785. #if 0
  3786.  
  3787. int
  3788. i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
  3789.                        struct drm_file *file_priv)
  3790. {
  3791.         drm_i915_private_t *dev_priv = dev->dev_private;
  3792.         int ret;
  3793.  
  3794.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  3795.                 return 0;
  3796.  
  3797.         if (atomic_read(&dev_priv->mm.wedged)) {
  3798.                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
  3799.                 atomic_set(&dev_priv->mm.wedged, 0);
  3800.         }
  3801.  
  3802.         mutex_lock(&dev->struct_mutex);
  3803.         dev_priv->mm.suspended = 0;
  3804.  
  3805.         ret = i915_gem_init_hw(dev);
  3806.         if (ret != 0) {
  3807.                 mutex_unlock(&dev->struct_mutex);
  3808.                 return ret;
  3809.         }
  3810.  
  3811.         BUG_ON(!list_empty(&dev_priv->mm.active_list));
  3812.         mutex_unlock(&dev->struct_mutex);
  3813.  
  3814.         ret = drm_irq_install(dev);
  3815.         if (ret)
  3816.                 goto cleanup_ringbuffer;
  3817.  
  3818.         return 0;
  3819.  
  3820. cleanup_ringbuffer:
  3821.         mutex_lock(&dev->struct_mutex);
  3822.         i915_gem_cleanup_ringbuffer(dev);
  3823.         dev_priv->mm.suspended = 1;
  3824.         mutex_unlock(&dev->struct_mutex);
  3825.  
  3826.         return ret;
  3827. }
  3828.  
  3829. int
  3830. i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
  3831.                        struct drm_file *file_priv)
  3832. {
  3833.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  3834.                 return 0;
  3835.  
  3836.         drm_irq_uninstall(dev);
  3837.         return i915_gem_idle(dev);
  3838. }
  3839.  
  3840. void
  3841. i915_gem_lastclose(struct drm_device *dev)
  3842. {
  3843.         int ret;
  3844.  
  3845.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  3846.                 return;
  3847.  
  3848.         ret = i915_gem_idle(dev);
  3849.         if (ret)
  3850.                 DRM_ERROR("failed to idle hardware: %d\n", ret);
  3851. }
  3852. #endif
  3853.  
  3854. static void
  3855. init_ring_lists(struct intel_ring_buffer *ring)
  3856. {
  3857.     INIT_LIST_HEAD(&ring->active_list);
  3858.     INIT_LIST_HEAD(&ring->request_list);
  3859. }
  3860.  
  3861. void
  3862. i915_gem_load(struct drm_device *dev)
  3863. {
  3864.     int i;
  3865.     drm_i915_private_t *dev_priv = dev->dev_private;
  3866.  
  3867.     INIT_LIST_HEAD(&dev_priv->mm.active_list);
  3868.     INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
  3869.         INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
  3870.         INIT_LIST_HEAD(&dev_priv->mm.bound_list);
  3871.     INIT_LIST_HEAD(&dev_priv->mm.fence_list);
  3872.     for (i = 0; i < I915_NUM_RINGS; i++)
  3873.         init_ring_lists(&dev_priv->ring[i]);
  3874.         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
  3875.         INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
  3876.         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
  3877.                           i915_gem_retire_work_handler);
  3878.  
  3879.     /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
  3880.     if (IS_GEN3(dev)) {
  3881.                 I915_WRITE(MI_ARB_STATE,
  3882.                            _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
  3883.     }
  3884.  
  3885.     dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
  3886.  
  3887.     if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
  3888.         dev_priv->num_fence_regs = 16;
  3889.     else
  3890.         dev_priv->num_fence_regs = 8;
  3891.  
  3892.     /* Initialize fence registers to zero */
  3893.         i915_gem_reset_fences(dev);
  3894.  
  3895.     i915_gem_detect_bit_6_swizzle(dev);
  3896.  
  3897.     dev_priv->mm.interruptible = true;
  3898.  
  3899. //    dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
  3900. //    dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
  3901. //    register_shrinker(&dev_priv->mm.inactive_shrinker);
  3902. }
  3903.  
  3904.  
  3905.