Subversion Repositories Kolibri OS

Rev

Rev 5354 | Rev 6084 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright © 2008 Intel Corporation
  3.  *
  4.  * Permission is hereby granted, free of charge, to any person obtaining a
  5.  * copy of this software and associated documentation files (the "Software"),
  6.  * to deal in the Software without restriction, including without limitation
  7.  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8.  * and/or sell copies of the Software, and to permit persons to whom the
  9.  * Software is furnished to do so, subject to the following conditions:
  10.  *
  11.  * The above copyright notice and this permission notice (including the next
  12.  * paragraph) shall be included in all copies or substantial portions of the
  13.  * Software.
  14.  *
  15.  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16.  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17.  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18.  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19.  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20.  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21.  * IN THE SOFTWARE.
  22.  *
  23.  * Authors:
  24.  *    Eric Anholt <eric@anholt.net>
  25.  *
  26.  */
  27.  
  28. #include <drm/drmP.h>
  29. #include <drm/drm_vma_manager.h>
  30. #include <drm/i915_drm.h>
  31. #include "i915_drv.h"
  32. #include "i915_trace.h"
  33. #include "intel_drv.h"
  34. #include <linux/shmem_fs.h>
  35. #include <linux/slab.h>
  36. //#include <linux/swap.h>
  37. #include <linux/scatterlist.h>
  38. #include <linux/pci.h>
  39.  
  40. extern int x86_clflush_size;
  41.  
  42. #define PROT_READ       0x1             /* page can be read */
  43. #define PROT_WRITE      0x2             /* page can be written */
  44. #define MAP_SHARED      0x01            /* Share changes */
  45.  
  46.  
  47. u64 nsecs_to_jiffies64(u64 n)
  48. {
  49. #if (NSEC_PER_SEC % HZ) == 0
  50.         /* Common case, HZ = 100, 128, 200, 250, 256, 500, 512, 1000 etc. */
  51.         return div_u64(n, NSEC_PER_SEC / HZ);
  52. #elif (HZ % 512) == 0
  53.         /* overflow after 292 years if HZ = 1024 */
  54.         return div_u64(n * HZ / 512, NSEC_PER_SEC / 512);
  55. #else
  56.         /*
  57.          * Generic case - optimized for cases where HZ is a multiple of 3.
  58.          * overflow after 64.99 years, exact for HZ = 60, 72, 90, 120 etc.
  59.          */
  60.         return div_u64(n * 9, (9ull * NSEC_PER_SEC + HZ / 2) / HZ);
  61. #endif
  62. }
  63.  
  64. unsigned long nsecs_to_jiffies(u64 n)
  65. {
  66.     return (unsigned long)nsecs_to_jiffies64(n);
  67. }
  68.  
  69.  
  70. struct drm_i915_gem_object *get_fb_obj();
  71.  
  72. unsigned long vm_mmap(struct file *file, unsigned long addr,
  73.          unsigned long len, unsigned long prot,
  74.          unsigned long flag, unsigned long offset);
  75.  
  76.  
  77. #define MAX_ERRNO       4095
  78.  
  79. #define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
  80.  
  81.  
  82. static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  83. static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
  84.                                                    bool force);
  85. static __must_check int
  86. i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
  87.                                bool readonly);
  88. static void
  89. i915_gem_object_retire(struct drm_i915_gem_object *obj);
  90.  
  91. static void i915_gem_write_fence(struct drm_device *dev, int reg,
  92.                                  struct drm_i915_gem_object *obj);
  93. static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
  94.                                          struct drm_i915_fence_reg *fence,
  95.                                          bool enable);
  96.  
  97.  
  98. static unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv);
  99.  
  100. static bool cpu_cache_is_coherent(struct drm_device *dev,
  101.                                   enum i915_cache_level level)
  102. {
  103.         return HAS_LLC(dev) || level != I915_CACHE_NONE;
  104. }
  105.  
  106. static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
  107. {
  108.         if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
  109.                 return true;
  110.  
  111.         return obj->pin_display;
  112. }
  113.  
  114. static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
  115. {
  116.         if (obj->tiling_mode)
  117.                 i915_gem_release_mmap(obj);
  118.  
  119.         /* As we do not have an associated fence register, we will force
  120.          * a tiling change if we ever need to acquire one.
  121.          */
  122.         obj->fence_dirty = false;
  123.         obj->fence_reg = I915_FENCE_REG_NONE;
  124. }
  125.  
  126. /* some bookkeeping */
  127. static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  128.                                   size_t size)
  129. {
  130.         spin_lock(&dev_priv->mm.object_stat_lock);
  131.         dev_priv->mm.object_count++;
  132.         dev_priv->mm.object_memory += size;
  133.         spin_unlock(&dev_priv->mm.object_stat_lock);
  134. }
  135.  
  136. static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  137.                                      size_t size)
  138. {
  139.         spin_lock(&dev_priv->mm.object_stat_lock);
  140.         dev_priv->mm.object_count--;
  141.         dev_priv->mm.object_memory -= size;
  142.         spin_unlock(&dev_priv->mm.object_stat_lock);
  143. }
  144.  
  145. static int
  146. i915_gem_wait_for_error(struct i915_gpu_error *error)
  147. {
  148.         int ret;
  149.  
  150. #define EXIT_COND (!i915_reset_in_progress(error))
  151.         if (EXIT_COND)
  152.                 return 0;
  153. #if 0
  154.         /*
  155.          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
  156.          * userspace. If it takes that long something really bad is going on and
  157.          * we should simply try to bail out and fail as gracefully as possible.
  158.          */
  159.         ret = wait_event_interruptible_timeout(error->reset_queue,
  160.                                                EXIT_COND,
  161.                                                10*HZ);
  162.         if (ret == 0) {
  163.                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
  164.                 return -EIO;
  165.         } else if (ret < 0) {
  166.                 return ret;
  167.         }
  168.  
  169. #endif
  170. #undef EXIT_COND
  171.  
  172.         return 0;
  173. }
  174.  
  175. int i915_mutex_lock_interruptible(struct drm_device *dev)
  176. {
  177.         struct drm_i915_private *dev_priv = dev->dev_private;
  178.         int ret;
  179.  
  180.         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
  181.         if (ret)
  182.                 return ret;
  183.  
  184.         ret = mutex_lock_interruptible(&dev->struct_mutex);
  185.         if (ret)
  186.                 return ret;
  187.  
  188.         WARN_ON(i915_verify_lists(dev));
  189.         return 0;
  190. }
  191.  
  192. static inline bool
  193. i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
  194. {
  195.         return i915_gem_obj_bound_any(obj) && !obj->active;
  196. }
  197.  
  198. int
  199. i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  200.                             struct drm_file *file)
  201. {
  202.         struct drm_i915_private *dev_priv = dev->dev_private;
  203.         struct drm_i915_gem_get_aperture *args = data;
  204.         struct drm_i915_gem_object *obj;
  205.         size_t pinned;
  206.  
  207.         pinned = 0;
  208.         mutex_lock(&dev->struct_mutex);
  209.         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
  210.                 if (i915_gem_obj_is_pinned(obj))
  211.                         pinned += i915_gem_obj_ggtt_size(obj);
  212.         mutex_unlock(&dev->struct_mutex);
  213.  
  214.         args->aper_size = dev_priv->gtt.base.total;
  215.         args->aper_available_size = args->aper_size - pinned;
  216.  
  217.         return 0;
  218. }
  219.  
  220. void *i915_gem_object_alloc(struct drm_device *dev)
  221. {
  222.         struct drm_i915_private *dev_priv = dev->dev_private;
  223.     return kzalloc(sizeof(struct drm_i915_gem_object), 0);
  224. }
  225.  
  226. void i915_gem_object_free(struct drm_i915_gem_object *obj)
  227. {
  228.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  229.         kfree(obj);
  230. }
  231.  
  232. static int
  233. i915_gem_create(struct drm_file *file,
  234.                 struct drm_device *dev,
  235.                 uint64_t size,
  236.                 uint32_t *handle_p)
  237. {
  238.         struct drm_i915_gem_object *obj;
  239.         int ret;
  240.         u32 handle;
  241.  
  242.         size = roundup(size, PAGE_SIZE);
  243.         if (size == 0)
  244.                 return -EINVAL;
  245.  
  246.         /* Allocate the new object */
  247.         obj = i915_gem_alloc_object(dev, size);
  248.         if (obj == NULL)
  249.                 return -ENOMEM;
  250.  
  251.         ret = drm_gem_handle_create(file, &obj->base, &handle);
  252.         /* drop reference from allocate - handle holds it now */
  253.         drm_gem_object_unreference_unlocked(&obj->base);
  254.         if (ret)
  255.                 return ret;
  256.  
  257.         *handle_p = handle;
  258.         return 0;
  259. }
  260.  
  261. int
  262. i915_gem_dumb_create(struct drm_file *file,
  263.                      struct drm_device *dev,
  264.                      struct drm_mode_create_dumb *args)
  265. {
  266.         /* have to work out size/pitch and return them */
  267.         args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
  268.         args->size = args->pitch * args->height;
  269.         return i915_gem_create(file, dev,
  270.                                args->size, &args->handle);
  271. }
  272.  
  273. /**
  274.  * Creates a new mm object and returns a handle to it.
  275.  */
  276. int
  277. i915_gem_create_ioctl(struct drm_device *dev, void *data,
  278.                       struct drm_file *file)
  279. {
  280.         struct drm_i915_gem_create *args = data;
  281.  
  282.         return i915_gem_create(file, dev,
  283.                                args->size, &args->handle);
  284. }
  285.  
  286.  
  287. #if 0
  288.  
  289. static inline int
  290. __copy_to_user_swizzled(char __user *cpu_vaddr,
  291.                         const char *gpu_vaddr, int gpu_offset,
  292.                 int length)
  293. {
  294.         int ret, cpu_offset = 0;
  295.  
  296.         while (length > 0) {
  297.                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
  298.                 int this_length = min(cacheline_end - gpu_offset, length);
  299.                 int swizzled_gpu_offset = gpu_offset ^ 64;
  300.  
  301.                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
  302.                                      gpu_vaddr + swizzled_gpu_offset,
  303.                                      this_length);
  304.                 if (ret)
  305.                         return ret + length;
  306.  
  307.                 cpu_offset += this_length;
  308.                 gpu_offset += this_length;
  309.                 length -= this_length;
  310.         }
  311.  
  312.         return 0;
  313. }
  314.  
  315. static inline int
  316. __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
  317.                           const char __user *cpu_vaddr,
  318.                           int length)
  319. {
  320.         int ret, cpu_offset = 0;
  321.  
  322.         while (length > 0) {
  323.                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
  324.                 int this_length = min(cacheline_end - gpu_offset, length);
  325.                 int swizzled_gpu_offset = gpu_offset ^ 64;
  326.  
  327.                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
  328.                                cpu_vaddr + cpu_offset,
  329.                                this_length);
  330.                 if (ret)
  331.                         return ret + length;
  332.  
  333.                 cpu_offset += this_length;
  334.                 gpu_offset += this_length;
  335.                 length -= this_length;
  336.         }
  337.  
  338.         return 0;
  339. }
  340.  
  341. /* Per-page copy function for the shmem pread fastpath.
  342.  * Flushes invalid cachelines before reading the target if
  343.  * needs_clflush is set. */
  344. static int
  345. shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
  346.                  char __user *user_data,
  347.                  bool page_do_bit17_swizzling, bool needs_clflush)
  348. {
  349.                 char *vaddr;
  350.                 int ret;
  351.  
  352.         if (unlikely(page_do_bit17_swizzling))
  353.                 return -EINVAL;
  354.  
  355.                 vaddr = kmap_atomic(page);
  356.         if (needs_clflush)
  357.                 drm_clflush_virt_range(vaddr + shmem_page_offset,
  358.                                        page_length);
  359.                 ret = __copy_to_user_inatomic(user_data,
  360.                                       vaddr + shmem_page_offset,
  361.                                               page_length);
  362.                 kunmap_atomic(vaddr);
  363.  
  364.         return ret ? -EFAULT : 0;
  365. }
  366.  
  367. static void
  368. shmem_clflush_swizzled_range(char *addr, unsigned long length,
  369.                              bool swizzled)
  370. {
  371.         if (unlikely(swizzled)) {
  372.                 unsigned long start = (unsigned long) addr;
  373.                 unsigned long end = (unsigned long) addr + length;
  374.  
  375.                 /* For swizzling simply ensure that we always flush both
  376.                  * channels. Lame, but simple and it works. Swizzled
  377.                  * pwrite/pread is far from a hotpath - current userspace
  378.                  * doesn't use it at all. */
  379.                 start = round_down(start, 128);
  380.                 end = round_up(end, 128);
  381.  
  382.                 drm_clflush_virt_range((void *)start, end - start);
  383.         } else {
  384.                 drm_clflush_virt_range(addr, length);
  385.         }
  386.  
  387. }
  388.  
  389. /* Only difference to the fast-path function is that this can handle bit17
  390.  * and uses non-atomic copy and kmap functions. */
  391. static int
  392. shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
  393.                  char __user *user_data,
  394.                  bool page_do_bit17_swizzling, bool needs_clflush)
  395. {
  396.         char *vaddr;
  397.         int ret;
  398.  
  399.         vaddr = kmap(page);
  400.         if (needs_clflush)
  401.                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  402.                                              page_length,
  403.                                              page_do_bit17_swizzling);
  404.  
  405.         if (page_do_bit17_swizzling)
  406.                 ret = __copy_to_user_swizzled(user_data,
  407.                                               vaddr, shmem_page_offset,
  408.                                               page_length);
  409.         else
  410.                 ret = __copy_to_user(user_data,
  411.                                      vaddr + shmem_page_offset,
  412.                                      page_length);
  413.         kunmap(page);
  414.  
  415.         return ret ? - EFAULT : 0;
  416. }
  417.  
  418. static int
  419. i915_gem_shmem_pread(struct drm_device *dev,
  420.                           struct drm_i915_gem_object *obj,
  421.                           struct drm_i915_gem_pread *args,
  422.                           struct drm_file *file)
  423. {
  424.         char __user *user_data;
  425.         ssize_t remain;
  426.         loff_t offset;
  427.         int shmem_page_offset, page_length, ret = 0;
  428.         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
  429.         int prefaulted = 0;
  430.         int needs_clflush = 0;
  431.         struct sg_page_iter sg_iter;
  432.  
  433.         user_data = to_user_ptr(args->data_ptr);
  434.         remain = args->size;
  435.  
  436.         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
  437.  
  438.         ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
  439.         if (ret)
  440.                 return ret;
  441.  
  442.         offset = args->offset;
  443.  
  444.         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
  445.                          offset >> PAGE_SHIFT) {
  446.                 struct page *page = sg_page_iter_page(&sg_iter);
  447.  
  448.                 if (remain <= 0)
  449.                         break;
  450.  
  451.                 /* Operation in this page
  452.                  *
  453.                  * shmem_page_offset = offset within page in shmem file
  454.                  * page_length = bytes to copy for this page
  455.                  */
  456.                 shmem_page_offset = offset_in_page(offset);
  457.                 page_length = remain;
  458.                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
  459.                         page_length = PAGE_SIZE - shmem_page_offset;
  460.  
  461.                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
  462.                         (page_to_phys(page) & (1 << 17)) != 0;
  463.  
  464.                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
  465.                                        user_data, page_do_bit17_swizzling,
  466.                                        needs_clflush);
  467.                 if (ret == 0)
  468.                         goto next_page;
  469.  
  470.                 mutex_unlock(&dev->struct_mutex);
  471.  
  472.                 if (likely(!i915.prefault_disable) && !prefaulted) {
  473.                         ret = fault_in_multipages_writeable(user_data, remain);
  474.                         /* Userspace is tricking us, but we've already clobbered
  475.                          * its pages with the prefault and promised to write the
  476.                          * data up to the first fault. Hence ignore any errors
  477.                          * and just continue. */
  478.                         (void)ret;
  479.                         prefaulted = 1;
  480.                 }
  481.  
  482.                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
  483.                                        user_data, page_do_bit17_swizzling,
  484.                                        needs_clflush);
  485.  
  486.                 mutex_lock(&dev->struct_mutex);
  487.  
  488.                 if (ret)
  489.                         goto out;
  490.  
  491. next_page:
  492.                 remain -= page_length;
  493.                 user_data += page_length;
  494.                 offset += page_length;
  495.         }
  496.  
  497. out:
  498.         i915_gem_object_unpin_pages(obj);
  499.  
  500.         return ret;
  501. }
  502.  
  503. /**
  504.  * Reads data from the object referenced by handle.
  505.  *
  506.  * On error, the contents of *data are undefined.
  507.  */
  508. int
  509. i915_gem_pread_ioctl(struct drm_device *dev, void *data,
  510.                      struct drm_file *file)
  511. {
  512.         struct drm_i915_gem_pread *args = data;
  513.         struct drm_i915_gem_object *obj;
  514.         int ret = 0;
  515.  
  516.         if (args->size == 0)
  517.                 return 0;
  518.  
  519.         if (!access_ok(VERIFY_WRITE,
  520.                        to_user_ptr(args->data_ptr),
  521.                        args->size))
  522.                 return -EFAULT;
  523.  
  524.         ret = i915_mutex_lock_interruptible(dev);
  525.         if (ret)
  526.                 return ret;
  527.  
  528.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  529.         if (&obj->base == NULL) {
  530.                 ret = -ENOENT;
  531.                 goto unlock;
  532.         }
  533.  
  534.         /* Bounds check source.  */
  535.         if (args->offset > obj->base.size ||
  536.             args->size > obj->base.size - args->offset) {
  537.                 ret = -EINVAL;
  538.                 goto out;
  539.         }
  540.  
  541.         /* prime objects have no backing filp to GEM pread/pwrite
  542.          * pages from.
  543.          */
  544.         if (!obj->base.filp) {
  545.                 ret = -EINVAL;
  546.                 goto out;
  547.         }
  548.  
  549.         trace_i915_gem_object_pread(obj, args->offset, args->size);
  550.  
  551.         ret = i915_gem_shmem_pread(dev, obj, args, file);
  552.  
  553. out:
  554.         drm_gem_object_unreference(&obj->base);
  555. unlock:
  556.         mutex_unlock(&dev->struct_mutex);
  557.         return ret;
  558. }
  559.  
  560. /* This is the fast write path which cannot handle
  561.  * page faults in the source data
  562.  */
  563.  
  564. static inline int
  565. fast_user_write(struct io_mapping *mapping,
  566.                 loff_t page_base, int page_offset,
  567.                 char __user *user_data,
  568.                 int length)
  569. {
  570.         void __iomem *vaddr_atomic;
  571.         void *vaddr;
  572.         unsigned long unwritten;
  573.  
  574.         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
  575.         /* We can use the cpu mem copy function because this is X86. */
  576.         vaddr = (void __force*)vaddr_atomic + page_offset;
  577.         unwritten = __copy_from_user_inatomic_nocache(vaddr,
  578.                                                       user_data, length);
  579.         io_mapping_unmap_atomic(vaddr_atomic);
  580.         return unwritten;
  581. }
  582. #endif
  583.  
  584. #define offset_in_page(p)       ((unsigned long)(p) & ~PAGE_MASK)
  585. /**
  586.  * This is the fast pwrite path, where we copy the data directly from the
  587.  * user into the GTT, uncached.
  588.  */
  589. static int
  590. i915_gem_gtt_pwrite_fast(struct drm_device *dev,
  591.                          struct drm_i915_gem_object *obj,
  592.                          struct drm_i915_gem_pwrite *args,
  593.                          struct drm_file *file)
  594. {
  595.         struct drm_i915_private *dev_priv = dev->dev_private;
  596.         ssize_t remain;
  597.         loff_t offset, page_base;
  598.         char __user *user_data;
  599.         int page_offset, page_length, ret;
  600.  
  601.         ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK);
  602.         if (ret)
  603.                 goto out;
  604.  
  605.         ret = i915_gem_object_set_to_gtt_domain(obj, true);
  606.         if (ret)
  607.                 goto out_unpin;
  608.  
  609.         ret = i915_gem_object_put_fence(obj);
  610.         if (ret)
  611.                 goto out_unpin;
  612.  
  613.         user_data = to_user_ptr(args->data_ptr);
  614.         remain = args->size;
  615.  
  616.         offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
  617.  
  618.         while (remain > 0) {
  619.                 /* Operation in this page
  620.                  *
  621.                  * page_base = page offset within aperture
  622.                  * page_offset = offset within page
  623.                  * page_length = bytes to copy for this page
  624.                  */
  625.                 page_base = offset & PAGE_MASK;
  626.                 page_offset = offset_in_page(offset);
  627.                 page_length = remain;
  628.                 if ((page_offset + remain) > PAGE_SIZE)
  629.                         page_length = PAGE_SIZE - page_offset;
  630.  
  631.         MapPage(dev_priv->gtt.mappable, dev_priv->gtt.mappable_base+page_base, PG_SW);
  632.  
  633.         memcpy((char*)dev_priv->gtt.mappable+page_offset, user_data, page_length);
  634.  
  635.                 remain -= page_length;
  636.                 user_data += page_length;
  637.                 offset += page_length;
  638.         }
  639.  
  640. out_unpin:
  641.         i915_gem_object_ggtt_unpin(obj);
  642. out:
  643.     return ret;
  644. }
  645.  
  646. /* Per-page copy function for the shmem pwrite fastpath.
  647.  * Flushes invalid cachelines before writing to the target if
  648.  * needs_clflush_before is set and flushes out any written cachelines after
  649.  * writing if needs_clflush is set. */
  650. static int
  651. shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
  652.                   char __user *user_data,
  653.                   bool page_do_bit17_swizzling,
  654.                   bool needs_clflush_before,
  655.                   bool needs_clflush_after)
  656. {
  657.         char *vaddr;
  658.         int ret;
  659.  
  660.         if (unlikely(page_do_bit17_swizzling))
  661.                 return -EINVAL;
  662.  
  663.         vaddr = kmap_atomic(page);
  664.         if (needs_clflush_before)
  665.                 drm_clflush_virt_range(vaddr + shmem_page_offset,
  666.                                        page_length);
  667.         memcpy(vaddr + shmem_page_offset,
  668.                                                 user_data,
  669.                                                 page_length);
  670.         if (needs_clflush_after)
  671.                 drm_clflush_virt_range(vaddr + shmem_page_offset,
  672.                                        page_length);
  673.         kunmap_atomic(vaddr);
  674.  
  675.         return ret ? -EFAULT : 0;
  676. }
  677. #if 0
  678.  
  679. /* Only difference to the fast-path function is that this can handle bit17
  680.  * and uses non-atomic copy and kmap functions. */
  681. static int
  682. shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
  683.                   char __user *user_data,
  684.                   bool page_do_bit17_swizzling,
  685.                   bool needs_clflush_before,
  686.                   bool needs_clflush_after)
  687. {
  688.         char *vaddr;
  689.         int ret;
  690.  
  691.         vaddr = kmap(page);
  692.         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
  693.                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  694.                                              page_length,
  695.                                              page_do_bit17_swizzling);
  696.         if (page_do_bit17_swizzling)
  697.                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
  698.                                                 user_data,
  699.                                                 page_length);
  700.         else
  701.                 ret = __copy_from_user(vaddr + shmem_page_offset,
  702.                                        user_data,
  703.                                        page_length);
  704.         if (needs_clflush_after)
  705.                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  706.                                              page_length,
  707.                                              page_do_bit17_swizzling);
  708.         kunmap(page);
  709.  
  710.         return ret ? -EFAULT : 0;
  711. }
  712. #endif
  713.  
  714.  
  715. static int
  716. i915_gem_shmem_pwrite(struct drm_device *dev,
  717.                       struct drm_i915_gem_object *obj,
  718.                       struct drm_i915_gem_pwrite *args,
  719.                       struct drm_file *file)
  720. {
  721.         ssize_t remain;
  722.         loff_t offset;
  723.         char __user *user_data;
  724.         int shmem_page_offset, page_length, ret = 0;
  725.         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
  726.         int hit_slowpath = 0;
  727.         int needs_clflush_after = 0;
  728.         int needs_clflush_before = 0;
  729.         struct sg_page_iter sg_iter;
  730.  
  731.         user_data = to_user_ptr(args->data_ptr);
  732.         remain = args->size;
  733.  
  734.         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
  735.  
  736.         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  737.                 /* If we're not in the cpu write domain, set ourself into the gtt
  738.                  * write domain and manually flush cachelines (if required). This
  739.                  * optimizes for the case when the gpu will use the data
  740.                  * right away and we therefore have to clflush anyway. */
  741.                 needs_clflush_after = cpu_write_needs_clflush(obj);
  742.                 ret = i915_gem_object_wait_rendering(obj, false);
  743.                         if (ret)
  744.                                 return ret;
  745.  
  746.                 i915_gem_object_retire(obj);
  747.                 }
  748.         /* Same trick applies to invalidate partially written cachelines read
  749.          * before writing. */
  750.         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
  751.                 needs_clflush_before =
  752.                         !cpu_cache_is_coherent(dev, obj->cache_level);
  753.  
  754.         ret = i915_gem_object_get_pages(obj);
  755.         if (ret)
  756.                 return ret;
  757.  
  758.         i915_gem_object_pin_pages(obj);
  759.  
  760.         offset = args->offset;
  761.         obj->dirty = 1;
  762.  
  763.         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
  764.                          offset >> PAGE_SHIFT) {
  765.                 struct page *page = sg_page_iter_page(&sg_iter);
  766.                 int partial_cacheline_write;
  767.  
  768.                 if (remain <= 0)
  769.                         break;
  770.  
  771.                 /* Operation in this page
  772.                  *
  773.                  * shmem_page_offset = offset within page in shmem file
  774.                  * page_length = bytes to copy for this page
  775.                  */
  776.                 shmem_page_offset = offset_in_page(offset);
  777.  
  778.                 page_length = remain;
  779.                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
  780.                         page_length = PAGE_SIZE - shmem_page_offset;
  781.  
  782.                 /* If we don't overwrite a cacheline completely we need to be
  783.                  * careful to have up-to-date data by first clflushing. Don't
  784.                  * overcomplicate things and flush the entire patch. */
  785.                 partial_cacheline_write = needs_clflush_before &&
  786.                         ((shmem_page_offset | page_length)
  787.                                 & (x86_clflush_size - 1));
  788.  
  789.                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
  790.                         (page_to_phys(page) & (1 << 17)) != 0;
  791.  
  792.                 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
  793.                                         user_data, page_do_bit17_swizzling,
  794.                                         partial_cacheline_write,
  795.                                         needs_clflush_after);
  796.                 if (ret == 0)
  797.                         goto next_page;
  798.  
  799.                 hit_slowpath = 1;
  800.                 mutex_unlock(&dev->struct_mutex);
  801.                 dbgprintf("%s need shmem_pwrite_slow\n",__FUNCTION__);
  802.  
  803. //              ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
  804. //                                      user_data, page_do_bit17_swizzling,
  805. //                                      partial_cacheline_write,
  806. //                                      needs_clflush_after);
  807.  
  808.                 mutex_lock(&dev->struct_mutex);
  809.  
  810.                 if (ret)
  811.                         goto out;
  812.  
  813. next_page:
  814.                 remain -= page_length;
  815.                 user_data += page_length;
  816.                 offset += page_length;
  817.         }
  818.  
  819. out:
  820.         i915_gem_object_unpin_pages(obj);
  821.  
  822.         if (hit_slowpath) {
  823.                 /*
  824.                  * Fixup: Flush cpu caches in case we didn't flush the dirty
  825.                  * cachelines in-line while writing and the object moved
  826.                  * out of the cpu write domain while we've dropped the lock.
  827.                  */
  828.                 if (!needs_clflush_after &&
  829.                     obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  830.                         if (i915_gem_clflush_object(obj, obj->pin_display))
  831.                         i915_gem_chipset_flush(dev);
  832.                 }
  833.         }
  834.  
  835.         if (needs_clflush_after)
  836.                 i915_gem_chipset_flush(dev);
  837.  
  838.         return ret;
  839. }
  840.  
  841. /**
  842.  * Writes data to the object referenced by handle.
  843.  *
  844.  * On error, the contents of the buffer that were to be modified are undefined.
  845.  */
  846. int
  847. i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
  848.                       struct drm_file *file)
  849. {
  850.         struct drm_i915_gem_pwrite *args = data;
  851.         struct drm_i915_gem_object *obj;
  852.         int ret;
  853.  
  854.         if (args->size == 0)
  855.                 return 0;
  856.  
  857.  
  858.         ret = i915_mutex_lock_interruptible(dev);
  859.         if (ret)
  860.                 return ret;
  861.  
  862.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  863.         if (&obj->base == NULL) {
  864.                 ret = -ENOENT;
  865.                 goto unlock;
  866.         }
  867.  
  868.         /* Bounds check destination. */
  869.         if (args->offset > obj->base.size ||
  870.             args->size > obj->base.size - args->offset) {
  871.                 ret = -EINVAL;
  872.                 goto out;
  873.         }
  874.  
  875.         /* prime objects have no backing filp to GEM pread/pwrite
  876.          * pages from.
  877.          */
  878.         if (!obj->base.filp) {
  879.                 ret = -EINVAL;
  880.                 goto out;
  881.         }
  882.  
  883.         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
  884.  
  885.         ret = -EFAULT;
  886.         /* We can only do the GTT pwrite on untiled buffers, as otherwise
  887.          * it would end up going through the fenced access, and we'll get
  888.          * different detiling behavior between reading and writing.
  889.          * pread/pwrite currently are reading and writing from the CPU
  890.          * perspective, requiring manual detiling by the client.
  891.          */
  892.         if (obj->tiling_mode == I915_TILING_NONE &&
  893.             obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
  894.             cpu_write_needs_clflush(obj)) {
  895.                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
  896.                 /* Note that the gtt paths might fail with non-page-backed user
  897.                  * pointers (e.g. gtt mappings when moving data between
  898.                  * textures). Fallback to the shmem path in that case. */
  899.         }
  900.  
  901.         if (ret == -EFAULT || ret == -ENOSPC)
  902.        ret = i915_gem_shmem_pwrite(dev, obj, args, file);
  903.  
  904. out:
  905.         drm_gem_object_unreference(&obj->base);
  906. unlock:
  907.         mutex_unlock(&dev->struct_mutex);
  908.         return ret;
  909. }
  910.  
  911. int
  912. i915_gem_check_wedge(struct i915_gpu_error *error,
  913.                      bool interruptible)
  914. {
  915.         if (i915_reset_in_progress(error)) {
  916.                 /* Non-interruptible callers can't handle -EAGAIN, hence return
  917.                  * -EIO unconditionally for these. */
  918.                 if (!interruptible)
  919.                         return -EIO;
  920.  
  921.                 /* Recovery complete, but the reset failed ... */
  922.                 if (i915_terminally_wedged(error))
  923.                         return -EIO;
  924.  
  925.                 return -EAGAIN;
  926.         }
  927.  
  928.         return 0;
  929. }
  930.  
  931. /*
  932.  * Compare seqno against outstanding lazy request. Emit a request if they are
  933.  * equal.
  934.  */
  935. int
  936. i915_gem_check_olr(struct intel_engine_cs *ring, u32 seqno)
  937. {
  938.         int ret;
  939.  
  940.         BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
  941.  
  942.         ret = 0;
  943.         if (seqno == ring->outstanding_lazy_seqno)
  944.                 ret = i915_add_request(ring, NULL);
  945.  
  946.         return ret;
  947. }
  948.  
  949. static void fake_irq(unsigned long data)
  950. {
  951. //      wake_up_process((struct task_struct *)data);
  952. }
  953.  
  954. static bool missed_irq(struct drm_i915_private *dev_priv,
  955.                        struct intel_engine_cs *ring)
  956. {
  957.         return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings);
  958. }
  959.  
  960. static bool can_wait_boost(struct drm_i915_file_private *file_priv)
  961. {
  962.         if (file_priv == NULL)
  963.                 return true;
  964.  
  965.         return !atomic_xchg(&file_priv->rps_wait_boost, true);
  966. }
  967.  
  968. /**
  969.  * __i915_wait_seqno - wait until execution of seqno has finished
  970.  * @ring: the ring expected to report seqno
  971.  * @seqno: duh!
  972.  * @reset_counter: reset sequence associated with the given seqno
  973.  * @interruptible: do an interruptible wait (normally yes)
  974.  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
  975.  *
  976.  * Note: It is of utmost importance that the passed in seqno and reset_counter
  977.  * values have been read by the caller in an smp safe manner. Where read-side
  978.  * locks are involved, it is sufficient to read the reset_counter before
  979.  * unlocking the lock that protects the seqno. For lockless tricks, the
  980.  * reset_counter _must_ be read before, and an appropriate smp_rmb must be
  981.  * inserted.
  982.  *
  983.  * Returns 0 if the seqno was found within the alloted time. Else returns the
  984.  * errno with remaining time filled in timeout argument.
  985.  */
  986. int __i915_wait_seqno(struct intel_engine_cs *ring, u32 seqno,
  987.                         unsigned reset_counter,
  988.                         bool interruptible,
  989.                         s64 *timeout,
  990.                         struct drm_i915_file_private *file_priv)
  991. {
  992.         struct drm_device *dev = ring->dev;
  993.         struct drm_i915_private *dev_priv = dev->dev_private;
  994.         const bool irq_test_in_progress =
  995.                 ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_ring_flag(ring);
  996.         unsigned long timeout_expire;
  997.         s64 before, now;
  998.  
  999.     wait_queue_t __wait;
  1000.         int ret;
  1001.  
  1002.         WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled");
  1003.  
  1004.         if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
  1005.                 return 0;
  1006.  
  1007.         timeout_expire = timeout ?
  1008.                 jiffies + nsecs_to_jiffies_timeout((u64)*timeout) : 0;
  1009.  
  1010.         if (INTEL_INFO(dev)->gen >= 6 && ring->id == RCS && can_wait_boost(file_priv)) {
  1011.                 gen6_rps_boost(dev_priv);
  1012.                 if (file_priv)
  1013.                         mod_delayed_work(dev_priv->wq,
  1014.                                          &file_priv->mm.idle_work,
  1015.                                          msecs_to_jiffies(100));
  1016.         }
  1017.  
  1018.         if (!irq_test_in_progress && WARN_ON(!ring->irq_get(ring)))
  1019.                 return -ENODEV;
  1020.  
  1021.     INIT_LIST_HEAD(&__wait.task_list);
  1022.     __wait.evnt = CreateEvent(NULL, MANUAL_DESTROY);
  1023.  
  1024.         /* Record current time in case interrupted by signal, or wedged */
  1025.         trace_i915_gem_request_wait_begin(ring, seqno);
  1026.  
  1027.         for (;;) {
  1028.         unsigned long flags;
  1029.  
  1030.                 /* We need to check whether any gpu reset happened in between
  1031.                  * the caller grabbing the seqno and now ... */
  1032.                 if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) {
  1033.                         /* ... but upgrade the -EAGAIN to an -EIO if the gpu
  1034.                          * is truely gone. */
  1035.                         ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
  1036.                         if (ret == 0)
  1037.                                 ret = -EAGAIN;
  1038.                         break;
  1039.                 }
  1040.  
  1041.                 if (i915_seqno_passed(ring->get_seqno(ring, false), seqno)) {
  1042.                         ret = 0;
  1043.                         break;
  1044.                 }
  1045.  
  1046.         if (timeout && time_after_eq(jiffies, timeout_expire)) {
  1047.                         ret = -ETIME;
  1048.                         break;
  1049.                 }
  1050.  
  1051.         spin_lock_irqsave(&ring->irq_queue.lock, flags);
  1052.         if (list_empty(&__wait.task_list))
  1053.             __add_wait_queue(&ring->irq_queue, &__wait);
  1054.         spin_unlock_irqrestore(&ring->irq_queue.lock, flags);
  1055.  
  1056.         WaitEventTimeout(__wait.evnt, 1);
  1057.  
  1058.         if (!list_empty(&__wait.task_list)) {
  1059.             spin_lock_irqsave(&ring->irq_queue.lock, flags);
  1060.             list_del_init(&__wait.task_list);
  1061.             spin_unlock_irqrestore(&ring->irq_queue.lock, flags);
  1062.         }
  1063.     };
  1064.     trace_i915_gem_request_wait_end(ring, seqno);
  1065.  
  1066.     DestroyEvent(__wait.evnt);
  1067.  
  1068.         if (!irq_test_in_progress)
  1069.         ring->irq_put(ring);
  1070.  
  1071. //      finish_wait(&ring->irq_queue, &wait);
  1072.         return ret;
  1073. }
  1074.  
  1075. /**
  1076.  * Waits for a sequence number to be signaled, and cleans up the
  1077.  * request and object lists appropriately for that event.
  1078.  */
  1079. int
  1080. i915_wait_seqno(struct intel_engine_cs *ring, uint32_t seqno)
  1081. {
  1082.         struct drm_device *dev = ring->dev;
  1083.         struct drm_i915_private *dev_priv = dev->dev_private;
  1084.         bool interruptible = dev_priv->mm.interruptible;
  1085.         unsigned reset_counter;
  1086.         int ret;
  1087.  
  1088.         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
  1089.         BUG_ON(seqno == 0);
  1090.  
  1091.         ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
  1092.         if (ret)
  1093.                 return ret;
  1094.  
  1095.         ret = i915_gem_check_olr(ring, seqno);
  1096.         if (ret)
  1097.                 return ret;
  1098.  
  1099.         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
  1100.         return __i915_wait_seqno(ring, seqno, reset_counter, interruptible,
  1101.                                  NULL, NULL);
  1102. }
  1103.  
  1104. static int
  1105. i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj)
  1106. {
  1107.         if (!obj->active)
  1108.                 return 0;
  1109.  
  1110.         /* Manually manage the write flush as we may have not yet
  1111.          * retired the buffer.
  1112.          *
  1113.          * Note that the last_write_seqno is always the earlier of
  1114.          * the two (read/write) seqno, so if we haved successfully waited,
  1115.          * we know we have passed the last write.
  1116.          */
  1117.         obj->last_write_seqno = 0;
  1118.  
  1119.         return 0;
  1120. }
  1121.  
  1122. /**
  1123.  * Ensures that all rendering to the object has completed and the object is
  1124.  * safe to unbind from the GTT or access from the CPU.
  1125.  */
  1126. static __must_check int
  1127. i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
  1128.                                bool readonly)
  1129. {
  1130.         struct intel_engine_cs *ring = obj->ring;
  1131.         u32 seqno;
  1132.         int ret;
  1133.  
  1134.         seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
  1135.         if (seqno == 0)
  1136.                 return 0;
  1137.  
  1138.         ret = i915_wait_seqno(ring, seqno);
  1139.     if (ret)
  1140.         return ret;
  1141.  
  1142.         return i915_gem_object_wait_rendering__tail(obj);
  1143. }
  1144.  
  1145. /* A nonblocking variant of the above wait. This is a highly dangerous routine
  1146.  * as the object state may change during this call.
  1147.  */
  1148. static __must_check int
  1149. i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
  1150.                                             struct drm_i915_file_private *file_priv,
  1151.                                             bool readonly)
  1152. {
  1153.         struct drm_device *dev = obj->base.dev;
  1154.         struct drm_i915_private *dev_priv = dev->dev_private;
  1155.         struct intel_engine_cs *ring = obj->ring;
  1156.         unsigned reset_counter;
  1157.         u32 seqno;
  1158.         int ret;
  1159.  
  1160.         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
  1161.         BUG_ON(!dev_priv->mm.interruptible);
  1162.  
  1163.         seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
  1164.         if (seqno == 0)
  1165.                 return 0;
  1166.  
  1167.         ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
  1168.         if (ret)
  1169.                 return ret;
  1170.  
  1171.         ret = i915_gem_check_olr(ring, seqno);
  1172.         if (ret)
  1173.                 return ret;
  1174.  
  1175.         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
  1176.         mutex_unlock(&dev->struct_mutex);
  1177.         ret = __i915_wait_seqno(ring, seqno, reset_counter, true, NULL,
  1178.                                 file_priv);
  1179.         mutex_lock(&dev->struct_mutex);
  1180.         if (ret)
  1181.                 return ret;
  1182.  
  1183.         return i915_gem_object_wait_rendering__tail(obj);
  1184. }
  1185.  
  1186. /**
  1187.  * Called when user space prepares to use an object with the CPU, either
  1188.  * through the mmap ioctl's mapping or a GTT mapping.
  1189.  */
  1190. int
  1191. i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
  1192.                           struct drm_file *file)
  1193. {
  1194.         struct drm_i915_gem_set_domain *args = data;
  1195.         struct drm_i915_gem_object *obj;
  1196.         uint32_t read_domains = args->read_domains;
  1197.         uint32_t write_domain = args->write_domain;
  1198.         int ret;
  1199.  
  1200.         /* Only handle setting domains to types used by the CPU. */
  1201.         if (write_domain & I915_GEM_GPU_DOMAINS)
  1202.                 return -EINVAL;
  1203.  
  1204.         if (read_domains & I915_GEM_GPU_DOMAINS)
  1205.                 return -EINVAL;
  1206.  
  1207.         /* Having something in the write domain implies it's in the read
  1208.          * domain, and only that read domain.  Enforce that in the request.
  1209.          */
  1210.         if (write_domain != 0 && read_domains != write_domain)
  1211.                 return -EINVAL;
  1212.  
  1213.         ret = i915_mutex_lock_interruptible(dev);
  1214.         if (ret)
  1215.                 return ret;
  1216.  
  1217.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  1218.         if (&obj->base == NULL) {
  1219.                 ret = -ENOENT;
  1220.                 goto unlock;
  1221.         }
  1222.  
  1223.         /* Try to flush the object off the GPU without holding the lock.
  1224.          * We will repeat the flush holding the lock in the normal manner
  1225.          * to catch cases where we are gazumped.
  1226.          */
  1227.         ret = i915_gem_object_wait_rendering__nonblocking(obj,
  1228.                                                           file->driver_priv,
  1229.                                                           !write_domain);
  1230.         if (ret)
  1231.                 goto unref;
  1232.  
  1233.         if (read_domains & I915_GEM_DOMAIN_GTT) {
  1234.                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
  1235.  
  1236.                 /* Silently promote "you're not bound, there was nothing to do"
  1237.                  * to success, since the client was just asking us to
  1238.                  * make sure everything was done.
  1239.                  */
  1240.                 if (ret == -EINVAL)
  1241.                         ret = 0;
  1242.         } else {
  1243.                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
  1244.         }
  1245.  
  1246. unref:
  1247.         drm_gem_object_unreference(&obj->base);
  1248. unlock:
  1249.         mutex_unlock(&dev->struct_mutex);
  1250.         return ret;
  1251. }
  1252.  
  1253. /**
  1254.  * Called when user space has done writes to this buffer
  1255.  */
  1256. int
  1257. i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
  1258.                          struct drm_file *file)
  1259. {
  1260.         struct drm_i915_gem_sw_finish *args = data;
  1261.         struct drm_i915_gem_object *obj;
  1262.         int ret = 0;
  1263.  
  1264.         ret = i915_mutex_lock_interruptible(dev);
  1265.         if (ret)
  1266.                 return ret;
  1267.  
  1268.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  1269.         if (&obj->base == NULL) {
  1270.                 ret = -ENOENT;
  1271.                 goto unlock;
  1272.         }
  1273.  
  1274.         /* Pinned buffers may be scanout, so flush the cache */
  1275.         if (obj->pin_display)
  1276.                 i915_gem_object_flush_cpu_write_domain(obj, true);
  1277.  
  1278.         drm_gem_object_unreference(&obj->base);
  1279. unlock:
  1280.         mutex_unlock(&dev->struct_mutex);
  1281.         return ret;
  1282. }
  1283.  
  1284. /**
  1285.  * Maps the contents of an object, returning the address it is mapped
  1286.  * into.
  1287.  *
  1288.  * While the mapping holds a reference on the contents of the object, it doesn't
  1289.  * imply a ref on the object itself.
  1290.  *
  1291.  * IMPORTANT:
  1292.  *
  1293.  * DRM driver writers who look a this function as an example for how to do GEM
  1294.  * mmap support, please don't implement mmap support like here. The modern way
  1295.  * to implement DRM mmap support is with an mmap offset ioctl (like
  1296.  * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
  1297.  * That way debug tooling like valgrind will understand what's going on, hiding
  1298.  * the mmap call in a driver private ioctl will break that. The i915 driver only
  1299.  * does cpu mmaps this way because we didn't know better.
  1300.  */
  1301. int
  1302. i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
  1303.                     struct drm_file *file)
  1304. {
  1305.         struct drm_i915_gem_mmap *args = data;
  1306.         struct drm_gem_object *obj;
  1307.         unsigned long addr;
  1308.  
  1309.         obj = drm_gem_object_lookup(dev, file, args->handle);
  1310.         if (obj == NULL)
  1311.                 return -ENOENT;
  1312.  
  1313.         /* prime objects have no backing filp to GEM mmap
  1314.          * pages from.
  1315.          */
  1316.         if (!obj->filp) {
  1317.                 drm_gem_object_unreference_unlocked(obj);
  1318.                 return -EINVAL;
  1319.         }
  1320.  
  1321.     addr = vm_mmap(obj->filp, 0, args->size,
  1322.               PROT_READ | PROT_WRITE, MAP_SHARED,
  1323.               args->offset);
  1324.         drm_gem_object_unreference_unlocked(obj);
  1325.     if (IS_ERR((void *)addr))
  1326.         return addr;
  1327.  
  1328.         args->addr_ptr = (uint64_t) addr;
  1329.  
  1330.     return 0;
  1331. }
  1332.  
  1333.  
  1334.  
  1335.  
  1336.  
  1337.  
  1338.  
  1339.  
  1340.  
  1341.  
  1342.  
  1343.  
  1344.  
  1345. /**
  1346.  * i915_gem_release_mmap - remove physical page mappings
  1347.  * @obj: obj in question
  1348.  *
  1349.  * Preserve the reservation of the mmapping with the DRM core code, but
  1350.  * relinquish ownership of the pages back to the system.
  1351.  *
  1352.  * It is vital that we remove the page mapping if we have mapped a tiled
  1353.  * object through the GTT and then lose the fence register due to
  1354.  * resource pressure. Similarly if the object has been moved out of the
  1355.  * aperture, than pages mapped into userspace must be revoked. Removing the
  1356.  * mapping will then trigger a page fault on the next user access, allowing
  1357.  * fixup by i915_gem_fault().
  1358.  */
  1359. void
  1360. i915_gem_release_mmap(struct drm_i915_gem_object *obj)
  1361. {
  1362.         if (!obj->fault_mappable)
  1363.                 return;
  1364.  
  1365. //      drm_vma_node_unmap(&obj->base.vma_node, obj->base.dev->dev_mapping);
  1366.         obj->fault_mappable = false;
  1367. }
  1368.  
  1369. uint32_t
  1370. i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
  1371. {
  1372.         uint32_t gtt_size;
  1373.  
  1374.         if (INTEL_INFO(dev)->gen >= 4 ||
  1375.             tiling_mode == I915_TILING_NONE)
  1376.                 return size;
  1377.  
  1378.         /* Previous chips need a power-of-two fence region when tiling */
  1379.         if (INTEL_INFO(dev)->gen == 3)
  1380.                 gtt_size = 1024*1024;
  1381.         else
  1382.                 gtt_size = 512*1024;
  1383.  
  1384.         while (gtt_size < size)
  1385.                 gtt_size <<= 1;
  1386.  
  1387.         return gtt_size;
  1388. }
  1389.  
  1390. /**
  1391.  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
  1392.  * @obj: object to check
  1393.  *
  1394.  * Return the required GTT alignment for an object, taking into account
  1395.  * potential fence register mapping.
  1396.  */
  1397. uint32_t
  1398. i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
  1399.                            int tiling_mode, bool fenced)
  1400. {
  1401.         /*
  1402.          * Minimum alignment is 4k (GTT page size), but might be greater
  1403.          * if a fence register is needed for the object.
  1404.          */
  1405.         if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
  1406.             tiling_mode == I915_TILING_NONE)
  1407.                 return 4096;
  1408.  
  1409.         /*
  1410.          * Previous chips need to be aligned to the size of the smallest
  1411.          * fence register that can contain the object.
  1412.          */
  1413.         return i915_gem_get_gtt_size(dev, size, tiling_mode);
  1414. }
  1415.  
  1416.  
  1417.  
  1418. int
  1419. i915_gem_mmap_gtt(struct drm_file *file,
  1420.           struct drm_device *dev,
  1421.           uint32_t handle,
  1422.           uint64_t *offset)
  1423. {
  1424.     struct drm_i915_private *dev_priv = dev->dev_private;
  1425.     struct drm_i915_gem_object *obj;
  1426.     unsigned long pfn;
  1427.     char *mem, *ptr;
  1428.     int ret;
  1429.  
  1430.     ret = i915_mutex_lock_interruptible(dev);
  1431.     if (ret)
  1432.         return ret;
  1433.  
  1434.     obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
  1435.     if (&obj->base == NULL) {
  1436.         ret = -ENOENT;
  1437.         goto unlock;
  1438.     }
  1439.  
  1440.     if (obj->base.size > dev_priv->gtt.mappable_end) {
  1441.         ret = -E2BIG;
  1442.         goto out;
  1443.     }
  1444.  
  1445.     if (obj->madv != I915_MADV_WILLNEED) {
  1446.                 DRM_DEBUG("Attempting to mmap a purgeable buffer\n");
  1447.                 ret = -EFAULT;
  1448.         goto out;
  1449.     }
  1450.     /* Now bind it into the GTT if needed */
  1451.         ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK);
  1452.     if (ret)
  1453.         goto out;
  1454.  
  1455.     ret = i915_gem_object_set_to_gtt_domain(obj, 1);
  1456.     if (ret)
  1457.         goto unpin;
  1458.  
  1459.     ret = i915_gem_object_get_fence(obj);
  1460.     if (ret)
  1461.         goto unpin;
  1462.  
  1463.     obj->fault_mappable = true;
  1464.  
  1465.     pfn = dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj);
  1466.  
  1467.     /* Finally, remap it using the new GTT offset */
  1468.  
  1469.     mem = UserAlloc(obj->base.size);
  1470.     if(unlikely(mem == NULL))
  1471.     {
  1472.         ret = -ENOMEM;
  1473.         goto unpin;
  1474.     }
  1475.  
  1476.     for(ptr = mem; ptr < mem + obj->base.size; ptr+= 4096, pfn+= 4096)
  1477.         MapPage(ptr, pfn, PG_SHARED|PG_UW);
  1478.  
  1479. unpin:
  1480.     i915_gem_object_unpin_pages(obj);
  1481.  
  1482.  
  1483.     *offset = (uint32_t)mem;
  1484.  
  1485. out:
  1486.     drm_gem_object_unreference(&obj->base);
  1487. unlock:
  1488.     mutex_unlock(&dev->struct_mutex);
  1489.     return ret;
  1490. }
  1491.  
  1492. /**
  1493.  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
  1494.  * @dev: DRM device
  1495.  * @data: GTT mapping ioctl data
  1496.  * @file: GEM object info
  1497.  *
  1498.  * Simply returns the fake offset to userspace so it can mmap it.
  1499.  * The mmap call will end up in drm_gem_mmap(), which will set things
  1500.  * up so we can get faults in the handler above.
  1501.  *
  1502.  * The fault handler will take care of binding the object into the GTT
  1503.  * (since it may have been evicted to make room for something), allocating
  1504.  * a fence register, and mapping the appropriate aperture address into
  1505.  * userspace.
  1506.  */
  1507. int
  1508. i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
  1509.             struct drm_file *file)
  1510. {
  1511.     struct drm_i915_gem_mmap_gtt *args = data;
  1512.  
  1513.     return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
  1514. }
  1515.  
  1516. static inline int
  1517. i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
  1518. {
  1519.         return obj->madv == I915_MADV_DONTNEED;
  1520. }
  1521.  
  1522. /* Immediately discard the backing storage */
  1523. static void
  1524. i915_gem_object_truncate(struct drm_i915_gem_object *obj)
  1525. {
  1526. //      i915_gem_object_free_mmap_offset(obj);
  1527.  
  1528.         if (obj->base.filp == NULL)
  1529.                 return;
  1530.  
  1531.         /* Our goal here is to return as much of the memory as
  1532.          * is possible back to the system as we are called from OOM.
  1533.          * To do this we must instruct the shmfs to drop all of its
  1534.          * backing pages, *now*.
  1535.          */
  1536. //      shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
  1537.         obj->madv = __I915_MADV_PURGED;
  1538. }
  1539.  
  1540. /* Try to discard unwanted pages */
  1541. static void
  1542. i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
  1543. {
  1544.         struct address_space *mapping;
  1545.  
  1546.         switch (obj->madv) {
  1547.         case I915_MADV_DONTNEED:
  1548.                 i915_gem_object_truncate(obj);
  1549.         case __I915_MADV_PURGED:
  1550.                 return;
  1551.         }
  1552.  
  1553.         if (obj->base.filp == NULL)
  1554.                 return;
  1555.  
  1556. }
  1557.  
  1558. static void
  1559. i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
  1560. {
  1561.         struct sg_page_iter sg_iter;
  1562.         int ret;
  1563.  
  1564.         BUG_ON(obj->madv == __I915_MADV_PURGED);
  1565.  
  1566.         ret = i915_gem_object_set_to_cpu_domain(obj, true);
  1567.         if (ret) {
  1568.                 /* In the event of a disaster, abandon all caches and
  1569.                  * hope for the best.
  1570.                  */
  1571.                 WARN_ON(ret != -EIO);
  1572.                 i915_gem_clflush_object(obj, true);
  1573.                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  1574.         }
  1575.  
  1576.         if (obj->madv == I915_MADV_DONTNEED)
  1577.                 obj->dirty = 0;
  1578.  
  1579.         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
  1580.                 struct page *page = sg_page_iter_page(&sg_iter);
  1581.  
  1582.         page_cache_release(page);
  1583.         }
  1584.     obj->dirty = 0;
  1585.  
  1586.         sg_free_table(obj->pages);
  1587.         kfree(obj->pages);
  1588. }
  1589.  
  1590. int
  1591. i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
  1592. {
  1593.         const struct drm_i915_gem_object_ops *ops = obj->ops;
  1594.  
  1595.         if (obj->pages == NULL)
  1596.                 return 0;
  1597.  
  1598.         if (obj->pages_pin_count)
  1599.                 return -EBUSY;
  1600.  
  1601.         BUG_ON(i915_gem_obj_bound_any(obj));
  1602.  
  1603.         /* ->put_pages might need to allocate memory for the bit17 swizzle
  1604.          * array, hence protect them from being reaped by removing them from gtt
  1605.          * lists early. */
  1606.         list_del(&obj->global_list);
  1607.  
  1608.         ops->put_pages(obj);
  1609.         obj->pages = NULL;
  1610.  
  1611.         i915_gem_object_invalidate(obj);
  1612.  
  1613.         return 0;
  1614. }
  1615.  
  1616.  
  1617.  
  1618.  
  1619.  
  1620.  
  1621.  
  1622.  
  1623. static int
  1624. i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
  1625. {
  1626.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  1627.     int page_count, i;
  1628.     struct sg_table *st;
  1629.         struct scatterlist *sg;
  1630.         struct sg_page_iter sg_iter;
  1631.         struct page *page;
  1632.         unsigned long last_pfn = 0;     /* suppress gcc warning */
  1633.         gfp_t gfp;
  1634.  
  1635.         /* Assert that the object is not currently in any GPU domain. As it
  1636.          * wasn't in the GTT, there shouldn't be any way it could have been in
  1637.          * a GPU cache
  1638.          */
  1639.         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
  1640.         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
  1641.  
  1642.         st = kmalloc(sizeof(*st), GFP_KERNEL);
  1643.         if (st == NULL)
  1644.                 return -ENOMEM;
  1645.  
  1646.         page_count = obj->base.size / PAGE_SIZE;
  1647.         if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
  1648.                 kfree(st);
  1649.                 return -ENOMEM;
  1650.         }
  1651.  
  1652.         /* Get the list of pages out of our struct file.  They'll be pinned
  1653.          * at this point until we release them.
  1654.          *
  1655.          * Fail silently without starting the shrinker
  1656.          */
  1657.         sg = st->sgl;
  1658.         st->nents = 0;
  1659.         for (i = 0; i < page_count; i++) {
  1660.         page = shmem_read_mapping_page_gfp(obj->base.filp, i, gfp);
  1661.                 if (IS_ERR(page)) {
  1662.             dbgprintf("%s invalid page %p\n", __FUNCTION__, page);
  1663.                         goto err_pages;
  1664.                 }
  1665. #ifdef CONFIG_SWIOTLB
  1666.                 if (swiotlb_nr_tbl()) {
  1667.                         st->nents++;
  1668.                         sg_set_page(sg, page, PAGE_SIZE, 0);
  1669.                         sg = sg_next(sg);
  1670.                         continue;
  1671.                 }
  1672. #endif
  1673.                 if (!i || page_to_pfn(page) != last_pfn + 1) {
  1674.                         if (i)
  1675.                                 sg = sg_next(sg);
  1676.                         st->nents++;
  1677.                 sg_set_page(sg, page, PAGE_SIZE, 0);
  1678.                 } else {
  1679.                         sg->length += PAGE_SIZE;
  1680.                 }
  1681.                 last_pfn = page_to_pfn(page);
  1682.         }
  1683. #ifdef CONFIG_SWIOTLB
  1684.         if (!swiotlb_nr_tbl())
  1685. #endif
  1686.                 sg_mark_end(sg);
  1687.         obj->pages = st;
  1688.  
  1689.  
  1690.         if (obj->tiling_mode != I915_TILING_NONE &&
  1691.             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
  1692.                 i915_gem_object_pin_pages(obj);
  1693.  
  1694.         return 0;
  1695.  
  1696. err_pages:
  1697.         sg_mark_end(sg);
  1698.         for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
  1699.                 page_cache_release(sg_page_iter_page(&sg_iter));
  1700.         sg_free_table(st);
  1701.         kfree(st);
  1702.  
  1703.         return PTR_ERR(page);
  1704. }
  1705.  
  1706. /* Ensure that the associated pages are gathered from the backing storage
  1707.  * and pinned into our object. i915_gem_object_get_pages() may be called
  1708.  * multiple times before they are released by a single call to
  1709.  * i915_gem_object_put_pages() - once the pages are no longer referenced
  1710.  * either as a result of memory pressure (reaping pages under the shrinker)
  1711.  * or as the object is itself released.
  1712.  */
  1713. int
  1714. i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
  1715. {
  1716.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  1717.         const struct drm_i915_gem_object_ops *ops = obj->ops;
  1718.         int ret;
  1719.  
  1720.         if (obj->pages)
  1721.                 return 0;
  1722.  
  1723.         if (obj->madv != I915_MADV_WILLNEED) {
  1724.                 DRM_DEBUG("Attempting to obtain a purgeable object\n");
  1725.                 return -EFAULT;
  1726.         }
  1727.  
  1728.         BUG_ON(obj->pages_pin_count);
  1729.  
  1730.         ret = ops->get_pages(obj);
  1731.         if (ret)
  1732.                 return ret;
  1733.  
  1734.         list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
  1735.     return 0;
  1736. }
  1737.  
  1738. static void
  1739. i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
  1740.                                struct intel_engine_cs *ring)
  1741. {
  1742.         u32 seqno = intel_ring_get_seqno(ring);
  1743.  
  1744.         BUG_ON(ring == NULL);
  1745.         if (obj->ring != ring && obj->last_write_seqno) {
  1746.                 /* Keep the seqno relative to the current ring */
  1747.                 obj->last_write_seqno = seqno;
  1748.         }
  1749.         obj->ring = ring;
  1750.  
  1751.         /* Add a reference if we're newly entering the active list. */
  1752.         if (!obj->active) {
  1753.                 drm_gem_object_reference(&obj->base);
  1754.                 obj->active = 1;
  1755.         }
  1756.  
  1757.         list_move_tail(&obj->ring_list, &ring->active_list);
  1758.  
  1759.         obj->last_read_seqno = seqno;
  1760. }
  1761.  
  1762. void i915_vma_move_to_active(struct i915_vma *vma,
  1763.                              struct intel_engine_cs *ring)
  1764. {
  1765.         list_move_tail(&vma->mm_list, &vma->vm->active_list);
  1766.         return i915_gem_object_move_to_active(vma->obj, ring);
  1767. }
  1768.  
  1769. static void
  1770. i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
  1771. {
  1772.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  1773.         struct i915_address_space *vm;
  1774.         struct i915_vma *vma;
  1775.  
  1776.         BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
  1777.         BUG_ON(!obj->active);
  1778.  
  1779.         list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
  1780.                 vma = i915_gem_obj_to_vma(obj, vm);
  1781.                 if (vma && !list_empty(&vma->mm_list))
  1782.                         list_move_tail(&vma->mm_list, &vm->inactive_list);
  1783.         }
  1784.  
  1785.         intel_fb_obj_flush(obj, true);
  1786.  
  1787.         list_del_init(&obj->ring_list);
  1788.         obj->ring = NULL;
  1789.  
  1790.         obj->last_read_seqno = 0;
  1791.         obj->last_write_seqno = 0;
  1792.         obj->base.write_domain = 0;
  1793.  
  1794.         obj->last_fenced_seqno = 0;
  1795.  
  1796.         obj->active = 0;
  1797.         drm_gem_object_unreference(&obj->base);
  1798.  
  1799.         WARN_ON(i915_verify_lists(dev));
  1800. }
  1801.  
  1802. static void
  1803. i915_gem_object_retire(struct drm_i915_gem_object *obj)
  1804. {
  1805.         struct intel_engine_cs *ring = obj->ring;
  1806.  
  1807.         if (ring == NULL)
  1808.                 return;
  1809.  
  1810.         if (i915_seqno_passed(ring->get_seqno(ring, true),
  1811.                               obj->last_read_seqno))
  1812.                 i915_gem_object_move_to_inactive(obj);
  1813. }
  1814.  
  1815. static int
  1816. i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
  1817. {
  1818.         struct drm_i915_private *dev_priv = dev->dev_private;
  1819.         struct intel_engine_cs *ring;
  1820.         int ret, i, j;
  1821.  
  1822.         /* Carefully retire all requests without writing to the rings */
  1823.         for_each_ring(ring, dev_priv, i) {
  1824.                 ret = intel_ring_idle(ring);
  1825.         if (ret)
  1826.                 return ret;
  1827.         }
  1828.         i915_gem_retire_requests(dev);
  1829.  
  1830.         /* Finally reset hw state */
  1831.         for_each_ring(ring, dev_priv, i) {
  1832.                 intel_ring_init_seqno(ring, seqno);
  1833.  
  1834.                 for (j = 0; j < ARRAY_SIZE(ring->semaphore.sync_seqno); j++)
  1835.                         ring->semaphore.sync_seqno[j] = 0;
  1836.         }
  1837.  
  1838.         return 0;
  1839. }
  1840.  
  1841. int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
  1842. {
  1843.         struct drm_i915_private *dev_priv = dev->dev_private;
  1844.         int ret;
  1845.  
  1846.         if (seqno == 0)
  1847.                 return -EINVAL;
  1848.  
  1849.         /* HWS page needs to be set less than what we
  1850.          * will inject to ring
  1851.          */
  1852.         ret = i915_gem_init_seqno(dev, seqno - 1);
  1853.         if (ret)
  1854.                 return ret;
  1855.  
  1856.         /* Carefully set the last_seqno value so that wrap
  1857.          * detection still works
  1858.          */
  1859.         dev_priv->next_seqno = seqno;
  1860.         dev_priv->last_seqno = seqno - 1;
  1861.         if (dev_priv->last_seqno == 0)
  1862.                 dev_priv->last_seqno--;
  1863.  
  1864.         return 0;
  1865. }
  1866.  
  1867. int
  1868. i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
  1869. {
  1870.         struct drm_i915_private *dev_priv = dev->dev_private;
  1871.  
  1872.         /* reserve 0 for non-seqno */
  1873.         if (dev_priv->next_seqno == 0) {
  1874.                 int ret = i915_gem_init_seqno(dev, 0);
  1875.                 if (ret)
  1876.                         return ret;
  1877.  
  1878.                 dev_priv->next_seqno = 1;
  1879.         }
  1880.  
  1881.         *seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
  1882.         return 0;
  1883. }
  1884.  
  1885. int __i915_add_request(struct intel_engine_cs *ring,
  1886.                  struct drm_file *file,
  1887.                        struct drm_i915_gem_object *obj,
  1888.                  u32 *out_seqno)
  1889. {
  1890.         struct drm_i915_private *dev_priv = ring->dev->dev_private;
  1891.         struct drm_i915_gem_request *request;
  1892.         struct intel_ringbuffer *ringbuf;
  1893.         u32 request_ring_position, request_start;
  1894.         int ret;
  1895.  
  1896.         request = ring->preallocated_lazy_request;
  1897.         if (WARN_ON(request == NULL))
  1898.                 return -ENOMEM;
  1899.  
  1900.         if (i915.enable_execlists) {
  1901.                 struct intel_context *ctx = request->ctx;
  1902.                 ringbuf = ctx->engine[ring->id].ringbuf;
  1903.         } else
  1904.                 ringbuf = ring->buffer;
  1905.  
  1906.         request_start = intel_ring_get_tail(ringbuf);
  1907.         /*
  1908.          * Emit any outstanding flushes - execbuf can fail to emit the flush
  1909.          * after having emitted the batchbuffer command. Hence we need to fix
  1910.          * things up similar to emitting the lazy request. The difference here
  1911.          * is that the flush _must_ happen before the next request, no matter
  1912.          * what.
  1913.          */
  1914.         if (i915.enable_execlists) {
  1915.                 ret = logical_ring_flush_all_caches(ringbuf);
  1916.                 if (ret)
  1917.                         return ret;
  1918.         } else {
  1919.    ret = intel_ring_flush_all_caches(ring);
  1920.    if (ret)
  1921.        return ret;
  1922.         }
  1923.  
  1924.         /* Record the position of the start of the request so that
  1925.          * should we detect the updated seqno part-way through the
  1926.     * GPU processing the request, we never over-estimate the
  1927.          * position of the head.
  1928.          */
  1929.         request_ring_position = intel_ring_get_tail(ringbuf);
  1930.  
  1931.         if (i915.enable_execlists) {
  1932.                 ret = ring->emit_request(ringbuf);
  1933.                 if (ret)
  1934.                         return ret;
  1935.         } else {
  1936.         ret = ring->add_request(ring);
  1937.         if (ret)
  1938.                 return ret;
  1939.         }
  1940.  
  1941.         request->seqno = intel_ring_get_seqno(ring);
  1942.         request->ring = ring;
  1943.         request->head = request_start;
  1944.         request->tail = request_ring_position;
  1945.  
  1946.         /* Whilst this request exists, batch_obj will be on the
  1947.          * active_list, and so will hold the active reference. Only when this
  1948.          * request is retired will the the batch_obj be moved onto the
  1949.          * inactive_list and lose its active reference. Hence we do not need
  1950.          * to explicitly hold another reference here.
  1951.          */
  1952.         request->batch_obj = obj;
  1953.  
  1954.         if (!i915.enable_execlists) {
  1955.         /* Hold a reference to the current context so that we can inspect
  1956.          * it later in case a hangcheck error event fires.
  1957.          */
  1958.         request->ctx = ring->last_context;
  1959.         if (request->ctx)
  1960.                 i915_gem_context_reference(request->ctx);
  1961.         }
  1962.  
  1963.         request->emitted_jiffies = jiffies;
  1964.         list_add_tail(&request->list, &ring->request_list);
  1965.         request->file_priv = NULL;
  1966.  
  1967.         if (file) {
  1968.                 struct drm_i915_file_private *file_priv = file->driver_priv;
  1969.  
  1970.                 spin_lock(&file_priv->mm.lock);
  1971.                 request->file_priv = file_priv;
  1972.                 list_add_tail(&request->client_list,
  1973.                               &file_priv->mm.request_list);
  1974.                 spin_unlock(&file_priv->mm.lock);
  1975.         }
  1976.  
  1977.         trace_i915_gem_request_add(ring, request->seqno);
  1978.         ring->outstanding_lazy_seqno = 0;
  1979.         ring->preallocated_lazy_request = NULL;
  1980.  
  1981. //              i915_queue_hangcheck(ring->dev);
  1982.  
  1983.            queue_delayed_work(dev_priv->wq,
  1984.                                            &dev_priv->mm.retire_work,
  1985.                                            round_jiffies_up_relative(HZ));
  1986.            intel_mark_busy(dev_priv->dev);
  1987.  
  1988.         if (out_seqno)
  1989.                 *out_seqno = request->seqno;
  1990.         return 0;
  1991. }
  1992.  
  1993. static inline void
  1994. i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
  1995. {
  1996.         struct drm_i915_file_private *file_priv = request->file_priv;
  1997.  
  1998.         if (!file_priv)
  1999.                 return;
  2000.  
  2001.         spin_lock(&file_priv->mm.lock);
  2002.                 list_del(&request->client_list);
  2003.                 request->file_priv = NULL;
  2004.         spin_unlock(&file_priv->mm.lock);
  2005. }
  2006.  
  2007. static bool i915_context_is_banned(struct drm_i915_private *dev_priv,
  2008.                                    const struct intel_context *ctx)
  2009. {
  2010.         unsigned long elapsed;
  2011.  
  2012.     elapsed = GetTimerTicks()/100 - ctx->hang_stats.guilty_ts;
  2013.  
  2014.         if (ctx->hang_stats.banned)
  2015.                 return true;
  2016.  
  2017.         if (elapsed <= DRM_I915_CTX_BAN_PERIOD) {
  2018.                 if (!i915_gem_context_is_default(ctx)) {
  2019.                         DRM_DEBUG("context hanging too fast, banning!\n");
  2020.                         return true;
  2021.                 } else if (i915_stop_ring_allow_ban(dev_priv)) {
  2022.                         if (i915_stop_ring_allow_warn(dev_priv))
  2023.                         DRM_ERROR("gpu hanging too fast, banning!\n");
  2024.                         return true;
  2025.         }
  2026.         }
  2027.  
  2028.         return false;
  2029. }
  2030.  
  2031. static void i915_set_reset_status(struct drm_i915_private *dev_priv,
  2032.                                   struct intel_context *ctx,
  2033.                                   const bool guilty)
  2034. {
  2035.         struct i915_ctx_hang_stats *hs;
  2036.  
  2037.         if (WARN_ON(!ctx))
  2038.                 return;
  2039.  
  2040.         hs = &ctx->hang_stats;
  2041.  
  2042.         if (guilty) {
  2043.                 hs->banned = i915_context_is_banned(dev_priv, ctx);
  2044.                 hs->batch_active++;
  2045.         hs->guilty_ts = GetTimerTicks()/100;
  2046.         } else {
  2047.                 hs->batch_pending++;
  2048.         }
  2049. }
  2050.  
  2051. static void i915_gem_free_request(struct drm_i915_gem_request *request)
  2052. {
  2053.         struct intel_context *ctx = request->ctx;
  2054.  
  2055.         list_del(&request->list);
  2056.         i915_gem_request_remove_from_client(request);
  2057.  
  2058.         if (ctx) {
  2059.                 if (i915.enable_execlists) {
  2060.                         struct intel_engine_cs *ring = request->ring;
  2061.  
  2062.                         if (ctx != ring->default_context)
  2063.                                 intel_lr_context_unpin(ring, ctx);
  2064.                 }
  2065.                 i915_gem_context_unreference(ctx);
  2066.         }
  2067.         kfree(request);
  2068. }
  2069.  
  2070. struct drm_i915_gem_request *
  2071. i915_gem_find_active_request(struct intel_engine_cs *ring)
  2072. {
  2073.         struct drm_i915_gem_request *request;
  2074.         u32 completed_seqno;
  2075.  
  2076.         completed_seqno = ring->get_seqno(ring, false);
  2077.  
  2078.         list_for_each_entry(request, &ring->request_list, list) {
  2079.                 if (i915_seqno_passed(completed_seqno, request->seqno))
  2080.                         continue;
  2081.  
  2082.                 return request;
  2083.         }
  2084.  
  2085.         return NULL;
  2086. }
  2087.  
  2088. static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv,
  2089.                                        struct intel_engine_cs *ring)
  2090. {
  2091.         struct drm_i915_gem_request *request;
  2092.         bool ring_hung;
  2093.  
  2094.         request = i915_gem_find_active_request(ring);
  2095.  
  2096.         if (request == NULL)
  2097.                 return;
  2098.  
  2099.         ring_hung = ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
  2100.  
  2101.         i915_set_reset_status(dev_priv, request->ctx, ring_hung);
  2102.  
  2103.         list_for_each_entry_continue(request, &ring->request_list, list)
  2104.                 i915_set_reset_status(dev_priv, request->ctx, false);
  2105. }
  2106.  
  2107. static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
  2108.                                         struct intel_engine_cs *ring)
  2109. {
  2110.         while (!list_empty(&ring->active_list)) {
  2111.                 struct drm_i915_gem_object *obj;
  2112.  
  2113.                 obj = list_first_entry(&ring->active_list,
  2114.                                        struct drm_i915_gem_object,
  2115.                                        ring_list);
  2116.  
  2117.                 i915_gem_object_move_to_inactive(obj);
  2118.         }
  2119.  
  2120.         /*
  2121.          * Clear the execlists queue up before freeing the requests, as those
  2122.          * are the ones that keep the context and ringbuffer backing objects
  2123.          * pinned in place.
  2124.          */
  2125.         while (!list_empty(&ring->execlist_queue)) {
  2126.                 struct intel_ctx_submit_request *submit_req;
  2127.  
  2128.                 submit_req = list_first_entry(&ring->execlist_queue,
  2129.                                 struct intel_ctx_submit_request,
  2130.                                 execlist_link);
  2131.                 list_del(&submit_req->execlist_link);
  2132.                 intel_runtime_pm_put(dev_priv);
  2133.                 i915_gem_context_unreference(submit_req->ctx);
  2134.                 kfree(submit_req);
  2135.         }
  2136.  
  2137.         /*
  2138.          * We must free the requests after all the corresponding objects have
  2139.          * been moved off active lists. Which is the same order as the normal
  2140.          * retire_requests function does. This is important if object hold
  2141.          * implicit references on things like e.g. ppgtt address spaces through
  2142.          * the request.
  2143.          */
  2144.         while (!list_empty(&ring->request_list)) {
  2145.                 struct drm_i915_gem_request *request;
  2146.  
  2147.                 request = list_first_entry(&ring->request_list,
  2148.                                            struct drm_i915_gem_request,
  2149.                                            list);
  2150.  
  2151.                 i915_gem_free_request(request);
  2152.         }
  2153.  
  2154.         /* These may not have been flush before the reset, do so now */
  2155.         kfree(ring->preallocated_lazy_request);
  2156.         ring->preallocated_lazy_request = NULL;
  2157.         ring->outstanding_lazy_seqno = 0;
  2158. }
  2159.  
  2160. void i915_gem_restore_fences(struct drm_device *dev)
  2161. {
  2162.         struct drm_i915_private *dev_priv = dev->dev_private;
  2163.         int i;
  2164.  
  2165.         for (i = 0; i < dev_priv->num_fence_regs; i++) {
  2166.                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
  2167.  
  2168.                 /*
  2169.                  * Commit delayed tiling changes if we have an object still
  2170.                  * attached to the fence, otherwise just clear the fence.
  2171.                  */
  2172.                 if (reg->obj) {
  2173.                         i915_gem_object_update_fence(reg->obj, reg,
  2174.                                                      reg->obj->tiling_mode);
  2175.                 } else {
  2176.                         i915_gem_write_fence(dev, i, NULL);
  2177.                 }
  2178.         }
  2179. }
  2180.  
  2181. void i915_gem_reset(struct drm_device *dev)
  2182. {
  2183.         struct drm_i915_private *dev_priv = dev->dev_private;
  2184.         struct intel_engine_cs *ring;
  2185.         int i;
  2186.  
  2187.         /*
  2188.          * Before we free the objects from the requests, we need to inspect
  2189.          * them for finding the guilty party. As the requests only borrow
  2190.          * their reference to the objects, the inspection must be done first.
  2191.          */
  2192.         for_each_ring(ring, dev_priv, i)
  2193.                 i915_gem_reset_ring_status(dev_priv, ring);
  2194.  
  2195.         for_each_ring(ring, dev_priv, i)
  2196.                 i915_gem_reset_ring_cleanup(dev_priv, ring);
  2197.  
  2198.         i915_gem_context_reset(dev);
  2199.  
  2200.         i915_gem_restore_fences(dev);
  2201. }
  2202.  
  2203. /**
  2204.  * This function clears the request list as sequence numbers are passed.
  2205.  */
  2206. void
  2207. i915_gem_retire_requests_ring(struct intel_engine_cs *ring)
  2208. {
  2209.         uint32_t seqno;
  2210.  
  2211.         if (list_empty(&ring->request_list))
  2212.                 return;
  2213.  
  2214.         WARN_ON(i915_verify_lists(ring->dev));
  2215.  
  2216.         seqno = ring->get_seqno(ring, true);
  2217.  
  2218.         /* Move any buffers on the active list that are no longer referenced
  2219.          * by the ringbuffer to the flushing/inactive lists as appropriate,
  2220.          * before we free the context associated with the requests.
  2221.          */
  2222.         while (!list_empty(&ring->active_list)) {
  2223.                 struct drm_i915_gem_object *obj;
  2224.  
  2225.                 obj = list_first_entry(&ring->active_list,
  2226.                                       struct drm_i915_gem_object,
  2227.                                       ring_list);
  2228.  
  2229.                 if (!i915_seqno_passed(seqno, obj->last_read_seqno))
  2230.                         break;
  2231.  
  2232.                 i915_gem_object_move_to_inactive(obj);
  2233.         }
  2234.  
  2235.  
  2236.         while (!list_empty(&ring->request_list)) {
  2237.                 struct drm_i915_gem_request *request;
  2238.                 struct intel_ringbuffer *ringbuf;
  2239.  
  2240.                 request = list_first_entry(&ring->request_list,
  2241.                                            struct drm_i915_gem_request,
  2242.                                            list);
  2243.  
  2244.                 if (!i915_seqno_passed(seqno, request->seqno))
  2245.                         break;
  2246.  
  2247.                 trace_i915_gem_request_retire(ring, request->seqno);
  2248.  
  2249.                 /* This is one of the few common intersection points
  2250.                  * between legacy ringbuffer submission and execlists:
  2251.                  * we need to tell them apart in order to find the correct
  2252.                  * ringbuffer to which the request belongs to.
  2253.                  */
  2254.                 if (i915.enable_execlists) {
  2255.                         struct intel_context *ctx = request->ctx;
  2256.                         ringbuf = ctx->engine[ring->id].ringbuf;
  2257.                 } else
  2258.                         ringbuf = ring->buffer;
  2259.  
  2260.                 /* We know the GPU must have read the request to have
  2261.                  * sent us the seqno + interrupt, so use the position
  2262.                  * of tail of the request to update the last known position
  2263.                  * of the GPU head.
  2264.                  */
  2265.                 ringbuf->last_retired_head = request->tail;
  2266.  
  2267.                 i915_gem_free_request(request);
  2268.         }
  2269.  
  2270.         if (unlikely(ring->trace_irq_seqno &&
  2271.                      i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
  2272.                 ring->irq_put(ring);
  2273.                 ring->trace_irq_seqno = 0;
  2274.         }
  2275.  
  2276.         WARN_ON(i915_verify_lists(ring->dev));
  2277. }
  2278.  
  2279. bool
  2280. i915_gem_retire_requests(struct drm_device *dev)
  2281. {
  2282.         struct drm_i915_private *dev_priv = dev->dev_private;
  2283.         struct intel_engine_cs *ring;
  2284.         bool idle = true;
  2285.         int i;
  2286.  
  2287.         for_each_ring(ring, dev_priv, i) {
  2288.                 i915_gem_retire_requests_ring(ring);
  2289.                 idle &= list_empty(&ring->request_list);
  2290.                 if (i915.enable_execlists) {
  2291.                         unsigned long flags;
  2292.  
  2293.                         spin_lock_irqsave(&ring->execlist_lock, flags);
  2294.                         idle &= list_empty(&ring->execlist_queue);
  2295.                         spin_unlock_irqrestore(&ring->execlist_lock, flags);
  2296.  
  2297.                         intel_execlists_retire_requests(ring);
  2298.                 }
  2299.         }
  2300.  
  2301.         if (idle)
  2302.                 mod_delayed_work(dev_priv->wq,
  2303.                                    &dev_priv->mm.idle_work,
  2304.                                    msecs_to_jiffies(100));
  2305.  
  2306.         return idle;
  2307. }
  2308.  
  2309. static void
  2310. i915_gem_retire_work_handler(struct work_struct *work)
  2311. {
  2312.         struct drm_i915_private *dev_priv =
  2313.                 container_of(work, typeof(*dev_priv), mm.retire_work.work);
  2314.         struct drm_device *dev = dev_priv->dev;
  2315.         bool idle;
  2316.  
  2317.         /* Come back later if the device is busy... */
  2318.         idle = false;
  2319.         if (mutex_trylock(&dev->struct_mutex)) {
  2320.                 idle = i915_gem_retire_requests(dev);
  2321.                 mutex_unlock(&dev->struct_mutex);
  2322.         }
  2323.         if (!idle)
  2324.                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
  2325.                                    round_jiffies_up_relative(HZ));
  2326. }
  2327.  
  2328. static void
  2329. i915_gem_idle_work_handler(struct work_struct *work)
  2330. {
  2331.         struct drm_i915_private *dev_priv =
  2332.                 container_of(work, typeof(*dev_priv), mm.idle_work.work);
  2333.  
  2334.         intel_mark_idle(dev_priv->dev);
  2335. }
  2336.  
  2337. /**
  2338.  * Ensures that an object will eventually get non-busy by flushing any required
  2339.  * write domains, emitting any outstanding lazy request and retiring and
  2340.  * completed requests.
  2341.  */
  2342. static int
  2343. i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
  2344. {
  2345.         int ret;
  2346.  
  2347.         if (obj->active) {
  2348.                 ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
  2349.                 if (ret)
  2350.                         return ret;
  2351.  
  2352.                 i915_gem_retire_requests_ring(obj->ring);
  2353.         }
  2354.  
  2355.         return 0;
  2356. }
  2357.  
  2358. /**
  2359.  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
  2360.  * @DRM_IOCTL_ARGS: standard ioctl arguments
  2361.  *
  2362.  * Returns 0 if successful, else an error is returned with the remaining time in
  2363.  * the timeout parameter.
  2364.  *  -ETIME: object is still busy after timeout
  2365.  *  -ERESTARTSYS: signal interrupted the wait
  2366.  *  -ENONENT: object doesn't exist
  2367.  * Also possible, but rare:
  2368.  *  -EAGAIN: GPU wedged
  2369.  *  -ENOMEM: damn
  2370.  *  -ENODEV: Internal IRQ fail
  2371.  *  -E?: The add request failed
  2372.  *
  2373.  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
  2374.  * non-zero timeout parameter the wait ioctl will wait for the given number of
  2375.  * nanoseconds on an object becoming unbusy. Since the wait itself does so
  2376.  * without holding struct_mutex the object may become re-busied before this
  2377.  * function completes. A similar but shorter * race condition exists in the busy
  2378.  * ioctl
  2379.  */
  2380. int
  2381. i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
  2382. {
  2383.         struct drm_i915_private *dev_priv = dev->dev_private;
  2384.         struct drm_i915_gem_wait *args = data;
  2385.         struct drm_i915_gem_object *obj;
  2386.         struct intel_engine_cs *ring = NULL;
  2387.         unsigned reset_counter;
  2388.         u32 seqno = 0;
  2389.         int ret = 0;
  2390.  
  2391.         if (args->flags != 0)
  2392.                 return -EINVAL;
  2393.  
  2394.         ret = i915_mutex_lock_interruptible(dev);
  2395.         if (ret)
  2396.                 return ret;
  2397.  
  2398.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
  2399.         if (&obj->base == NULL) {
  2400.                 mutex_unlock(&dev->struct_mutex);
  2401.                 return -ENOENT;
  2402.         }
  2403.  
  2404.         /* Need to make sure the object gets inactive eventually. */
  2405.         ret = i915_gem_object_flush_active(obj);
  2406.         if (ret)
  2407.                 goto out;
  2408.  
  2409.         if (obj->active) {
  2410.                 seqno = obj->last_read_seqno;
  2411.                 ring = obj->ring;
  2412.         }
  2413.  
  2414.         if (seqno == 0)
  2415.                  goto out;
  2416.  
  2417.         /* Do this after OLR check to make sure we make forward progress polling
  2418.          * on this IOCTL with a timeout <=0 (like busy ioctl)
  2419.          */
  2420.         if (args->timeout_ns <= 0) {
  2421.                 ret = -ETIME;
  2422.                 goto out;
  2423.         }
  2424.  
  2425.         drm_gem_object_unreference(&obj->base);
  2426.         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
  2427.         mutex_unlock(&dev->struct_mutex);
  2428.  
  2429.         return __i915_wait_seqno(ring, seqno, reset_counter, true,
  2430.                                  &args->timeout_ns, file->driver_priv);
  2431.  
  2432. out:
  2433.         drm_gem_object_unreference(&obj->base);
  2434.         mutex_unlock(&dev->struct_mutex);
  2435.         return ret;
  2436. }
  2437.  
  2438. /**
  2439.  * i915_gem_object_sync - sync an object to a ring.
  2440.  *
  2441.  * @obj: object which may be in use on another ring.
  2442.  * @to: ring we wish to use the object on. May be NULL.
  2443.  *
  2444.  * This code is meant to abstract object synchronization with the GPU.
  2445.  * Calling with NULL implies synchronizing the object with the CPU
  2446.  * rather than a particular GPU ring.
  2447.  *
  2448.  * Returns 0 if successful, else propagates up the lower layer error.
  2449.  */
  2450. int
  2451. i915_gem_object_sync(struct drm_i915_gem_object *obj,
  2452.                      struct intel_engine_cs *to)
  2453. {
  2454.         struct intel_engine_cs *from = obj->ring;
  2455.         u32 seqno;
  2456.         int ret, idx;
  2457.  
  2458.         if (from == NULL || to == from)
  2459.                 return 0;
  2460.  
  2461.         if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
  2462.                 return i915_gem_object_wait_rendering(obj, false);
  2463.  
  2464.         idx = intel_ring_sync_index(from, to);
  2465.  
  2466.         seqno = obj->last_read_seqno;
  2467.         /* Optimization: Avoid semaphore sync when we are sure we already
  2468.          * waited for an object with higher seqno */
  2469.         if (seqno <= from->semaphore.sync_seqno[idx])
  2470.                 return 0;
  2471.  
  2472.         ret = i915_gem_check_olr(obj->ring, seqno);
  2473.         if (ret)
  2474.                 return ret;
  2475.  
  2476.         trace_i915_gem_ring_sync_to(from, to, seqno);
  2477.         ret = to->semaphore.sync_to(to, from, seqno);
  2478.         if (!ret)
  2479.                 /* We use last_read_seqno because sync_to()
  2480.                  * might have just caused seqno wrap under
  2481.                  * the radar.
  2482.                  */
  2483.                 from->semaphore.sync_seqno[idx] = obj->last_read_seqno;
  2484.  
  2485.         return ret;
  2486. }
  2487.  
  2488. static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
  2489. {
  2490.         u32 old_write_domain, old_read_domains;
  2491.  
  2492.         /* Force a pagefault for domain tracking on next user access */
  2493. //      i915_gem_release_mmap(obj);
  2494.  
  2495.         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
  2496.                 return;
  2497.  
  2498.         /* Wait for any direct GTT access to complete */
  2499.         mb();
  2500.  
  2501.         old_read_domains = obj->base.read_domains;
  2502.         old_write_domain = obj->base.write_domain;
  2503.  
  2504.         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
  2505.         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
  2506.  
  2507.         trace_i915_gem_object_change_domain(obj,
  2508.                                             old_read_domains,
  2509.                                             old_write_domain);
  2510. }
  2511.  
  2512. int i915_vma_unbind(struct i915_vma *vma)
  2513. {
  2514.         struct drm_i915_gem_object *obj = vma->obj;
  2515.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  2516.         int ret;
  2517.     if(obj == get_fb_obj())
  2518.     {
  2519.         WARN(1,"attempt to unbind fb object\n");
  2520.         return 0;
  2521.     };
  2522.  
  2523.         if (list_empty(&vma->vma_link))
  2524.                 return 0;
  2525.  
  2526.         if (!drm_mm_node_allocated(&vma->node)) {
  2527.                 i915_gem_vma_destroy(vma);
  2528.                 return 0;
  2529.         }
  2530.  
  2531.         if (vma->pin_count)
  2532.                 return -EBUSY;
  2533.  
  2534.         BUG_ON(obj->pages == NULL);
  2535.  
  2536.         ret = i915_gem_object_finish_gpu(obj);
  2537.         if (ret)
  2538.                 return ret;
  2539.         /* Continue on if we fail due to EIO, the GPU is hung so we
  2540.          * should be safe and we need to cleanup or else we might
  2541.          * cause memory corruption through use-after-free.
  2542.          */
  2543.  
  2544.         /* Throw away the active reference before moving to the unbound list */
  2545.         i915_gem_object_retire(obj);
  2546.  
  2547.         if (i915_is_ggtt(vma->vm)) {
  2548.         i915_gem_object_finish_gtt(obj);
  2549.  
  2550.         /* release the fence reg _after_ flushing */
  2551.         ret = i915_gem_object_put_fence(obj);
  2552.         if (ret)
  2553.                 return ret;
  2554.         }
  2555.  
  2556.         trace_i915_vma_unbind(vma);
  2557.  
  2558.         vma->unbind_vma(vma);
  2559.  
  2560.         list_del_init(&vma->mm_list);
  2561.         if (i915_is_ggtt(vma->vm))
  2562.                 obj->map_and_fenceable = false;
  2563.  
  2564.         drm_mm_remove_node(&vma->node);
  2565.         i915_gem_vma_destroy(vma);
  2566.  
  2567.         /* Since the unbound list is global, only move to that list if
  2568.          * no more VMAs exist. */
  2569.         if (list_empty(&obj->vma_list)) {
  2570.                 i915_gem_gtt_finish_object(obj);
  2571.                 list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
  2572.         }
  2573.  
  2574.         /* And finally now the object is completely decoupled from this vma,
  2575.          * we can drop its hold on the backing storage and allow it to be
  2576.          * reaped by the shrinker.
  2577.          */
  2578.         i915_gem_object_unpin_pages(obj);
  2579.  
  2580.         return 0;
  2581. }
  2582.  
  2583. int i915_gpu_idle(struct drm_device *dev)
  2584. {
  2585.         struct drm_i915_private *dev_priv = dev->dev_private;
  2586.         struct intel_engine_cs *ring;
  2587.         int ret, i;
  2588.  
  2589.         /* Flush everything onto the inactive list. */
  2590.         for_each_ring(ring, dev_priv, i) {
  2591.                 if (!i915.enable_execlists) {
  2592.                 ret = i915_switch_context(ring, ring->default_context);
  2593.                 if (ret)
  2594.                         return ret;
  2595.                 }
  2596.  
  2597.                 ret = intel_ring_idle(ring);
  2598.                 if (ret)
  2599.                         return ret;
  2600.         }
  2601.  
  2602.         return 0;
  2603. }
  2604.  
  2605. static void i965_write_fence_reg(struct drm_device *dev, int reg,
  2606.                                         struct drm_i915_gem_object *obj)
  2607. {
  2608.         struct drm_i915_private *dev_priv = dev->dev_private;
  2609.         int fence_reg;
  2610.         int fence_pitch_shift;
  2611.  
  2612.         if (INTEL_INFO(dev)->gen >= 6) {
  2613.                 fence_reg = FENCE_REG_SANDYBRIDGE_0;
  2614.                 fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
  2615.         } else {
  2616.                 fence_reg = FENCE_REG_965_0;
  2617.                 fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
  2618.         }
  2619.  
  2620.         fence_reg += reg * 8;
  2621.  
  2622.         /* To w/a incoherency with non-atomic 64-bit register updates,
  2623.          * we split the 64-bit update into two 32-bit writes. In order
  2624.          * for a partial fence not to be evaluated between writes, we
  2625.          * precede the update with write to turn off the fence register,
  2626.          * and only enable the fence as the last step.
  2627.          *
  2628.          * For extra levels of paranoia, we make sure each step lands
  2629.          * before applying the next step.
  2630.          */
  2631.         I915_WRITE(fence_reg, 0);
  2632.         POSTING_READ(fence_reg);
  2633.  
  2634.         if (obj) {
  2635.                 u32 size = i915_gem_obj_ggtt_size(obj);
  2636.                 uint64_t val;
  2637.  
  2638.                 val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) &
  2639.                                  0xfffff000) << 32;
  2640.                 val |= i915_gem_obj_ggtt_offset(obj) & 0xfffff000;
  2641.                 val |= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
  2642.                 if (obj->tiling_mode == I915_TILING_Y)
  2643.                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
  2644.                 val |= I965_FENCE_REG_VALID;
  2645.  
  2646.                 I915_WRITE(fence_reg + 4, val >> 32);
  2647.                 POSTING_READ(fence_reg + 4);
  2648.  
  2649.                 I915_WRITE(fence_reg + 0, val);
  2650.                 POSTING_READ(fence_reg);
  2651.         } else {
  2652.                 I915_WRITE(fence_reg + 4, 0);
  2653.                 POSTING_READ(fence_reg + 4);
  2654.         }
  2655. }
  2656.  
  2657. static void i915_write_fence_reg(struct drm_device *dev, int reg,
  2658.                                  struct drm_i915_gem_object *obj)
  2659. {
  2660.         struct drm_i915_private *dev_priv = dev->dev_private;
  2661.         u32 val;
  2662.  
  2663.         if (obj) {
  2664.                 u32 size = i915_gem_obj_ggtt_size(obj);
  2665.                 int pitch_val;
  2666.                 int tile_width;
  2667.  
  2668.                 WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) ||
  2669.                      (size & -size) != size ||
  2670.                      (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
  2671.                      "object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
  2672.                      i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size);
  2673.  
  2674.                 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
  2675.                         tile_width = 128;
  2676.                 else
  2677.                         tile_width = 512;
  2678.  
  2679.                 /* Note: pitch better be a power of two tile widths */
  2680.                 pitch_val = obj->stride / tile_width;
  2681.                 pitch_val = ffs(pitch_val) - 1;
  2682.  
  2683.                 val = i915_gem_obj_ggtt_offset(obj);
  2684.                 if (obj->tiling_mode == I915_TILING_Y)
  2685.                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
  2686.                 val |= I915_FENCE_SIZE_BITS(size);
  2687.                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
  2688.                 val |= I830_FENCE_REG_VALID;
  2689.         } else
  2690.                 val = 0;
  2691.  
  2692.         if (reg < 8)
  2693.                 reg = FENCE_REG_830_0 + reg * 4;
  2694.         else
  2695.                 reg = FENCE_REG_945_8 + (reg - 8) * 4;
  2696.  
  2697.         I915_WRITE(reg, val);
  2698.         POSTING_READ(reg);
  2699. }
  2700.  
  2701. static void i830_write_fence_reg(struct drm_device *dev, int reg,
  2702.                                 struct drm_i915_gem_object *obj)
  2703. {
  2704.         struct drm_i915_private *dev_priv = dev->dev_private;
  2705.         uint32_t val;
  2706.  
  2707.         if (obj) {
  2708.                 u32 size = i915_gem_obj_ggtt_size(obj);
  2709.                 uint32_t pitch_val;
  2710.  
  2711.                 WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) ||
  2712.                      (size & -size) != size ||
  2713.                      (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
  2714.                      "object 0x%08lx not 512K or pot-size 0x%08x aligned\n",
  2715.                      i915_gem_obj_ggtt_offset(obj), size);
  2716.  
  2717.                 pitch_val = obj->stride / 128;
  2718.                 pitch_val = ffs(pitch_val) - 1;
  2719.  
  2720.                 val = i915_gem_obj_ggtt_offset(obj);
  2721.                 if (obj->tiling_mode == I915_TILING_Y)
  2722.                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
  2723.                 val |= I830_FENCE_SIZE_BITS(size);
  2724.                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
  2725.                 val |= I830_FENCE_REG_VALID;
  2726.         } else
  2727.                 val = 0;
  2728.  
  2729.         I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
  2730.         POSTING_READ(FENCE_REG_830_0 + reg * 4);
  2731. }
  2732.  
  2733. inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
  2734. {
  2735.         return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT;
  2736. }
  2737.  
  2738. static void i915_gem_write_fence(struct drm_device *dev, int reg,
  2739.                                  struct drm_i915_gem_object *obj)
  2740. {
  2741.         struct drm_i915_private *dev_priv = dev->dev_private;
  2742.  
  2743.         /* Ensure that all CPU reads are completed before installing a fence
  2744.          * and all writes before removing the fence.
  2745.          */
  2746.         if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
  2747.                 mb();
  2748.  
  2749.         WARN(obj && (!obj->stride || !obj->tiling_mode),
  2750.              "bogus fence setup with stride: 0x%x, tiling mode: %i\n",
  2751.              obj->stride, obj->tiling_mode);
  2752.  
  2753.         switch (INTEL_INFO(dev)->gen) {
  2754.         case 9:
  2755.         case 8:
  2756.         case 7:
  2757.         case 6:
  2758.         case 5:
  2759.         case 4: i965_write_fence_reg(dev, reg, obj); break;
  2760.         case 3: i915_write_fence_reg(dev, reg, obj); break;
  2761.         case 2: i830_write_fence_reg(dev, reg, obj); break;
  2762.         default: BUG();
  2763.         }
  2764.  
  2765.         /* And similarly be paranoid that no direct access to this region
  2766.          * is reordered to before the fence is installed.
  2767.          */
  2768.         if (i915_gem_object_needs_mb(obj))
  2769.                 mb();
  2770. }
  2771.  
  2772. static inline int fence_number(struct drm_i915_private *dev_priv,
  2773.                                struct drm_i915_fence_reg *fence)
  2774. {
  2775.         return fence - dev_priv->fence_regs;
  2776. }
  2777.  
  2778. static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
  2779.                                          struct drm_i915_fence_reg *fence,
  2780.                                          bool enable)
  2781. {
  2782.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  2783.         int reg = fence_number(dev_priv, fence);
  2784.  
  2785.         i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
  2786.  
  2787.         if (enable) {
  2788.                 obj->fence_reg = reg;
  2789.                 fence->obj = obj;
  2790.                 list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
  2791.         } else {
  2792.                 obj->fence_reg = I915_FENCE_REG_NONE;
  2793.                 fence->obj = NULL;
  2794.                 list_del_init(&fence->lru_list);
  2795.         }
  2796.         obj->fence_dirty = false;
  2797. }
  2798.  
  2799. static int
  2800. i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
  2801. {
  2802.         if (obj->last_fenced_seqno) {
  2803.                 int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
  2804.                         if (ret)
  2805.                                 return ret;
  2806.  
  2807.                 obj->last_fenced_seqno = 0;
  2808.         }
  2809.  
  2810.         return 0;
  2811. }
  2812.  
  2813. int
  2814. i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
  2815. {
  2816.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  2817.         struct drm_i915_fence_reg *fence;
  2818.         int ret;
  2819.  
  2820.         ret = i915_gem_object_wait_fence(obj);
  2821.     if (ret)
  2822.        return ret;
  2823.  
  2824.         if (obj->fence_reg == I915_FENCE_REG_NONE)
  2825.                 return 0;
  2826.  
  2827.         fence = &dev_priv->fence_regs[obj->fence_reg];
  2828.  
  2829.         if (WARN_ON(fence->pin_count))
  2830.                 return -EBUSY;
  2831.  
  2832.         i915_gem_object_fence_lost(obj);
  2833.         i915_gem_object_update_fence(obj, fence, false);
  2834.  
  2835.         return 0;
  2836. }
  2837.  
  2838. static struct drm_i915_fence_reg *
  2839. i915_find_fence_reg(struct drm_device *dev)
  2840. {
  2841.         struct drm_i915_private *dev_priv = dev->dev_private;
  2842.         struct drm_i915_fence_reg *reg, *avail;
  2843.         int i;
  2844.  
  2845.         /* First try to find a free reg */
  2846.         avail = NULL;
  2847.         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
  2848.                 reg = &dev_priv->fence_regs[i];
  2849.                 if (!reg->obj)
  2850.                         return reg;
  2851.  
  2852.                 if (!reg->pin_count)
  2853.                         avail = reg;
  2854.         }
  2855.  
  2856.         if (avail == NULL)
  2857.                 goto deadlock;
  2858.  
  2859.         /* None available, try to steal one or wait for a user to finish */
  2860.         list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
  2861.                 if (reg->pin_count)
  2862.                         continue;
  2863.  
  2864.                 return reg;
  2865.         }
  2866.  
  2867. deadlock:
  2868.         /* Wait for completion of pending flips which consume fences */
  2869. //   if (intel_has_pending_fb_unpin(dev))
  2870. //       return ERR_PTR(-EAGAIN);
  2871.  
  2872.         return ERR_PTR(-EDEADLK);
  2873. }
  2874.  
  2875. /**
  2876.  * i915_gem_object_get_fence - set up fencing for an object
  2877.  * @obj: object to map through a fence reg
  2878.  *
  2879.  * When mapping objects through the GTT, userspace wants to be able to write
  2880.  * to them without having to worry about swizzling if the object is tiled.
  2881.  * This function walks the fence regs looking for a free one for @obj,
  2882.  * stealing one if it can't find any.
  2883.  *
  2884.  * It then sets up the reg based on the object's properties: address, pitch
  2885.  * and tiling format.
  2886.  *
  2887.  * For an untiled surface, this removes any existing fence.
  2888.  */
  2889. int
  2890. i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
  2891. {
  2892.         struct drm_device *dev = obj->base.dev;
  2893.         struct drm_i915_private *dev_priv = dev->dev_private;
  2894.         bool enable = obj->tiling_mode != I915_TILING_NONE;
  2895.         struct drm_i915_fence_reg *reg;
  2896.         int ret;
  2897.  
  2898.         /* Have we updated the tiling parameters upon the object and so
  2899.          * will need to serialise the write to the associated fence register?
  2900.          */
  2901.         if (obj->fence_dirty) {
  2902.                 ret = i915_gem_object_wait_fence(obj);
  2903.                 if (ret)
  2904.                         return ret;
  2905.         }
  2906.  
  2907.         /* Just update our place in the LRU if our fence is getting reused. */
  2908.         if (obj->fence_reg != I915_FENCE_REG_NONE) {
  2909.                 reg = &dev_priv->fence_regs[obj->fence_reg];
  2910.                 if (!obj->fence_dirty) {
  2911.                         list_move_tail(&reg->lru_list,
  2912.                                        &dev_priv->mm.fence_list);
  2913.                         return 0;
  2914.                 }
  2915.         } else if (enable) {
  2916.                 if (WARN_ON(!obj->map_and_fenceable))
  2917.                         return -EINVAL;
  2918.  
  2919.                 reg = i915_find_fence_reg(dev);
  2920.                 if (IS_ERR(reg))
  2921.                         return PTR_ERR(reg);
  2922.  
  2923.                 if (reg->obj) {
  2924.                         struct drm_i915_gem_object *old = reg->obj;
  2925.  
  2926.                         ret = i915_gem_object_wait_fence(old);
  2927.                         if (ret)
  2928.                                 return ret;
  2929.  
  2930.                         i915_gem_object_fence_lost(old);
  2931.                 }
  2932.         } else
  2933.                 return 0;
  2934.  
  2935.         i915_gem_object_update_fence(obj, reg, enable);
  2936.  
  2937.         return 0;
  2938. }
  2939.  
  2940. static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
  2941.                                      unsigned long cache_level)
  2942. {
  2943.         struct drm_mm_node *gtt_space = &vma->node;
  2944.         struct drm_mm_node *other;
  2945.  
  2946.         /*
  2947.          * On some machines we have to be careful when putting differing types
  2948.          * of snoopable memory together to avoid the prefetcher crossing memory
  2949.          * domains and dying. During vm initialisation, we decide whether or not
  2950.          * these constraints apply and set the drm_mm.color_adjust
  2951.          * appropriately.
  2952.          */
  2953.         if (vma->vm->mm.color_adjust == NULL)
  2954.                 return true;
  2955.  
  2956.         if (!drm_mm_node_allocated(gtt_space))
  2957.                 return true;
  2958.  
  2959.         if (list_empty(&gtt_space->node_list))
  2960.                 return true;
  2961.  
  2962.         other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
  2963.         if (other->allocated && !other->hole_follows && other->color != cache_level)
  2964.                 return false;
  2965.  
  2966.         other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
  2967.         if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
  2968.                 return false;
  2969.  
  2970.         return true;
  2971. }
  2972.  
  2973. /**
  2974.  * Finds free space in the GTT aperture and binds the object there.
  2975.  */
  2976. static struct i915_vma *
  2977. i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
  2978.                            struct i915_address_space *vm,
  2979.                             unsigned alignment,
  2980.                            uint64_t flags)
  2981. {
  2982.         struct drm_device *dev = obj->base.dev;
  2983.         struct drm_i915_private *dev_priv = dev->dev_private;
  2984.         u32 size, fence_size, fence_alignment, unfenced_alignment;
  2985.         unsigned long start =
  2986.                 flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
  2987.         unsigned long end =
  2988.                 flags & PIN_MAPPABLE ? dev_priv->gtt.mappable_end : vm->total;
  2989.         struct i915_vma *vma;
  2990.         int ret;
  2991.  
  2992.         fence_size = i915_gem_get_gtt_size(dev,
  2993.                                            obj->base.size,
  2994.                                            obj->tiling_mode);
  2995.         fence_alignment = i915_gem_get_gtt_alignment(dev,
  2996.                                                      obj->base.size,
  2997.                                                      obj->tiling_mode, true);
  2998.         unfenced_alignment =
  2999.                 i915_gem_get_gtt_alignment(dev,
  3000.                                                     obj->base.size,
  3001.                                                     obj->tiling_mode, false);
  3002.  
  3003.         if (alignment == 0)
  3004.                 alignment = flags & PIN_MAPPABLE ? fence_alignment :
  3005.                                                 unfenced_alignment;
  3006.