Subversion Repositories Kolibri OS

Rev

Rev 6937 | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright © 2008-2015 Intel Corporation
  3.  *
  4.  * Permission is hereby granted, free of charge, to any person obtaining a
  5.  * copy of this software and associated documentation files (the "Software"),
  6.  * to deal in the Software without restriction, including without limitation
  7.  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8.  * and/or sell copies of the Software, and to permit persons to whom the
  9.  * Software is furnished to do so, subject to the following conditions:
  10.  *
  11.  * The above copyright notice and this permission notice (including the next
  12.  * paragraph) shall be included in all copies or substantial portions of the
  13.  * Software.
  14.  *
  15.  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16.  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17.  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18.  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19.  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20.  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21.  * IN THE SOFTWARE.
  22.  *
  23.  * Authors:
  24.  *    Eric Anholt <eric@anholt.net>
  25.  *
  26.  */
  27.  
  28. #include <drm/drmP.h>
  29. #include <drm/drm_vma_manager.h>
  30. #include <drm/i915_drm.h>
  31. #include "i915_drv.h"
  32. #include "i915_vgpu.h"
  33. #include "i915_trace.h"
  34. #include "intel_drv.h"
  35. #include <linux/shmem_fs.h>
  36. #include <linux/slab.h>
  37. #include <linux/swap.h>
  38. #include <linux/pci.h>
  39. #include <linux/dma-buf.h>
  40.  
  41. #define RQ_BUG_ON(expr)
  42.  
  43. extern int x86_clflush_size;
  44.  
  45. #define PROT_READ       0x1             /* page can be read */
  46. #define PROT_WRITE      0x2             /* page can be written */
  47. #define MAP_SHARED      0x01            /* Share changes */
  48.  
  49. struct drm_i915_gem_object *get_fb_obj();
  50.  
  51. unsigned long vm_mmap(struct file *file, unsigned long addr,
  52.          unsigned long len, unsigned long prot,
  53.          unsigned long flag, unsigned long offset);
  54.  
  55.  
  56. static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  57. static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  58. static void
  59. i915_gem_object_retire__write(struct drm_i915_gem_object *obj);
  60. static void
  61. i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring);
  62.  
  63. static bool cpu_cache_is_coherent(struct drm_device *dev,
  64.                                   enum i915_cache_level level)
  65. {
  66.         return HAS_LLC(dev) || level != I915_CACHE_NONE;
  67. }
  68.  
  69. static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
  70. {
  71.         if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
  72.                 return true;
  73.  
  74.         return obj->pin_display;
  75. }
  76.  
  77. /* some bookkeeping */
  78. static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  79.                                   size_t size)
  80. {
  81.         spin_lock(&dev_priv->mm.object_stat_lock);
  82.         dev_priv->mm.object_count++;
  83.         dev_priv->mm.object_memory += size;
  84.         spin_unlock(&dev_priv->mm.object_stat_lock);
  85. }
  86.  
  87. static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  88.                                      size_t size)
  89. {
  90.         spin_lock(&dev_priv->mm.object_stat_lock);
  91.         dev_priv->mm.object_count--;
  92.         dev_priv->mm.object_memory -= size;
  93.         spin_unlock(&dev_priv->mm.object_stat_lock);
  94. }
  95.  
  96. static int
  97. i915_gem_wait_for_error(struct i915_gpu_error *error)
  98. {
  99.         int ret;
  100.  
  101. #define EXIT_COND (!i915_reset_in_progress(error))
  102.         if (EXIT_COND)
  103.                 return 0;
  104. #if 0
  105.         /*
  106.          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
  107.          * userspace. If it takes that long something really bad is going on and
  108.          * we should simply try to bail out and fail as gracefully as possible.
  109.          */
  110.         ret = wait_event_interruptible_timeout(error->reset_queue,
  111.                                                EXIT_COND,
  112.                                                10*HZ);
  113.         if (ret == 0) {
  114.                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
  115.                 return -EIO;
  116.         } else if (ret < 0) {
  117.                 return ret;
  118.         }
  119.  
  120. #endif
  121. #undef EXIT_COND
  122.  
  123.         return 0;
  124. }
  125.  
  126. int i915_mutex_lock_interruptible(struct drm_device *dev)
  127. {
  128.         struct drm_i915_private *dev_priv = dev->dev_private;
  129.         int ret;
  130.  
  131.         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
  132.         if (ret)
  133.                 return ret;
  134.  
  135.         ret = mutex_lock_interruptible(&dev->struct_mutex);
  136.         if (ret)
  137.                 return ret;
  138.  
  139.         WARN_ON(i915_verify_lists(dev));
  140.         return 0;
  141. }
  142.  
  143. int
  144. i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  145.                             struct drm_file *file)
  146. {
  147.         struct drm_i915_private *dev_priv = dev->dev_private;
  148.         struct drm_i915_gem_get_aperture *args = data;
  149.         struct i915_gtt *ggtt = &dev_priv->gtt;
  150.         struct i915_vma *vma;
  151.         size_t pinned;
  152.  
  153.         pinned = 0;
  154.         mutex_lock(&dev->struct_mutex);
  155.         list_for_each_entry(vma, &ggtt->base.active_list, vm_link)
  156.                 if (vma->pin_count)
  157.                         pinned += vma->node.size;
  158.         list_for_each_entry(vma, &ggtt->base.inactive_list, vm_link)
  159.                 if (vma->pin_count)
  160.                         pinned += vma->node.size;
  161.         mutex_unlock(&dev->struct_mutex);
  162.  
  163.         args->aper_size = dev_priv->gtt.base.total;
  164.         args->aper_available_size = args->aper_size - pinned;
  165.  
  166.         return 0;
  167. }
  168.  
  169. static int
  170. i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
  171. {
  172.         char *vaddr = obj->phys_handle->vaddr;
  173.         struct sg_table *st;
  174.         struct scatterlist *sg;
  175.         int i;
  176.  
  177.         if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
  178.                 return -EINVAL;
  179.  
  180.  
  181.         st = kmalloc(sizeof(*st), GFP_KERNEL);
  182.         if (st == NULL)
  183.                 return -ENOMEM;
  184.  
  185.         if (sg_alloc_table(st, 1, GFP_KERNEL)) {
  186.                 kfree(st);
  187.                 return -ENOMEM;
  188.         }
  189.  
  190.         sg = st->sgl;
  191.         sg->offset = 0;
  192.         sg->length = obj->base.size;
  193.  
  194.         sg_dma_address(sg) = obj->phys_handle->busaddr;
  195.         sg_dma_len(sg) = obj->base.size;
  196.  
  197.         obj->pages = st;
  198.         return 0;
  199. }
  200.  
  201. static void
  202. i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
  203. {
  204.         int ret;
  205.  
  206.         BUG_ON(obj->madv == __I915_MADV_PURGED);
  207.  
  208.         ret = i915_gem_object_set_to_cpu_domain(obj, true);
  209.         if (ret) {
  210.                 /* In the event of a disaster, abandon all caches and
  211.                  * hope for the best.
  212.                  */
  213.                 WARN_ON(ret != -EIO);
  214.                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  215.         }
  216.  
  217.         if (obj->madv == I915_MADV_DONTNEED)
  218.                 obj->dirty = 0;
  219.  
  220.         if (obj->dirty) {
  221.                 obj->dirty = 0;
  222.         }
  223.  
  224.         sg_free_table(obj->pages);
  225.         kfree(obj->pages);
  226. }
  227.  
  228. static void
  229. i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
  230. {
  231.         drm_pci_free(obj->base.dev, obj->phys_handle);
  232. }
  233.  
  234. static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
  235.         .get_pages = i915_gem_object_get_pages_phys,
  236.         .put_pages = i915_gem_object_put_pages_phys,
  237.         .release = i915_gem_object_release_phys,
  238. };
  239.  
  240. static int
  241. drop_pages(struct drm_i915_gem_object *obj)
  242. {
  243.         struct i915_vma *vma, *next;
  244.         int ret;
  245.  
  246.         drm_gem_object_reference(&obj->base);
  247.         list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link)
  248.                 if (i915_vma_unbind(vma))
  249.                         break;
  250.  
  251.         ret = i915_gem_object_put_pages(obj);
  252.         drm_gem_object_unreference(&obj->base);
  253.  
  254.         return ret;
  255. }
  256.  
  257. int
  258. i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
  259.                             int align)
  260. {
  261.         drm_dma_handle_t *phys;
  262.         int ret;
  263.  
  264.         if (obj->phys_handle) {
  265.                 if ((unsigned long)obj->phys_handle->vaddr & (align -1))
  266.                         return -EBUSY;
  267.  
  268.                 return 0;
  269.         }
  270.  
  271.         if (obj->madv != I915_MADV_WILLNEED)
  272.                 return -EFAULT;
  273.  
  274.         if (obj->base.filp == NULL)
  275.                 return -EINVAL;
  276.  
  277.         ret = drop_pages(obj);
  278.         if (ret)
  279.                 return ret;
  280.  
  281.         /* create a new object */
  282.         phys = drm_pci_alloc(obj->base.dev, obj->base.size, align);
  283.         if (!phys)
  284.                 return -ENOMEM;
  285.  
  286.         obj->phys_handle = phys;
  287.         obj->ops = &i915_gem_phys_ops;
  288.  
  289.         return i915_gem_object_get_pages(obj);
  290. }
  291. void *i915_gem_object_alloc(struct drm_device *dev)
  292. {
  293.         struct drm_i915_private *dev_priv = dev->dev_private;
  294.     return kzalloc(sizeof(struct drm_i915_gem_object), 0);
  295. }
  296.  
  297. void i915_gem_object_free(struct drm_i915_gem_object *obj)
  298. {
  299.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  300.         kfree(obj);
  301. }
  302.  
  303. static int
  304. i915_gem_create(struct drm_file *file,
  305.                 struct drm_device *dev,
  306.                 uint64_t size,
  307.                 uint32_t *handle_p)
  308. {
  309.         struct drm_i915_gem_object *obj;
  310.         int ret;
  311.         u32 handle;
  312.  
  313.         size = roundup(size, PAGE_SIZE);
  314.         if (size == 0)
  315.                 return -EINVAL;
  316.  
  317.         /* Allocate the new object */
  318.         obj = i915_gem_alloc_object(dev, size);
  319.         if (obj == NULL)
  320.                 return -ENOMEM;
  321.  
  322.         ret = drm_gem_handle_create(file, &obj->base, &handle);
  323.         /* drop reference from allocate - handle holds it now */
  324.         drm_gem_object_unreference_unlocked(&obj->base);
  325.         if (ret)
  326.                 return ret;
  327.  
  328.         *handle_p = handle;
  329.         return 0;
  330. }
  331.  
  332. int
  333. i915_gem_dumb_create(struct drm_file *file,
  334.                      struct drm_device *dev,
  335.                      struct drm_mode_create_dumb *args)
  336. {
  337.         /* have to work out size/pitch and return them */
  338.         args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
  339.         args->size = args->pitch * args->height;
  340.         return i915_gem_create(file, dev,
  341.                                args->size, &args->handle);
  342. }
  343.  
  344. /**
  345.  * Creates a new mm object and returns a handle to it.
  346.  */
  347. int
  348. i915_gem_create_ioctl(struct drm_device *dev, void *data,
  349.                       struct drm_file *file)
  350. {
  351.         struct drm_i915_gem_create *args = data;
  352.  
  353.         return i915_gem_create(file, dev,
  354.                                args->size, &args->handle);
  355. }
  356.  
  357. static inline int
  358. __copy_to_user_swizzled(char __user *cpu_vaddr,
  359.                         const char *gpu_vaddr, int gpu_offset,
  360.                         int length)
  361. {
  362.         int ret, cpu_offset = 0;
  363.  
  364.         while (length > 0) {
  365.                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
  366.                 int this_length = min(cacheline_end - gpu_offset, length);
  367.                 int swizzled_gpu_offset = gpu_offset ^ 64;
  368.  
  369.                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
  370.                                      gpu_vaddr + swizzled_gpu_offset,
  371.                                      this_length);
  372.                 if (ret)
  373.                         return ret + length;
  374.  
  375.                 cpu_offset += this_length;
  376.                 gpu_offset += this_length;
  377.                 length -= this_length;
  378.         }
  379.  
  380.         return 0;
  381. }
  382.  
  383. static inline int
  384. __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
  385.                           const char __user *cpu_vaddr,
  386.                           int length)
  387. {
  388.         int ret, cpu_offset = 0;
  389.  
  390.         while (length > 0) {
  391.                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
  392.                 int this_length = min(cacheline_end - gpu_offset, length);
  393.                 int swizzled_gpu_offset = gpu_offset ^ 64;
  394.  
  395.                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
  396.                                        cpu_vaddr + cpu_offset,
  397.                                        this_length);
  398.                 if (ret)
  399.                         return ret + length;
  400.  
  401.                 cpu_offset += this_length;
  402.                 gpu_offset += this_length;
  403.                 length -= this_length;
  404.         }
  405.  
  406.         return 0;
  407. }
  408.  
  409. /*
  410.  * Pins the specified object's pages and synchronizes the object with
  411.  * GPU accesses. Sets needs_clflush to non-zero if the caller should
  412.  * flush the object from the CPU cache.
  413.  */
  414. int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
  415.                                     int *needs_clflush)
  416. {
  417.         int ret;
  418.  
  419.         *needs_clflush = 0;
  420.  
  421.         if (!obj->base.filp)
  422.                 return -EINVAL;
  423.  
  424.         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
  425.                 /* If we're not in the cpu read domain, set ourself into the gtt
  426.                  * read domain and manually flush cachelines (if required). This
  427.                  * optimizes for the case when the gpu will dirty the data
  428.                  * anyway again before the next pread happens. */
  429.                 *needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
  430.                                                         obj->cache_level);
  431.                 ret = i915_gem_object_wait_rendering(obj, true);
  432.                 if (ret)
  433.                         return ret;
  434.         }
  435.  
  436.         ret = i915_gem_object_get_pages(obj);
  437.         if (ret)
  438.                 return ret;
  439.  
  440.         i915_gem_object_pin_pages(obj);
  441.  
  442.         return ret;
  443. }
  444.  
  445. /* Per-page copy function for the shmem pread fastpath.
  446.  * Flushes invalid cachelines before reading the target if
  447.  * needs_clflush is set. */
  448. static int
  449. shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
  450.                  char __user *user_data,
  451.                  bool page_do_bit17_swizzling, bool needs_clflush)
  452. {
  453.         char *vaddr;
  454.         int ret;
  455.  
  456.         if (unlikely(page_do_bit17_swizzling))
  457.                 return -EINVAL;
  458.  
  459.         vaddr = kmap_atomic(page);
  460.         if (needs_clflush)
  461.                 drm_clflush_virt_range(vaddr + shmem_page_offset,
  462.                                        page_length);
  463.         ret = __copy_to_user_inatomic(user_data,
  464.                                       vaddr + shmem_page_offset,
  465.                                       page_length);
  466.         kunmap_atomic(vaddr);
  467.  
  468.         return ret ? -EFAULT : 0;
  469. }
  470.  
  471. static void
  472. shmem_clflush_swizzled_range(char *addr, unsigned long length,
  473.                              bool swizzled)
  474. {
  475.         if (unlikely(swizzled)) {
  476.                 unsigned long start = (unsigned long) addr;
  477.                 unsigned long end = (unsigned long) addr + length;
  478.  
  479.                 /* For swizzling simply ensure that we always flush both
  480.                  * channels. Lame, but simple and it works. Swizzled
  481.                  * pwrite/pread is far from a hotpath - current userspace
  482.                  * doesn't use it at all. */
  483.                 start = round_down(start, 128);
  484.                 end = round_up(end, 128);
  485.  
  486.                 drm_clflush_virt_range((void *)start, end - start);
  487.         } else {
  488.                 drm_clflush_virt_range(addr, length);
  489.         }
  490.  
  491. }
  492.  
  493. /* Only difference to the fast-path function is that this can handle bit17
  494.  * and uses non-atomic copy and kmap functions. */
  495. static int
  496. shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
  497.                  char __user *user_data,
  498.                  bool page_do_bit17_swizzling, bool needs_clflush)
  499. {
  500.         char *vaddr;
  501.         int ret;
  502.  
  503.         vaddr = kmap(page);
  504.         if (needs_clflush)
  505.                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  506.                                              page_length,
  507.                                              page_do_bit17_swizzling);
  508.  
  509.         if (page_do_bit17_swizzling)
  510.                 ret = __copy_to_user_swizzled(user_data,
  511.                                               vaddr, shmem_page_offset,
  512.                                               page_length);
  513.         else
  514.                 ret = __copy_to_user(user_data,
  515.                                      vaddr + shmem_page_offset,
  516.                                      page_length);
  517.         kunmap(page);
  518.  
  519.         return ret ? - EFAULT : 0;
  520. }
  521.  
  522. static int
  523. i915_gem_shmem_pread(struct drm_device *dev,
  524.                      struct drm_i915_gem_object *obj,
  525.                      struct drm_i915_gem_pread *args,
  526.                      struct drm_file *file)
  527. {
  528.         char __user *user_data;
  529.         ssize_t remain;
  530.         loff_t offset;
  531.         int shmem_page_offset, page_length, ret = 0;
  532.         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
  533.         int prefaulted = 0;
  534.         int needs_clflush = 0;
  535.         struct sg_page_iter sg_iter;
  536.  
  537.         user_data = to_user_ptr(args->data_ptr);
  538.         remain = args->size;
  539.  
  540.         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
  541.  
  542.         ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
  543.         if (ret)
  544.                 return ret;
  545.  
  546.         offset = args->offset;
  547.  
  548.         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
  549.                          offset >> PAGE_SHIFT) {
  550.                 struct page *page = sg_page_iter_page(&sg_iter);
  551.  
  552.                 if (remain <= 0)
  553.                         break;
  554.  
  555.                 /* Operation in this page
  556.                  *
  557.                  * shmem_page_offset = offset within page in shmem file
  558.                  * page_length = bytes to copy for this page
  559.                  */
  560.                 shmem_page_offset = offset_in_page(offset);
  561.                 page_length = remain;
  562.                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
  563.                         page_length = PAGE_SIZE - shmem_page_offset;
  564.  
  565.                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
  566.                         (page_to_phys(page) & (1 << 17)) != 0;
  567.  
  568.                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
  569.                                        user_data, page_do_bit17_swizzling,
  570.                                        needs_clflush);
  571.                 if (ret == 0)
  572.                         goto next_page;
  573.  
  574.                 mutex_unlock(&dev->struct_mutex);
  575.  
  576.                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
  577.                                        user_data, page_do_bit17_swizzling,
  578.                                        needs_clflush);
  579.  
  580.                 mutex_lock(&dev->struct_mutex);
  581.  
  582.                 if (ret)
  583.                         goto out;
  584.  
  585. next_page:
  586.                 remain -= page_length;
  587.                 user_data += page_length;
  588.                 offset += page_length;
  589.         }
  590.  
  591. out:
  592.         i915_gem_object_unpin_pages(obj);
  593.  
  594.         return ret;
  595. }
  596.  
  597. /**
  598.  * Reads data from the object referenced by handle.
  599.  *
  600.  * On error, the contents of *data are undefined.
  601.  */
  602. int
  603. i915_gem_pread_ioctl(struct drm_device *dev, void *data,
  604.                      struct drm_file *file)
  605. {
  606.         struct drm_i915_gem_pread *args = data;
  607.         struct drm_i915_gem_object *obj;
  608.         int ret = 0;
  609.  
  610.         if (args->size == 0)
  611.                 return 0;
  612.  
  613.         ret = i915_mutex_lock_interruptible(dev);
  614.         if (ret)
  615.                 return ret;
  616.  
  617.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  618.         if (&obj->base == NULL) {
  619.                 ret = -ENOENT;
  620.                 goto unlock;
  621.         }
  622.  
  623.         /* Bounds check source.  */
  624.         if (args->offset > obj->base.size ||
  625.             args->size > obj->base.size - args->offset) {
  626.                 ret = -EINVAL;
  627.                 goto out;
  628.         }
  629.  
  630.         /* prime objects have no backing filp to GEM pread/pwrite
  631.          * pages from.
  632.          */
  633.         if (!obj->base.filp) {
  634.                 ret = -EINVAL;
  635.                 goto out;
  636.         }
  637.  
  638.         trace_i915_gem_object_pread(obj, args->offset, args->size);
  639.  
  640.         ret = i915_gem_shmem_pread(dev, obj, args, file);
  641.  
  642. out:
  643.         drm_gem_object_unreference(&obj->base);
  644. unlock:
  645.         mutex_unlock(&dev->struct_mutex);
  646.         return ret;
  647. }
  648.  
  649. /* This is the fast write path which cannot handle
  650.  * page faults in the source data
  651.  */
  652.  
  653. static inline int
  654. fast_user_write(struct io_mapping *mapping,
  655.                 loff_t page_base, int page_offset,
  656.                 char __user *user_data,
  657.                 int length)
  658. {
  659.         void __iomem *vaddr_atomic;
  660.         void *vaddr;
  661.         unsigned long unwritten;
  662.  
  663.         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
  664.         /* We can use the cpu mem copy function because this is X86. */
  665.         vaddr = (void __force*)vaddr_atomic + page_offset;
  666.         unwritten = __copy_from_user_inatomic_nocache(vaddr,
  667.                                                       user_data, length);
  668.         io_mapping_unmap_atomic(vaddr_atomic);
  669.         return unwritten;
  670. }
  671.  
  672. /**
  673.  * This is the fast pwrite path, where we copy the data directly from the
  674.  * user into the GTT, uncached.
  675.  */
  676. static int
  677. i915_gem_gtt_pwrite_fast(struct drm_device *dev,
  678.                          struct drm_i915_gem_object *obj,
  679.                          struct drm_i915_gem_pwrite *args,
  680.                          struct drm_file *file)
  681. {
  682.         struct drm_i915_private *dev_priv = dev->dev_private;
  683.         ssize_t remain;
  684.         loff_t offset, page_base;
  685.         char __user *user_data;
  686.         int page_offset, page_length, ret;
  687.  
  688.         ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK);
  689.         if (ret)
  690.                 goto out;
  691.  
  692.         ret = i915_gem_object_set_to_gtt_domain(obj, true);
  693.         if (ret)
  694.                 goto out_unpin;
  695.  
  696.         ret = i915_gem_object_put_fence(obj);
  697.         if (ret)
  698.                 goto out_unpin;
  699.  
  700.         user_data = to_user_ptr(args->data_ptr);
  701.         remain = args->size;
  702.  
  703.         offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
  704.  
  705.         intel_fb_obj_invalidate(obj, ORIGIN_GTT);
  706.  
  707.         while (remain > 0) {
  708.                 /* Operation in this page
  709.                  *
  710.                  * page_base = page offset within aperture
  711.                  * page_offset = offset within page
  712.                  * page_length = bytes to copy for this page
  713.                  */
  714.                 page_base = offset & PAGE_MASK;
  715.                 page_offset = offset_in_page(offset);
  716.                 page_length = remain;
  717.                 if ((page_offset + remain) > PAGE_SIZE)
  718.                         page_length = PAGE_SIZE - page_offset;
  719.  
  720.                 /* If we get a fault while copying data, then (presumably) our
  721.                  * source page isn't available.  Return the error and we'll
  722.                  * retry in the slow path.
  723.                  */
  724.                 if (fast_user_write(dev_priv->gtt.mappable, page_base,
  725.                                     page_offset, user_data, page_length)) {
  726.                         ret = -EFAULT;
  727.                         goto out_flush;
  728.                 }
  729.  
  730.                 remain -= page_length;
  731.                 user_data += page_length;
  732.                 offset += page_length;
  733.         }
  734.  
  735. out_flush:
  736.         intel_fb_obj_flush(obj, false, ORIGIN_GTT);
  737. out_unpin:
  738.         i915_gem_object_ggtt_unpin(obj);
  739. out:
  740.         return ret;
  741. }
  742.  
  743. /* Per-page copy function for the shmem pwrite fastpath.
  744.  * Flushes invalid cachelines before writing to the target if
  745.  * needs_clflush_before is set and flushes out any written cachelines after
  746.  * writing if needs_clflush is set. */
  747. static int
  748. shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
  749.                   char __user *user_data,
  750.                   bool page_do_bit17_swizzling,
  751.                   bool needs_clflush_before,
  752.                   bool needs_clflush_after)
  753. {
  754.         char *vaddr;
  755.         int ret;
  756.  
  757.         if (unlikely(page_do_bit17_swizzling))
  758.                 return -EINVAL;
  759.  
  760.         vaddr = kmap_atomic(page);
  761.         if (needs_clflush_before)
  762.                 drm_clflush_virt_range(vaddr + shmem_page_offset,
  763.                                        page_length);
  764.         ret = __copy_from_user_inatomic(vaddr + shmem_page_offset,
  765.                                         user_data, page_length);
  766.         if (needs_clflush_after)
  767.                 drm_clflush_virt_range(vaddr + shmem_page_offset,
  768.                                        page_length);
  769.         kunmap_atomic(vaddr);
  770.  
  771.         return ret ? -EFAULT : 0;
  772. }
  773.  
  774. /* Only difference to the fast-path function is that this can handle bit17
  775.  * and uses non-atomic copy and kmap functions. */
  776. static int
  777. shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
  778.                   char __user *user_data,
  779.                   bool page_do_bit17_swizzling,
  780.                   bool needs_clflush_before,
  781.                   bool needs_clflush_after)
  782. {
  783.         char *vaddr;
  784.         int ret;
  785.  
  786.         vaddr = kmap(page);
  787.         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
  788.                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  789.                                              page_length,
  790.                                              page_do_bit17_swizzling);
  791.         if (page_do_bit17_swizzling)
  792.                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
  793.                                                 user_data,
  794.                                                 page_length);
  795.         else
  796.                 ret = __copy_from_user(vaddr + shmem_page_offset,
  797.                                        user_data,
  798.                                        page_length);
  799.         if (needs_clflush_after)
  800.                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  801.                                              page_length,
  802.                                              page_do_bit17_swizzling);
  803.         kunmap(page);
  804.  
  805.         return ret ? -EFAULT : 0;
  806. }
  807.  
  808. static int
  809. i915_gem_shmem_pwrite(struct drm_device *dev,
  810.                       struct drm_i915_gem_object *obj,
  811.                       struct drm_i915_gem_pwrite *args,
  812.                       struct drm_file *file)
  813. {
  814.         ssize_t remain;
  815.         loff_t offset;
  816.         char __user *user_data;
  817.         int shmem_page_offset, page_length, ret = 0;
  818.         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
  819.         int hit_slowpath = 0;
  820.         int needs_clflush_after = 0;
  821.         int needs_clflush_before = 0;
  822.         struct sg_page_iter sg_iter;
  823.  
  824.         user_data = to_user_ptr(args->data_ptr);
  825.         remain = args->size;
  826.  
  827.         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
  828.  
  829.         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  830.                 /* If we're not in the cpu write domain, set ourself into the gtt
  831.                  * write domain and manually flush cachelines (if required). This
  832.                  * optimizes for the case when the gpu will use the data
  833.                  * right away and we therefore have to clflush anyway. */
  834.                 needs_clflush_after = cpu_write_needs_clflush(obj);
  835.                 ret = i915_gem_object_wait_rendering(obj, false);
  836.                 if (ret)
  837.                         return ret;
  838.         }
  839.         /* Same trick applies to invalidate partially written cachelines read
  840.          * before writing. */
  841.         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
  842.                 needs_clflush_before =
  843.                         !cpu_cache_is_coherent(dev, obj->cache_level);
  844.  
  845.         ret = i915_gem_object_get_pages(obj);
  846.         if (ret)
  847.                 return ret;
  848.  
  849.         intel_fb_obj_invalidate(obj, ORIGIN_CPU);
  850.  
  851.         i915_gem_object_pin_pages(obj);
  852.  
  853.         offset = args->offset;
  854.         obj->dirty = 1;
  855.  
  856.         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
  857.                          offset >> PAGE_SHIFT) {
  858.                 struct page *page = sg_page_iter_page(&sg_iter);
  859.                 int partial_cacheline_write;
  860.  
  861.                 if (remain <= 0)
  862.                         break;
  863.  
  864.                 /* Operation in this page
  865.                  *
  866.                  * shmem_page_offset = offset within page in shmem file
  867.                  * page_length = bytes to copy for this page
  868.                  */
  869.                 shmem_page_offset = offset_in_page(offset);
  870.  
  871.                 page_length = remain;
  872.                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
  873.                         page_length = PAGE_SIZE - shmem_page_offset;
  874.  
  875.                 /* If we don't overwrite a cacheline completely we need to be
  876.                  * careful to have up-to-date data by first clflushing. Don't
  877.                  * overcomplicate things and flush the entire patch. */
  878.                 partial_cacheline_write = needs_clflush_before &&
  879.                         ((shmem_page_offset | page_length)
  880.                                 & (x86_clflush_size - 1));
  881.  
  882.                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
  883.                         (page_to_phys(page) & (1 << 17)) != 0;
  884.  
  885.                 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
  886.                                         user_data, page_do_bit17_swizzling,
  887.                                         partial_cacheline_write,
  888.                                         needs_clflush_after);
  889.                 if (ret == 0)
  890.                         goto next_page;
  891.  
  892.                 hit_slowpath = 1;
  893.                 mutex_unlock(&dev->struct_mutex);
  894.                 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
  895.                                         user_data, page_do_bit17_swizzling,
  896.                                         partial_cacheline_write,
  897.                                         needs_clflush_after);
  898.  
  899.                 mutex_lock(&dev->struct_mutex);
  900.  
  901.                 if (ret)
  902.                         goto out;
  903.  
  904. next_page:
  905.                 remain -= page_length;
  906.                 user_data += page_length;
  907.                 offset += page_length;
  908.         }
  909.  
  910. out:
  911.         i915_gem_object_unpin_pages(obj);
  912.  
  913.         if (hit_slowpath) {
  914.                 /*
  915.                  * Fixup: Flush cpu caches in case we didn't flush the dirty
  916.                  * cachelines in-line while writing and the object moved
  917.                  * out of the cpu write domain while we've dropped the lock.
  918.                  */
  919.                 if (!needs_clflush_after &&
  920.                     obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  921.                         if (i915_gem_clflush_object(obj, obj->pin_display))
  922.                                 needs_clflush_after = true;
  923.                 }
  924.         }
  925.  
  926.         if (needs_clflush_after)
  927.                 i915_gem_chipset_flush(dev);
  928.         else
  929.                 obj->cache_dirty = true;
  930.  
  931.         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
  932.         return ret;
  933. }
  934.  
  935. /**
  936.  * Writes data to the object referenced by handle.
  937.  *
  938.  * On error, the contents of the buffer that were to be modified are undefined.
  939.  */
  940. int
  941. i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
  942.                       struct drm_file *file)
  943. {
  944.         struct drm_i915_private *dev_priv = dev->dev_private;
  945.         struct drm_i915_gem_pwrite *args = data;
  946.         struct drm_i915_gem_object *obj;
  947.         int ret;
  948.  
  949.         if (args->size == 0)
  950.                 return 0;
  951.  
  952.         intel_runtime_pm_get(dev_priv);
  953.  
  954.         ret = i915_mutex_lock_interruptible(dev);
  955.         if (ret)
  956.                 goto put_rpm;
  957.  
  958.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  959.         if (&obj->base == NULL) {
  960.                 ret = -ENOENT;
  961.                 goto unlock;
  962.         }
  963.  
  964.         /* Bounds check destination. */
  965.         if (args->offset > obj->base.size ||
  966.             args->size > obj->base.size - args->offset) {
  967.                 ret = -EINVAL;
  968.                 goto out;
  969.         }
  970.  
  971.         /* prime objects have no backing filp to GEM pread/pwrite
  972.          * pages from.
  973.          */
  974.         if (!obj->base.filp) {
  975.                 ret = -EINVAL;
  976.                 goto out;
  977.         }
  978.  
  979.         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
  980.  
  981.         ret = -EFAULT;
  982.         /* We can only do the GTT pwrite on untiled buffers, as otherwise
  983.          * it would end up going through the fenced access, and we'll get
  984.          * different detiling behavior between reading and writing.
  985.          * pread/pwrite currently are reading and writing from the CPU
  986.          * perspective, requiring manual detiling by the client.
  987.          */
  988.         if (obj->tiling_mode == I915_TILING_NONE &&
  989.             obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
  990.             cpu_write_needs_clflush(obj)) {
  991.                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
  992.                 /* Note that the gtt paths might fail with non-page-backed user
  993.                  * pointers (e.g. gtt mappings when moving data between
  994.                  * textures). Fallback to the shmem path in that case. */
  995.         }
  996.  
  997.         if (ret == -EFAULT || ret == -ENOSPC) {
  998.                         ret = i915_gem_shmem_pwrite(dev, obj, args, file);
  999.         }
  1000.  
  1001. out:
  1002.         drm_gem_object_unreference(&obj->base);
  1003. unlock:
  1004.         mutex_unlock(&dev->struct_mutex);
  1005. put_rpm:
  1006.         intel_runtime_pm_put(dev_priv);
  1007.  
  1008.         return ret;
  1009. }
  1010.  
  1011. int
  1012. i915_gem_check_wedge(struct i915_gpu_error *error,
  1013.                      bool interruptible)
  1014. {
  1015.         if (i915_reset_in_progress(error)) {
  1016.                 /* Non-interruptible callers can't handle -EAGAIN, hence return
  1017.                  * -EIO unconditionally for these. */
  1018.                 if (!interruptible)
  1019.                         return -EIO;
  1020.  
  1021.                 /* Recovery complete, but the reset failed ... */
  1022.                 if (i915_terminally_wedged(error))
  1023.                         return -EIO;
  1024.  
  1025.                 /*
  1026.                  * Check if GPU Reset is in progress - we need intel_ring_begin
  1027.                  * to work properly to reinit the hw state while the gpu is
  1028.                  * still marked as reset-in-progress. Handle this with a flag.
  1029.                  */
  1030.                 if (!error->reload_in_reset)
  1031.                         return -EAGAIN;
  1032.         }
  1033.  
  1034.         return 0;
  1035. }
  1036.  
  1037. static void fake_irq(unsigned long data)
  1038. {
  1039. //      wake_up_process((struct task_struct *)data);
  1040. }
  1041.  
  1042. static bool missed_irq(struct drm_i915_private *dev_priv,
  1043.                        struct intel_engine_cs *ring)
  1044. {
  1045.         return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings);
  1046. }
  1047.  
  1048. static unsigned long local_clock_us(unsigned *cpu)
  1049. {
  1050.         unsigned long t;
  1051.  
  1052.         /* Cheaply and approximately convert from nanoseconds to microseconds.
  1053.          * The result and subsequent calculations are also defined in the same
  1054.          * approximate microseconds units. The principal source of timing
  1055.          * error here is from the simple truncation.
  1056.          *
  1057.          * Note that local_clock() is only defined wrt to the current CPU;
  1058.          * the comparisons are no longer valid if we switch CPUs. Instead of
  1059.          * blocking preemption for the entire busywait, we can detect the CPU
  1060.          * switch and use that as indicator of system load and a reason to
  1061.          * stop busywaiting, see busywait_stop().
  1062.          */
  1063.         t = GetClockNs() >> 10;
  1064.  
  1065.         return t;
  1066. }
  1067.  
  1068. static bool busywait_stop(unsigned long timeout, unsigned cpu)
  1069. {
  1070.         unsigned this_cpu = 0;
  1071.  
  1072.         if (time_after(local_clock_us(&this_cpu), timeout))
  1073.                 return true;
  1074.  
  1075.         return this_cpu != cpu;
  1076. }
  1077.  
  1078. static int __i915_spin_request(struct drm_i915_gem_request *req, int state)
  1079. {
  1080.         unsigned long timeout;
  1081.         unsigned cpu;
  1082.  
  1083.         /* When waiting for high frequency requests, e.g. during synchronous
  1084.          * rendering split between the CPU and GPU, the finite amount of time
  1085.          * required to set up the irq and wait upon it limits the response
  1086.          * rate. By busywaiting on the request completion for a short while we
  1087.          * can service the high frequency waits as quick as possible. However,
  1088.          * if it is a slow request, we want to sleep as quickly as possible.
  1089.          * The tradeoff between waiting and sleeping is roughly the time it
  1090.          * takes to sleep on a request, on the order of a microsecond.
  1091.          */
  1092.  
  1093.         if (req->ring->irq_refcount)
  1094.                 return -EBUSY;
  1095.  
  1096.         /* Only spin if we know the GPU is processing this request */
  1097.         if (!i915_gem_request_started(req, true))
  1098.                 return -EAGAIN;
  1099.  
  1100.         timeout = local_clock_us(&cpu) + 5;
  1101.         while (1 /*!need_resched()*/) {
  1102.                 if (i915_gem_request_completed(req, true))
  1103.                         return 0;
  1104.  
  1105.                 if (busywait_stop(timeout, cpu))
  1106.                         break;
  1107.  
  1108.                 cpu_relax_lowlatency();
  1109.         }
  1110.  
  1111.         if (i915_gem_request_completed(req, false))
  1112.                 return 0;
  1113.  
  1114.         return -EAGAIN;
  1115. }
  1116.  
  1117. /**
  1118.  * __i915_wait_request - wait until execution of request has finished
  1119.  * @req: duh!
  1120.  * @reset_counter: reset sequence associated with the given request
  1121.  * @interruptible: do an interruptible wait (normally yes)
  1122.  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
  1123.  *
  1124.  * Note: It is of utmost importance that the passed in seqno and reset_counter
  1125.  * values have been read by the caller in an smp safe manner. Where read-side
  1126.  * locks are involved, it is sufficient to read the reset_counter before
  1127.  * unlocking the lock that protects the seqno. For lockless tricks, the
  1128.  * reset_counter _must_ be read before, and an appropriate smp_rmb must be
  1129.  * inserted.
  1130.  *
  1131.  * Returns 0 if the request was found within the alloted time. Else returns the
  1132.  * errno with remaining time filled in timeout argument.
  1133.  */
  1134. int __i915_wait_request(struct drm_i915_gem_request *req,
  1135.                         unsigned reset_counter,
  1136.                         bool interruptible,
  1137.                         s64 *timeout,
  1138.                         struct intel_rps_client *rps)
  1139. {
  1140.         struct intel_engine_cs *ring = i915_gem_request_get_ring(req);
  1141.         struct drm_device *dev = ring->dev;
  1142.         struct drm_i915_private *dev_priv = dev->dev_private;
  1143.         const bool irq_test_in_progress =
  1144.                 ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_ring_flag(ring);
  1145.         int state = interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
  1146.         wait_queue_t wait;
  1147.         unsigned long timeout_expire;
  1148.         s64 before = 0; /* Only to silence a compiler warning. */
  1149.         int ret;
  1150.  
  1151.         WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled");
  1152.  
  1153.         if (list_empty(&req->list))
  1154.                 return 0;
  1155.  
  1156.         if (i915_gem_request_completed(req, true))
  1157.                 return 0;
  1158.  
  1159.         timeout_expire = 0;
  1160.         if (timeout) {
  1161.                 if (WARN_ON(*timeout < 0))
  1162.                         return -EINVAL;
  1163.  
  1164.                 if (*timeout == 0)
  1165.                         return -ETIME;
  1166.  
  1167.                 timeout_expire = jiffies + nsecs_to_jiffies_timeout(*timeout);
  1168.  
  1169.                 /*
  1170.                  * Record current time in case interrupted by signal, or wedged.
  1171.                  */
  1172.                 before = ktime_get_raw_ns();
  1173.         }
  1174.  
  1175.         if (INTEL_INFO(dev_priv)->gen >= 6)
  1176.                 gen6_rps_boost(dev_priv, rps, req->emitted_jiffies);
  1177.  
  1178.         trace_i915_gem_request_wait_begin(req);
  1179.  
  1180.         /* Optimistic spin for the next jiffie before touching IRQs */
  1181.         ret = __i915_spin_request(req, state);
  1182.         if (ret == 0)
  1183.                 goto out;
  1184.  
  1185.         if (!irq_test_in_progress && WARN_ON(!ring->irq_get(ring))) {
  1186.                 ret = -ENODEV;
  1187.                 goto out;
  1188.         }
  1189.  
  1190.         INIT_LIST_HEAD(&wait.task_list);
  1191.         wait.evnt = CreateEvent(NULL, MANUAL_DESTROY);
  1192.  
  1193.         for (;;) {
  1194.                 unsigned long flags;
  1195.  
  1196.                 /* We need to check whether any gpu reset happened in between
  1197.                  * the caller grabbing the seqno and now ... */
  1198.                 if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) {
  1199.                         /* ... but upgrade the -EAGAIN to an -EIO if the gpu
  1200.                          * is truely gone. */
  1201.                         ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
  1202.                         if (ret == 0)
  1203.                                 ret = -EAGAIN;
  1204.                         break;
  1205.                 }
  1206.  
  1207.                 if (i915_gem_request_completed(req, false)) {
  1208.                         ret = 0;
  1209.                         break;
  1210.                 }
  1211.  
  1212.                 if (timeout && time_after_eq(jiffies, timeout_expire)) {
  1213.                         ret = -ETIME;
  1214.                         break;
  1215.                 }
  1216.  
  1217.         spin_lock_irqsave(&ring->irq_queue.lock, flags);
  1218.         if (list_empty(&wait.task_list))
  1219.             __add_wait_queue(&ring->irq_queue, &wait);
  1220.         spin_unlock_irqrestore(&ring->irq_queue.lock, flags);
  1221.  
  1222.             WaitEventTimeout(wait.evnt, 1);
  1223.  
  1224.         if (!list_empty(&wait.task_list)) {
  1225.             spin_lock_irqsave(&ring->irq_queue.lock, flags);
  1226.             list_del_init(&wait.task_list);
  1227.             spin_unlock_irqrestore(&ring->irq_queue.lock, flags);
  1228.         }
  1229.  
  1230.         };
  1231.  
  1232.         if (!irq_test_in_progress)
  1233.                 ring->irq_put(ring);
  1234.  
  1235.     DestroyEvent(wait.evnt);
  1236.  
  1237. out:
  1238.         trace_i915_gem_request_wait_end(req);
  1239.  
  1240.         if (timeout) {
  1241.                 s64 tres = *timeout - (ktime_get_raw_ns() - before);
  1242.  
  1243.                 *timeout = tres < 0 ? 0 : tres;
  1244.  
  1245.                 /*
  1246.                  * Apparently ktime isn't accurate enough and occasionally has a
  1247.                  * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
  1248.                  * things up to make the test happy. We allow up to 1 jiffy.
  1249.                  *
  1250.                  * This is a regrssion from the timespec->ktime conversion.
  1251.                  */
  1252.                 if (ret == -ETIME && *timeout < jiffies_to_usecs(1)*1000)
  1253.                         *timeout = 0;
  1254.         }
  1255.  
  1256.         return ret;
  1257. }
  1258.  
  1259. int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
  1260.                                    struct drm_file *file)
  1261. {
  1262.         struct drm_i915_private *dev_private;
  1263.         struct drm_i915_file_private *file_priv;
  1264.  
  1265.         WARN_ON(!req || !file || req->file_priv);
  1266.  
  1267.         if (!req || !file)
  1268.                 return -EINVAL;
  1269.  
  1270.         if (req->file_priv)
  1271.                 return -EINVAL;
  1272.  
  1273.         dev_private = req->ring->dev->dev_private;
  1274.         file_priv = file->driver_priv;
  1275.  
  1276.         spin_lock(&file_priv->mm.lock);
  1277.         req->file_priv = file_priv;
  1278.         list_add_tail(&req->client_list, &file_priv->mm.request_list);
  1279.         spin_unlock(&file_priv->mm.lock);
  1280.  
  1281.     req->pid = (struct pid*)1;
  1282.  
  1283.         return 0;
  1284. }
  1285.  
  1286. static inline void
  1287. i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
  1288. {
  1289.         struct drm_i915_file_private *file_priv = request->file_priv;
  1290.  
  1291.         if (!file_priv)
  1292.                 return;
  1293.  
  1294.         spin_lock(&file_priv->mm.lock);
  1295.         list_del(&request->client_list);
  1296.         request->file_priv = NULL;
  1297.         spin_unlock(&file_priv->mm.lock);
  1298.         request->pid = NULL;
  1299. }
  1300.  
  1301. static void i915_gem_request_retire(struct drm_i915_gem_request *request)
  1302. {
  1303.         trace_i915_gem_request_retire(request);
  1304.  
  1305.         /* We know the GPU must have read the request to have
  1306.          * sent us the seqno + interrupt, so use the position
  1307.          * of tail of the request to update the last known position
  1308.          * of the GPU head.
  1309.          *
  1310.          * Note this requires that we are always called in request
  1311.          * completion order.
  1312.          */
  1313.         request->ringbuf->last_retired_head = request->postfix;
  1314.  
  1315.         list_del_init(&request->list);
  1316.         i915_gem_request_remove_from_client(request);
  1317.  
  1318.         i915_gem_request_unreference(request);
  1319. }
  1320.  
  1321. static void
  1322. __i915_gem_request_retire__upto(struct drm_i915_gem_request *req)
  1323. {
  1324.         struct intel_engine_cs *engine = req->ring;
  1325.         struct drm_i915_gem_request *tmp;
  1326.  
  1327.         lockdep_assert_held(&engine->dev->struct_mutex);
  1328.  
  1329.         if (list_empty(&req->list))
  1330.                 return;
  1331.  
  1332.         do {
  1333.                 tmp = list_first_entry(&engine->request_list,
  1334.                                        typeof(*tmp), list);
  1335.  
  1336.                 i915_gem_request_retire(tmp);
  1337.         } while (tmp != req);
  1338.  
  1339.         WARN_ON(i915_verify_lists(engine->dev));
  1340. }
  1341.  
  1342. /**
  1343.  * Waits for a request to be signaled, and cleans up the
  1344.  * request and object lists appropriately for that event.
  1345.  */
  1346. int
  1347. i915_wait_request(struct drm_i915_gem_request *req)
  1348. {
  1349.         struct drm_device *dev;
  1350.         struct drm_i915_private *dev_priv;
  1351.         bool interruptible;
  1352.         int ret;
  1353.  
  1354.         BUG_ON(req == NULL);
  1355.  
  1356.         dev = req->ring->dev;
  1357.         dev_priv = dev->dev_private;
  1358.         interruptible = dev_priv->mm.interruptible;
  1359.  
  1360.         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
  1361.  
  1362.         ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
  1363.         if (ret)
  1364.                 return ret;
  1365.  
  1366.         ret = __i915_wait_request(req,
  1367.                                   atomic_read(&dev_priv->gpu_error.reset_counter),
  1368.                                   interruptible, NULL, NULL);
  1369.         if (ret)
  1370.                 return ret;
  1371.  
  1372.         __i915_gem_request_retire__upto(req);
  1373.         return 0;
  1374. }
  1375.  
  1376. /**
  1377.  * Ensures that all rendering to the object has completed and the object is
  1378.  * safe to unbind from the GTT or access from the CPU.
  1379.  */
  1380. int
  1381. i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
  1382.                                bool readonly)
  1383. {
  1384.         int ret, i;
  1385.  
  1386.         if (!obj->active)
  1387.                 return 0;
  1388.  
  1389.         if (readonly) {
  1390.                 if (obj->last_write_req != NULL) {
  1391.                         ret = i915_wait_request(obj->last_write_req);
  1392.                         if (ret)
  1393.                                 return ret;
  1394.  
  1395.                         i = obj->last_write_req->ring->id;
  1396.                         if (obj->last_read_req[i] == obj->last_write_req)
  1397.                                 i915_gem_object_retire__read(obj, i);
  1398.                         else
  1399.                                 i915_gem_object_retire__write(obj);
  1400.                 }
  1401.         } else {
  1402.                 for (i = 0; i < I915_NUM_RINGS; i++) {
  1403.                         if (obj->last_read_req[i] == NULL)
  1404.                                 continue;
  1405.  
  1406.                         ret = i915_wait_request(obj->last_read_req[i]);
  1407.                         if (ret)
  1408.                                 return ret;
  1409.  
  1410.                         i915_gem_object_retire__read(obj, i);
  1411.                 }
  1412.                 RQ_BUG_ON(obj->active);
  1413.         }
  1414.  
  1415.         return 0;
  1416. }
  1417.  
  1418. static void
  1419. i915_gem_object_retire_request(struct drm_i915_gem_object *obj,
  1420.                                struct drm_i915_gem_request *req)
  1421. {
  1422.         int ring = req->ring->id;
  1423.  
  1424.         if (obj->last_read_req[ring] == req)
  1425.                 i915_gem_object_retire__read(obj, ring);
  1426.         else if (obj->last_write_req == req)
  1427.                 i915_gem_object_retire__write(obj);
  1428.  
  1429.         __i915_gem_request_retire__upto(req);
  1430. }
  1431.  
  1432. /* A nonblocking variant of the above wait. This is a highly dangerous routine
  1433.  * as the object state may change during this call.
  1434.  */
  1435. static __must_check int
  1436. i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
  1437.                                             struct intel_rps_client *rps,
  1438.                                             bool readonly)
  1439. {
  1440.         struct drm_device *dev = obj->base.dev;
  1441.         struct drm_i915_private *dev_priv = dev->dev_private;
  1442.         struct drm_i915_gem_request *requests[I915_NUM_RINGS];
  1443.         unsigned reset_counter;
  1444.         int ret, i, n = 0;
  1445.  
  1446.         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
  1447.         BUG_ON(!dev_priv->mm.interruptible);
  1448.  
  1449.         if (!obj->active)
  1450.                 return 0;
  1451.  
  1452.         ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
  1453.         if (ret)
  1454.                 return ret;
  1455.  
  1456.         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
  1457.  
  1458.         if (readonly) {
  1459.                 struct drm_i915_gem_request *req;
  1460.  
  1461.                 req = obj->last_write_req;
  1462.                 if (req == NULL)
  1463.                         return 0;
  1464.  
  1465.                 requests[n++] = i915_gem_request_reference(req);
  1466.         } else {
  1467.                 for (i = 0; i < I915_NUM_RINGS; i++) {
  1468.                         struct drm_i915_gem_request *req;
  1469.  
  1470.                         req = obj->last_read_req[i];
  1471.                         if (req == NULL)
  1472.                                 continue;
  1473.  
  1474.                         requests[n++] = i915_gem_request_reference(req);
  1475.                 }
  1476.         }
  1477.  
  1478.         mutex_unlock(&dev->struct_mutex);
  1479.         for (i = 0; ret == 0 && i < n; i++)
  1480.                 ret = __i915_wait_request(requests[i], reset_counter, true,
  1481.                                           NULL, rps);
  1482.         mutex_lock(&dev->struct_mutex);
  1483.  
  1484.         for (i = 0; i < n; i++) {
  1485.                 if (ret == 0)
  1486.                         i915_gem_object_retire_request(obj, requests[i]);
  1487.                 i915_gem_request_unreference(requests[i]);
  1488.         }
  1489.  
  1490.         return ret;
  1491. }
  1492.  
  1493. static struct intel_rps_client *to_rps_client(struct drm_file *file)
  1494. {
  1495.         struct drm_i915_file_private *fpriv = file->driver_priv;
  1496.         return &fpriv->rps;
  1497. }
  1498.  
  1499. /**
  1500.  * Called when user space prepares to use an object with the CPU, either
  1501.  * through the mmap ioctl's mapping or a GTT mapping.
  1502.  */
  1503. int
  1504. i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
  1505.                           struct drm_file *file)
  1506. {
  1507.         struct drm_i915_gem_set_domain *args = data;
  1508.         struct drm_i915_gem_object *obj;
  1509.         uint32_t read_domains = args->read_domains;
  1510.         uint32_t write_domain = args->write_domain;
  1511.         int ret;
  1512.  
  1513.         /* Only handle setting domains to types used by the CPU. */
  1514.         if (write_domain & I915_GEM_GPU_DOMAINS)
  1515.                 return -EINVAL;
  1516.  
  1517.         if (read_domains & I915_GEM_GPU_DOMAINS)
  1518.                 return -EINVAL;
  1519.  
  1520.         /* Having something in the write domain implies it's in the read
  1521.          * domain, and only that read domain.  Enforce that in the request.
  1522.          */
  1523.         if (write_domain != 0 && read_domains != write_domain)
  1524.                 return -EINVAL;
  1525.  
  1526.         ret = i915_mutex_lock_interruptible(dev);
  1527.         if (ret)
  1528.                 return ret;
  1529.  
  1530.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  1531.         if (&obj->base == NULL) {
  1532.                 ret = -ENOENT;
  1533.                 goto unlock;
  1534.         }
  1535.  
  1536.         /* Try to flush the object off the GPU without holding the lock.
  1537.          * We will repeat the flush holding the lock in the normal manner
  1538.          * to catch cases where we are gazumped.
  1539.          */
  1540.         ret = i915_gem_object_wait_rendering__nonblocking(obj,
  1541.                                                           to_rps_client(file),
  1542.                                                           !write_domain);
  1543.         if (ret)
  1544.                 goto unref;
  1545.  
  1546.         if (read_domains & I915_GEM_DOMAIN_GTT)
  1547.                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
  1548.         else
  1549.                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
  1550.  
  1551.         if (write_domain != 0)
  1552.                 intel_fb_obj_invalidate(obj,
  1553.                                         write_domain == I915_GEM_DOMAIN_GTT ?
  1554.                                         ORIGIN_GTT : ORIGIN_CPU);
  1555.  
  1556. unref:
  1557.         drm_gem_object_unreference(&obj->base);
  1558. unlock:
  1559.         mutex_unlock(&dev->struct_mutex);
  1560.         return ret;
  1561. }
  1562.  
  1563. /**
  1564.  * Called when user space has done writes to this buffer
  1565.  */
  1566. int
  1567. i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
  1568.                          struct drm_file *file)
  1569. {
  1570.         struct drm_i915_gem_sw_finish *args = data;
  1571.         struct drm_i915_gem_object *obj;
  1572.         int ret = 0;
  1573.  
  1574.         ret = i915_mutex_lock_interruptible(dev);
  1575.         if (ret)
  1576.                 return ret;
  1577.  
  1578.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  1579.         if (&obj->base == NULL) {
  1580.                 ret = -ENOENT;
  1581.                 goto unlock;
  1582.         }
  1583.  
  1584.         /* Pinned buffers may be scanout, so flush the cache */
  1585.         if (obj->pin_display)
  1586.                 i915_gem_object_flush_cpu_write_domain(obj);
  1587.  
  1588.         drm_gem_object_unreference(&obj->base);
  1589. unlock:
  1590.         mutex_unlock(&dev->struct_mutex);
  1591.         return ret;
  1592. }
  1593.  
  1594. /**
  1595.  * Maps the contents of an object, returning the address it is mapped
  1596.  * into.
  1597.  *
  1598.  * While the mapping holds a reference on the contents of the object, it doesn't
  1599.  * imply a ref on the object itself.
  1600.  *
  1601.  * IMPORTANT:
  1602.  *
  1603.  * DRM driver writers who look a this function as an example for how to do GEM
  1604.  * mmap support, please don't implement mmap support like here. The modern way
  1605.  * to implement DRM mmap support is with an mmap offset ioctl (like
  1606.  * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
  1607.  * That way debug tooling like valgrind will understand what's going on, hiding
  1608.  * the mmap call in a driver private ioctl will break that. The i915 driver only
  1609.  * does cpu mmaps this way because we didn't know better.
  1610.  */
  1611. int
  1612. i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
  1613.                     struct drm_file *file)
  1614. {
  1615.         struct drm_i915_gem_mmap *args = data;
  1616.         struct drm_gem_object *obj;
  1617.         unsigned long addr;
  1618.  
  1619. //      if (args->flags & ~(I915_MMAP_WC))
  1620. //              return -EINVAL;
  1621.         obj = drm_gem_object_lookup(dev, file, args->handle);
  1622.         if (obj == NULL)
  1623.                 return -ENOENT;
  1624.  
  1625.         /* prime objects have no backing filp to GEM mmap
  1626.          * pages from.
  1627.          */
  1628.         if (!obj->filp) {
  1629.                 drm_gem_object_unreference_unlocked(obj);
  1630.                 return -EINVAL;
  1631.         }
  1632.  
  1633.         addr = vm_mmap(obj->filp, 0, args->size,
  1634.                        PROT_READ | PROT_WRITE, MAP_SHARED,
  1635.                        args->offset);
  1636.         drm_gem_object_unreference_unlocked(obj);
  1637.         if (IS_ERR((void *)addr))
  1638.                 return addr;
  1639.  
  1640.         args->addr_ptr = (uint64_t) addr;
  1641.  
  1642.         return 0;
  1643. }
  1644.  
  1645.  
  1646.  
  1647.  
  1648.  
  1649.  
  1650.  
  1651.  
  1652.  
  1653.  
  1654.  
  1655.  
  1656.  
  1657. /**
  1658.  * i915_gem_release_mmap - remove physical page mappings
  1659.  * @obj: obj in question
  1660.  *
  1661.  * Preserve the reservation of the mmapping with the DRM core code, but
  1662.  * relinquish ownership of the pages back to the system.
  1663.  *
  1664.  * It is vital that we remove the page mapping if we have mapped a tiled
  1665.  * object through the GTT and then lose the fence register due to
  1666.  * resource pressure. Similarly if the object has been moved out of the
  1667.  * aperture, than pages mapped into userspace must be revoked. Removing the
  1668.  * mapping will then trigger a page fault on the next user access, allowing
  1669.  * fixup by i915_gem_fault().
  1670.  */
  1671. void
  1672. i915_gem_release_mmap(struct drm_i915_gem_object *obj)
  1673. {
  1674.         if (!obj->fault_mappable)
  1675.                 return;
  1676.  
  1677. //      drm_vma_node_unmap(&obj->base.vma_node, obj->base.dev->dev_mapping);
  1678.         obj->fault_mappable = false;
  1679. }
  1680.  
  1681. void
  1682. i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv)
  1683. {
  1684.         struct drm_i915_gem_object *obj;
  1685.  
  1686.         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
  1687.                 i915_gem_release_mmap(obj);
  1688. }
  1689.  
  1690. uint32_t
  1691. i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
  1692. {
  1693.         uint32_t gtt_size;
  1694.  
  1695.         if (INTEL_INFO(dev)->gen >= 4 ||
  1696.             tiling_mode == I915_TILING_NONE)
  1697.                 return size;
  1698.  
  1699.         /* Previous chips need a power-of-two fence region when tiling */
  1700.         if (INTEL_INFO(dev)->gen == 3)
  1701.                 gtt_size = 1024*1024;
  1702.         else
  1703.                 gtt_size = 512*1024;
  1704.  
  1705.         while (gtt_size < size)
  1706.                 gtt_size <<= 1;
  1707.  
  1708.         return gtt_size;
  1709. }
  1710.  
  1711. /**
  1712.  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
  1713.  * @obj: object to check
  1714.  *
  1715.  * Return the required GTT alignment for an object, taking into account
  1716.  * potential fence register mapping.
  1717.  */
  1718. uint32_t
  1719. i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
  1720.                            int tiling_mode, bool fenced)
  1721. {
  1722.         /*
  1723.          * Minimum alignment is 4k (GTT page size), but might be greater
  1724.          * if a fence register is needed for the object.
  1725.          */
  1726.         if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
  1727.             tiling_mode == I915_TILING_NONE)
  1728.                 return 4096;
  1729.  
  1730.         /*
  1731.          * Previous chips need to be aligned to the size of the smallest
  1732.          * fence register that can contain the object.
  1733.          */
  1734.         return i915_gem_get_gtt_size(dev, size, tiling_mode);
  1735. }
  1736.  
  1737.  
  1738.  
  1739. int
  1740. i915_gem_mmap_gtt(struct drm_file *file,
  1741.           struct drm_device *dev,
  1742.                   uint32_t handle,
  1743.           uint64_t *offset)
  1744. {
  1745.     struct drm_i915_private *dev_priv = dev->dev_private;
  1746.     struct drm_i915_gem_object *obj;
  1747.     unsigned long pfn;
  1748.     char *mem, *ptr;
  1749.     int ret;
  1750.  
  1751.     ret = i915_mutex_lock_interruptible(dev);
  1752.     if (ret)
  1753.         return ret;
  1754.  
  1755.     obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
  1756.     if (&obj->base == NULL) {
  1757.         ret = -ENOENT;
  1758.         goto unlock;
  1759.     }
  1760.  
  1761.     if (obj->madv != I915_MADV_WILLNEED) {
  1762.                 DRM_DEBUG("Attempting to mmap a purgeable buffer\n");
  1763.                 ret = -EFAULT;
  1764.         goto out;
  1765.     }
  1766.     /* Now bind it into the GTT if needed */
  1767.         ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK);
  1768.     if (ret)
  1769.         goto out;
  1770.  
  1771.     ret = i915_gem_object_set_to_gtt_domain(obj, 1);
  1772.     if (ret)
  1773.         goto unpin;
  1774.  
  1775.     ret = i915_gem_object_get_fence(obj);
  1776.     if (ret)
  1777.         goto unpin;
  1778.  
  1779.     obj->fault_mappable = true;
  1780.  
  1781.     pfn = dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj);
  1782.  
  1783.     /* Finally, remap it using the new GTT offset */
  1784.  
  1785.     mem = UserAlloc(obj->base.size);
  1786.     if(unlikely(mem == NULL))
  1787.     {
  1788.         ret = -ENOMEM;
  1789.         goto unpin;
  1790.     }
  1791.  
  1792.     for(ptr = mem; ptr < mem + obj->base.size; ptr+= 4096, pfn+= 4096)
  1793.         MapPage(ptr, pfn, PG_SHARED|PG_UW);
  1794.  
  1795. unpin:
  1796.     i915_gem_object_unpin_pages(obj);
  1797.  
  1798.  
  1799.     *offset = (uint32_t)mem;
  1800.  
  1801. out:
  1802.         drm_gem_object_unreference(&obj->base);
  1803. unlock:
  1804.         mutex_unlock(&dev->struct_mutex);
  1805.         return ret;
  1806. }
  1807.  
  1808. /**
  1809.  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
  1810.  * @dev: DRM device
  1811.  * @data: GTT mapping ioctl data
  1812.  * @file: GEM object info
  1813.  *
  1814.  * Simply returns the fake offset to userspace so it can mmap it.
  1815.  * The mmap call will end up in drm_gem_mmap(), which will set things
  1816.  * up so we can get faults in the handler above.
  1817.  *
  1818.  * The fault handler will take care of binding the object into the GTT
  1819.  * (since it may have been evicted to make room for something), allocating
  1820.  * a fence register, and mapping the appropriate aperture address into
  1821.  * userspace.
  1822.  */
  1823. int
  1824. i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
  1825.                         struct drm_file *file)
  1826. {
  1827.         struct drm_i915_gem_mmap_gtt *args = data;
  1828.  
  1829.         return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
  1830. }
  1831.  
  1832. /* Immediately discard the backing storage */
  1833. static void
  1834. i915_gem_object_truncate(struct drm_i915_gem_object *obj)
  1835. {
  1836. //      i915_gem_object_free_mmap_offset(obj);
  1837.  
  1838.         if (obj->base.filp == NULL)
  1839.                 return;
  1840.  
  1841.         /* Our goal here is to return as much of the memory as
  1842.          * is possible back to the system as we are called from OOM.
  1843.          * To do this we must instruct the shmfs to drop all of its
  1844.          * backing pages, *now*.
  1845.          */
  1846. //      shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
  1847.         obj->madv = __I915_MADV_PURGED;
  1848. }
  1849.  
  1850. /* Try to discard unwanted pages */
  1851. static void
  1852. i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
  1853. {
  1854.         struct address_space *mapping;
  1855.  
  1856.         switch (obj->madv) {
  1857.         case I915_MADV_DONTNEED:
  1858.                 i915_gem_object_truncate(obj);
  1859.         case __I915_MADV_PURGED:
  1860.                 return;
  1861.         }
  1862.  
  1863.         if (obj->base.filp == NULL)
  1864.                 return;
  1865.  
  1866. }
  1867.  
  1868. static void
  1869. i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
  1870. {
  1871.         struct sg_page_iter sg_iter;
  1872.         int ret;
  1873.  
  1874.         BUG_ON(obj->madv == __I915_MADV_PURGED);
  1875.  
  1876.         ret = i915_gem_object_set_to_cpu_domain(obj, true);
  1877.         if (ret) {
  1878.                 /* In the event of a disaster, abandon all caches and
  1879.                  * hope for the best.
  1880.                  */
  1881.                 WARN_ON(ret != -EIO);
  1882.                 i915_gem_clflush_object(obj, true);
  1883.                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  1884.         }
  1885.  
  1886.         i915_gem_gtt_finish_object(obj);
  1887.  
  1888.         if (i915_gem_object_needs_bit17_swizzle(obj))
  1889.                 i915_gem_object_save_bit_17_swizzle(obj);
  1890.  
  1891.         if (obj->madv == I915_MADV_DONTNEED)
  1892.                 obj->dirty = 0;
  1893.  
  1894.         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
  1895.                 struct page *page = sg_page_iter_page(&sg_iter);
  1896.  
  1897.                 page_cache_release(page);
  1898.         }
  1899.         obj->dirty = 0;
  1900.  
  1901.         sg_free_table(obj->pages);
  1902.         kfree(obj->pages);
  1903. }
  1904.  
  1905. int
  1906. i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
  1907. {
  1908.         const struct drm_i915_gem_object_ops *ops = obj->ops;
  1909.  
  1910.         if (obj->pages == NULL)
  1911.                 return 0;
  1912.  
  1913.         if (obj->pages_pin_count)
  1914.                 return -EBUSY;
  1915.  
  1916.         BUG_ON(i915_gem_obj_bound_any(obj));
  1917.  
  1918.         /* ->put_pages might need to allocate memory for the bit17 swizzle
  1919.          * array, hence protect them from being reaped by removing them from gtt
  1920.          * lists early. */
  1921.         list_del(&obj->global_list);
  1922.  
  1923.         ops->put_pages(obj);
  1924.         obj->pages = NULL;
  1925.  
  1926.         i915_gem_object_invalidate(obj);
  1927.  
  1928.         return 0;
  1929. }
  1930.  
  1931. static int
  1932. i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
  1933. {
  1934.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  1935.         int page_count, i;
  1936.         struct address_space *mapping;
  1937.         struct sg_table *st;
  1938.         struct scatterlist *sg;
  1939.         struct sg_page_iter sg_iter;
  1940.         struct page *page;
  1941.         unsigned long last_pfn = 0;     /* suppress gcc warning */
  1942.         int ret;
  1943.         gfp_t gfp;
  1944.  
  1945.         /* Assert that the object is not currently in any GPU domain. As it
  1946.          * wasn't in the GTT, there shouldn't be any way it could have been in
  1947.          * a GPU cache
  1948.          */
  1949.         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
  1950.         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
  1951.  
  1952.         st = kmalloc(sizeof(*st), GFP_KERNEL);
  1953.         if (st == NULL)
  1954.                 return -ENOMEM;
  1955.  
  1956.         page_count = obj->base.size / PAGE_SIZE;
  1957.         if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
  1958.                 kfree(st);
  1959.                 return -ENOMEM;
  1960.         }
  1961.  
  1962.         /* Get the list of pages out of our struct file.  They'll be pinned
  1963.          * at this point until we release them.
  1964.          *
  1965.          * Fail silently without starting the shrinker
  1966.          */
  1967.         sg = st->sgl;
  1968.         st->nents = 0;
  1969.         for (i = 0; i < page_count; i++) {
  1970.         page = shmem_read_mapping_page_gfp(obj->base.filp, i, gfp);
  1971.                 if (IS_ERR(page)) {
  1972.             dbgprintf("%s invalid page %p\n", __FUNCTION__, page);
  1973.                         goto err_pages;
  1974.                 }
  1975. #ifdef CONFIG_SWIOTLB
  1976.                 if (swiotlb_nr_tbl()) {
  1977.                         st->nents++;
  1978.                         sg_set_page(sg, page, PAGE_SIZE, 0);
  1979.                         sg = sg_next(sg);
  1980.                         continue;
  1981.                 }
  1982. #endif
  1983.                 if (!i || page_to_pfn(page) != last_pfn + 1) {
  1984.                         if (i)
  1985.                                 sg = sg_next(sg);
  1986.                         st->nents++;
  1987.                         sg_set_page(sg, page, PAGE_SIZE, 0);
  1988.                 } else {
  1989.                         sg->length += PAGE_SIZE;
  1990.                 }
  1991.                 last_pfn = page_to_pfn(page);
  1992.  
  1993.                 /* Check that the i965g/gm workaround works. */
  1994.                 WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
  1995.         }
  1996. #ifdef CONFIG_SWIOTLB
  1997.         if (!swiotlb_nr_tbl())
  1998. #endif
  1999.                 sg_mark_end(sg);
  2000.         obj->pages = st;
  2001.  
  2002.         ret = i915_gem_gtt_prepare_object(obj);
  2003.         if (ret)
  2004.                 goto err_pages;
  2005.  
  2006.         if (i915_gem_object_needs_bit17_swizzle(obj))
  2007.                 i915_gem_object_do_bit_17_swizzle(obj);
  2008.  
  2009.         if (obj->tiling_mode != I915_TILING_NONE &&
  2010.             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
  2011.                 i915_gem_object_pin_pages(obj);
  2012.  
  2013.         return 0;
  2014.  
  2015. err_pages:
  2016.         sg_mark_end(sg);
  2017.         for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
  2018.                 page_cache_release(sg_page_iter_page(&sg_iter));
  2019.         sg_free_table(st);
  2020.         kfree(st);
  2021.  
  2022.         return PTR_ERR(page);
  2023. }
  2024.  
  2025. /* Ensure that the associated pages are gathered from the backing storage
  2026.  * and pinned into our object. i915_gem_object_get_pages() may be called
  2027.  * multiple times before they are released by a single call to
  2028.  * i915_gem_object_put_pages() - once the pages are no longer referenced
  2029.  * either as a result of memory pressure (reaping pages under the shrinker)
  2030.  * or as the object is itself released.
  2031.  */
  2032. int
  2033. i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
  2034. {
  2035.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  2036.         const struct drm_i915_gem_object_ops *ops = obj->ops;
  2037.         int ret;
  2038.  
  2039.         if (obj->pages)
  2040.                 return 0;
  2041.  
  2042.         if (obj->madv != I915_MADV_WILLNEED) {
  2043.                 DRM_DEBUG("Attempting to obtain a purgeable object\n");
  2044.                 return -EFAULT;
  2045.         }
  2046.  
  2047.         BUG_ON(obj->pages_pin_count);
  2048.  
  2049.         ret = ops->get_pages(obj);
  2050.         if (ret)
  2051.                 return ret;
  2052.  
  2053.         list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
  2054.  
  2055.         obj->get_page.sg = obj->pages->sgl;
  2056.         obj->get_page.last = 0;
  2057.  
  2058.         return 0;
  2059. }
  2060.  
  2061. void i915_vma_move_to_active(struct i915_vma *vma,
  2062.                              struct drm_i915_gem_request *req)
  2063. {
  2064.         struct drm_i915_gem_object *obj = vma->obj;
  2065.         struct intel_engine_cs *ring;
  2066.  
  2067.         ring = i915_gem_request_get_ring(req);
  2068.  
  2069.         /* Add a reference if we're newly entering the active list. */
  2070.         if (obj->active == 0)
  2071.                 drm_gem_object_reference(&obj->base);
  2072.         obj->active |= intel_ring_flag(ring);
  2073.  
  2074.         list_move_tail(&obj->ring_list[ring->id], &ring->active_list);
  2075.         i915_gem_request_assign(&obj->last_read_req[ring->id], req);
  2076.  
  2077.         list_move_tail(&vma->vm_link, &vma->vm->active_list);
  2078. }
  2079.  
  2080. static void
  2081. i915_gem_object_retire__write(struct drm_i915_gem_object *obj)
  2082. {
  2083.         RQ_BUG_ON(obj->last_write_req == NULL);
  2084.         RQ_BUG_ON(!(obj->active & intel_ring_flag(obj->last_write_req->ring)));
  2085.  
  2086.         i915_gem_request_assign(&obj->last_write_req, NULL);
  2087.         intel_fb_obj_flush(obj, true, ORIGIN_CS);
  2088. }
  2089.  
  2090. static void
  2091. i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring)
  2092. {
  2093.         struct i915_vma *vma;
  2094.  
  2095.         RQ_BUG_ON(obj->last_read_req[ring] == NULL);
  2096.         RQ_BUG_ON(!(obj->active & (1 << ring)));
  2097.  
  2098.         list_del_init(&obj->ring_list[ring]);
  2099.         i915_gem_request_assign(&obj->last_read_req[ring], NULL);
  2100.  
  2101.         if (obj->last_write_req && obj->last_write_req->ring->id == ring)
  2102.                 i915_gem_object_retire__write(obj);
  2103.  
  2104.         obj->active &= ~(1 << ring);
  2105.         if (obj->active)
  2106.                 return;
  2107.  
  2108.         /* Bump our place on the bound list to keep it roughly in LRU order
  2109.          * so that we don't steal from recently used but inactive objects
  2110.          * (unless we are forced to ofc!)
  2111.          */
  2112.         list_move_tail(&obj->global_list,
  2113.                        &to_i915(obj->base.dev)->mm.bound_list);
  2114.  
  2115.         list_for_each_entry(vma, &obj->vma_list, obj_link) {
  2116.                 if (!list_empty(&vma->vm_link))
  2117.                         list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
  2118.         }
  2119.  
  2120.         i915_gem_request_assign(&obj->last_fenced_req, NULL);
  2121.         drm_gem_object_unreference(&obj->base);
  2122. }
  2123.  
  2124. static int
  2125. i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
  2126. {
  2127.         struct drm_i915_private *dev_priv = dev->dev_private;
  2128.         struct intel_engine_cs *ring;
  2129.         int ret, i, j;
  2130.  
  2131.         /* Carefully retire all requests without writing to the rings */
  2132.         for_each_ring(ring, dev_priv, i) {
  2133.                 ret = intel_ring_idle(ring);
  2134.                 if (ret)
  2135.                         return ret;
  2136.         }
  2137.         i915_gem_retire_requests(dev);
  2138.  
  2139.         /* Finally reset hw state */
  2140.         for_each_ring(ring, dev_priv, i) {
  2141.                 intel_ring_init_seqno(ring, seqno);
  2142.  
  2143.                 for (j = 0; j < ARRAY_SIZE(ring->semaphore.sync_seqno); j++)
  2144.                         ring->semaphore.sync_seqno[j] = 0;
  2145.         }
  2146.  
  2147.         return 0;
  2148. }
  2149.  
  2150. int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
  2151. {
  2152.         struct drm_i915_private *dev_priv = dev->dev_private;
  2153.         int ret;
  2154.  
  2155.         if (seqno == 0)
  2156.                 return -EINVAL;
  2157.  
  2158.         /* HWS page needs to be set less than what we
  2159.          * will inject to ring
  2160.          */
  2161.         ret = i915_gem_init_seqno(dev, seqno - 1);
  2162.         if (ret)
  2163.                 return ret;
  2164.  
  2165.         /* Carefully set the last_seqno value so that wrap
  2166.          * detection still works
  2167.          */
  2168.         dev_priv->next_seqno = seqno;
  2169.         dev_priv->last_seqno = seqno - 1;
  2170.         if (dev_priv->last_seqno == 0)
  2171.                 dev_priv->last_seqno--;
  2172.  
  2173.         return 0;
  2174. }
  2175.  
  2176. int
  2177. i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
  2178. {
  2179.         struct drm_i915_private *dev_priv = dev->dev_private;
  2180.  
  2181.         /* reserve 0 for non-seqno */
  2182.         if (dev_priv->next_seqno == 0) {
  2183.                 int ret = i915_gem_init_seqno(dev, 0);
  2184.                 if (ret)
  2185.                         return ret;
  2186.  
  2187.                 dev_priv->next_seqno = 1;
  2188.         }
  2189.  
  2190.         *seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
  2191.         return 0;
  2192. }
  2193.  
  2194. /*
  2195.  * NB: This function is not allowed to fail. Doing so would mean the the
  2196.  * request is not being tracked for completion but the work itself is
  2197.  * going to happen on the hardware. This would be a Bad Thing(tm).
  2198.  */
  2199. void __i915_add_request(struct drm_i915_gem_request *request,
  2200.                         struct drm_i915_gem_object *obj,
  2201.                         bool flush_caches)
  2202. {
  2203.         struct intel_engine_cs *ring;
  2204.         struct drm_i915_private *dev_priv;
  2205.         struct intel_ringbuffer *ringbuf;
  2206.         u32 request_start;
  2207.         int ret;
  2208.  
  2209.         if (WARN_ON(request == NULL))
  2210.                 return;
  2211.  
  2212.         ring = request->ring;
  2213.         dev_priv = ring->dev->dev_private;
  2214.         ringbuf = request->ringbuf;
  2215.  
  2216.         /*
  2217.          * To ensure that this call will not fail, space for its emissions
  2218.          * should already have been reserved in the ring buffer. Let the ring
  2219.          * know that it is time to use that space up.
  2220.          */
  2221.         intel_ring_reserved_space_use(ringbuf);
  2222.  
  2223.         request_start = intel_ring_get_tail(ringbuf);
  2224.         /*
  2225.          * Emit any outstanding flushes - execbuf can fail to emit the flush
  2226.          * after having emitted the batchbuffer command. Hence we need to fix
  2227.          * things up similar to emitting the lazy request. The difference here
  2228.          * is that the flush _must_ happen before the next request, no matter
  2229.          * what.
  2230.          */
  2231.         if (flush_caches) {
  2232.                 if (i915.enable_execlists)
  2233.                         ret = logical_ring_flush_all_caches(request);
  2234.                 else
  2235.                         ret = intel_ring_flush_all_caches(request);
  2236.                 /* Not allowed to fail! */
  2237.                 WARN(ret, "*_ring_flush_all_caches failed: %d!\n", ret);
  2238.         }
  2239.  
  2240.         /* Record the position of the start of the request so that
  2241.          * should we detect the updated seqno part-way through the
  2242.          * GPU processing the request, we never over-estimate the
  2243.          * position of the head.
  2244.          */
  2245.         request->postfix = intel_ring_get_tail(ringbuf);
  2246.  
  2247.         if (i915.enable_execlists)
  2248.                 ret = ring->emit_request(request);
  2249.         else {
  2250.                 ret = ring->add_request(request);
  2251.  
  2252.                 request->tail = intel_ring_get_tail(ringbuf);
  2253.         }
  2254.         /* Not allowed to fail! */
  2255.         WARN(ret, "emit|add_request failed: %d!\n", ret);
  2256.  
  2257.         request->head = request_start;
  2258.  
  2259.         /* Whilst this request exists, batch_obj will be on the
  2260.          * active_list, and so will hold the active reference. Only when this
  2261.          * request is retired will the the batch_obj be moved onto the
  2262.          * inactive_list and lose its active reference. Hence we do not need
  2263.          * to explicitly hold another reference here.
  2264.          */
  2265.         request->batch_obj = obj;
  2266.  
  2267.         request->emitted_jiffies = jiffies;
  2268.         request->previous_seqno = ring->last_submitted_seqno;
  2269.         ring->last_submitted_seqno = request->seqno;
  2270.         list_add_tail(&request->list, &ring->request_list);
  2271.  
  2272.         trace_i915_gem_request_add(request);
  2273.  
  2274.         i915_queue_hangcheck(ring->dev);
  2275.  
  2276.         queue_delayed_work(dev_priv->wq,
  2277.                            &dev_priv->mm.retire_work,
  2278.                            round_jiffies_up_relative(HZ));
  2279.         intel_mark_busy(dev_priv->dev);
  2280.  
  2281.         /* Sanity check that the reserved size was large enough. */
  2282.         intel_ring_reserved_space_end(ringbuf);
  2283. }
  2284.  
  2285. static bool i915_context_is_banned(struct drm_i915_private *dev_priv,
  2286.                                    const struct intel_context *ctx)
  2287. {
  2288.         unsigned long elapsed;
  2289.  
  2290.     elapsed = GetTimerTicks()/100 - ctx->hang_stats.guilty_ts;
  2291.  
  2292.         if (ctx->hang_stats.banned)
  2293.                 return true;
  2294.  
  2295.         if (ctx->hang_stats.ban_period_seconds &&
  2296.             elapsed <= ctx->hang_stats.ban_period_seconds) {
  2297.                 if (!i915_gem_context_is_default(ctx)) {
  2298.                         DRM_DEBUG("context hanging too fast, banning!\n");
  2299.                         return true;
  2300.                 } else if (i915_stop_ring_allow_ban(dev_priv)) {
  2301.                         if (i915_stop_ring_allow_warn(dev_priv))
  2302.                                 DRM_ERROR("gpu hanging too fast, banning!\n");
  2303.                         return true;
  2304.                 }
  2305.         }
  2306.  
  2307.         return false;
  2308. }
  2309.  
  2310. static void i915_set_reset_status(struct drm_i915_private *dev_priv,
  2311.                                   struct intel_context *ctx,
  2312.                                   const bool guilty)
  2313. {
  2314.         struct i915_ctx_hang_stats *hs;
  2315.  
  2316.         if (WARN_ON(!ctx))
  2317.                 return;
  2318.  
  2319.         hs = &ctx->hang_stats;
  2320.  
  2321.         if (guilty) {
  2322.                 hs->banned = i915_context_is_banned(dev_priv, ctx);
  2323.                 hs->batch_active++;
  2324.         hs->guilty_ts = GetTimerTicks()/100;
  2325.         } else {
  2326.                 hs->batch_pending++;
  2327.         }
  2328. }
  2329.  
  2330. void i915_gem_request_free(struct kref *req_ref)
  2331. {
  2332.         struct drm_i915_gem_request *req = container_of(req_ref,
  2333.                                                  typeof(*req), ref);
  2334.         struct intel_context *ctx = req->ctx;
  2335.  
  2336.         if (req->file_priv)
  2337.                 i915_gem_request_remove_from_client(req);
  2338.  
  2339.         if (ctx) {
  2340.                 if (i915.enable_execlists && ctx != req->i915->kernel_context)
  2341.                         intel_lr_context_unpin(ctx, req->ring);
  2342.  
  2343.                 i915_gem_context_unreference(ctx);
  2344.         }
  2345.  
  2346.         kfree(req);
  2347. }
  2348.  
  2349. static inline int
  2350. __i915_gem_request_alloc(struct intel_engine_cs *ring,
  2351.                          struct intel_context *ctx,
  2352.                          struct drm_i915_gem_request **req_out)
  2353. {
  2354.         struct drm_i915_private *dev_priv = to_i915(ring->dev);
  2355.         struct drm_i915_gem_request *req;
  2356.         int ret;
  2357.  
  2358.         if (!req_out)
  2359.                 return -EINVAL;
  2360.  
  2361.         *req_out = NULL;
  2362.  
  2363. //      req = kmem_cache_zalloc(dev_priv->requests, GFP_KERNEL);
  2364.         req = kzalloc(sizeof(*req),0);
  2365.         if (req == NULL)
  2366.                 return -ENOMEM;
  2367.  
  2368.         ret = i915_gem_get_seqno(ring->dev, &req->seqno);
  2369.         if (ret)
  2370.                 goto err;
  2371.  
  2372.         kref_init(&req->ref);
  2373.         req->i915 = dev_priv;
  2374.         req->ring = ring;
  2375.         req->ctx  = ctx;
  2376.         i915_gem_context_reference(req->ctx);
  2377.  
  2378.         if (i915.enable_execlists)
  2379.                 ret = intel_logical_ring_alloc_request_extras(req);
  2380.         else
  2381.                 ret = intel_ring_alloc_request_extras(req);
  2382.         if (ret) {
  2383.                 i915_gem_context_unreference(req->ctx);
  2384.                 goto err;
  2385.         }
  2386.  
  2387.         /*
  2388.          * Reserve space in the ring buffer for all the commands required to
  2389.          * eventually emit this request. This is to guarantee that the
  2390.          * i915_add_request() call can't fail. Note that the reserve may need
  2391.          * to be redone if the request is not actually submitted straight
  2392.          * away, e.g. because a GPU scheduler has deferred it.
  2393.          */
  2394.         if (i915.enable_execlists)
  2395.                 ret = intel_logical_ring_reserve_space(req);
  2396.         else
  2397.                 ret = intel_ring_reserve_space(req);
  2398.         if (ret) {
  2399.                 /*
  2400.                  * At this point, the request is fully allocated even if not
  2401.                  * fully prepared. Thus it can be cleaned up using the proper
  2402.                  * free code.
  2403.                  */
  2404.                 i915_gem_request_cancel(req);
  2405.                 return ret;
  2406.         }
  2407.  
  2408.         *req_out = req;
  2409.         return 0;
  2410.  
  2411. err:
  2412.         kfree(req);
  2413.         return ret;
  2414. }
  2415.  
  2416. /**
  2417.  * i915_gem_request_alloc - allocate a request structure
  2418.  *
  2419.  * @engine: engine that we wish to issue the request on.
  2420.  * @ctx: context that the request will be associated with.
  2421.  *       This can be NULL if the request is not directly related to
  2422.  *       any specific user context, in which case this function will
  2423.  *       choose an appropriate context to use.
  2424.  *
  2425.  * Returns a pointer to the allocated request if successful,
  2426.  * or an error code if not.
  2427.  */
  2428. struct drm_i915_gem_request *
  2429. i915_gem_request_alloc(struct intel_engine_cs *engine,
  2430.                        struct intel_context *ctx)
  2431. {
  2432.         struct drm_i915_gem_request *req;
  2433.         int err;
  2434.  
  2435.         if (ctx == NULL)
  2436.                 ctx = to_i915(engine->dev)->kernel_context;
  2437.         err = __i915_gem_request_alloc(engine, ctx, &req);
  2438.         return err ? ERR_PTR(err) : req;
  2439. }
  2440.  
  2441. void i915_gem_request_cancel(struct drm_i915_gem_request *req)
  2442. {
  2443.         intel_ring_reserved_space_cancel(req->ringbuf);
  2444.  
  2445.         i915_gem_request_unreference(req);
  2446. }
  2447.  
  2448. struct drm_i915_gem_request *
  2449. i915_gem_find_active_request(struct intel_engine_cs *ring)
  2450. {
  2451.         struct drm_i915_gem_request *request;
  2452.  
  2453.         list_for_each_entry(request, &ring->request_list, list) {
  2454.                 if (i915_gem_request_completed(request, false))
  2455.                         continue;
  2456.  
  2457.                 return request;
  2458.         }
  2459.  
  2460.         return NULL;
  2461. }
  2462.  
  2463. static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv,
  2464.                                        struct intel_engine_cs *ring)
  2465. {
  2466.         struct drm_i915_gem_request *request;
  2467.         bool ring_hung;
  2468.  
  2469.         request = i915_gem_find_active_request(ring);
  2470.  
  2471.         if (request == NULL)
  2472.                 return;
  2473.  
  2474.         ring_hung = ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
  2475.  
  2476.         i915_set_reset_status(dev_priv, request->ctx, ring_hung);
  2477.  
  2478.         list_for_each_entry_continue(request, &ring->request_list, list)
  2479.                 i915_set_reset_status(dev_priv, request->ctx, false);
  2480. }
  2481.  
  2482. static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
  2483.                                         struct intel_engine_cs *ring)
  2484. {
  2485.         struct intel_ringbuffer *buffer;
  2486.  
  2487.         while (!list_empty(&ring->active_list)) {
  2488.                 struct drm_i915_gem_object *obj;
  2489.  
  2490.                 obj = list_first_entry(&ring->active_list,
  2491.                                        struct drm_i915_gem_object,
  2492.                                        ring_list[ring->id]);
  2493.  
  2494.                 i915_gem_object_retire__read(obj, ring->id);
  2495.         }
  2496.  
  2497.         /*
  2498.          * Clear the execlists queue up before freeing the requests, as those
  2499.          * are the ones that keep the context and ringbuffer backing objects
  2500.          * pinned in place.
  2501.          */
  2502.  
  2503.         if (i915.enable_execlists) {
  2504.                 spin_lock_irq(&ring->execlist_lock);
  2505.  
  2506.                 /* list_splice_tail_init checks for empty lists */
  2507.                 list_splice_tail_init(&ring->execlist_queue,
  2508.                                       &ring->execlist_retired_req_list);
  2509.  
  2510.                 spin_unlock_irq(&ring->execlist_lock);
  2511.                 intel_execlists_retire_requests(ring);
  2512.         }
  2513.  
  2514.         /*
  2515.          * We must free the requests after all the corresponding objects have
  2516.          * been moved off active lists. Which is the same order as the normal
  2517.          * retire_requests function does. This is important if object hold
  2518.          * implicit references on things like e.g. ppgtt address spaces through
  2519.          * the request.
  2520.          */
  2521.         while (!list_empty(&ring->request_list)) {
  2522.                 struct drm_i915_gem_request *request;
  2523.  
  2524.                 request = list_first_entry(&ring->request_list,
  2525.                                            struct drm_i915_gem_request,
  2526.                                            list);
  2527.  
  2528.                 i915_gem_request_retire(request);
  2529.         }
  2530.  
  2531.         /* Having flushed all requests from all queues, we know that all
  2532.          * ringbuffers must now be empty. However, since we do not reclaim
  2533.          * all space when retiring the request (to prevent HEADs colliding
  2534.          * with rapid ringbuffer wraparound) the amount of available space
  2535.          * upon reset is less than when we start. Do one more pass over
  2536.          * all the ringbuffers to reset last_retired_head.
  2537.          */
  2538.         list_for_each_entry(buffer, &ring->buffers, link) {
  2539.                 buffer->last_retired_head = buffer->tail;
  2540.                 intel_ring_update_space(buffer);
  2541.         }
  2542. }
  2543.  
  2544. void i915_gem_reset(struct drm_device *dev)
  2545. {
  2546.         struct drm_i915_private *dev_priv = dev->dev_private;
  2547.         struct intel_engine_cs *ring;
  2548.         int i;
  2549.  
  2550.         /*
  2551.          * Before we free the objects from the requests, we need to inspect
  2552.          * them for finding the guilty party. As the requests only borrow
  2553.          * their reference to the objects, the inspection must be done first.
  2554.          */
  2555.         for_each_ring(ring, dev_priv, i)
  2556.                 i915_gem_reset_ring_status(dev_priv, ring);
  2557.  
  2558.         for_each_ring(ring, dev_priv, i)
  2559.                 i915_gem_reset_ring_cleanup(dev_priv, ring);
  2560.  
  2561.         i915_gem_context_reset(dev);
  2562.  
  2563.         i915_gem_restore_fences(dev);
  2564.  
  2565.         WARN_ON(i915_verify_lists(dev));
  2566. }
  2567.  
  2568. /**
  2569.  * This function clears the request list as sequence numbers are passed.
  2570.  */
  2571. void
  2572. i915_gem_retire_requests_ring(struct intel_engine_cs *ring)
  2573. {
  2574.         WARN_ON(i915_verify_lists(ring->dev));
  2575.  
  2576.         /* Retire requests first as we use it above for the early return.
  2577.          * If we retire requests last, we may use a later seqno and so clear
  2578.          * the requests lists without clearing the active list, leading to
  2579.          * confusion.
  2580.          */
  2581.         while (!list_empty(&ring->request_list)) {
  2582.                 struct drm_i915_gem_request *request;
  2583.  
  2584.                 request = list_first_entry(&ring->request_list,
  2585.                                            struct drm_i915_gem_request,
  2586.                                            list);
  2587.  
  2588.                 if (!i915_gem_request_completed(request, true))
  2589.                         break;
  2590.  
  2591.                 i915_gem_request_retire(request);
  2592.         }
  2593.  
  2594.         /* Move any buffers on the active list that are no longer referenced
  2595.          * by the ringbuffer to the flushing/inactive lists as appropriate,
  2596.          * before we free the context associated with the requests.
  2597.          */
  2598.         while (!list_empty(&ring->active_list)) {
  2599.                 struct drm_i915_gem_object *obj;
  2600.  
  2601.                 obj = list_first_entry(&ring->active_list,
  2602.                                       struct drm_i915_gem_object,
  2603.                                       ring_list[ring->id]);
  2604.  
  2605.                 if (!list_empty(&obj->last_read_req[ring->id]->list))
  2606.                         break;
  2607.  
  2608.                 i915_gem_object_retire__read(obj, ring->id);
  2609.         }
  2610.  
  2611.         if (unlikely(ring->trace_irq_req &&
  2612.                      i915_gem_request_completed(ring->trace_irq_req, true))) {
  2613.                 ring->irq_put(ring);
  2614.                 i915_gem_request_assign(&ring->trace_irq_req, NULL);
  2615.         }
  2616.  
  2617.         WARN_ON(i915_verify_lists(ring->dev));
  2618. }
  2619.  
  2620. bool
  2621. i915_gem_retire_requests(struct drm_device *dev)
  2622. {
  2623.         struct drm_i915_private *dev_priv = dev->dev_private;
  2624.         struct intel_engine_cs *ring;
  2625.         bool idle = true;
  2626.         int i;
  2627.  
  2628.         for_each_ring(ring, dev_priv, i) {
  2629.                 i915_gem_retire_requests_ring(ring);
  2630.                 idle &= list_empty(&ring->request_list);
  2631.                 if (i915.enable_execlists) {
  2632.                         spin_lock_irq(&ring->execlist_lock);
  2633.                         idle &= list_empty(&ring->execlist_queue);
  2634.                         spin_unlock_irq(&ring->execlist_lock);
  2635.  
  2636.                         intel_execlists_retire_requests(ring);
  2637.                 }
  2638.         }
  2639.  
  2640. //      if (idle)
  2641. //              mod_delayed_work(dev_priv->wq,
  2642. //                                 &dev_priv->mm.idle_work,
  2643. //                                 msecs_to_jiffies(100));
  2644.  
  2645.         return idle;
  2646. }
  2647.  
  2648. static void
  2649. i915_gem_retire_work_handler(struct work_struct *work)
  2650. {
  2651.         struct drm_i915_private *dev_priv =
  2652.                 container_of(work, typeof(*dev_priv), mm.retire_work.work);
  2653.         struct drm_device *dev = dev_priv->dev;
  2654.         bool idle;
  2655.  
  2656.         /* Come back later if the device is busy... */
  2657.         idle = false;
  2658.         if (mutex_trylock(&dev->struct_mutex)) {
  2659.                 idle = i915_gem_retire_requests(dev);
  2660.                 mutex_unlock(&dev->struct_mutex);
  2661.         }
  2662.         if (!idle)
  2663.                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
  2664.                                    round_jiffies_up_relative(HZ));
  2665. }
  2666.  
  2667. static void
  2668. i915_gem_idle_work_handler(struct work_struct *work)
  2669. {
  2670.         struct drm_i915_private *dev_priv =
  2671.                 container_of(work, typeof(*dev_priv), mm.idle_work.work);
  2672.         struct drm_device *dev = dev_priv->dev;
  2673.         struct intel_engine_cs *ring;
  2674.         int i;
  2675.  
  2676.         for_each_ring(ring, dev_priv, i)
  2677.                 if (!list_empty(&ring->request_list))
  2678.                         return;
  2679.  
  2680.         /* we probably should sync with hangcheck here, using cancel_work_sync.
  2681.          * Also locking seems to be fubar here, ring->request_list is protected
  2682.          * by dev->struct_mutex. */
  2683.  
  2684.         intel_mark_idle(dev);
  2685.  
  2686.         if (mutex_trylock(&dev->struct_mutex)) {
  2687.                 struct intel_engine_cs *ring;
  2688.                 int i;
  2689.  
  2690.                 for_each_ring(ring, dev_priv, i)
  2691.                         i915_gem_batch_pool_fini(&ring->batch_pool);
  2692.  
  2693.                 mutex_unlock(&dev->struct_mutex);
  2694.         }
  2695. }
  2696.  
  2697. /**
  2698.  * Ensures that an object will eventually get non-busy by flushing any required
  2699.  * write domains, emitting any outstanding lazy request and retiring and
  2700.  * completed requests.
  2701.  */
  2702. static int
  2703. i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
  2704. {
  2705.         int i;
  2706.  
  2707.         if (!obj->active)
  2708.                 return 0;
  2709.  
  2710.         for (i = 0; i < I915_NUM_RINGS; i++) {
  2711.                 struct drm_i915_gem_request *req;
  2712.  
  2713.                 req = obj->last_read_req[i];
  2714.                 if (req == NULL)
  2715.                         continue;
  2716.  
  2717.                 if (list_empty(&req->list))
  2718.                         goto retire;
  2719.  
  2720.                 if (i915_gem_request_completed(req, true)) {
  2721.                         __i915_gem_request_retire__upto(req);
  2722. retire:
  2723.                         i915_gem_object_retire__read(obj, i);
  2724.                 }
  2725.         }
  2726.  
  2727.         return 0;
  2728. }
  2729.  
  2730. /**
  2731.  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
  2732.  * @DRM_IOCTL_ARGS: standard ioctl arguments
  2733.  *
  2734.  * Returns 0 if successful, else an error is returned with the remaining time in
  2735.  * the timeout parameter.
  2736.  *  -ETIME: object is still busy after timeout
  2737.  *  -ERESTARTSYS: signal interrupted the wait
  2738.  *  -ENONENT: object doesn't exist
  2739.  * Also possible, but rare:
  2740.  *  -EAGAIN: GPU wedged
  2741.  *  -ENOMEM: damn
  2742.  *  -ENODEV: Internal IRQ fail
  2743.  *  -E?: The add request failed
  2744.  *
  2745.  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
  2746.  * non-zero timeout parameter the wait ioctl will wait for the given number of
  2747.  * nanoseconds on an object becoming unbusy. Since the wait itself does so
  2748.  * without holding struct_mutex the object may become re-busied before this
  2749.  * function completes. A similar but shorter * race condition exists in the busy
  2750.  * ioctl
  2751.  */
  2752. int
  2753. i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
  2754. {
  2755.         struct drm_i915_private *dev_priv = dev->dev_private;
  2756.         struct drm_i915_gem_wait *args = data;
  2757.         struct drm_i915_gem_object *obj;
  2758.         struct drm_i915_gem_request *req[I915_NUM_RINGS];
  2759.         unsigned reset_counter;
  2760.         int i, n = 0;
  2761.         int ret;
  2762.  
  2763.         if (args->flags != 0)
  2764.                 return -EINVAL;
  2765.  
  2766.         ret = i915_mutex_lock_interruptible(dev);
  2767.         if (ret)
  2768.                 return ret;
  2769.  
  2770.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
  2771.         if (&obj->base == NULL) {
  2772.                 mutex_unlock(&dev->struct_mutex);
  2773.                 return -ENOENT;
  2774.         }
  2775.  
  2776.         /* Need to make sure the object gets inactive eventually. */
  2777.         ret = i915_gem_object_flush_active(obj);
  2778.         if (ret)
  2779.                 goto out;
  2780.  
  2781.         if (!obj->active)
  2782.                 goto out;
  2783.  
  2784.         /* Do this after OLR check to make sure we make forward progress polling
  2785.          * on this IOCTL with a timeout == 0 (like busy ioctl)
  2786.          */
  2787.         if (args->timeout_ns == 0) {
  2788.                 ret = -ETIME;
  2789.                 goto out;
  2790.         }
  2791.  
  2792.         drm_gem_object_unreference(&obj->base);
  2793.         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
  2794.  
  2795.         for (i = 0; i < I915_NUM_RINGS; i++) {
  2796.                 if (obj->last_read_req[i] == NULL)
  2797.                         continue;
  2798.  
  2799.                 req[n++] = i915_gem_request_reference(obj->last_read_req[i]);
  2800.         }
  2801.  
  2802.         mutex_unlock(&dev->struct_mutex);
  2803.  
  2804.         for (i = 0; i < n; i++) {
  2805.                 if (ret == 0)
  2806.                         ret = __i915_wait_request(req[i], reset_counter, true,
  2807.                                                   args->timeout_ns > 0 ? &args->timeout_ns : NULL,
  2808.                                                   to_rps_client(file));
  2809.                 i915_gem_request_unreference__unlocked(req[i]);
  2810.         }
  2811.         return ret;
  2812.  
  2813. out:
  2814.         drm_gem_object_unreference(&obj->base);
  2815.         mutex_unlock(&dev->struct_mutex);
  2816.         return ret;
  2817. }
  2818.  
  2819. static int
  2820. __i915_gem_object_sync(struct drm_i915_gem_object *obj,
  2821.                        struct intel_engine_cs *to,
  2822.                        struct drm_i915_gem_request *from_req,
  2823.                        struct drm_i915_gem_request **to_req)
  2824. {
  2825.         struct intel_engine_cs *from;
  2826.         int ret;
  2827.  
  2828.         from = i915_gem_request_get_ring(from_req);
  2829.         if (to == from)
  2830.                 return 0;
  2831.  
  2832.         if (i915_gem_request_completed(from_req, true))
  2833.                 return 0;
  2834.  
  2835.         if (!i915_semaphore_is_enabled(obj->base.dev)) {
  2836.                 struct drm_i915_private *i915 = to_i915(obj->base.dev);
  2837.                 ret = __i915_wait_request(from_req,
  2838.                                           atomic_read(&i915->gpu_error.reset_counter),
  2839.                                           i915->mm.interruptible,
  2840.                                           NULL,
  2841.                                           &i915->rps.semaphores);
  2842.                 if (ret)
  2843.                         return ret;
  2844.  
  2845.                 i915_gem_object_retire_request(obj, from_req);
  2846.         } else {
  2847.                 int idx = intel_ring_sync_index(from, to);
  2848.                 u32 seqno = i915_gem_request_get_seqno(from_req);
  2849.  
  2850.                 WARN_ON(!to_req);
  2851.  
  2852.                 if (seqno <= from->semaphore.sync_seqno[idx])
  2853.                         return 0;
  2854.  
  2855.                 if (*to_req == NULL) {
  2856.                         struct drm_i915_gem_request *req;
  2857.  
  2858.                         req = i915_gem_request_alloc(to, NULL);
  2859.                         if (IS_ERR(req))
  2860.                                 return PTR_ERR(req);
  2861.  
  2862.                         *to_req = req;
  2863.                 }
  2864.  
  2865.                 trace_i915_gem_ring_sync_to(*to_req, from, from_req);
  2866.                 ret = to->semaphore.sync_to(*to_req, from, seqno);
  2867.                 if (ret)
  2868.                         return ret;
  2869.  
  2870.                 /* We use last_read_req because sync_to()
  2871.                  * might have just caused seqno wrap under
  2872.                  * the radar.
  2873.                  */
  2874.                 from->semaphore.sync_seqno[idx] =
  2875.                         i915_gem_request_get_seqno(obj->last_read_req[from->id]);
  2876.         }
  2877.  
  2878.         return 0;
  2879. }
  2880.  
  2881. /**
  2882.  * i915_gem_object_sync - sync an object to a ring.
  2883.  *
  2884.  * @obj: object which may be in use on another ring.
  2885.  * @to: ring we wish to use the object on. May be NULL.
  2886.  * @to_req: request we wish to use the object for. See below.
  2887.  *          This will be allocated and returned if a request is
  2888.  *          required but not passed in.
  2889.  *
  2890.  * This code is meant to abstract object synchronization with the GPU.
  2891.  * Calling with NULL implies synchronizing the object with the CPU
  2892.  * rather than a particular GPU ring. Conceptually we serialise writes
  2893.  * between engines inside the GPU. We only allow one engine to write
  2894.  * into a buffer at any time, but multiple readers. To ensure each has
  2895.  * a coherent view of memory, we must:
  2896.  *
  2897.  * - If there is an outstanding write request to the object, the new
  2898.  *   request must wait for it to complete (either CPU or in hw, requests
  2899.  *   on the same ring will be naturally ordered).
  2900.  *
  2901.  * - If we are a write request (pending_write_domain is set), the new
  2902.  *   request must wait for outstanding read requests to complete.
  2903.  *
  2904.  * For CPU synchronisation (NULL to) no request is required. For syncing with
  2905.  * rings to_req must be non-NULL. However, a request does not have to be
  2906.  * pre-allocated. If *to_req is NULL and sync commands will be emitted then a
  2907.  * request will be allocated automatically and returned through *to_req. Note
  2908.  * that it is not guaranteed that commands will be emitted (because the system
  2909.  * might already be idle). Hence there is no need to create a request that
  2910.  * might never have any work submitted. Note further that if a request is
  2911.  * returned in *to_req, it is the responsibility of the caller to submit
  2912.  * that request (after potentially adding more work to it).
  2913.  *
  2914.  * Returns 0 if successful, else propagates up the lower layer error.
  2915.  */
  2916. int
  2917. i915_gem_object_sync(struct drm_i915_gem_object *obj,
  2918.                      struct intel_engine_cs *to,
  2919.                      struct drm_i915_gem_request **to_req)
  2920. {
  2921.         const bool readonly = obj->base.pending_write_domain == 0;
  2922.         struct drm_i915_gem_request *req[I915_NUM_RINGS];
  2923.         int ret, i, n;
  2924.  
  2925.         if (!obj->active)
  2926.                 return 0;
  2927.  
  2928.         if (to == NULL)
  2929.                 return i915_gem_object_wait_rendering(obj, readonly);
  2930.  
  2931.         n = 0;
  2932.         if (readonly) {
  2933.                 if (obj->last_write_req)
  2934.                         req[n++] = obj->last_write_req;
  2935.         } else {
  2936.                 for (i = 0; i < I915_NUM_RINGS; i++)
  2937.                         if (obj->last_read_req[i])
  2938.                                 req[n++] = obj->last_read_req[i];
  2939.         }
  2940.         for (i = 0; i < n; i++) {
  2941.                 ret = __i915_gem_object_sync(obj, to, req[i], to_req);
  2942.                 if (ret)
  2943.                         return ret;
  2944.         }
  2945.  
  2946.         return 0;
  2947. }
  2948.  
  2949. static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
  2950. {
  2951.         u32 old_write_domain, old_read_domains;
  2952.  
  2953.         /* Force a pagefault for domain tracking on next user access */
  2954.         i915_gem_release_mmap(obj);
  2955.  
  2956.         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
  2957.                 return;
  2958.  
  2959.         /* Wait for any direct GTT access to complete */
  2960.         mb();
  2961.  
  2962.         old_read_domains = obj->base.read_domains;
  2963.         old_write_domain = obj->base.write_domain;
  2964.  
  2965.         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
  2966.         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
  2967.  
  2968.         trace_i915_gem_object_change_domain(obj,
  2969.                                             old_read_domains,
  2970.                                             old_write_domain);
  2971. }
  2972.  
  2973. static int __i915_vma_unbind(struct i915_vma *vma, bool wait)
  2974. {
  2975.         struct drm_i915_gem_object *obj = vma->obj;
  2976.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  2977.         int ret;
  2978.  
  2979.         if (list_empty(&vma->obj_link))
  2980.                 return 0;
  2981.  
  2982.         if (!drm_mm_node_allocated(&vma->node)) {
  2983.                 i915_gem_vma_destroy(vma);
  2984.                 return 0;
  2985.         }
  2986.  
  2987.         if (vma->pin_count)
  2988.                 return -EBUSY;
  2989.  
  2990.         BUG_ON(obj->pages == NULL);
  2991.  
  2992.         if (wait) {
  2993.                 ret = i915_gem_object_wait_rendering(obj, false);
  2994.                 if (ret)
  2995.                         return ret;
  2996.         }
  2997.  
  2998.         if (vma->is_ggtt && vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
  2999.                 i915_gem_object_finish_gtt(obj);
  3000.  
  3001.                 /* release the fence reg _after_ flushing */
  3002.                 ret = i915_gem_object_put_fence(obj);
  3003.                 if (ret)
  3004.                         return ret;
  3005.         }
  3006.  
  3007.         trace_i915_vma_unbind(vma);
  3008.  
  3009.         vma->vm->unbind_vma(vma);
  3010.         vma->bound = 0;
  3011.  
  3012.         list_del_init(&vma->vm_link);
  3013.         if (vma->is_ggtt) {
  3014.                 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
  3015.                         obj->map_and_fenceable = false;
  3016.                 } else if (vma->ggtt_view.pages) {
  3017.                         sg_free_table(vma->ggtt_view.pages);
  3018.                         kfree(vma->ggtt_view.pages);
  3019.                 }
  3020.                 vma->ggtt_view.pages = NULL;
  3021.         }
  3022.  
  3023.         drm_mm_remove_node(&vma->node);
  3024.         i915_gem_vma_destroy(vma);
  3025.  
  3026.         /* Since the unbound list is global, only move to that list if
  3027.          * no more VMAs exist. */
  3028.         if (list_empty(&obj->vma_list))
  3029.                 list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
  3030.  
  3031.         /* And finally now the object is completely decoupled from this vma,
  3032.          * we can drop its hold on the backing storage and allow it to be
  3033.          * reaped by the shrinker.
  3034.          */
  3035.         i915_gem_object_unpin_pages(obj);
  3036.  
  3037.         return 0;
  3038. }
  3039.  
  3040. int i915_vma_unbind(struct i915_vma *vma)
  3041. {
  3042.         return __i915_vma_unbind(vma, true);
  3043. }
  3044.  
  3045. int __i915_vma_unbind_no_wait(struct i915_vma *vma)
  3046. {
  3047.         return __i915_vma_unbind(vma, false);
  3048. }
  3049.  
  3050. int i915_gpu_idle(struct drm_device *dev)
  3051. {
  3052.         struct drm_i915_private *dev_priv = dev->dev_private;
  3053.         struct intel_engine_cs *ring;
  3054.         int ret, i;
  3055.  
  3056.         /* Flush everything onto the inactive list. */
  3057.         for_each_ring(ring, dev_priv, i) {
  3058.                 if (!i915.enable_execlists) {
  3059.                         struct drm_i915_gem_request *req;
  3060.  
  3061.                         req = i915_gem_request_alloc(ring, NULL);
  3062.                         if (IS_ERR(req))
  3063.                                 return PTR_ERR(req);
  3064.  
  3065.                         ret = i915_switch_context(req);
  3066.                         if (ret) {
  3067.                                 i915_gem_request_cancel(req);
  3068.                                 return ret;
  3069.                         }
  3070.  
  3071.                         i915_add_request_no_flush(req);
  3072.                 }
  3073.  
  3074.                 ret = intel_ring_idle(ring);
  3075.                 if (ret)
  3076.                         return ret;
  3077.         }
  3078.  
  3079.         WARN_ON(i915_verify_lists(dev));
  3080.         return 0;
  3081. }
  3082.  
  3083. static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
  3084.                                      unsigned long cache_level)
  3085. {
  3086.         struct drm_mm_node *gtt_space = &vma->node;
  3087.         struct drm_mm_node *other;
  3088.  
  3089.         /*
  3090.          * On some machines we have to be careful when putting differing types
  3091.          * of snoopable memory together to avoid the prefetcher crossing memory
  3092.          * domains and dying. During vm initialisation, we decide whether or not
  3093.          * these constraints apply and set the drm_mm.color_adjust
  3094.          * appropriately.
  3095.          */
  3096.         if (vma->vm->mm.color_adjust == NULL)
  3097.                 return true;
  3098.  
  3099.         if (!drm_mm_node_allocated(gtt_space))
  3100.                 return true;
  3101.  
  3102.         if (list_empty(&gtt_space->node_list))
  3103.                 return true;
  3104.  
  3105.         other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
  3106.         if (other->allocated && !other->hole_follows && other->color != cache_level)
  3107.                 return false;
  3108.  
  3109.         other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
  3110.         if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
  3111.                 return false;
  3112.  
  3113.         return true;
  3114. }
  3115.  
  3116. /**
  3117.  * Finds free space in the GTT aperture and binds the object or a view of it
  3118.  * there.
  3119.  */
  3120. static struct i915_vma *
  3121. i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
  3122.                            struct i915_address_space *vm,
  3123.                            const struct i915_ggtt_view *ggtt_view,
  3124.                            unsigned alignment,
  3125.                            uint64_t flags)
  3126. {
  3127.         struct drm_device *dev = obj->base.dev;
  3128.         struct drm_i915_private *dev_priv = dev->dev_private;
  3129.         u32 fence_alignment, unfenced_alignment;
  3130.         u32 search_flag, alloc_flag;
  3131.         u64 start, end;
  3132.         u64 size, fence_size;
  3133.         struct i915_vma *vma;
  3134.         int ret;
  3135.  
  3136.         if (i915_is_ggtt(vm)) {
  3137.                 u32 view_size;
  3138.  
  3139.                 if (WARN_ON(!ggtt_view))
  3140.                         return ERR_PTR(-EINVAL);
  3141.  
  3142.                 view_size = i915_ggtt_view_size(obj, ggtt_view);
  3143.  
  3144.                 fence_size = i915_gem_get_gtt_size(dev,
  3145.                                                    view_size,
  3146.                                                    obj->tiling_mode);
  3147.                 fence_alignment = i915_gem_get_gtt_alignment(dev,
  3148.                                                              view_size,
  3149.                                                              obj->tiling_mode,
  3150.                                                              true);
  3151.                 unfenced_alignment = i915_gem_get_gtt_alignment(dev,
  3152.                                                                 view_size,
  3153.                                                                 obj->tiling_mode,
  3154.                                                                 false);
  3155.                 size = flags & PIN_MAPPABLE ? fence_size : view_size;
  3156.         } else {
  3157.                 fence_size = i915_gem_get_gtt_size(dev,
  3158.                                                    obj->base.size,
  3159.                                                    obj->tiling_mode);
  3160.                 fence_alignment = i915_gem_get_gtt_alignment(dev,
  3161.                                                              obj->base.size,
  3162.                                                              obj->tiling_mode,
  3163.                                                              true);
  3164.                 unfenced_alignment =
  3165.                         i915_gem_get_gtt_alignment(dev,
  3166.                                                    obj->base.size,
  3167.                                                    obj->tiling_mode,
  3168.                                                    false);
  3169.                 size = flags & PIN_MAPPABLE ? fence_size : obj->base.size;
  3170.         }
  3171.  
  3172.         start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
  3173.         end = vm->total;
  3174.         if (flags & PIN_MAPPABLE)
  3175.                 end = min_t(u64, end, dev_priv->gtt.mappable_end);
  3176.         if (flags & PIN_ZONE_4G)
  3177.                 end = min_t(u64, end, (1ULL << 32) - PAGE_SIZE);
  3178.  
  3179.         if (alignment == 0)
  3180.                 alignment = flags & PIN_MAPPABLE ? fence_alignment :
  3181.                                                 unfenced_alignment;
  3182.         if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) {
  3183.                 DRM_DEBUG("Invalid object (view type=%u) alignment requested %u\n",
  3184.                           ggtt_view ? ggtt_view->type : 0,
  3185.                           alignment);
  3186.                 return ERR_PTR(-EINVAL);
  3187.         }
  3188.  
  3189.         /* If binding the object/GGTT view requires more space than the entire
  3190.          * aperture has, reject it early before evicting everything in a vain
  3191.          * attempt to find space.
  3192.          */
  3193.         if (size > end) {
  3194.                 DRM_DEBUG("Attempting to bind an object (view type=%u) larger than the aperture: size=%llu > %s aperture=%llu\n",
  3195.                           ggtt_view ? ggtt_view->type : 0,
  3196.                           size,
  3197.                           flags & PIN_MAPPABLE ? "mappable" : "total",
  3198.                           end);
  3199.                 return ERR_PTR(-E2BIG);
  3200.         }
  3201.  
  3202.         ret = i915_gem_object_get_pages(obj);
  3203.         if (ret)
  3204.                 return ERR_PTR(ret);
  3205.  
  3206.         i915_gem_object_pin_pages(obj);
  3207.  
  3208.         vma = ggtt_view ? i915_gem_obj_lookup_or_create_ggtt_vma(obj, ggtt_view) :
  3209.                           i915_gem_obj_lookup_or_create_vma(obj, vm);
  3210.  
  3211.         if (IS_ERR(vma))
  3212.                 goto err_unpin;
  3213.  
  3214.         if (flags & PIN_OFFSET_FIXED) {
  3215.                 uint64_t offset = flags & PIN_OFFSET_MASK;
  3216.  
  3217.                 if (offset & (alignment - 1) || offset + size > end) {
  3218.                         ret = -EINVAL;
  3219.                         goto err_free_vma;
  3220.                 }
  3221.                 vma->node.start = offset;
  3222.                 vma->node.size = size;
  3223.                 vma->node.color = obj->cache_level;
  3224.                 ret = drm_mm_reserve_node(&vm->mm, &vma->node);
  3225.                 if (ret)
  3226.                         goto err_free_vma;
  3227.         } else {
  3228.                 if (flags & PIN_HIGH) {
  3229.                         search_flag = DRM_MM_SEARCH_BELOW;
  3230.                         alloc_flag = DRM_MM_CREATE_TOP;
  3231.                 } else {
  3232.                         search_flag = DRM_MM_SEARCH_DEFAULT;
  3233.                         alloc_flag = DRM_MM_CREATE_DEFAULT;
  3234.                 }
  3235.  
  3236. search_free:
  3237.                 ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
  3238.                                                           size, alignment,
  3239.                                                           obj->cache_level,
  3240.                                                           start, end,
  3241.                                                           search_flag,
  3242.                                                           alloc_flag);
  3243.                 if (ret) {
  3244.  
  3245.                         goto err_free_vma;
  3246.                 }
  3247.         }
  3248.         if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) {
  3249.                 ret = -EINVAL;
  3250.                 goto err_remove_node;
  3251.         }
  3252.  
  3253.         trace_i915_vma_bind(vma, flags);
  3254.         ret = i915_vma_bind(vma, obj->cache_level, flags);
  3255.         if (ret)
  3256.                 goto err_remove_node;
  3257.  
  3258.         list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
  3259.         list_add_tail(&vma->vm_link, &vm->inactive_list);
  3260.  
  3261.         return vma;
  3262.  
  3263. err_remove_node:
  3264.         drm_mm_remove_node(&vma->node);
  3265. err_free_vma:
  3266.         i915_gem_vma_destroy(vma);
  3267.         vma = ERR_PTR(ret);
  3268. err_unpin:
  3269.         i915_gem_object_unpin_pages(obj);
  3270.         return vma;
  3271. }
  3272.  
  3273. bool
  3274. i915_gem_clflush_object(struct drm_i915_gem_object *obj,
  3275.                         bool force)
  3276. {
  3277.         /* If we don't have a page list set up, then we're not pinned
  3278.          * to GPU, and we can ignore the cache flush because it'll happen
  3279.          * again at bind time.
  3280.          */
  3281.         if (obj->pages == NULL)
  3282.                 return false;
  3283.  
  3284.         /*
  3285.          * Stolen memory is always coherent with the GPU as it is explicitly
  3286.          * marked as wc by the system, or the system is cache-coherent.
  3287.          */
  3288.         if (obj->stolen || obj->phys_handle)
  3289.                 return false;
  3290.  
  3291.         /* If the GPU is snooping the contents of the CPU cache,
  3292.          * we do not need to manually clear the CPU cache lines.  However,
  3293.          * the caches are only snooped when the render cache is
  3294.          * flushed/invalidated.  As we always have to emit invalidations
  3295.          * and flushes when moving into and out of the RENDER domain, correct
  3296.          * snooping behaviour occurs naturally as the result of our domain
  3297.          * tracking.
  3298.          */
  3299.         if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) {
  3300.                 obj->cache_dirty = true;
  3301.                 return false;
  3302.         }
  3303.  
  3304.         trace_i915_gem_object_clflush(obj);
  3305.         drm_clflush_sg(obj->pages);
  3306.         obj->cache_dirty = false;
  3307.  
  3308.         return true;
  3309. }
  3310.  
  3311. /** Flushes the GTT write domain for the object if it's dirty. */
  3312. static void
  3313. i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
  3314. {
  3315.         uint32_t old_write_domain;
  3316.  
  3317.         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
  3318.                 return;
  3319.  
  3320.         /* No actual flushing is required for the GTT write domain.  Writes
  3321.          * to it immediately go to main memory as far as we know, so there's
  3322.          * no chipset flush.  It also doesn't land in render cache.
  3323.          *
  3324.          * However, we do have to enforce the order so that all writes through
  3325.          * the GTT land before any writes to the device, such as updates to
  3326.          * the GATT itself.
  3327.          */
  3328.         wmb();
  3329.  
  3330.         old_write_domain = obj->base.write_domain;
  3331.         obj->base.write_domain = 0;
  3332.  
  3333.         intel_fb_obj_flush(obj, false, ORIGIN_GTT);
  3334.  
  3335.         trace_i915_gem_object_change_domain(obj,
  3336.                                             obj->base.read_domains,
  3337.                                             old_write_domain);
  3338. }
  3339.  
  3340. /** Flushes the CPU write domain for the object if it's dirty. */
  3341. static void
  3342. i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
  3343. {
  3344.         uint32_t old_write_domain;
  3345.  
  3346.         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
  3347.                 return;
  3348.  
  3349.         if (i915_gem_clflush_object(obj, obj->pin_display))
  3350.                 i915_gem_chipset_flush(obj->base.dev);
  3351.  
  3352.         old_write_domain = obj->base.write_domain;
  3353.         obj->base.write_domain = 0;
  3354.  
  3355.         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
  3356.  
  3357.         trace_i915_gem_object_change_domain(obj,
  3358.                                             obj->base.read_domains,
  3359.                                             old_write_domain);
  3360. }
  3361.  
  3362. /**
  3363.  * Moves a single object to the GTT read, and possibly write domain.
  3364.  *
  3365.  * This function returns when the move is complete, including waiting on
  3366.  * flushes to occur.
  3367.  */
  3368. int
  3369. i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
  3370. {
  3371.         uint32_t old_write_domain, old_read_domains;
  3372.         struct i915_vma *vma;
  3373.         int ret;
  3374.  
  3375.         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
  3376.                 return 0;
  3377.  
  3378.         ret = i915_gem_object_wait_rendering(obj, !write);
  3379.         if (ret)
  3380.                 return ret;
  3381.  
  3382.         /* Flush and acquire obj->pages so that we are coherent through
  3383.          * direct access in memory with previous cached writes through
  3384.          * shmemfs and that our cache domain tracking remains valid.
  3385.          * For example, if the obj->filp was moved to swap without us
  3386.          * being notified and releasing the pages, we would mistakenly
  3387.          * continue to assume that the obj remained out of the CPU cached
  3388.          * domain.
  3389.          */
  3390.         ret = i915_gem_object_get_pages(obj);
  3391.         if (ret)
  3392.                 return ret;
  3393.  
  3394.         i915_gem_object_flush_cpu_write_domain(obj);
  3395.  
  3396.         /* Serialise direct access to this object with the barriers for
  3397.          * coherent writes from the GPU, by effectively invalidating the
  3398.          * GTT domain upon first access.
  3399.          */
  3400.         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
  3401.                 mb();
  3402.  
  3403.         old_write_domain = obj->base.write_domain;
  3404.         old_read_domains = obj->base.read_domains;
  3405.  
  3406.         /* It should now be out of any other write domains, and we can update
  3407.          * the domain values for our changes.
  3408.          */
  3409.         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
  3410.         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
  3411.         if (write) {
  3412.                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
  3413.                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
  3414.                 obj->dirty = 1;
  3415.         }
  3416.  
  3417.         trace_i915_gem_object_change_domain(obj,
  3418.                                             old_read_domains,
  3419.                                             old_write_domain);
  3420.  
  3421.         /* And bump the LRU for this access */
  3422.         vma = i915_gem_obj_to_ggtt(obj);
  3423.         if (vma && drm_mm_node_allocated(&vma->node) && !obj->active)
  3424.                 list_move_tail(&vma->vm_link,
  3425.                                &to_i915(obj->base.dev)->gtt.base.inactive_list);
  3426.  
  3427.         return 0;
  3428. }
  3429.  
  3430. /**
  3431.  * Changes the cache-level of an object across all VMA.
  3432.  *
  3433.  * After this function returns, the object will be in the new cache-level
  3434.  * across all GTT and the contents of the backing storage will be coherent,
  3435.  * with respect to the new cache-level. In order to keep the backing storage
  3436.  * coherent for all users, we only allow a single cache level to be set
  3437.  * globally on the object and prevent it from being changed whilst the
  3438.  * hardware is reading from the object. That is if the object is currently
  3439.  * on the scanout it will be set to uncached (or equivalent display
  3440.  * cache coherency) and all non-MOCS GPU access will also be uncached so
  3441.  * that all direct access to the scanout remains coherent.
  3442.  */
  3443. int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
  3444.                                     enum i915_cache_level cache_level)
  3445. {
  3446.         struct drm_device *dev = obj->base.dev;
  3447.         struct i915_vma *vma, *next;
  3448.         bool bound = false;
  3449.         int ret = 0;
  3450.  
  3451.         if (obj->cache_level == cache_level)
  3452.                 goto out;
  3453.  
  3454.         /* Inspect the list of currently bound VMA and unbind any that would
  3455.          * be invalid given the new cache-level. This is principally to
  3456.          * catch the issue of the CS prefetch crossing page boundaries and
  3457.          * reading an invalid PTE on older architectures.
  3458.          */
  3459.         list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) {
  3460.                 if (!drm_mm_node_allocated(&vma->node))
  3461.                         continue;
  3462.  
  3463.                 if (vma->pin_count) {
  3464.                         DRM_DEBUG("can not change the cache level of pinned objects\n");
  3465.                         return -EBUSY;
  3466.                 }
  3467.  
  3468.                 if (!i915_gem_valid_gtt_space(vma, cache_level)) {
  3469.                         ret = i915_vma_unbind(vma);
  3470.                         if (ret)
  3471.                                 return ret;
  3472.                 } else
  3473.                         bound = true;
  3474.         }
  3475.  
  3476.         /* We can reuse the existing drm_mm nodes but need to change the
  3477.          * cache-level on the PTE. We could simply unbind them all and
  3478.          * rebind with the correct cache-level on next use. However since
  3479.          * we already have a valid slot, dma mapping, pages etc, we may as
  3480.          * rewrite the PTE in the belief that doing so tramples upon less
  3481.          * state and so involves less work.
  3482.          */
  3483.         if (bound) {
  3484.                 /* Before we change the PTE, the GPU must not be accessing it.
  3485.                  * If we wait upon the object, we know that all the bound
  3486.                  * VMA are no longer active.
  3487.                  */
  3488.                 ret = i915_gem_object_wait_rendering(obj, false);
  3489.                 if (ret)
  3490.                         return ret;
  3491.  
  3492.                 if (!HAS_LLC(dev) && cache_level != I915_CACHE_NONE) {
  3493.                         /* Access to snoopable pages through the GTT is
  3494.                          * incoherent and on some machines causes a hard
  3495.                          * lockup. Relinquish the CPU mmaping to force
  3496.                          * userspace to refault in the pages and we can
  3497.                          * then double check if the GTT mapping is still
  3498.                          * valid for that pointer access.
  3499.                          */
  3500.                         i915_gem_release_mmap(obj);
  3501.  
  3502.                         /* As we no longer need a fence for GTT access,
  3503.                          * we can relinquish it now (and so prevent having
  3504.                          * to steal a fence from someone else on the next
  3505.                          * fence request). Note GPU activity would have
  3506.                          * dropped the fence as all snoopable access is
  3507.                          * supposed to be linear.
  3508.                          */
  3509.                         ret = i915_gem_object_put_fence(obj);
  3510.                         if (ret)
  3511.                                 return ret;
  3512.                 } else {
  3513.                         /* We either have incoherent backing store and
  3514.                          * so no GTT access or the architecture is fully
  3515.                          * coherent. In such cases, existing GTT mmaps
  3516.                          * ignore the cache bit in the PTE and we can
  3517.                          * rewrite it without confusing the GPU or having
  3518.                          * to force userspace to fault back in its mmaps.
  3519.                          */
  3520.                 }
  3521.  
  3522.                 list_for_each_entry(vma, &obj->vma_list, obj_link) {
  3523.                         if (!drm_mm_node_allocated(&vma->node))
  3524.                                 continue;
  3525.  
  3526.                         ret = i915_vma_bind(vma, cache_level, PIN_UPDATE);
  3527.                         if (ret)
  3528.                                 return ret;
  3529.                 }
  3530.         }
  3531.  
  3532.         list_for_each_entry(vma, &obj->vma_list, obj_link)
  3533.                 vma->node.color = cache_level;
  3534.         obj->cache_level = cache_level;
  3535.  
  3536. out:
  3537.         /* Flush the dirty CPU caches to the backing storage so that the
  3538.          * object is now coherent at its new cache level (with respect
  3539.          * to the access domain).
  3540.          */
  3541.         if (obj->cache_dirty &&
  3542.             obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
  3543.             cpu_write_needs_clflush(obj)) {
  3544.                 if (i915_gem_clflush_object(obj, true))
  3545.                         i915_gem_chipset_flush(obj->base.dev);
  3546.         }
  3547.  
  3548.         return 0;
  3549. }
  3550.  
  3551. int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
  3552.                                struct drm_file *file)
  3553. {
  3554.         struct drm_i915_gem_caching *args = data;
  3555.         struct drm_i915_gem_object *obj;
  3556.  
  3557.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  3558.         if (&obj->base == NULL)
  3559.                 return -ENOENT;
  3560.  
  3561.         switch (obj->cache_level) {
  3562.         case I915_CACHE_LLC:
  3563.         case I915_CACHE_L3_LLC:
  3564.                 args->caching = I915_CACHING_CACHED;
  3565.                 break;
  3566.  
  3567.         case I915_CACHE_WT:
  3568.                 args->caching = I915_CACHING_DISPLAY;
  3569.                 break;
  3570.  
  3571.         default:
  3572.                 args->caching = I915_CACHING_NONE;
  3573.                 break;
  3574.         }
  3575.  
  3576.         drm_gem_object_unreference_unlocked(&obj->base);
  3577.         return 0;
  3578. }
  3579.  
  3580. int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
  3581.                                struct drm_file *file)
  3582. {
  3583.         struct drm_i915_private *dev_priv = dev->dev_private;
  3584.         struct drm_i915_gem_caching *args = data;
  3585.         struct drm_i915_gem_object *obj;
  3586.         enum i915_cache_level level;
  3587.         int ret;
  3588.  
  3589.         switch (args->caching) {
  3590.         case I915_CACHING_NONE:
  3591.                 level = I915_CACHE_NONE;
  3592.                 break;
  3593.         case I915_CACHING_CACHED:
  3594.                 /*
  3595.                  * Due to a HW issue on BXT A stepping, GPU stores via a
  3596.                  * snooped mapping may leave stale data in a corresponding CPU
  3597.                  * cacheline, whereas normally such cachelines would get
  3598.                  * invalidated.
  3599.                  */
  3600.                 if (IS_BXT_REVID(dev, 0, BXT_REVID_A1))
  3601.                         return -ENODEV;
  3602.  
  3603.                 level = I915_CACHE_LLC;
  3604.                 break;
  3605.         case I915_CACHING_DISPLAY:
  3606.                 level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
  3607.                 break;
  3608.         default:
  3609.                 return -EINVAL;
  3610.         }
  3611.  
  3612.         intel_runtime_pm_get(dev_priv);
  3613.  
  3614.         ret = i915_mutex_lock_interruptible(dev);
  3615.         if (ret)
  3616.                 goto rpm_put;
  3617.  
  3618.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  3619.         if (&obj->base == NULL) {
  3620.                 ret = -ENOENT;
  3621.                 goto unlock;
  3622.         }
  3623.  
  3624.         ret = i915_gem_object_set_cache_level(obj, level);
  3625.  
  3626.         drm_gem_object_unreference(&obj->base);
  3627. unlock:
  3628.         mutex_unlock(&dev->struct_mutex);
  3629. rpm_put:
  3630.         intel_runtime_pm_put(dev_priv);
  3631.  
  3632.         return ret;
  3633. }
  3634.  
  3635. /*
  3636.  * Prepare buffer for display plane (scanout, cursors, etc).
  3637.  * Can be called from an uninterruptible phase (modesetting) and allows
  3638.  * any flushes to be pipelined (for pageflips).
  3639.  */
  3640. int
  3641. i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
  3642.                                      u32 alignment,
  3643.                                      const struct i915_ggtt_view *view)
  3644. {
  3645.         u32 old_read_domains, old_write_domain;
  3646.         int ret;
  3647.  
  3648.         /* Mark the pin_display early so that we account for the
  3649.          * display coherency whilst setting up the cache domains.
  3650.          */
  3651.         obj->pin_display++;
  3652.  
  3653.         /* The display engine is not coherent with the LLC cache on gen6.  As
  3654.          * a result, we make sure that the pinning that is about to occur is
  3655.          * done with uncached PTEs. This is lowest common denominator for all
  3656.          * chipsets.
  3657.          *
  3658.          * However for gen6+, we could do better by using the GFDT bit instead
  3659.          * of uncaching, which would allow us to flush all the LLC-cached data
  3660.          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
  3661.          */
  3662.         ret = i915_gem_object_set_cache_level(obj,
  3663.                                               HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
  3664.         if (ret)
  3665.                 goto err_unpin_display;
  3666.  
  3667.         /* As the user may map the buffer once pinned in the display plane
  3668.          * (e.g. libkms for the bootup splash), we have to ensure that we
  3669.          * always use map_and_fenceable for all scanout buffers.
  3670.          */
  3671.         ret = i915_gem_object_ggtt_pin(obj, view, alignment,
  3672.                                        view->type == I915_GGTT_VIEW_NORMAL ?
  3673.                                        PIN_MAPPABLE : 0);
  3674.         if (ret)
  3675.                 goto err_unpin_display;
  3676.  
  3677.         i915_gem_object_flush_cpu_write_domain(obj);
  3678.  
  3679.         old_write_domain = obj->base.write_domain;
  3680.         old_read_domains = obj->base.read_domains;
  3681.  
  3682.         /* It should now be out of any other write domains, and we can update
  3683.          * the domain values for our changes.
  3684.          */
  3685.         obj->base.write_domain = 0;
  3686.         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
  3687.  
  3688.         trace_i915_gem_object_change_domain(obj,
  3689.                                             old_read_domains,
  3690.                                             old_write_domain);
  3691.  
  3692.         return 0;
  3693.  
  3694. err_unpin_display:
  3695.         obj->pin_display--;
  3696.         return ret;
  3697. }
  3698.  
  3699. void
  3700. i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj,
  3701.                                          const struct i915_ggtt_view *view)
  3702. {
  3703.         if (WARN_ON(obj->pin_display == 0))
  3704.                 return;
  3705.  
  3706.         i915_gem_object_ggtt_unpin_view(obj, view);
  3707.  
  3708.         obj->pin_display--;
  3709. }
  3710.  
  3711. /**
  3712.  * Moves a single object to the CPU read, and possibly write domain.
  3713.  *
  3714.  * This function returns when the move is complete, including waiting on
  3715.  * flushes to occur.
  3716.  */
  3717. int
  3718. i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
  3719. {
  3720.         uint32_t old_write_domain, old_read_domains;
  3721.         int ret;
  3722.  
  3723.         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
  3724.                 return 0;
  3725.  
  3726.         ret = i915_gem_object_wait_rendering(obj, !write);
  3727.         if (ret)
  3728.                 return ret;
  3729.  
  3730.         i915_gem_object_flush_gtt_write_domain(obj);
  3731.  
  3732.         old_write_domain = obj->base.write_domain;
  3733.         old_read_domains = obj->base.read_domains;
  3734.  
  3735.         /* Flush the CPU cache if it's still invalid. */
  3736.         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
  3737.                 i915_gem_clflush_object(obj, false);
  3738.  
  3739.                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
  3740.         }
  3741.  
  3742.         /* It should now be out of any other write domains, and we can update
  3743.          * the domain values for our changes.
  3744.          */
  3745.         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
  3746.  
  3747.         /* If we're writing through the CPU, then the GPU read domains will
  3748.          * need to be invalidated at next use.
  3749.          */
  3750.         if (write) {
  3751.                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  3752.                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  3753.         }
  3754.  
  3755.         trace_i915_gem_object_change_domain(obj,
  3756.                                             old_read_domains,
  3757.                                             old_write_domain);
  3758.  
  3759.         return 0;
  3760. }
  3761.  
  3762. /* Throttle our rendering by waiting until the ring has completed our requests
  3763.  * emitted over 20 msec ago.
  3764.  *
  3765.  * Note that if we were to use the current jiffies each time around the loop,
  3766.  * we wouldn't escape the function with any frames outstanding if the time to
  3767.  * render a frame was over 20ms.
  3768.  *
  3769.  * This should get us reasonable parallelism between CPU and GPU but also
  3770.  * relatively low latency when blocking on a particular request to finish.
  3771.  */
  3772. static int
  3773. i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
  3774. {
  3775.         struct drm_i915_private *dev_priv = dev->dev_private;
  3776.         struct drm_i915_file_private *file_priv = file->driver_priv;
  3777.         unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
  3778.         struct drm_i915_gem_request *request, *target = NULL;
  3779.         unsigned reset_counter;
  3780.         int ret;
  3781.  
  3782.         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
  3783.         if (ret)
  3784.                 return ret;
  3785.  
  3786.         ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
  3787.         if (ret)
  3788.                 return ret;
  3789.  
  3790.         spin_lock(&file_priv->mm.lock);
  3791.         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
  3792.                 if (time_after_eq(request->emitted_jiffies, recent_enough))
  3793.                         break;
  3794.  
  3795.                 /*
  3796.                  * Note that the request might not have been submitted yet.
  3797.                  * In which case emitted_jiffies will be zero.
  3798.                  */
  3799.                 if (!request->emitted_jiffies)
  3800.                         continue;
  3801.  
  3802.                 target = request;
  3803.         }
  3804.         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
  3805.         if (target)
  3806.                 i915_gem_request_reference(target);
  3807.         spin_unlock(&file_priv->mm.lock);
  3808.  
  3809.         if (target == NULL)
  3810.                 return 0;
  3811.  
  3812.         ret = __i915_wait_request(target, reset_counter, true, NULL, NULL);
  3813.         if (ret == 0)
  3814.                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
  3815.  
  3816.         i915_gem_request_unreference__unlocked(target);
  3817.  
  3818.         return ret;
  3819. }
  3820.  
  3821. static bool
  3822. i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags)
  3823. {
  3824.         struct drm_i915_gem_object *obj = vma->obj;
  3825.  
  3826.         if (alignment &&
  3827.             vma->node.start & (alignment - 1))
  3828.                 return true;
  3829.  
  3830.         if (flags & PIN_MAPPABLE && !obj->map_and_fenceable)
  3831.                 return true;
  3832.  
  3833.         if (flags & PIN_OFFSET_BIAS &&
  3834.             vma->node.start < (flags & PIN_OFFSET_MASK))
  3835.                 return true;
  3836.  
  3837.         if (flags & PIN_OFFSET_FIXED &&
  3838.             vma->node.start != (flags & PIN_OFFSET_MASK))
  3839.                 return true;
  3840.  
  3841.         return false;
  3842. }
  3843.  
  3844. void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
  3845. {
  3846.         struct drm_i915_gem_object *obj = vma->obj;
  3847.         bool mappable, fenceable;
  3848.         u32 fence_size, fence_alignment;
  3849.  
  3850.         fence_size = i915_gem_get_gtt_size(obj->base.dev,
  3851.                                            obj->base.size,
  3852.                                            obj->tiling_mode);
  3853.         fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev,
  3854.                                                      obj->base.size,
  3855.                                                      obj->tiling_mode,
  3856.                                                      true);
  3857.  
  3858.         fenceable = (vma->node.size == fence_size &&
  3859.                      (vma->node.start & (fence_alignment - 1)) == 0);
  3860.  
  3861.         mappable = (vma->node.start + fence_size <=
  3862.                     to_i915(obj->base.dev)->gtt.mappable_end);
  3863.  
  3864.         obj->map_and_fenceable = mappable && fenceable;
  3865. }
  3866.  
  3867. static int
  3868. i915_gem_object_do_pin(struct drm_i915_gem_object *obj,
  3869.                        struct i915_address_space *vm,
  3870.                        const struct i915_ggtt_view *ggtt_view,
  3871.                        uint32_t alignment,
  3872.                        uint64_t flags)
  3873. {
  3874.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  3875.         struct i915_vma *vma;
  3876.         unsigned bound;
  3877.         int ret;
  3878.  
  3879.         if (WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base))
  3880.                 return -ENODEV;
  3881.  
  3882.         if (WARN_ON(flags & (PIN_GLOBAL | PIN_MAPPABLE) && !i915_is_ggtt(vm)))
  3883.                 return -EINVAL;
  3884.  
  3885.         if (WARN_ON((flags & (PIN_MAPPABLE | PIN_GLOBAL)) == PIN_MAPPABLE))
  3886.                 return -EINVAL;
  3887.  
  3888.         if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view))
  3889.                 return -EINVAL;
  3890.  
  3891.         vma = ggtt_view ? i915_gem_obj_to_ggtt_view(obj, ggtt_view) :
  3892.                           i915_gem_obj_to_vma(obj, vm);
  3893.  
  3894.         if (IS_ERR(vma))
  3895.                 return PTR_ERR(vma);
  3896.  
  3897.         if (vma) {
  3898.                 if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
  3899.                         return -EBUSY;
  3900.  
  3901.                 if (i915_vma_misplaced(vma, alignment, flags)) {
  3902.                         WARN(vma->pin_count,
  3903.                              "bo is already pinned in %s with incorrect alignment:"
  3904.                              " offset=%08x %08x, req.alignment=%x, req.map_and_fenceable=%d,"
  3905.                              " obj->map_and_fenceable=%d\n",
  3906.                              ggtt_view ? "ggtt" : "ppgtt",
  3907.                              upper_32_bits(vma->node.start),
  3908.                              lower_32_bits(vma->node.start),
  3909.                              alignment,
  3910.                              !!(flags & PIN_MAPPABLE),
  3911.                              obj->map_and_fenceable);
  3912.                         ret = i915_vma_unbind(vma);
  3913.                         if (ret)
  3914.                                 return ret;
  3915.  
  3916.                         vma = NULL;
  3917.                 }
  3918.         }
  3919.  
  3920.         bound = vma ? vma->bound : 0;
  3921.         if (vma == NULL || !drm_mm_node_allocated(&vma->node)) {
  3922.                 vma = i915_gem_object_bind_to_vm(obj, vm, ggtt_view, alignment,
  3923.                                                  flags);
  3924.                 if (IS_ERR(vma))
  3925.                         return PTR_ERR(vma);
  3926.         } else {
  3927.                 ret = i915_vma_bind(vma, obj->cache_level, flags);
  3928.                 if (ret)
  3929.                         return ret;
  3930.         }
  3931.  
  3932.         if (ggtt_view && ggtt_view->type == I915_GGTT_VIEW_NORMAL &&
  3933.             (bound ^ vma->bound) & GLOBAL_BIND) {
  3934.                 __i915_vma_set_map_and_fenceable(vma);
  3935.                 WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
  3936.         }
  3937.  
  3938.         vma->pin_count++;
  3939.         return 0;
  3940. }
  3941.  
  3942. int
  3943. i915_gem_object_pin(struct drm_i915_gem_object *obj,
  3944.                     struct i915_address_space *vm,
  3945.                     uint32_t alignment,
  3946.                     uint64_t flags)
  3947. {
  3948.         return i915_gem_object_do_pin(obj, vm,
  3949.                                       i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL,
  3950.                                       alignment, flags);
  3951. }
  3952.  
  3953. int
  3954. i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
  3955.                          const struct i915_ggtt_view *view,
  3956.                          uint32_t alignment,
  3957.                          uint64_t flags)
  3958. {
  3959.         if (WARN_ONCE(!view, "no view specified"))
  3960.                 return -EINVAL;
  3961.  
  3962.         return i915_gem_object_do_pin(obj, i915_obj_to_ggtt(obj), view,
  3963.                                       alignment, flags | PIN_GLOBAL);
  3964. }
  3965.  
  3966. void
  3967. i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj,
  3968.                                 const struct i915_ggtt_view *view)
  3969. {
  3970.         struct i915_vma *vma = i915_gem_obj_to_ggtt_view(obj, view);
  3971.  
  3972.         BUG_ON(!vma);
  3973.         WARN_ON(vma->pin_count == 0);
  3974.         WARN_ON(!i915_gem_obj_ggtt_bound_view(obj, view));
  3975.  
  3976.         --vma->pin_count;
  3977. }
  3978.  
  3979. int
  3980. i915_gem_busy_ioctl(struct drm_device *dev, void *data,
  3981.                     struct drm_file *file)
  3982. {
  3983.         struct drm_i915_gem_busy *args = data;
  3984.         struct drm_i915_gem_object *obj;
  3985.         int ret;
  3986.  
  3987.         ret = i915_mutex_lock_interruptible(dev);
  3988.         if (ret)
  3989.                 return ret;
  3990.  
  3991.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  3992.         if (&obj->base == NULL) {
  3993.                 ret = -ENOENT;
  3994.                 goto unlock;
  3995.         }
  3996.  
  3997.         /* Count all active objects as busy, even if they are currently not used
  3998.          * by the gpu. Users of this interface expect objects to eventually
  3999.          * become non-busy without any further actions, therefore emit any
  4000.          * necessary flushes here.
  4001.          */
  4002.         ret = i915_gem_object_flush_active(obj);
  4003.         if (ret)
  4004.                 goto unref;
  4005.  
  4006.         args->busy = 0;
  4007.         if (obj->active) {
  4008.                 int i;
  4009.  
  4010.                 for (i = 0; i < I915_NUM_RINGS; i++) {
  4011.                         struct drm_i915_gem_request *req;
  4012.  
  4013.                         req = obj->last_read_req[i];
  4014.                         if (req)
  4015.                                 args->busy |= 1 << (16 + req->ring->exec_id);
  4016.                 }
  4017.                 if (obj->last_write_req)
  4018.                         args->busy |= obj->last_write_req->ring->exec_id;
  4019.         }
  4020.  
  4021. unref:
  4022.         drm_gem_object_unreference(&obj->base);
  4023. unlock:
  4024.         mutex_unlock(&dev->struct_mutex);
  4025.         return ret;
  4026. }
  4027.  
  4028. int
  4029. i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
  4030.                         struct drm_file *file_priv)
  4031. {
  4032.         return i915_gem_ring_throttle(dev, file_priv);
  4033. }
  4034.  
  4035. #if 0
  4036.  
  4037. int
  4038. i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
  4039.                        struct drm_file *file_priv)
  4040. {
  4041.         struct drm_i915_private *dev_priv = dev->dev_private;
  4042.         struct drm_i915_gem_madvise *args = data;
  4043.         struct drm_i915_gem_object *obj;
  4044.         int ret;
  4045.  
  4046.         switch (args->madv) {
  4047.         case I915_MADV_DONTNEED:
  4048.         case I915_MADV_WILLNEED:
  4049.             break;
  4050.         default:
  4051.             return -EINVAL;
  4052.         }
  4053.  
  4054.         ret = i915_mutex_lock_interruptible(dev);
  4055.         if (ret)
  4056.                 return ret;
  4057.  
  4058.         obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
  4059.         if (&obj->base == NULL) {
  4060.                 ret = -ENOENT;
  4061.                 goto unlock;
  4062.         }
  4063.  
  4064.         if (i915_gem_obj_is_pinned(obj)) {
  4065.                 ret = -EINVAL;
  4066.                 goto out;
  4067.         }
  4068.  
  4069.         if (obj->pages &&
  4070.             obj->tiling_mode != I915_TILING_NONE &&
  4071.             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
  4072.                 if (obj->madv == I915_MADV_WILLNEED)
  4073.                         i915_gem_object_unpin_pages(obj);
  4074.                 if (args->madv == I915_MADV_WILLNEED)
  4075.                         i915_gem_object_pin_pages(obj);
  4076.         }
  4077.  
  4078.         if (obj->madv != __I915_MADV_PURGED)
  4079.                 obj->madv = args->madv;
  4080.  
  4081.         /* if the object is no longer attached, discard its backing storage */
  4082.         if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL)
  4083.                 i915_gem_object_truncate(obj);
  4084.  
  4085.         args->retained = obj->madv != __I915_MADV_PURGED;
  4086.  
  4087. out:
  4088.         drm_gem_object_unreference(&obj->base);
  4089. unlock:
  4090.         mutex_unlock(&dev->struct_mutex);
  4091.         return ret;
  4092. }
  4093. #endif
  4094.  
  4095. void i915_gem_object_init(struct drm_i915_gem_object *obj,
  4096.                           const struct drm_i915_gem_object_ops *ops)
  4097. {
  4098.         int i;
  4099.  
  4100.         INIT_LIST_HEAD(&obj->global_list);
  4101.         for (i = 0; i < I915_NUM_RINGS; i++)
  4102.                 INIT_LIST_HEAD(&obj->ring_list[i]);
  4103.         INIT_LIST_HEAD(&obj->obj_exec_link);
  4104.         INIT_LIST_HEAD(&obj->vma_list);
  4105.         INIT_LIST_HEAD(&obj->batch_pool_link);
  4106.  
  4107.         obj->ops = ops;
  4108.  
  4109.         obj->fence_reg = I915_FENCE_REG_NONE;
  4110.         obj->madv = I915_MADV_WILLNEED;
  4111.  
  4112.         i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
  4113. }
  4114.  
  4115. static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
  4116.         .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE,
  4117.         .get_pages = i915_gem_object_get_pages_gtt,
  4118.         .put_pages = i915_gem_object_put_pages_gtt,
  4119. };
  4120.  
  4121. struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
  4122.                                                   size_t size)
  4123. {
  4124.         struct drm_i915_gem_object *obj;
  4125.         struct address_space *mapping;
  4126.         gfp_t mask;
  4127.  
  4128.         obj = i915_gem_object_alloc(dev);
  4129.         if (obj == NULL)
  4130.                 return NULL;
  4131.  
  4132.         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
  4133.                 i915_gem_object_free(obj);
  4134.                 return NULL;
  4135.         }
  4136.  
  4137.  
  4138.         i915_gem_object_init(obj, &i915_gem_object_ops);
  4139.  
  4140.         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  4141.         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  4142.  
  4143.         if (HAS_LLC(dev)) {
  4144.                 /* On some devices, we can have the GPU use the LLC (the CPU
  4145.                  * cache) for about a 10% performance improvement
  4146.                  * compared to uncached.  Graphics requests other than
  4147.                  * display scanout are coherent with the CPU in
  4148.                  * accessing this cache.  This means in this mode we
  4149.                  * don't need to clflush on the CPU side, and on the
  4150.                  * GPU side we only need to flush internal caches to
  4151.                  * get data visible to the CPU.
  4152.                  *
  4153.                  * However, we maintain the display planes as UC, and so
  4154.                  * need to rebind when first used as such.
  4155.                  */
  4156.                 obj->cache_level = I915_CACHE_LLC;
  4157.         } else
  4158.                 obj->cache_level = I915_CACHE_NONE;
  4159.  
  4160.         trace_i915_gem_object_create(obj);
  4161.  
  4162.         return obj;
  4163. }
  4164.  
  4165. static bool discard_backing_storage(struct drm_i915_gem_object *obj)
  4166. {
  4167.         /* If we are the last user of the backing storage (be it shmemfs
  4168.          * pages or stolen etc), we know that the pages are going to be
  4169.          * immediately released. In this case, we can then skip copying
  4170.          * back the contents from the GPU.
  4171.          */
  4172.  
  4173.         if (obj->madv != I915_MADV_WILLNEED)
  4174.                 return false;
  4175.  
  4176.         if (obj->base.filp == NULL)
  4177.                 return true;
  4178.  
  4179. //        printf("filp %p\n", obj->base.filp);
  4180.         shmem_file_delete(obj->base.filp);
  4181.         return true;
  4182. }
  4183.  
  4184. void i915_gem_free_object(struct drm_gem_object *gem_obj)
  4185. {
  4186.         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
  4187.         struct drm_device *dev = obj->base.dev;
  4188.         struct drm_i915_private *dev_priv = dev->dev_private;
  4189.         struct i915_vma *vma, *next;
  4190.  
  4191.         intel_runtime_pm_get(dev_priv);
  4192.  
  4193.         trace_i915_gem_object_destroy(obj);
  4194.  
  4195.         list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) {
  4196.                 int ret;
  4197.  
  4198.                 vma->pin_count = 0;
  4199.                 ret = i915_vma_unbind(vma);
  4200.                 if (WARN_ON(ret == -ERESTARTSYS)) {
  4201.                         bool was_interruptible;
  4202.  
  4203.                         was_interruptible = dev_priv->mm.interruptible;
  4204.                         dev_priv->mm.interruptible = false;
  4205.  
  4206.                         WARN_ON(i915_vma_unbind(vma));
  4207.  
  4208.                         dev_priv->mm.interruptible = was_interruptible;
  4209.                 }
  4210.         }
  4211.  
  4212.         /* Stolen objects don't hold a ref, but do hold pin count. Fix that up
  4213.          * before progressing. */
  4214.         if (obj->stolen)
  4215.                 i915_gem_object_unpin_pages(obj);
  4216.  
  4217.         WARN_ON(obj->frontbuffer_bits);
  4218.  
  4219.         if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
  4220.             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
  4221.             obj->tiling_mode != I915_TILING_NONE)
  4222.                 i915_gem_object_unpin_pages(obj);
  4223.  
  4224.         if (WARN_ON(obj->pages_pin_count))
  4225.                 obj->pages_pin_count = 0;
  4226.         if (discard_backing_storage(obj))
  4227.                 obj->madv = I915_MADV_DONTNEED;
  4228.         i915_gem_object_put_pages(obj);
  4229. //   i915_gem_object_free_mmap_offset(obj);
  4230.  
  4231.         BUG_ON(obj->pages);
  4232.  
  4233.         if (obj->ops->release)
  4234.                 obj->ops->release(obj);
  4235.  
  4236.         drm_gem_object_release(&obj->base);
  4237.         i915_gem_info_remove_obj(dev_priv, obj->base.size);
  4238.  
  4239.         kfree(obj->bit_17);
  4240.         i915_gem_object_free(obj);
  4241.  
  4242.         intel_runtime_pm_put(dev_priv);
  4243. }
  4244.  
  4245. struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
  4246.                                      struct i915_address_space *vm)
  4247. {
  4248.         struct i915_vma *vma;
  4249.         list_for_each_entry(vma, &obj->vma_list, obj_link) {
  4250.                 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL &&
  4251.                     vma->vm == vm)
  4252.                         return vma;
  4253.         }
  4254.         return NULL;
  4255. }
  4256.  
  4257. struct i915_vma *i915_gem_obj_to_ggtt_view(struct drm_i915_gem_object *obj,
  4258.                                            const struct i915_ggtt_view *view)
  4259. {
  4260.         struct i915_address_space *ggtt = i915_obj_to_ggtt(obj);
  4261.         struct i915_vma *vma;
  4262.  
  4263.         if (WARN_ONCE(!view, "no view specified"))
  4264.                 return ERR_PTR(-EINVAL);
  4265.  
  4266.         list_for_each_entry(vma, &obj->vma_list, obj_link)
  4267.                 if (vma->vm == ggtt &&
  4268.                     i915_ggtt_view_equal(&vma->ggtt_view, view))
  4269.                         return vma;
  4270.         return NULL;
  4271. }
  4272.  
  4273. void i915_gem_vma_destroy(struct i915_vma *vma)
  4274. {
  4275.         WARN_ON(vma->node.allocated);
  4276.  
  4277.         /* Keep the vma as a placeholder in the execbuffer reservation lists */
  4278.         if (!list_empty(&vma->exec_list))
  4279.                 return;
  4280.  
  4281.         if (!vma->is_ggtt)
  4282.                 i915_ppgtt_put(i915_vm_to_ppgtt(vma->vm));
  4283.  
  4284.         list_del(&vma->obj_link);
  4285.  
  4286.         kfree(vma);
  4287. }
  4288.  
  4289. static void
  4290. i915_gem_stop_ringbuffers(struct drm_device *dev)
  4291. {
  4292.         struct drm_i915_private *dev_priv = dev->dev_private;
  4293.         struct intel_engine_cs *ring;
  4294.         int i;
  4295.  
  4296.         for_each_ring(ring, dev_priv, i)
  4297.                 dev_priv->gt.stop_ring(ring);
  4298. }
  4299.  
  4300. #if 0
  4301. int
  4302. i915_gem_suspend(struct drm_device *dev)
  4303. {
  4304.         struct drm_i915_private *dev_priv = dev->dev_private;
  4305.         int ret = 0;
  4306.  
  4307.         mutex_lock(&dev->struct_mutex);
  4308.         ret = i915_gpu_idle(dev);
  4309.         if (ret)
  4310.                 goto err;
  4311.  
  4312.         i915_gem_retire_requests(dev);
  4313.  
  4314.         i915_gem_stop_ringbuffers(dev);
  4315.         mutex_unlock(&dev->struct_mutex);
  4316.  
  4317.         cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
  4318.         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
  4319.         flush_delayed_work(&dev_priv->mm.idle_work);
  4320.  
  4321.         /* Assert that we sucessfully flushed all the work and
  4322.          * reset the GPU back to its idle, low power state.
  4323.          */
  4324.         WARN_ON(dev_priv->mm.busy);
  4325.  
  4326.         return 0;
  4327.  
  4328. err:
  4329.         mutex_unlock(&dev->struct_mutex);
  4330.         return ret;
  4331. }
  4332. #endif
  4333.  
  4334. int i915_gem_l3_remap(struct drm_i915_gem_request *req, int slice)
  4335. {
  4336.         struct intel_engine_cs *ring = req->ring;
  4337.         struct drm_device *dev = ring->dev;
  4338.         struct drm_i915_private *dev_priv = dev->dev_private;
  4339.         u32 *remap_info = dev_priv->l3_parity.remap_info[slice];
  4340.         int i, ret;
  4341.  
  4342.         if (!HAS_L3_DPF(dev) || !remap_info)
  4343.                 return 0;
  4344.  
  4345.         ret = intel_ring_begin(req, GEN7_L3LOG_SIZE / 4 * 3);
  4346.         if (ret)
  4347.                 return ret;
  4348.  
  4349.         /*
  4350.          * Note: We do not worry about the concurrent register cacheline hang
  4351.          * here because no other code should access these registers other than
  4352.          * at initialization time.
  4353.          */
  4354.         for (i = 0; i < GEN7_L3LOG_SIZE / 4; i++) {
  4355.                 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
  4356.                 intel_ring_emit_reg(ring, GEN7_L3LOG(slice, i));
  4357.                 intel_ring_emit(ring, remap_info[i]);
  4358.         }
  4359.  
  4360.         intel_ring_advance(ring);
  4361.  
  4362.         return ret;
  4363. }
  4364.  
  4365. void i915_gem_init_swizzling(struct drm_device *dev)
  4366. {
  4367.         struct drm_i915_private *dev_priv = dev->dev_private;
  4368.  
  4369.         if (INTEL_INFO(dev)->gen < 5 ||
  4370.             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
  4371.                 return;
  4372.  
  4373.         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
  4374.                                  DISP_TILE_SURFACE_SWIZZLING);
  4375.  
  4376.         if (IS_GEN5(dev))
  4377.                 return;
  4378.  
  4379.         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
  4380.         if (IS_GEN6(dev))
  4381.                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
  4382.         else if (IS_GEN7(dev))
  4383.                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
  4384.         else if (IS_GEN8(dev))
  4385.                 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
  4386.         else
  4387.                 BUG();
  4388. }
  4389.  
  4390. static void init_unused_ring(struct drm_device *dev, u32 base)
  4391. {
  4392.         struct drm_i915_private *dev_priv = dev->dev_private;
  4393.  
  4394.         I915_WRITE(RING_CTL(base), 0);
  4395.         I915_WRITE(RING_HEAD(base), 0);
  4396.         I915_WRITE(RING_TAIL(base), 0);
  4397.         I915_WRITE(RING_START(base), 0);
  4398. }
  4399.  
  4400. static void init_unused_rings(struct drm_device *dev)
  4401. {
  4402.         if (IS_I830(dev)) {
  4403.                 init_unused_ring(dev, PRB1_BASE);
  4404.                 init_unused_ring(dev, SRB0_BASE);
  4405.                 init_unused_ring(dev, SRB1_BASE);
  4406.                 init_unused_ring(dev, SRB2_BASE);
  4407.                 init_unused_ring(dev, SRB3_BASE);
  4408.         } else if (IS_GEN2(dev)) {
  4409.                 init_unused_ring(dev, SRB0_BASE);
  4410.                 init_unused_ring(dev, SRB1_BASE);
  4411.         } else if (IS_GEN3(dev)) {
  4412.                 init_unused_ring(dev, PRB1_BASE);
  4413.                 init_unused_ring(dev, PRB2_BASE);
  4414.         }
  4415. }
  4416.  
  4417. int i915_gem_init_rings(struct drm_device *dev)
  4418. {
  4419.         struct drm_i915_private *dev_priv = dev->dev_private;
  4420.         int ret;
  4421.  
  4422.         ret = intel_init_render_ring_buffer(dev);
  4423.         if (ret)
  4424.                 return ret;
  4425.  
  4426.         if (HAS_BSD(dev)) {
  4427.                 ret = intel_init_bsd_ring_buffer(dev);
  4428.                 if (ret)
  4429.                         goto cleanup_render_ring;
  4430.         }
  4431.  
  4432.         if (HAS_BLT(dev)) {
  4433.                 ret = intel_init_blt_ring_buffer(dev);
  4434.                 if (ret)
  4435.                         goto cleanup_bsd_ring;
  4436.         }
  4437.  
  4438.         if (HAS_VEBOX(dev)) {
  4439.                 ret = intel_init_vebox_ring_buffer(dev);
  4440.                 if (ret)
  4441.                         goto cleanup_blt_ring;
  4442.         }
  4443.  
  4444.         if (HAS_BSD2(dev)) {
  4445.                 ret = intel_init_bsd2_ring_buffer(dev);
  4446.                 if (ret)
  4447.                         goto cleanup_vebox_ring;
  4448.         }
  4449.  
  4450.         return 0;
  4451.  
  4452. cleanup_vebox_ring:
  4453.         intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
  4454. cleanup_blt_ring:
  4455.         intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
  4456. cleanup_bsd_ring:
  4457.         intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
  4458. cleanup_render_ring:
  4459.         intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
  4460.  
  4461.         return ret;
  4462. }
  4463.  
  4464. int
  4465. i915_gem_init_hw(struct drm_device *dev)
  4466. {
  4467.         struct drm_i915_private *dev_priv = dev->dev_private;
  4468.         struct intel_engine_cs *ring;
  4469.         int ret, i, j;
  4470.  
  4471.         if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
  4472.                 return -EIO;
  4473.  
  4474.         /* Double layer security blanket, see i915_gem_init() */
  4475.         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
  4476.  
  4477.         if (dev_priv->ellc_size)
  4478.                 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
  4479.  
  4480.         if (IS_HASWELL(dev))
  4481.                 I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
  4482.                            LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
  4483.  
  4484.         if (HAS_PCH_NOP(dev)) {
  4485.                 if (IS_IVYBRIDGE(dev)) {
  4486.                         u32 temp = I915_READ(GEN7_MSG_CTL);
  4487.                         temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
  4488.                         I915_WRITE(GEN7_MSG_CTL, temp);
  4489.                 } else if (INTEL_INFO(dev)->gen >= 7) {
  4490.                         u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
  4491.                         temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
  4492.                         I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
  4493.                 }
  4494.         }
  4495.  
  4496.         i915_gem_init_swizzling(dev);
  4497.  
  4498.         /*
  4499.          * At least 830 can leave some of the unused rings
  4500.          * "active" (ie. head != tail) after resume which
  4501.          * will prevent c3 entry. Makes sure all unused rings
  4502.          * are totally idle.
  4503.          */
  4504.         init_unused_rings(dev);
  4505.  
  4506.         BUG_ON(!dev_priv->kernel_context);
  4507.  
  4508.         ret = i915_ppgtt_init_hw(dev);
  4509.         if (ret) {
  4510.                 DRM_ERROR("PPGTT enable HW failed %d\n", ret);
  4511.                 goto out;
  4512.         }
  4513.  
  4514.         /* Need to do basic initialisation of all rings first: */
  4515.         for_each_ring(ring, dev_priv, i) {
  4516.                 ret = ring->init_hw(ring);
  4517.                 if (ret)
  4518.                         goto out;
  4519.         }
  4520.  
  4521.         /* We can't enable contexts until all firmware is loaded */
  4522.         if (HAS_GUC_UCODE(dev)) {
  4523.                 ret = intel_guc_ucode_load(dev);
  4524.                 if (ret) {
  4525.                         DRM_ERROR("Failed to initialize GuC, error %d\n", ret);
  4526.                         ret = -EIO;
  4527.                         goto out;
  4528.                 }
  4529.         }
  4530.  
  4531.         /*
  4532.          * Increment the next seqno by 0x100 so we have a visible break
  4533.          * on re-initialisation
  4534.          */
  4535.         ret = i915_gem_set_seqno(dev, dev_priv->next_seqno+0x100);
  4536.         if (ret)
  4537.                 goto out;
  4538.  
  4539.         /* Now it is safe to go back round and do everything else: */
  4540.         for_each_ring(ring, dev_priv, i) {
  4541.                 struct drm_i915_gem_request *req;
  4542.  
  4543.                 req = i915_gem_request_alloc(ring, NULL);
  4544.                 if (IS_ERR(req)) {
  4545.                         ret = PTR_ERR(req);
  4546.                         i915_gem_cleanup_ringbuffer(dev);
  4547.                         goto out;
  4548.                 }
  4549.  
  4550.                 if (ring->id == RCS) {
  4551.                         for (j = 0; j < NUM_L3_SLICES(dev); j++)
  4552.                                 i915_gem_l3_remap(req, j);
  4553.                 }
  4554.  
  4555.                 ret = i915_ppgtt_init_ring(req);
  4556.                 if (ret && ret != -EIO) {
  4557.                         DRM_ERROR("PPGTT enable ring #%d failed %d\n", i, ret);
  4558.                         i915_gem_request_cancel(req);
  4559.                         i915_gem_cleanup_ringbuffer(dev);
  4560.                         goto out;
  4561.                 }
  4562.  
  4563.                 ret = i915_gem_context_enable(req);
  4564.                 if (ret && ret != -EIO) {
  4565.                         DRM_ERROR("Context enable ring #%d failed %d\n", i, ret);
  4566.                         i915_gem_request_cancel(req);
  4567.                         i915_gem_cleanup_ringbuffer(dev);
  4568.                         goto out;
  4569.                 }
  4570.  
  4571.                 i915_add_request_no_flush(req);
  4572.         }
  4573.  
  4574. out:
  4575.         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
  4576.         return ret;
  4577. }
  4578.  
  4579. int i915_gem_init(struct drm_device *dev)
  4580. {
  4581.         struct drm_i915_private *dev_priv = dev->dev_private;
  4582.         int ret;
  4583.  
  4584.         i915.enable_execlists = intel_sanitize_enable_execlists(dev,
  4585.                         i915.enable_execlists);
  4586.  
  4587.         mutex_lock(&dev->struct_mutex);
  4588.  
  4589.         if (!i915.enable_execlists) {
  4590.                 dev_priv->gt.execbuf_submit = i915_gem_ringbuffer_submission;
  4591.                 dev_priv->gt.init_rings = i915_gem_init_rings;
  4592.                 dev_priv->gt.cleanup_ring = intel_cleanup_ring_buffer;
  4593.                 dev_priv->gt.stop_ring = intel_stop_ring_buffer;
  4594.         } else {
  4595.                 dev_priv->gt.execbuf_submit = intel_execlists_submission;
  4596.                 dev_priv->gt.init_rings = intel_logical_rings_init;
  4597.                 dev_priv->gt.cleanup_ring = intel_logical_ring_cleanup;
  4598.                 dev_priv->gt.stop_ring = intel_logical_ring_stop;
  4599.         }
  4600.  
  4601.         /* This is just a security blanket to placate dragons.
  4602.          * On some systems, we very sporadically observe that the first TLBs
  4603.          * used by the CS may be stale, despite us poking the TLB reset. If
  4604.          * we hold the forcewake during initialisation these problems
  4605.          * just magically go away.
  4606.          */
  4607.         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
  4608.  
  4609. //      ret = i915_gem_init_userptr(dev);
  4610. //      if (ret)
  4611. //              goto out_unlock;
  4612.  
  4613.         i915_gem_init_global_gtt(dev);
  4614.  
  4615.         ret = i915_gem_context_init(dev);
  4616.         if (ret)
  4617.                 goto out_unlock;
  4618.  
  4619.         ret = dev_priv->gt.init_rings(dev);
  4620.         if (ret)
  4621.                 goto out_unlock;
  4622.  
  4623.         ret = i915_gem_init_hw(dev);
  4624.         if (ret == -EIO) {
  4625.                 /* Allow ring initialisation to fail by marking the GPU as
  4626.                  * wedged. But we only want to do this where the GPU is angry,
  4627.                  * for all other failure, such as an allocation failure, bail.
  4628.                  */
  4629.                 DRM_ERROR("Failed to initialize GPU, declaring it wedged\n");
  4630.                 atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
  4631.                 ret = 0;
  4632.         }
  4633.  
  4634. out_unlock:
  4635.         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
  4636.         mutex_unlock(&dev->struct_mutex);
  4637.  
  4638.         return ret;
  4639. }
  4640.  
  4641. void
  4642. i915_gem_cleanup_ringbuffer(struct drm_device *dev)
  4643. {
  4644.         struct drm_i915_private *dev_priv = dev->dev_private;
  4645.         struct intel_engine_cs *ring;
  4646.         int i;
  4647.  
  4648.         for_each_ring(ring, dev_priv, i)
  4649.                 dev_priv->gt.cleanup_ring(ring);
  4650.  
  4651.     if (i915.enable_execlists)
  4652.             /*
  4653.              * Neither the BIOS, ourselves or any other kernel
  4654.              * expects the system to be in execlists mode on startup,
  4655.              * so we need to reset the GPU back to legacy mode.
  4656.              */
  4657.             intel_gpu_reset(dev);
  4658. }
  4659.  
  4660. static void
  4661. init_ring_lists(struct intel_engine_cs *ring)
  4662. {
  4663.         INIT_LIST_HEAD(&ring->active_list);
  4664.         INIT_LIST_HEAD(&ring->request_list);
  4665. }
  4666.  
  4667. void
  4668. i915_gem_load_init(struct drm_device *dev)
  4669. {
  4670.         struct drm_i915_private *dev_priv = dev->dev_private;
  4671.         int i;
  4672.  
  4673.         INIT_LIST_HEAD(&dev_priv->vm_list);
  4674.         INIT_LIST_HEAD(&dev_priv->context_list);
  4675.         INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
  4676.         INIT_LIST_HEAD(&dev_priv->mm.bound_list);
  4677.         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
  4678.         for (i = 0; i < I915_NUM_RINGS; i++)
  4679.                 init_ring_lists(&dev_priv->ring[i]);
  4680.         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
  4681.                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
  4682.         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
  4683.                           i915_gem_retire_work_handler);
  4684.         INIT_DELAYED_WORK(&dev_priv->mm.idle_work,
  4685.                           i915_gem_idle_work_handler);
  4686.         init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
  4687.  
  4688.         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
  4689.  
  4690.         if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev))
  4691.                 dev_priv->num_fence_regs = 32;
  4692.         else if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
  4693.                 dev_priv->num_fence_regs = 16;
  4694.         else
  4695.                 dev_priv->num_fence_regs = 8;
  4696.  
  4697.         if (intel_vgpu_active(dev))
  4698.                 dev_priv->num_fence_regs =
  4699.                                 I915_READ(vgtif_reg(avail_rs.fence_num));
  4700.  
  4701.         /*
  4702.          * Set initial sequence number for requests.
  4703.          * Using this number allows the wraparound to happen early,
  4704.          * catching any obvious problems.
  4705.          */
  4706.         dev_priv->next_seqno = ((u32)~0 - 0x1100);
  4707.         dev_priv->last_seqno = ((u32)~0 - 0x1101);
  4708.  
  4709.         /* Initialize fence registers to zero */
  4710.         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
  4711.         i915_gem_restore_fences(dev);
  4712.  
  4713.         i915_gem_detect_bit_6_swizzle(dev);
  4714.         init_waitqueue_head(&dev_priv->pending_flip_queue);
  4715.  
  4716.         dev_priv->mm.interruptible = true;
  4717.  
  4718.         mutex_init(&dev_priv->fb_tracking.lock);
  4719. }
  4720.  
  4721. void i915_gem_release(struct drm_device *dev, struct drm_file *file)
  4722. {
  4723.         struct drm_i915_file_private *file_priv = file->driver_priv;
  4724.  
  4725.         /* Clean up our request list when the client is going away, so that
  4726.          * later retire_requests won't dereference our soon-to-be-gone
  4727.          * file_priv.
  4728.          */
  4729.         spin_lock(&file_priv->mm.lock);
  4730.         while (!list_empty(&file_priv->mm.request_list)) {
  4731.                 struct drm_i915_gem_request *request;
  4732.  
  4733.                 request = list_first_entry(&file_priv->mm.request_list,
  4734.                                            struct drm_i915_gem_request,
  4735.                                            client_list);
  4736.                 list_del(&request->client_list);
  4737.                 request->file_priv = NULL;
  4738.         }
  4739.         spin_unlock(&file_priv->mm.lock);
  4740.  
  4741.         if (!list_empty(&file_priv->rps.link)) {
  4742.                 spin_lock(&to_i915(dev)->rps.client_lock);
  4743.                 list_del(&file_priv->rps.link);
  4744.                 spin_unlock(&to_i915(dev)->rps.client_lock);
  4745.         }
  4746. }
  4747.  
  4748. int i915_gem_open(struct drm_device *dev, struct drm_file *file)
  4749. {
  4750.         struct drm_i915_file_private *file_priv;
  4751.         int ret;
  4752.  
  4753.         DRM_DEBUG_DRIVER("\n");
  4754.  
  4755.         file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
  4756.         if (!file_priv)
  4757.                 return -ENOMEM;
  4758.  
  4759.         file->driver_priv = file_priv;
  4760.         file_priv->dev_priv = dev->dev_private;
  4761.         file_priv->file = file;
  4762.         INIT_LIST_HEAD(&file_priv->rps.link);
  4763.  
  4764.         spin_lock_init(&file_priv->mm.lock);
  4765.         INIT_LIST_HEAD(&file_priv->mm.request_list);
  4766.  
  4767.         file_priv->bsd_ring = -1;
  4768.  
  4769.         ret = i915_gem_context_open(dev, file);
  4770.         if (ret)
  4771.                 kfree(file_priv);
  4772.  
  4773.         return ret;
  4774. }
  4775.  
  4776. /**
  4777.  * i915_gem_track_fb - update frontbuffer tracking
  4778.  * @old: current GEM buffer for the frontbuffer slots
  4779.  * @new: new GEM buffer for the frontbuffer slots
  4780.  * @frontbuffer_bits: bitmask of frontbuffer slots
  4781.  *
  4782.  * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
  4783.  * from @old and setting them in @new. Both @old and @new can be NULL.
  4784.  */
  4785. void i915_gem_track_fb(struct drm_i915_gem_object *old,
  4786.                        struct drm_i915_gem_object *new,
  4787.                        unsigned frontbuffer_bits)
  4788. {
  4789.         if (old) {
  4790.                 WARN_ON(!mutex_is_locked(&old->base.dev->struct_mutex));
  4791.                 WARN_ON(!(old->frontbuffer_bits & frontbuffer_bits));
  4792.                 old->frontbuffer_bits &= ~frontbuffer_bits;
  4793.         }
  4794.  
  4795.         if (new) {
  4796.                 WARN_ON(!mutex_is_locked(&new->base.dev->struct_mutex));
  4797.                 WARN_ON(new->frontbuffer_bits & frontbuffer_bits);
  4798.                 new->frontbuffer_bits |= frontbuffer_bits;
  4799.         }
  4800. }
  4801.  
  4802. /* All the new VM stuff */
  4803. u64 i915_gem_obj_offset(struct drm_i915_gem_object *o,
  4804.                         struct i915_address_space *vm)
  4805. {
  4806.         struct drm_i915_private *dev_priv = o->base.dev->dev_private;
  4807.         struct i915_vma *vma;
  4808.  
  4809.         WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
  4810.  
  4811.         list_for_each_entry(vma, &o->vma_list, obj_link) {
  4812.                 if (vma->is_ggtt &&
  4813.                     vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
  4814.                         continue;
  4815.                 if (vma->vm == vm)
  4816.                         return vma->node.start;
  4817.         }
  4818.  
  4819.         WARN(1, "%s vma for this object not found.\n",
  4820.              i915_is_ggtt(vm) ? "global" : "ppgtt");
  4821.         return -1;
  4822. }
  4823.  
  4824. u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o,
  4825.                                   const struct i915_ggtt_view *view)
  4826. {
  4827.         struct i915_address_space *ggtt = i915_obj_to_ggtt(o);
  4828.         struct i915_vma *vma;
  4829.  
  4830.         list_for_each_entry(vma, &o->vma_list, obj_link)
  4831.                 if (vma->vm == ggtt &&
  4832.                     i915_ggtt_view_equal(&vma->ggtt_view, view))
  4833.                         return vma->node.start;
  4834.  
  4835.         WARN(1, "global vma for this object not found. (view=%u)\n", view->type);
  4836.         return -1;
  4837. }
  4838.  
  4839. bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
  4840.                         struct i915_address_space *vm)
  4841. {
  4842.         struct i915_vma *vma;
  4843.  
  4844.         list_for_each_entry(vma, &o->vma_list, obj_link) {
  4845.                 if (vma->is_ggtt &&
  4846.                     vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
  4847.                         continue;
  4848.                 if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
  4849.                         return true;
  4850.         }
  4851.  
  4852.         return false;
  4853. }
  4854.  
  4855. bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o,
  4856.                                   const struct i915_ggtt_view *view)
  4857. {
  4858.         struct i915_address_space *ggtt = i915_obj_to_ggtt(o);
  4859.         struct i915_vma *vma;
  4860.  
  4861.         list_for_each_entry(vma, &o->vma_list, obj_link)
  4862.                 if (vma->vm == ggtt &&
  4863.                     i915_ggtt_view_equal(&vma->ggtt_view, view) &&
  4864.                     drm_mm_node_allocated(&vma->node))
  4865.                         return true;
  4866.  
  4867.         return false;
  4868. }
  4869.  
  4870. bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
  4871. {
  4872.         struct i915_vma *vma;
  4873.  
  4874.         list_for_each_entry(vma, &o->vma_list, obj_link)
  4875.                 if (drm_mm_node_allocated(&vma->node))
  4876.                         return true;
  4877.  
  4878.         return false;
  4879. }
  4880.  
  4881. unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
  4882.                                 struct i915_address_space *vm)
  4883. {
  4884.         struct drm_i915_private *dev_priv = o->base.dev->dev_private;
  4885.         struct i915_vma *vma;
  4886.  
  4887.         WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
  4888.  
  4889.         BUG_ON(list_empty(&o->vma_list));
  4890.  
  4891.         list_for_each_entry(vma, &o->vma_list, obj_link) {
  4892.                 if (vma->is_ggtt &&
  4893.                     vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
  4894.                         continue;
  4895.                 if (vma->vm == vm)
  4896.                         return vma->node.size;
  4897.         }
  4898.         return 0;
  4899. }
  4900.  
  4901. bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj)
  4902. {
  4903.         struct i915_vma *vma;
  4904.         list_for_each_entry(vma, &obj->vma_list, obj_link)
  4905.                 if (vma->pin_count > 0)
  4906.                         return true;
  4907.  
  4908.         return false;
  4909. }
  4910.  
  4911. /* Like i915_gem_object_get_page(), but mark the returned page dirty */
  4912. struct page *
  4913. i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n)
  4914. {
  4915.         struct page *page;
  4916.  
  4917.         /* Only default objects have per-page dirty tracking */
  4918.         if (WARN_ON((obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE) == 0))
  4919.                 return NULL;
  4920.  
  4921.         page = i915_gem_object_get_page(obj, n);
  4922.         set_page_dirty(page);
  4923.         return page;
  4924. }
  4925.  
  4926. /* Allocate a new GEM object and fill it with the supplied data */
  4927. struct drm_i915_gem_object *
  4928. i915_gem_object_create_from_data(struct drm_device *dev,
  4929.                                  const void *data, size_t size)
  4930. {
  4931.         struct drm_i915_gem_object *obj;
  4932.         struct sg_table *sg;
  4933.         size_t bytes;
  4934.         int ret;
  4935.  
  4936.         obj = i915_gem_alloc_object(dev, round_up(size, PAGE_SIZE));
  4937.         if (IS_ERR_OR_NULL(obj))
  4938.                 return obj;
  4939.  
  4940.         ret = i915_gem_object_set_to_cpu_domain(obj, true);
  4941.         if (ret)
  4942.                 goto fail;
  4943.  
  4944.         ret = i915_gem_object_get_pages(obj);
  4945.         if (ret)
  4946.                 goto fail;
  4947.  
  4948.         i915_gem_object_pin_pages(obj);
  4949.         sg = obj->pages;
  4950.         bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size);
  4951.         obj->dirty = 1;         /* Backing store is now out of date */
  4952.         i915_gem_object_unpin_pages(obj);
  4953.  
  4954.         if (WARN_ON(bytes != size)) {
  4955.                 DRM_ERROR("Incomplete copy, wrote %zu of %zu", bytes, size);
  4956.                 ret = -EFAULT;
  4957.                 goto fail;
  4958.         }
  4959.  
  4960.         return obj;
  4961.  
  4962. fail:
  4963.         drm_gem_object_unreference(&obj->base);
  4964.         return ERR_PTR(ret);
  4965. }
  4966.