Subversion Repositories Kolibri OS

Rev

Rev 3263 | Rev 3290 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright © 2008 Intel Corporation
  3.  *
  4.  * Permission is hereby granted, free of charge, to any person obtaining a
  5.  * copy of this software and associated documentation files (the "Software"),
  6.  * to deal in the Software without restriction, including without limitation
  7.  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8.  * and/or sell copies of the Software, and to permit persons to whom the
  9.  * Software is furnished to do so, subject to the following conditions:
  10.  *
  11.  * The above copyright notice and this permission notice (including the next
  12.  * paragraph) shall be included in all copies or substantial portions of the
  13.  * Software.
  14.  *
  15.  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16.  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17.  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18.  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19.  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20.  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21.  * IN THE SOFTWARE.
  22.  *
  23.  * Authors:
  24.  *    Eric Anholt <eric@anholt.net>
  25.  *
  26.  */
  27.  
  28. #include <drm/drmP.h>
  29. #include <drm/i915_drm.h>
  30. #include "i915_drv.h"
  31. #include "i915_trace.h"
  32. #include "intel_drv.h"
  33. #include <linux/shmem_fs.h>
  34. #include <linux/slab.h>
  35. //#include <linux/swap.h>
  36. #include <linux/pci.h>
  37.  
  38. extern int x86_clflush_size;
  39.  
  40. #define PROT_READ       0x1             /* page can be read */
  41. #define PROT_WRITE      0x2             /* page can be written */
  42. #define MAP_SHARED      0x01            /* Share changes */
  43.  
  44. #undef mb
  45. #undef rmb
  46. #undef wmb
  47. #define mb() asm volatile("mfence")
  48. #define rmb() asm volatile ("lfence")
  49. #define wmb() asm volatile ("sfence")
  50.  
  51. struct drm_i915_gem_object *get_fb_obj();
  52.  
  53. unsigned long vm_mmap(struct file *file, unsigned long addr,
  54.          unsigned long len, unsigned long prot,
  55.          unsigned long flag, unsigned long offset);
  56.  
  57. static inline void clflush(volatile void *__p)
  58. {
  59.     asm volatile("clflush %0" : "+m" (*(volatile char*)__p));
  60. }
  61.  
  62. #define MAX_ERRNO       4095
  63.  
  64. #define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
  65.  
  66.  
  67. #define I915_EXEC_CONSTANTS_MASK        (3<<6)
  68. #define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */
  69. #define I915_EXEC_CONSTANTS_ABSOLUTE    (1<<6)
  70. #define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */
  71.  
  72. static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  73. static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  74. static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  75.                                                     unsigned alignment,
  76.                                                     bool map_and_fenceable,
  77.                                                     bool nonblocking);
  78. static int i915_gem_phys_pwrite(struct drm_device *dev,
  79.                                 struct drm_i915_gem_object *obj,
  80.                                 struct drm_i915_gem_pwrite *args,
  81.                                 struct drm_file *file);
  82.  
  83. static void i915_gem_write_fence(struct drm_device *dev, int reg,
  84.                                  struct drm_i915_gem_object *obj);
  85. static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
  86.                                          struct drm_i915_fence_reg *fence,
  87.                                          bool enable);
  88.  
  89. static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
  90. static void i915_gem_shrink_all(struct drm_i915_private *dev_priv);
  91. static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
  92.  
  93. static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
  94. {
  95.         if (obj->tiling_mode)
  96.                 i915_gem_release_mmap(obj);
  97.  
  98.         /* As we do not have an associated fence register, we will force
  99.          * a tiling change if we ever need to acquire one.
  100.          */
  101.         obj->fence_dirty = false;
  102.         obj->fence_reg = I915_FENCE_REG_NONE;
  103. }
  104.  
  105. /* some bookkeeping */
  106. static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  107.                                   size_t size)
  108. {
  109.         dev_priv->mm.object_count++;
  110.         dev_priv->mm.object_memory += size;
  111. }
  112.  
  113. static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  114.                                      size_t size)
  115. {
  116.         dev_priv->mm.object_count--;
  117.         dev_priv->mm.object_memory -= size;
  118. }
  119.  
  120. static int
  121. i915_gem_wait_for_error(struct drm_device *dev)
  122. {
  123.         struct drm_i915_private *dev_priv = dev->dev_private;
  124.         struct completion *x = &dev_priv->error_completion;
  125.         unsigned long flags;
  126.         int ret;
  127.  
  128.         if (!atomic_read(&dev_priv->mm.wedged))
  129.                 return 0;
  130. #if 0
  131.         /*
  132.          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
  133.          * userspace. If it takes that long something really bad is going on and
  134.          * we should simply try to bail out and fail as gracefully as possible.
  135.          */
  136.         ret = wait_for_completion_interruptible_timeout(x, 10*HZ);
  137.         if (ret == 0) {
  138.                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
  139.                 return -EIO;
  140.         } else if (ret < 0) {
  141.                 return ret;
  142.         }
  143.  
  144.         if (atomic_read(&dev_priv->mm.wedged)) {
  145.                 /* GPU is hung, bump the completion count to account for
  146.                  * the token we just consumed so that we never hit zero and
  147.                  * end up waiting upon a subsequent completion event that
  148.                  * will never happen.
  149.                  */
  150.                 spin_lock_irqsave(&x->wait.lock, flags);
  151.                 x->done++;
  152.                 spin_unlock_irqrestore(&x->wait.lock, flags);
  153.         }
  154. #endif
  155.  
  156.         return 0;
  157. }
  158.  
  159. int i915_mutex_lock_interruptible(struct drm_device *dev)
  160. {
  161.         int ret;
  162.  
  163.         ret = i915_gem_wait_for_error(dev);
  164.         if (ret)
  165.                 return ret;
  166.  
  167.     mutex_lock(&dev->struct_mutex);
  168.  
  169.         WARN_ON(i915_verify_lists(dev));
  170.         return 0;
  171. }
  172.  
  173. static inline bool
  174. i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
  175. {
  176.         return obj->gtt_space && !obj->active;
  177. }
  178.  
  179.  
  180. #if 0
  181.  
  182. int
  183. i915_gem_init_ioctl(struct drm_device *dev, void *data,
  184.                     struct drm_file *file)
  185. {
  186.         struct drm_i915_gem_init *args = data;
  187.  
  188.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  189.                 return -ENODEV;
  190.  
  191.         if (args->gtt_start >= args->gtt_end ||
  192.             (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
  193.                 return -EINVAL;
  194.  
  195.         /* GEM with user mode setting was never supported on ilk and later. */
  196.         if (INTEL_INFO(dev)->gen >= 5)
  197.                 return -ENODEV;
  198.  
  199.         mutex_lock(&dev->struct_mutex);
  200.         i915_gem_init_global_gtt(dev, args->gtt_start,
  201.                                  args->gtt_end, args->gtt_end);
  202.         mutex_unlock(&dev->struct_mutex);
  203.  
  204.         return 0;
  205. }
  206. #endif
  207.  
  208. int
  209. i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  210.                             struct drm_file *file)
  211. {
  212.         struct drm_i915_private *dev_priv = dev->dev_private;
  213.         struct drm_i915_gem_get_aperture *args = data;
  214.         struct drm_i915_gem_object *obj;
  215.         size_t pinned;
  216.  
  217.         pinned = 0;
  218.         mutex_lock(&dev->struct_mutex);
  219.         list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
  220.                 if (obj->pin_count)
  221.                 pinned += obj->gtt_space->size;
  222.         mutex_unlock(&dev->struct_mutex);
  223.  
  224.         args->aper_size = dev_priv->mm.gtt_total;
  225.         args->aper_available_size = args->aper_size - pinned;
  226.  
  227.         return 0;
  228. }
  229.  
  230. static int
  231. i915_gem_create(struct drm_file *file,
  232.                 struct drm_device *dev,
  233.                 uint64_t size,
  234.                 uint32_t *handle_p)
  235. {
  236.         struct drm_i915_gem_object *obj;
  237.         int ret;
  238.         u32 handle;
  239.  
  240.         size = roundup(size, PAGE_SIZE);
  241.         if (size == 0)
  242.                 return -EINVAL;
  243.  
  244.         /* Allocate the new object */
  245.         obj = i915_gem_alloc_object(dev, size);
  246.         if (obj == NULL)
  247.                 return -ENOMEM;
  248.  
  249.         ret = drm_gem_handle_create(file, &obj->base, &handle);
  250.         if (ret) {
  251.                 drm_gem_object_release(&obj->base);
  252.                 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
  253.                 kfree(obj);
  254.                 return ret;
  255.         }
  256.  
  257.         /* drop reference from allocate - handle holds it now */
  258.         drm_gem_object_unreference(&obj->base);
  259.         trace_i915_gem_object_create(obj);
  260.  
  261.         *handle_p = handle;
  262.         return 0;
  263. }
  264.  
  265. int
  266. i915_gem_dumb_create(struct drm_file *file,
  267.                      struct drm_device *dev,
  268.                      struct drm_mode_create_dumb *args)
  269. {
  270.         /* have to work out size/pitch and return them */
  271.         args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
  272.         args->size = args->pitch * args->height;
  273.         return i915_gem_create(file, dev,
  274.                                args->size, &args->handle);
  275. }
  276.  
  277. int i915_gem_dumb_destroy(struct drm_file *file,
  278.                           struct drm_device *dev,
  279.                           uint32_t handle)
  280. {
  281.         return drm_gem_handle_delete(file, handle);
  282. }
  283.  
  284. /**
  285.  * Creates a new mm object and returns a handle to it.
  286.  */
  287. int
  288. i915_gem_create_ioctl(struct drm_device *dev, void *data,
  289.                       struct drm_file *file)
  290. {
  291.         struct drm_i915_gem_create *args = data;
  292.  
  293.         return i915_gem_create(file, dev,
  294.                                args->size, &args->handle);
  295. }
  296.  
  297. static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
  298. {
  299.         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
  300.  
  301.         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
  302.                 obj->tiling_mode != I915_TILING_NONE;
  303. }
  304. #if 0
  305.  
  306. static inline int
  307. __copy_to_user_swizzled(char __user *cpu_vaddr,
  308.                         const char *gpu_vaddr, int gpu_offset,
  309.                 int length)
  310. {
  311.         int ret, cpu_offset = 0;
  312.  
  313.         while (length > 0) {
  314.                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
  315.                 int this_length = min(cacheline_end - gpu_offset, length);
  316.                 int swizzled_gpu_offset = gpu_offset ^ 64;
  317.  
  318.                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
  319.                                      gpu_vaddr + swizzled_gpu_offset,
  320.                                      this_length);
  321.                 if (ret)
  322.                         return ret + length;
  323.  
  324.                 cpu_offset += this_length;
  325.                 gpu_offset += this_length;
  326.                 length -= this_length;
  327.         }
  328.  
  329.         return 0;
  330. }
  331.  
  332. static inline int
  333. __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
  334.                           const char __user *cpu_vaddr,
  335.                           int length)
  336. {
  337.         int ret, cpu_offset = 0;
  338.  
  339.         while (length > 0) {
  340.                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
  341.                 int this_length = min(cacheline_end - gpu_offset, length);
  342.                 int swizzled_gpu_offset = gpu_offset ^ 64;
  343.  
  344.                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
  345.                                cpu_vaddr + cpu_offset,
  346.                                this_length);
  347.                 if (ret)
  348.                         return ret + length;
  349.  
  350.                 cpu_offset += this_length;
  351.                 gpu_offset += this_length;
  352.                 length -= this_length;
  353.         }
  354.  
  355.         return 0;
  356. }
  357.  
  358. /* Per-page copy function for the shmem pread fastpath.
  359.  * Flushes invalid cachelines before reading the target if
  360.  * needs_clflush is set. */
  361. static int
  362. shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
  363.                  char __user *user_data,
  364.                  bool page_do_bit17_swizzling, bool needs_clflush)
  365. {
  366.                 char *vaddr;
  367.                 int ret;
  368.  
  369.         if (unlikely(page_do_bit17_swizzling))
  370.                 return -EINVAL;
  371.  
  372.                 vaddr = kmap_atomic(page);
  373.         if (needs_clflush)
  374.                 drm_clflush_virt_range(vaddr + shmem_page_offset,
  375.                                        page_length);
  376.                 ret = __copy_to_user_inatomic(user_data,
  377.                                       vaddr + shmem_page_offset,
  378.                                               page_length);
  379.                 kunmap_atomic(vaddr);
  380.  
  381.         return ret ? -EFAULT : 0;
  382. }
  383.  
  384. static void
  385. shmem_clflush_swizzled_range(char *addr, unsigned long length,
  386.                              bool swizzled)
  387. {
  388.         if (unlikely(swizzled)) {
  389.                 unsigned long start = (unsigned long) addr;
  390.                 unsigned long end = (unsigned long) addr + length;
  391.  
  392.                 /* For swizzling simply ensure that we always flush both
  393.                  * channels. Lame, but simple and it works. Swizzled
  394.                  * pwrite/pread is far from a hotpath - current userspace
  395.                  * doesn't use it at all. */
  396.                 start = round_down(start, 128);
  397.                 end = round_up(end, 128);
  398.  
  399.                 drm_clflush_virt_range((void *)start, end - start);
  400.         } else {
  401.                 drm_clflush_virt_range(addr, length);
  402.         }
  403.  
  404. }
  405.  
  406. /* Only difference to the fast-path function is that this can handle bit17
  407.  * and uses non-atomic copy and kmap functions. */
  408. static int
  409. shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
  410.                  char __user *user_data,
  411.                  bool page_do_bit17_swizzling, bool needs_clflush)
  412. {
  413.         char *vaddr;
  414.         int ret;
  415.  
  416.         vaddr = kmap(page);
  417.         if (needs_clflush)
  418.                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  419.                                              page_length,
  420.                                              page_do_bit17_swizzling);
  421.  
  422.         if (page_do_bit17_swizzling)
  423.                 ret = __copy_to_user_swizzled(user_data,
  424.                                               vaddr, shmem_page_offset,
  425.                                               page_length);
  426.         else
  427.                 ret = __copy_to_user(user_data,
  428.                                      vaddr + shmem_page_offset,
  429.                                      page_length);
  430.         kunmap(page);
  431.  
  432.         return ret ? - EFAULT : 0;
  433. }
  434.  
  435. static int
  436. i915_gem_shmem_pread(struct drm_device *dev,
  437.                           struct drm_i915_gem_object *obj,
  438.                           struct drm_i915_gem_pread *args,
  439.                           struct drm_file *file)
  440. {
  441.         char __user *user_data;
  442.         ssize_t remain;
  443.         loff_t offset;
  444.         int shmem_page_offset, page_length, ret = 0;
  445.         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
  446.         int hit_slowpath = 0;
  447.         int prefaulted = 0;
  448.         int needs_clflush = 0;
  449.         struct scatterlist *sg;
  450.         int i;
  451.  
  452.         user_data = (char __user *) (uintptr_t) args->data_ptr;
  453.         remain = args->size;
  454.  
  455.         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
  456.  
  457.         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
  458.                 /* If we're not in the cpu read domain, set ourself into the gtt
  459.                  * read domain and manually flush cachelines (if required). This
  460.                  * optimizes for the case when the gpu will dirty the data
  461.                  * anyway again before the next pread happens. */
  462.                 if (obj->cache_level == I915_CACHE_NONE)
  463.                         needs_clflush = 1;
  464.                 if (obj->gtt_space) {
  465.                         ret = i915_gem_object_set_to_gtt_domain(obj, false);
  466.                         if (ret)
  467.                                 return ret;
  468.                 }
  469.         }
  470.  
  471.         ret = i915_gem_object_get_pages(obj);
  472.         if (ret)
  473.                 return ret;
  474.  
  475.         i915_gem_object_pin_pages(obj);
  476.  
  477.         offset = args->offset;
  478.  
  479.         for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
  480.                 struct page *page;
  481.  
  482.                 if (i < offset >> PAGE_SHIFT)
  483.                         continue;
  484.  
  485.                 if (remain <= 0)
  486.                         break;
  487.  
  488.                 /* Operation in this page
  489.                  *
  490.                  * shmem_page_offset = offset within page in shmem file
  491.                  * page_length = bytes to copy for this page
  492.                  */
  493.                 shmem_page_offset = offset_in_page(offset);
  494.                 page_length = remain;
  495.                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
  496.                         page_length = PAGE_SIZE - shmem_page_offset;
  497.  
  498.                 page = sg_page(sg);
  499.                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
  500.                         (page_to_phys(page) & (1 << 17)) != 0;
  501.  
  502.                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
  503.                                        user_data, page_do_bit17_swizzling,
  504.                                        needs_clflush);
  505.                 if (ret == 0)
  506.                         goto next_page;
  507.  
  508.                 hit_slowpath = 1;
  509.                 mutex_unlock(&dev->struct_mutex);
  510.  
  511.                 if (!prefaulted) {
  512.                         ret = fault_in_multipages_writeable(user_data, remain);
  513.                         /* Userspace is tricking us, but we've already clobbered
  514.                          * its pages with the prefault and promised to write the
  515.                          * data up to the first fault. Hence ignore any errors
  516.                          * and just continue. */
  517.                         (void)ret;
  518.                         prefaulted = 1;
  519.                 }
  520.  
  521.                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
  522.                                        user_data, page_do_bit17_swizzling,
  523.                                        needs_clflush);
  524.  
  525.                 mutex_lock(&dev->struct_mutex);
  526.  
  527. next_page:
  528.                 mark_page_accessed(page);
  529.  
  530.                 if (ret)
  531.                         goto out;
  532.  
  533.                 remain -= page_length;
  534.                 user_data += page_length;
  535.                 offset += page_length;
  536.         }
  537.  
  538. out:
  539.         i915_gem_object_unpin_pages(obj);
  540.  
  541.         if (hit_slowpath) {
  542.                 /* Fixup: Kill any reinstated backing storage pages */
  543.                 if (obj->madv == __I915_MADV_PURGED)
  544.                         i915_gem_object_truncate(obj);
  545.         }
  546.  
  547.         return ret;
  548. }
  549.  
  550. /**
  551.  * Reads data from the object referenced by handle.
  552.  *
  553.  * On error, the contents of *data are undefined.
  554.  */
  555. int
  556. i915_gem_pread_ioctl(struct drm_device *dev, void *data,
  557.                      struct drm_file *file)
  558. {
  559.         struct drm_i915_gem_pread *args = data;
  560.         struct drm_i915_gem_object *obj;
  561.         int ret = 0;
  562.  
  563.         if (args->size == 0)
  564.                 return 0;
  565.  
  566.         if (!access_ok(VERIFY_WRITE,
  567.                        (char __user *)(uintptr_t)args->data_ptr,
  568.                        args->size))
  569.                 return -EFAULT;
  570.  
  571.         ret = i915_mutex_lock_interruptible(dev);
  572.         if (ret)
  573.                 return ret;
  574.  
  575.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  576.         if (&obj->base == NULL) {
  577.                 ret = -ENOENT;
  578.                 goto unlock;
  579.         }
  580.  
  581.         /* Bounds check source.  */
  582.         if (args->offset > obj->base.size ||
  583.             args->size > obj->base.size - args->offset) {
  584.                 ret = -EINVAL;
  585.                 goto out;
  586.         }
  587.  
  588.         /* prime objects have no backing filp to GEM pread/pwrite
  589.          * pages from.
  590.          */
  591.         if (!obj->base.filp) {
  592.                 ret = -EINVAL;
  593.                 goto out;
  594.         }
  595.  
  596.         trace_i915_gem_object_pread(obj, args->offset, args->size);
  597.  
  598.         ret = i915_gem_shmem_pread(dev, obj, args, file);
  599.  
  600. out:
  601.         drm_gem_object_unreference(&obj->base);
  602. unlock:
  603.         mutex_unlock(&dev->struct_mutex);
  604.         return ret;
  605. }
  606.  
  607. /* This is the fast write path which cannot handle
  608.  * page faults in the source data
  609.  */
  610.  
  611. static inline int
  612. fast_user_write(struct io_mapping *mapping,
  613.                 loff_t page_base, int page_offset,
  614.                 char __user *user_data,
  615.                 int length)
  616. {
  617.         void __iomem *vaddr_atomic;
  618.         void *vaddr;
  619.         unsigned long unwritten;
  620.  
  621.         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
  622.         /* We can use the cpu mem copy function because this is X86. */
  623.         vaddr = (void __force*)vaddr_atomic + page_offset;
  624.         unwritten = __copy_from_user_inatomic_nocache(vaddr,
  625.                                                       user_data, length);
  626.         io_mapping_unmap_atomic(vaddr_atomic);
  627.         return unwritten;
  628. }
  629. #endif
  630.  
  631. #define offset_in_page(p)       ((unsigned long)(p) & ~PAGE_MASK)
  632. /**
  633.  * This is the fast pwrite path, where we copy the data directly from the
  634.  * user into the GTT, uncached.
  635.  */
  636. static int
  637. i915_gem_gtt_pwrite_fast(struct drm_device *dev,
  638.                          struct drm_i915_gem_object *obj,
  639.                          struct drm_i915_gem_pwrite *args,
  640.                          struct drm_file *file)
  641. {
  642.         drm_i915_private_t *dev_priv = dev->dev_private;
  643.         ssize_t remain;
  644.         loff_t offset, page_base;
  645.         char __user *user_data;
  646.         int page_offset, page_length, ret;
  647.     char *vaddr;
  648.  
  649.         ret = i915_gem_object_pin(obj, 0, true, true);
  650.         if (ret)
  651.                 goto out;
  652.  
  653.         ret = i915_gem_object_set_to_gtt_domain(obj, true);
  654.         if (ret)
  655.                 goto out_unpin;
  656.  
  657.         ret = i915_gem_object_put_fence(obj);
  658.         if (ret)
  659.                 goto out_unpin;
  660.  
  661.     vaddr = AllocKernelSpace(4096);
  662.     if(vaddr == NULL)
  663.     {
  664.         ret = -ENOSPC;
  665.         goto out_unpin;
  666.     };
  667.  
  668.         user_data = (char __user *) (uintptr_t) args->data_ptr;
  669.         remain = args->size;
  670.  
  671.         offset = obj->gtt_offset + args->offset;
  672.  
  673.         while (remain > 0) {
  674.                 /* Operation in this page
  675.                  *
  676.                  * page_base = page offset within aperture
  677.                  * page_offset = offset within page
  678.                  * page_length = bytes to copy for this page
  679.                  */
  680.                 page_base = offset & PAGE_MASK;
  681.                 page_offset = offset_in_page(offset);
  682.                 page_length = remain;
  683.                 if ((page_offset + remain) > PAGE_SIZE)
  684.                         page_length = PAGE_SIZE - page_offset;
  685.  
  686.         MapPage(vaddr, page_base, PG_SW|PG_NOCACHE);
  687.  
  688.         memcpy(vaddr+page_offset, user_data, page_length);
  689.  
  690.                 remain -= page_length;
  691.                 user_data += page_length;
  692.                 offset += page_length;
  693.         }
  694.  
  695.     FreeKernelSpace(vaddr);
  696.  
  697. out_unpin:
  698.         i915_gem_object_unpin(obj);
  699. out:
  700.     printf("% s ret = %d\n", __FUNCTION__, ret);
  701.  
  702.         return ret;
  703. }
  704.  
  705. /* Per-page copy function for the shmem pwrite fastpath.
  706.  * Flushes invalid cachelines before writing to the target if
  707.  * needs_clflush_before is set and flushes out any written cachelines after
  708.  * writing if needs_clflush is set. */
  709. static int
  710. shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
  711.                   char __user *user_data,
  712.                   bool page_do_bit17_swizzling,
  713.                   bool needs_clflush_before,
  714.                   bool needs_clflush_after)
  715. {
  716.         char *vaddr;
  717.         int ret = 0;
  718.  
  719.         if (unlikely(page_do_bit17_swizzling))
  720.                 return -EINVAL;
  721.  
  722.         vaddr = (char *)MapIoMem((addr_t)page, 4096, PG_SW);
  723.         if (needs_clflush_before)
  724.                 drm_clflush_virt_range(vaddr + shmem_page_offset,
  725.                                        page_length);
  726.         memcpy(vaddr + shmem_page_offset,
  727.                                                 user_data,
  728.                                                 page_length);
  729.         if (needs_clflush_after)
  730.                 drm_clflush_virt_range(vaddr + shmem_page_offset,
  731.                                        page_length);
  732.         FreeKernelSpace(vaddr);
  733.  
  734.         return ret ? -EFAULT : 0;
  735. }
  736. #if 0
  737.  
  738. /* Only difference to the fast-path function is that this can handle bit17
  739.  * and uses non-atomic copy and kmap functions. */
  740. static int
  741. shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
  742.                   char __user *user_data,
  743.                   bool page_do_bit17_swizzling,
  744.                   bool needs_clflush_before,
  745.                   bool needs_clflush_after)
  746. {
  747.         char *vaddr;
  748.         int ret;
  749.  
  750.         vaddr = kmap(page);
  751.         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
  752.                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  753.                                              page_length,
  754.                                              page_do_bit17_swizzling);
  755.         if (page_do_bit17_swizzling)
  756.                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
  757.                                                 user_data,
  758.                                                 page_length);
  759.         else
  760.                 ret = __copy_from_user(vaddr + shmem_page_offset,
  761.                                        user_data,
  762.                                        page_length);
  763.         if (needs_clflush_after)
  764.                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  765.                                              page_length,
  766.                                              page_do_bit17_swizzling);
  767.         kunmap(page);
  768.  
  769.         return ret ? -EFAULT : 0;
  770. }
  771. #endif
  772.  
  773.  
  774. static int
  775. i915_gem_shmem_pwrite(struct drm_device *dev,
  776.                       struct drm_i915_gem_object *obj,
  777.                       struct drm_i915_gem_pwrite *args,
  778.                       struct drm_file *file)
  779. {
  780.         ssize_t remain;
  781.         loff_t offset;
  782.         char __user *user_data;
  783.         int shmem_page_offset, page_length, ret = 0;
  784.         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
  785.         int hit_slowpath = 0;
  786.         int needs_clflush_after = 0;
  787.         int needs_clflush_before = 0;
  788.         int i;
  789.         struct scatterlist *sg;
  790.  
  791.         user_data = (char __user *) (uintptr_t) args->data_ptr;
  792.         remain = args->size;
  793.  
  794.         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
  795.  
  796.         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  797.                 /* If we're not in the cpu write domain, set ourself into the gtt
  798.                  * write domain and manually flush cachelines (if required). This
  799.                  * optimizes for the case when the gpu will use the data
  800.                  * right away and we therefore have to clflush anyway. */
  801.                 if (obj->cache_level == I915_CACHE_NONE)
  802.                         needs_clflush_after = 1;
  803.                 if (obj->gtt_space) {
  804.                         ret = i915_gem_object_set_to_gtt_domain(obj, true);
  805.                         if (ret)
  806.                                 return ret;
  807.                 }
  808.         }
  809.         /* Same trick applies for invalidate partially written cachelines before
  810.          * writing.  */
  811.         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
  812.             && obj->cache_level == I915_CACHE_NONE)
  813.                 needs_clflush_before = 1;
  814.  
  815.         ret = i915_gem_object_get_pages(obj);
  816.         if (ret)
  817.                 return ret;
  818.  
  819.         i915_gem_object_pin_pages(obj);
  820.  
  821.         offset = args->offset;
  822.         obj->dirty = 1;
  823.  
  824.         for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
  825.                 struct page *page;
  826.                 int partial_cacheline_write;
  827.  
  828.                 if (i < offset >> PAGE_SHIFT)
  829.                         continue;
  830.  
  831.                 if (remain <= 0)
  832.                         break;
  833.  
  834.                 /* Operation in this page
  835.                  *
  836.                  * shmem_page_offset = offset within page in shmem file
  837.                  * page_length = bytes to copy for this page
  838.                  */
  839.                 shmem_page_offset = offset_in_page(offset);
  840.  
  841.                 page_length = remain;
  842.                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
  843.                         page_length = PAGE_SIZE - shmem_page_offset;
  844.  
  845.                 /* If we don't overwrite a cacheline completely we need to be
  846.                  * careful to have up-to-date data by first clflushing. Don't
  847.                  * overcomplicate things and flush the entire patch. */
  848.                 partial_cacheline_write = needs_clflush_before &&
  849.                         ((shmem_page_offset | page_length)
  850.                                 & (x86_clflush_size - 1));
  851.  
  852.                 page = sg_page(sg);
  853.                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
  854.                         (page_to_phys(page) & (1 << 17)) != 0;
  855.  
  856.                 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
  857.                                         user_data, page_do_bit17_swizzling,
  858.                                         partial_cacheline_write,
  859.                                         needs_clflush_after);
  860.                 if (ret == 0)
  861.                         goto next_page;
  862.  
  863.                 hit_slowpath = 1;
  864.                 mutex_unlock(&dev->struct_mutex);
  865.                 dbgprintf("%s need shmem_pwrite_slow\n",__FUNCTION__);
  866.  
  867. //              ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
  868. //                                      user_data, page_do_bit17_swizzling,
  869. //                                      partial_cacheline_write,
  870. //                                      needs_clflush_after);
  871.  
  872.                 mutex_lock(&dev->struct_mutex);
  873.  
  874. next_page:
  875.  
  876.                 if (ret)
  877.                         goto out;
  878.  
  879.                 remain -= page_length;
  880.                 user_data += page_length;
  881.                 offset += page_length;
  882.         }
  883.  
  884. out:
  885.         i915_gem_object_unpin_pages(obj);
  886.  
  887.         if (hit_slowpath) {
  888.                 /* Fixup: Kill any reinstated backing storage pages */
  889.                 if (obj->madv == __I915_MADV_PURGED)
  890.                         i915_gem_object_truncate(obj);
  891.                 /* and flush dirty cachelines in case the object isn't in the cpu write
  892.                  * domain anymore. */
  893.                 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  894.                         i915_gem_clflush_object(obj);
  895.                         i915_gem_chipset_flush(dev);
  896.                 }
  897.         }
  898.  
  899.         if (needs_clflush_after)
  900.                 i915_gem_chipset_flush(dev);
  901.  
  902.         return ret;
  903. }
  904.  
  905. /**
  906.  * Writes data to the object referenced by handle.
  907.  *
  908.  * On error, the contents of the buffer that were to be modified are undefined.
  909.  */
  910. int
  911. i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
  912.                       struct drm_file *file)
  913. {
  914.         struct drm_i915_gem_pwrite *args = data;
  915.         struct drm_i915_gem_object *obj;
  916.         int ret;
  917.  
  918.         if (args->size == 0)
  919.                 return 0;
  920.  
  921.         ret = i915_mutex_lock_interruptible(dev);
  922.         if (ret)
  923.                 return ret;
  924.  
  925.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  926.         if (&obj->base == NULL) {
  927.                 ret = -ENOENT;
  928.                 goto unlock;
  929.         }
  930.  
  931.         /* Bounds check destination. */
  932.         if (args->offset > obj->base.size ||
  933.             args->size > obj->base.size - args->offset) {
  934.                 ret = -EINVAL;
  935.                 goto out;
  936.         }
  937.  
  938.         /* prime objects have no backing filp to GEM pread/pwrite
  939.          * pages from.
  940.          */
  941.         if (!obj->base.filp) {
  942.                 ret = -EINVAL;
  943.                 goto out;
  944.         }
  945.  
  946.         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
  947.  
  948.         ret = -EFAULT;
  949.         /* We can only do the GTT pwrite on untiled buffers, as otherwise
  950.          * it would end up going through the fenced access, and we'll get
  951.          * different detiling behavior between reading and writing.
  952.          * pread/pwrite currently are reading and writing from the CPU
  953.          * perspective, requiring manual detiling by the client.
  954.          */
  955. //   if (obj->phys_obj) {
  956. //       ret = i915_gem_phys_pwrite(dev, obj, args, file);
  957. //       goto out;
  958. //   }
  959.  
  960.         if (obj->cache_level == I915_CACHE_NONE &&
  961.             obj->tiling_mode == I915_TILING_NONE &&
  962.             obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  963.                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
  964.                 /* Note that the gtt paths might fail with non-page-backed user
  965.                  * pointers (e.g. gtt mappings when moving data between
  966.                  * textures). Fallback to the shmem path in that case. */
  967.         }
  968.  
  969.         if (ret == -EFAULT || ret == -ENOSPC)
  970.        ret = i915_gem_shmem_pwrite(dev, obj, args, file);
  971.  
  972. out:
  973.         drm_gem_object_unreference(&obj->base);
  974. unlock:
  975.         mutex_unlock(&dev->struct_mutex);
  976.         return ret;
  977. }
  978.  
  979. int
  980. i915_gem_check_wedge(struct drm_i915_private *dev_priv,
  981.                      bool interruptible)
  982. {
  983.         if (atomic_read(&dev_priv->mm.wedged)) {
  984.                 struct completion *x = &dev_priv->error_completion;
  985.                 bool recovery_complete;
  986.                 unsigned long flags;
  987.  
  988.                 /* Give the error handler a chance to run. */
  989.                 spin_lock_irqsave(&x->wait.lock, flags);
  990.                 recovery_complete = x->done > 0;
  991.                 spin_unlock_irqrestore(&x->wait.lock, flags);
  992.  
  993.                 /* Non-interruptible callers can't handle -EAGAIN, hence return
  994.                  * -EIO unconditionally for these. */
  995.                 if (!interruptible)
  996.                         return -EIO;
  997.  
  998.                 /* Recovery complete, but still wedged means reset failure. */
  999.                 if (recovery_complete)
  1000.                         return -EIO;
  1001.  
  1002.                 return -EAGAIN;
  1003.         }
  1004.  
  1005.         return 0;
  1006. }
  1007.  
  1008. /*
  1009.  * Compare seqno against outstanding lazy request. Emit a request if they are
  1010.  * equal.
  1011.  */
  1012. static int
  1013. i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
  1014. {
  1015.         int ret;
  1016.  
  1017.         BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
  1018.  
  1019.         ret = 0;
  1020.         if (seqno == ring->outstanding_lazy_request)
  1021.                 ret = i915_add_request(ring, NULL, NULL);
  1022.  
  1023.         return ret;
  1024. }
  1025.  
  1026. /**
  1027.  * __wait_seqno - wait until execution of seqno has finished
  1028.  * @ring: the ring expected to report seqno
  1029.  * @seqno: duh!
  1030.  * @interruptible: do an interruptible wait (normally yes)
  1031.  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
  1032.  *
  1033.  * Returns 0 if the seqno was found within the alloted time. Else returns the
  1034.  * errno with remaining time filled in timeout argument.
  1035.  */
  1036. static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
  1037.                         bool interruptible, struct timespec *timeout)
  1038. {
  1039.         drm_i915_private_t *dev_priv = ring->dev->dev_private;
  1040.         struct timespec before, now, wait_time={1,0};
  1041.         unsigned long timeout_jiffies;
  1042.         long end;
  1043.         bool wait_forever = true;
  1044.         int ret;
  1045.  
  1046.         if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
  1047.                 return 0;
  1048.  
  1049.         trace_i915_gem_request_wait_begin(ring, seqno);
  1050.  
  1051.         if (timeout != NULL) {
  1052.                 wait_time = *timeout;
  1053.                 wait_forever = false;
  1054.         }
  1055.  
  1056.         timeout_jiffies = timespec_to_jiffies(&wait_time);
  1057.  
  1058.         if (WARN_ON(!ring->irq_get(ring)))
  1059.                 return -ENODEV;
  1060.  
  1061.     /* Record current time in case interrupted by signal, or wedged * */
  1062.         getrawmonotonic(&before);
  1063.  
  1064. #define EXIT_COND \
  1065.         (i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \
  1066.         atomic_read(&dev_priv->mm.wedged))
  1067.         do {
  1068.                 if (interruptible)
  1069.                         end = wait_event_interruptible_timeout(ring->irq_queue,
  1070.                                                                EXIT_COND,
  1071.                                                                timeout_jiffies);
  1072.                 else
  1073.                         end = wait_event_timeout(ring->irq_queue, EXIT_COND,
  1074.                                                  timeout_jiffies);
  1075.  
  1076.                 ret = i915_gem_check_wedge(dev_priv, interruptible);
  1077.                 if (ret)
  1078.                         end = ret;
  1079.         } while (end == 0 && wait_forever);
  1080.  
  1081.         getrawmonotonic(&now);
  1082.  
  1083.         ring->irq_put(ring);
  1084.         trace_i915_gem_request_wait_end(ring, seqno);
  1085. #undef EXIT_COND
  1086.  
  1087.         if (timeout) {
  1088. //       struct timespec sleep_time = timespec_sub(now, before);
  1089. //       *timeout = timespec_sub(*timeout, sleep_time);
  1090.         }
  1091.  
  1092.         switch (end) {
  1093.         case -EIO:
  1094.         case -EAGAIN: /* Wedged */
  1095.         case -ERESTARTSYS: /* Signal */
  1096.                 return (int)end;
  1097.         case 0: /* Timeout */
  1098.                 if (timeout)
  1099.                         set_normalized_timespec(timeout, 0, 0);
  1100.                 return -ETIME;
  1101.         default: /* Completed */
  1102.                 WARN_ON(end < 0); /* We're not aware of other errors */
  1103.                 return 0;
  1104.         }
  1105. }
  1106.  
  1107. /**
  1108.  * Waits for a sequence number to be signaled, and cleans up the
  1109.  * request and object lists appropriately for that event.
  1110.  */
  1111. int
  1112. i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
  1113. {
  1114.         struct drm_device *dev = ring->dev;
  1115.         struct drm_i915_private *dev_priv = dev->dev_private;
  1116.         bool interruptible = dev_priv->mm.interruptible;
  1117.         int ret;
  1118.  
  1119.         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
  1120.         BUG_ON(seqno == 0);
  1121.  
  1122.         ret = i915_gem_check_wedge(dev_priv, interruptible);
  1123.         if (ret)
  1124.                 return ret;
  1125.  
  1126.         ret = i915_gem_check_olr(ring, seqno);
  1127.         if (ret)
  1128.                 return ret;
  1129.  
  1130.         return __wait_seqno(ring, seqno, interruptible, NULL);
  1131. }
  1132.  
  1133. /**
  1134.  * Ensures that all rendering to the object has completed and the object is
  1135.  * safe to unbind from the GTT or access from the CPU.
  1136.  */
  1137. static __must_check int
  1138. i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
  1139.                                bool readonly)
  1140. {
  1141.         struct intel_ring_buffer *ring = obj->ring;
  1142.         u32 seqno;
  1143.         int ret;
  1144.  
  1145.         seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
  1146.         if (seqno == 0)
  1147.                 return 0;
  1148.  
  1149.         ret = i915_wait_seqno(ring, seqno);
  1150.        if (ret)
  1151.            return ret;
  1152.  
  1153.         i915_gem_retire_requests_ring(ring);
  1154.  
  1155.         /* Manually manage the write flush as we may have not yet
  1156.          * retired the buffer.
  1157.          */
  1158.         if (obj->last_write_seqno &&
  1159.             i915_seqno_passed(seqno, obj->last_write_seqno)) {
  1160.                 obj->last_write_seqno = 0;
  1161.                 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
  1162.         }
  1163.  
  1164.         return 0;
  1165. }
  1166.  
  1167. /* A nonblocking variant of the above wait. This is a highly dangerous routine
  1168.  * as the object state may change during this call.
  1169.  */
  1170. static __must_check int
  1171. i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
  1172.                                             bool readonly)
  1173. {
  1174.         struct drm_device *dev = obj->base.dev;
  1175.         struct drm_i915_private *dev_priv = dev->dev_private;
  1176.         struct intel_ring_buffer *ring = obj->ring;
  1177.         u32 seqno;
  1178.         int ret;
  1179.  
  1180.         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
  1181.         BUG_ON(!dev_priv->mm.interruptible);
  1182.  
  1183.         seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
  1184.         if (seqno == 0)
  1185.                 return 0;
  1186.  
  1187.         ret = i915_gem_check_wedge(dev_priv, true);
  1188.         if (ret)
  1189.                 return ret;
  1190.  
  1191.         ret = i915_gem_check_olr(ring, seqno);
  1192.         if (ret)
  1193.                 return ret;
  1194.  
  1195.         mutex_unlock(&dev->struct_mutex);
  1196.         ret = __wait_seqno(ring, seqno, true, NULL);
  1197.         mutex_lock(&dev->struct_mutex);
  1198.  
  1199.         i915_gem_retire_requests_ring(ring);
  1200.  
  1201.         /* Manually manage the write flush as we may have not yet
  1202.          * retired the buffer.
  1203.          */
  1204.         if (obj->last_write_seqno &&
  1205.             i915_seqno_passed(seqno, obj->last_write_seqno)) {
  1206.                 obj->last_write_seqno = 0;
  1207.                 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
  1208.         }
  1209.  
  1210.         return ret;
  1211. }
  1212.  
  1213. /**
  1214.  * Called when user space prepares to use an object with the CPU, either
  1215.  * through the mmap ioctl's mapping or a GTT mapping.
  1216.  */
  1217. int
  1218. i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
  1219.                           struct drm_file *file)
  1220. {
  1221.         struct drm_i915_gem_set_domain *args = data;
  1222.         struct drm_i915_gem_object *obj;
  1223.         uint32_t read_domains = args->read_domains;
  1224.         uint32_t write_domain = args->write_domain;
  1225.         int ret;
  1226.  
  1227.         /* Only handle setting domains to types used by the CPU. */
  1228.         if (write_domain & I915_GEM_GPU_DOMAINS)
  1229.                 return -EINVAL;
  1230.  
  1231.         if (read_domains & I915_GEM_GPU_DOMAINS)
  1232.                 return -EINVAL;
  1233.  
  1234.         /* Having something in the write domain implies it's in the read
  1235.          * domain, and only that read domain.  Enforce that in the request.
  1236.          */
  1237.         if (write_domain != 0 && read_domains != write_domain)
  1238.                 return -EINVAL;
  1239.  
  1240.         ret = i915_mutex_lock_interruptible(dev);
  1241.         if (ret)
  1242.                 return ret;
  1243.  
  1244.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  1245.         if (&obj->base == NULL) {
  1246.                 ret = -ENOENT;
  1247.                 goto unlock;
  1248.         }
  1249.  
  1250.         /* Try to flush the object off the GPU without holding the lock.
  1251.          * We will repeat the flush holding the lock in the normal manner
  1252.          * to catch cases where we are gazumped.
  1253.          */
  1254.         ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
  1255.         if (ret)
  1256.                 goto unref;
  1257.  
  1258.         if (read_domains & I915_GEM_DOMAIN_GTT) {
  1259.                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
  1260.  
  1261.                 /* Silently promote "you're not bound, there was nothing to do"
  1262.                  * to success, since the client was just asking us to
  1263.                  * make sure everything was done.
  1264.                  */
  1265.                 if (ret == -EINVAL)
  1266.                         ret = 0;
  1267.         } else {
  1268.                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
  1269.         }
  1270.  
  1271. unref:
  1272.         drm_gem_object_unreference(&obj->base);
  1273. unlock:
  1274.         mutex_unlock(&dev->struct_mutex);
  1275.         return ret;
  1276. }
  1277.  
  1278.  
  1279.  
  1280.  
  1281.  
  1282.  
  1283. /**
  1284.  * Maps the contents of an object, returning the address it is mapped
  1285.  * into.
  1286.  *
  1287.  * While the mapping holds a reference on the contents of the object, it doesn't
  1288.  * imply a ref on the object itself.
  1289.  */
  1290. int
  1291. i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
  1292.                     struct drm_file *file)
  1293. {
  1294.         struct drm_i915_gem_mmap *args = data;
  1295.         struct drm_gem_object *obj;
  1296.         unsigned long addr = 0;
  1297.  
  1298.         obj = drm_gem_object_lookup(dev, file, args->handle);
  1299.         if (obj == NULL)
  1300.                 return -ENOENT;
  1301.        
  1302.     dbgprintf("%s offset %lld size %lld\n",
  1303.                 __FUNCTION__, args->offset, args->size);
  1304.         /* prime objects have no backing filp to GEM mmap
  1305.          * pages from.
  1306.          */
  1307.         if (!obj->filp) {
  1308.                 drm_gem_object_unreference_unlocked(obj);
  1309.                 return -EINVAL;
  1310.         }
  1311.  
  1312.     addr = vm_mmap(obj->filp, 0, args->size,
  1313.               PROT_READ | PROT_WRITE, MAP_SHARED,
  1314.               args->offset);
  1315.         drm_gem_object_unreference_unlocked(obj);
  1316.     if (IS_ERR((void *)addr))
  1317.         return addr;
  1318.  
  1319.         args->addr_ptr = (uint64_t) addr;
  1320.  
  1321.     return 0;
  1322. }
  1323.  
  1324.  
  1325.  
  1326.  
  1327.  
  1328.  
  1329.  
  1330.  
  1331.  
  1332.  
  1333.  
  1334.  
  1335.  
  1336. /**
  1337.  * i915_gem_release_mmap - remove physical page mappings
  1338.  * @obj: obj in question
  1339.  *
  1340.  * Preserve the reservation of the mmapping with the DRM core code, but
  1341.  * relinquish ownership of the pages back to the system.
  1342.  *
  1343.  * It is vital that we remove the page mapping if we have mapped a tiled
  1344.  * object through the GTT and then lose the fence register due to
  1345.  * resource pressure. Similarly if the object has been moved out of the
  1346.  * aperture, than pages mapped into userspace must be revoked. Removing the
  1347.  * mapping will then trigger a page fault on the next user access, allowing
  1348.  * fixup by i915_gem_fault().
  1349.  */
  1350. void
  1351. i915_gem_release_mmap(struct drm_i915_gem_object *obj)
  1352. {
  1353.         if (!obj->fault_mappable)
  1354.                 return;
  1355.  
  1356.         if (obj->base.dev->dev_mapping)
  1357. //              unmap_mapping_range(obj->base.dev->dev_mapping,
  1358. //                                  (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
  1359. //                                  obj->base.size, 1);
  1360.  
  1361.         obj->fault_mappable = false;
  1362. }
  1363.  
  1364. static uint32_t
  1365. i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
  1366. {
  1367.         uint32_t gtt_size;
  1368.  
  1369.         if (INTEL_INFO(dev)->gen >= 4 ||
  1370.             tiling_mode == I915_TILING_NONE)
  1371.                 return size;
  1372.  
  1373.         /* Previous chips need a power-of-two fence region when tiling */
  1374.         if (INTEL_INFO(dev)->gen == 3)
  1375.                 gtt_size = 1024*1024;
  1376.         else
  1377.                 gtt_size = 512*1024;
  1378.  
  1379.         while (gtt_size < size)
  1380.                 gtt_size <<= 1;
  1381.  
  1382.         return gtt_size;
  1383. }
  1384.  
  1385. /**
  1386.  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
  1387.  * @obj: object to check
  1388.  *
  1389.  * Return the required GTT alignment for an object, taking into account
  1390.  * potential fence register mapping.
  1391.  */
  1392. static uint32_t
  1393. i915_gem_get_gtt_alignment(struct drm_device *dev,
  1394.                            uint32_t size,
  1395.                            int tiling_mode)
  1396. {
  1397.         /*
  1398.          * Minimum alignment is 4k (GTT page size), but might be greater
  1399.          * if a fence register is needed for the object.
  1400.          */
  1401.         if (INTEL_INFO(dev)->gen >= 4 ||
  1402.             tiling_mode == I915_TILING_NONE)
  1403.                 return 4096;
  1404.  
  1405.         /*
  1406.          * Previous chips need to be aligned to the size of the smallest
  1407.          * fence register that can contain the object.
  1408.          */
  1409.         return i915_gem_get_gtt_size(dev, size, tiling_mode);
  1410. }
  1411.  
  1412. /**
  1413.  * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
  1414.  *                                       unfenced object
  1415.  * @dev: the device
  1416.  * @size: size of the object
  1417.  * @tiling_mode: tiling mode of the object
  1418.  *
  1419.  * Return the required GTT alignment for an object, only taking into account
  1420.  * unfenced tiled surface requirements.
  1421.  */
  1422. uint32_t
  1423. i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
  1424.                                     uint32_t size,
  1425.                                     int tiling_mode)
  1426. {
  1427.         /*
  1428.          * Minimum alignment is 4k (GTT page size) for sane hw.
  1429.          */
  1430.         if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
  1431.             tiling_mode == I915_TILING_NONE)
  1432.                 return 4096;
  1433.  
  1434.         /* Previous hardware however needs to be aligned to a power-of-two
  1435.          * tile height. The simplest method for determining this is to reuse
  1436.          * the power-of-tile object size.
  1437.          */
  1438.         return i915_gem_get_gtt_size(dev, size, tiling_mode);
  1439. }
  1440.  
  1441. /* Immediately discard the backing storage */
  1442. static void
  1443. i915_gem_object_truncate(struct drm_i915_gem_object *obj)
  1444. {
  1445. //      struct inode *inode;
  1446.  
  1447. //      i915_gem_object_free_mmap_offset(obj);
  1448.  
  1449.         if (obj->base.filp == NULL)
  1450.                 return;
  1451.  
  1452.         /* Our goal here is to return as much of the memory as
  1453.          * is possible back to the system as we are called from OOM.
  1454.          * To do this we must instruct the shmfs to drop all of its
  1455.          * backing pages, *now*.
  1456.          */
  1457. //      inode = obj->base.filp->f_path.dentry->d_inode;
  1458. //      shmem_truncate_range(inode, 0, (loff_t)-1);
  1459.  
  1460.         obj->madv = __I915_MADV_PURGED;
  1461. }
  1462.  
  1463. static inline int
  1464. i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
  1465. {
  1466.         return obj->madv == I915_MADV_DONTNEED;
  1467. }
  1468.  
  1469. static void
  1470. i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
  1471. {
  1472.         int page_count = obj->base.size / PAGE_SIZE;
  1473.         struct scatterlist *sg;
  1474.         int ret, i;
  1475.  
  1476.         BUG_ON(obj->madv == __I915_MADV_PURGED);
  1477.  
  1478.         ret = i915_gem_object_set_to_cpu_domain(obj, true);
  1479.         if (ret) {
  1480.                 /* In the event of a disaster, abandon all caches and
  1481.                  * hope for the best.
  1482.                  */
  1483.                 WARN_ON(ret != -EIO);
  1484.                 i915_gem_clflush_object(obj);
  1485.                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  1486.         }
  1487.  
  1488.         if (obj->madv == I915_MADV_DONTNEED)
  1489.                 obj->dirty = 0;
  1490.  
  1491.         for_each_sg(obj->pages->sgl, sg, page_count, i) {
  1492.                 struct page *page = sg_page(sg);
  1493.  
  1494.  
  1495.  
  1496. //       page_cache_release(page);
  1497.         }
  1498.     //DRM_DEBUG_KMS("%s release %d pages\n", __FUNCTION__, page_count);
  1499.         obj->dirty = 0;
  1500.  
  1501.         sg_free_table(obj->pages);
  1502.         kfree(obj->pages);
  1503. }
  1504.  
  1505. static int
  1506. i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
  1507. {
  1508.         const struct drm_i915_gem_object_ops *ops = obj->ops;
  1509.  
  1510.         if (obj->pages == NULL)
  1511.                 return 0;
  1512.  
  1513.         BUG_ON(obj->gtt_space);
  1514.  
  1515.         if (obj->pages_pin_count)
  1516.                 return -EBUSY;
  1517.  
  1518.         /* ->put_pages might need to allocate memory for the bit17 swizzle
  1519.          * array, hence protect them from being reaped by removing them from gtt
  1520.          * lists early. */
  1521.         list_del(&obj->gtt_list);
  1522.  
  1523.         ops->put_pages(obj);
  1524.         obj->pages = NULL;
  1525.  
  1526.         if (i915_gem_object_is_purgeable(obj))
  1527.                 i915_gem_object_truncate(obj);
  1528.  
  1529.         return 0;
  1530. }
  1531.  
  1532.  
  1533.  
  1534.  
  1535.  
  1536.  
  1537.  
  1538.  
  1539. static int
  1540. i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
  1541. {
  1542.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  1543.     int page_count, i;
  1544.         struct address_space *mapping;
  1545.         struct sg_table *st;
  1546.         struct scatterlist *sg;
  1547.         struct page *page;
  1548.         gfp_t gfp;
  1549.  
  1550.         /* Assert that the object is not currently in any GPU domain. As it
  1551.          * wasn't in the GTT, there shouldn't be any way it could have been in
  1552.          * a GPU cache
  1553.          */
  1554.         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
  1555.         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
  1556.  
  1557.         st = kmalloc(sizeof(*st), GFP_KERNEL);
  1558.         if (st == NULL)
  1559.                 return -ENOMEM;
  1560.  
  1561.         page_count = obj->base.size / PAGE_SIZE;
  1562.         if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
  1563.                 sg_free_table(st);
  1564.                 kfree(st);
  1565.                 return -ENOMEM;
  1566.         }
  1567.  
  1568.         /* Get the list of pages out of our struct file.  They'll be pinned
  1569.          * at this point until we release them.
  1570.          *
  1571.          * Fail silently without starting the shrinker
  1572.          */
  1573.         for_each_sg(st->sgl, sg, page_count, i) {
  1574.                 page = shmem_read_mapping_page_gfp(obj->base.filp, i, gfp);
  1575.                 if (IS_ERR(page)) {
  1576.             dbgprintf("%s invalid page %p\n", __FUNCTION__, page);
  1577.                         goto err_pages;
  1578.  
  1579.                 }
  1580.                 sg_set_page(sg, page, PAGE_SIZE, 0);
  1581.         }
  1582.  
  1583.         obj->pages = st;
  1584.  
  1585.     DRM_DEBUG_KMS("%s alloc %d pages\n", __FUNCTION__, page_count);
  1586.  
  1587.         return 0;
  1588.  
  1589. err_pages:
  1590.         for_each_sg(st->sgl, sg, i, page_count)
  1591.                 page_cache_release(sg_page(sg));
  1592.         sg_free_table(st);
  1593.         kfree(st);
  1594.         return PTR_ERR(page);
  1595. }
  1596.  
  1597. /* Ensure that the associated pages are gathered from the backing storage
  1598.  * and pinned into our object. i915_gem_object_get_pages() may be called
  1599.  * multiple times before they are released by a single call to
  1600.  * i915_gem_object_put_pages() - once the pages are no longer referenced
  1601.  * either as a result of memory pressure (reaping pages under the shrinker)
  1602.  * or as the object is itself released.
  1603.  */
  1604. int
  1605. i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
  1606. {
  1607.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  1608.         const struct drm_i915_gem_object_ops *ops = obj->ops;
  1609.         int ret;
  1610.  
  1611.         if (obj->pages)
  1612.                 return 0;
  1613.  
  1614.         BUG_ON(obj->pages_pin_count);
  1615.  
  1616.         ret = ops->get_pages(obj);
  1617.         if (ret)
  1618.                 return ret;
  1619.  
  1620.         list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
  1621.     return 0;
  1622. }
  1623.  
  1624. void
  1625. i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
  1626.                                struct intel_ring_buffer *ring)
  1627. {
  1628.         struct drm_device *dev = obj->base.dev;
  1629.         struct drm_i915_private *dev_priv = dev->dev_private;
  1630.         u32 seqno = intel_ring_get_seqno(ring);
  1631.  
  1632.         BUG_ON(ring == NULL);
  1633.         obj->ring = ring;
  1634.  
  1635.         /* Add a reference if we're newly entering the active list. */
  1636.         if (!obj->active) {
  1637.                 drm_gem_object_reference(&obj->base);
  1638.                 obj->active = 1;
  1639.         }
  1640.  
  1641.         /* Move from whatever list we were on to the tail of execution. */
  1642.         list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
  1643.         list_move_tail(&obj->ring_list, &ring->active_list);
  1644.  
  1645.         obj->last_read_seqno = seqno;
  1646.  
  1647.         if (obj->fenced_gpu_access) {
  1648.                 obj->last_fenced_seqno = seqno;
  1649.  
  1650.                 /* Bump MRU to take account of the delayed flush */
  1651.                 if (obj->fence_reg != I915_FENCE_REG_NONE) {
  1652.                 struct drm_i915_fence_reg *reg;
  1653.  
  1654.                 reg = &dev_priv->fence_regs[obj->fence_reg];
  1655.                         list_move_tail(&reg->lru_list,
  1656.                                        &dev_priv->mm.fence_list);
  1657.                 }
  1658.         }
  1659. }
  1660.  
  1661. static void
  1662. i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
  1663. {
  1664.         struct drm_device *dev = obj->base.dev;
  1665.         struct drm_i915_private *dev_priv = dev->dev_private;
  1666.  
  1667.         BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
  1668.         BUG_ON(!obj->active);
  1669.  
  1670.         if (obj->pin_count) /* are we a framebuffer? */
  1671.                 intel_mark_fb_idle(obj);
  1672.  
  1673.                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
  1674.  
  1675.         list_del_init(&obj->ring_list);
  1676.         obj->ring = NULL;
  1677.  
  1678.         obj->last_read_seqno = 0;
  1679.         obj->last_write_seqno = 0;
  1680.         obj->base.write_domain = 0;
  1681.  
  1682.         obj->last_fenced_seqno = 0;
  1683.         obj->fenced_gpu_access = false;
  1684.  
  1685.         obj->active = 0;
  1686.         drm_gem_object_unreference(&obj->base);
  1687.  
  1688.         WARN_ON(i915_verify_lists(dev));
  1689. }
  1690.  
  1691. static int
  1692. i915_gem_handle_seqno_wrap(struct drm_device *dev)
  1693. {
  1694.         struct drm_i915_private *dev_priv = dev->dev_private;
  1695.         struct intel_ring_buffer *ring;
  1696.         int ret, i, j;
  1697.  
  1698.         /* The hardware uses various monotonic 32-bit counters, if we
  1699.          * detect that they will wraparound we need to idle the GPU
  1700.          * and reset those counters.
  1701.          */
  1702.         ret = 0;
  1703.         for_each_ring(ring, dev_priv, i) {
  1704.                 for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
  1705.                         ret |= ring->sync_seqno[j] != 0;
  1706.         }
  1707.         if (ret == 0)
  1708.                 return ret;
  1709.  
  1710.         ret = i915_gpu_idle(dev);
  1711.         if (ret)
  1712.                 return ret;
  1713.  
  1714.         i915_gem_retire_requests(dev);
  1715.         for_each_ring(ring, dev_priv, i) {
  1716.                 for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
  1717.                         ring->sync_seqno[j] = 0;
  1718.         }
  1719.  
  1720.         return 0;
  1721. }
  1722.  
  1723. int
  1724. i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
  1725. {
  1726.         struct drm_i915_private *dev_priv = dev->dev_private;
  1727.  
  1728.         /* reserve 0 for non-seqno */
  1729.         if (dev_priv->next_seqno == 0) {
  1730.                 int ret = i915_gem_handle_seqno_wrap(dev);
  1731.                 if (ret)
  1732.                         return ret;
  1733.  
  1734.                 dev_priv->next_seqno = 1;
  1735.         }
  1736.  
  1737.         *seqno = dev_priv->next_seqno++;
  1738.         return 0;
  1739. }
  1740.  
  1741. int
  1742. i915_add_request(struct intel_ring_buffer *ring,
  1743.                  struct drm_file *file,
  1744.                  u32 *out_seqno)
  1745. {
  1746.         drm_i915_private_t *dev_priv = ring->dev->dev_private;
  1747.         struct drm_i915_gem_request *request;
  1748.         u32 request_ring_position;
  1749.         int was_empty;
  1750.         int ret;
  1751.  
  1752.         /*
  1753.          * Emit any outstanding flushes - execbuf can fail to emit the flush
  1754.          * after having emitted the batchbuffer command. Hence we need to fix
  1755.          * things up similar to emitting the lazy request. The difference here
  1756.          * is that the flush _must_ happen before the next request, no matter
  1757.          * what.
  1758.          */
  1759.         ret = intel_ring_flush_all_caches(ring);
  1760.         if (ret)
  1761.                 return ret;
  1762.  
  1763.         request = kmalloc(sizeof(*request), GFP_KERNEL);
  1764.         if (request == NULL)
  1765.                 return -ENOMEM;
  1766.  
  1767.  
  1768.         /* Record the position of the start of the request so that
  1769.          * should we detect the updated seqno part-way through the
  1770.          * GPU processing the request, we never over-estimate the
  1771.          * position of the head.
  1772.          */
  1773.         request_ring_position = intel_ring_get_tail(ring);
  1774.  
  1775.         ret = ring->add_request(ring);
  1776.         if (ret) {
  1777.                 kfree(request);
  1778.             return ret;
  1779.         }
  1780.  
  1781.         request->seqno = intel_ring_get_seqno(ring);
  1782.         request->ring = ring;
  1783.         request->tail = request_ring_position;
  1784.     request->emitted_jiffies = GetTimerTicks();
  1785.         was_empty = list_empty(&ring->request_list);
  1786.         list_add_tail(&request->list, &ring->request_list);
  1787.         request->file_priv = NULL;
  1788.  
  1789.         if (file) {
  1790.                 struct drm_i915_file_private *file_priv = file->driver_priv;
  1791.  
  1792.                 spin_lock(&file_priv->mm.lock);
  1793.                 request->file_priv = file_priv;
  1794.                 list_add_tail(&request->client_list,
  1795.                               &file_priv->mm.request_list);
  1796.                 spin_unlock(&file_priv->mm.lock);
  1797.         }
  1798.  
  1799.         trace_i915_gem_request_add(ring, request->seqno);
  1800.         ring->outstanding_lazy_request = 0;
  1801.  
  1802.         if (!dev_priv->mm.suspended) {
  1803.                 if (i915_enable_hangcheck) {
  1804. //                      mod_timer(&dev_priv->hangcheck_timer,
  1805. //                                jiffies +
  1806. //                                msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
  1807.                 }
  1808.                 if (was_empty) {
  1809.            queue_delayed_work(dev_priv->wq,
  1810.                       &dev_priv->mm.retire_work, HZ);
  1811.                         intel_mark_busy(dev_priv->dev);
  1812.                 }
  1813.         }
  1814.  
  1815.         if (out_seqno)
  1816.                 *out_seqno = request->seqno;
  1817.         return 0;
  1818. }
  1819.  
  1820. static inline void
  1821. i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
  1822. {
  1823.         struct drm_i915_file_private *file_priv = request->file_priv;
  1824.  
  1825.         if (!file_priv)
  1826.                 return;
  1827.  
  1828.         spin_lock(&file_priv->mm.lock);
  1829.         if (request->file_priv) {
  1830.                 list_del(&request->client_list);
  1831.                 request->file_priv = NULL;
  1832.         }
  1833.         spin_unlock(&file_priv->mm.lock);
  1834. }
  1835.  
  1836. static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
  1837.                                       struct intel_ring_buffer *ring)
  1838. {
  1839.         while (!list_empty(&ring->request_list)) {
  1840.                 struct drm_i915_gem_request *request;
  1841.  
  1842.                 request = list_first_entry(&ring->request_list,
  1843.                                            struct drm_i915_gem_request,
  1844.                                            list);
  1845.  
  1846.                 list_del(&request->list);
  1847.                 i915_gem_request_remove_from_client(request);
  1848.                 kfree(request);
  1849.         }
  1850.  
  1851.         while (!list_empty(&ring->active_list)) {
  1852.                 struct drm_i915_gem_object *obj;
  1853.  
  1854.                 obj = list_first_entry(&ring->active_list,
  1855.                                        struct drm_i915_gem_object,
  1856.                                        ring_list);
  1857.  
  1858.                 i915_gem_object_move_to_inactive(obj);
  1859.         }
  1860. }
  1861.  
  1862. static void i915_gem_reset_fences(struct drm_device *dev)
  1863. {
  1864.         struct drm_i915_private *dev_priv = dev->dev_private;
  1865.         int i;
  1866.  
  1867.         for (i = 0; i < dev_priv->num_fence_regs; i++) {
  1868.                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
  1869.  
  1870.                 i915_gem_write_fence(dev, i, NULL);
  1871.  
  1872.                 if (reg->obj)
  1873.                         i915_gem_object_fence_lost(reg->obj);
  1874.  
  1875.                 reg->pin_count = 0;
  1876.                 reg->obj = NULL;
  1877.                 INIT_LIST_HEAD(&reg->lru_list);
  1878.         }
  1879.  
  1880.         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
  1881. }
  1882.  
  1883. void i915_gem_reset(struct drm_device *dev)
  1884. {
  1885.         struct drm_i915_private *dev_priv = dev->dev_private;
  1886.         struct drm_i915_gem_object *obj;
  1887.         struct intel_ring_buffer *ring;
  1888.         int i;
  1889.  
  1890.         for_each_ring(ring, dev_priv, i)
  1891.                 i915_gem_reset_ring_lists(dev_priv, ring);
  1892.  
  1893.         /* Move everything out of the GPU domains to ensure we do any
  1894.          * necessary invalidation upon reuse.
  1895.          */
  1896.         list_for_each_entry(obj,
  1897.                             &dev_priv->mm.inactive_list,
  1898.                             mm_list)
  1899.         {
  1900.                 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
  1901.         }
  1902.  
  1903.         /* The fence registers are invalidated so clear them out */
  1904.         i915_gem_reset_fences(dev);
  1905. }
  1906.  
  1907. /**
  1908.  * This function clears the request list as sequence numbers are passed.
  1909.  */
  1910. void
  1911. i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
  1912. {
  1913.         uint32_t seqno;
  1914.  
  1915.         if (list_empty(&ring->request_list))
  1916.                 return;
  1917.  
  1918.         WARN_ON(i915_verify_lists(ring->dev));
  1919.  
  1920.         seqno = ring->get_seqno(ring, true);
  1921.  
  1922.         while (!list_empty(&ring->request_list)) {
  1923.                 struct drm_i915_gem_request *request;
  1924.  
  1925.                 request = list_first_entry(&ring->request_list,
  1926.                                            struct drm_i915_gem_request,
  1927.                                            list);
  1928.  
  1929.                 if (!i915_seqno_passed(seqno, request->seqno))
  1930.                         break;
  1931.  
  1932.                 trace_i915_gem_request_retire(ring, request->seqno);
  1933.                 /* We know the GPU must have read the request to have
  1934.                  * sent us the seqno + interrupt, so use the position
  1935.                  * of tail of the request to update the last known position
  1936.                  * of the GPU head.
  1937.                  */
  1938.                 ring->last_retired_head = request->tail;
  1939.  
  1940.                 list_del(&request->list);
  1941.                 i915_gem_request_remove_from_client(request);
  1942.                 kfree(request);
  1943.         }
  1944.  
  1945.         /* Move any buffers on the active list that are no longer referenced
  1946.          * by the ringbuffer to the flushing/inactive lists as appropriate.
  1947.          */
  1948.         while (!list_empty(&ring->active_list)) {
  1949.                 struct drm_i915_gem_object *obj;
  1950.  
  1951.                 obj = list_first_entry(&ring->active_list,
  1952.                                       struct drm_i915_gem_object,
  1953.                                       ring_list);
  1954.  
  1955.                 if (!i915_seqno_passed(seqno, obj->last_read_seqno))
  1956.                         break;
  1957.  
  1958.                         i915_gem_object_move_to_inactive(obj);
  1959.         }
  1960.  
  1961.         if (unlikely(ring->trace_irq_seqno &&
  1962.                      i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
  1963.                 ring->irq_put(ring);
  1964.                 ring->trace_irq_seqno = 0;
  1965.         }
  1966.  
  1967.         WARN_ON(i915_verify_lists(ring->dev));
  1968. }
  1969.  
  1970. void
  1971. i915_gem_retire_requests(struct drm_device *dev)
  1972. {
  1973.         drm_i915_private_t *dev_priv = dev->dev_private;
  1974.         struct intel_ring_buffer *ring;
  1975.         int i;
  1976.  
  1977.         for_each_ring(ring, dev_priv, i)
  1978.                 i915_gem_retire_requests_ring(ring);
  1979. }
  1980.  
  1981. static void
  1982. i915_gem_retire_work_handler(struct work_struct *work)
  1983. {
  1984.         drm_i915_private_t *dev_priv;
  1985.         struct drm_device *dev;
  1986.         struct intel_ring_buffer *ring;
  1987.         bool idle;
  1988.         int i;
  1989.  
  1990.         dev_priv = container_of(work, drm_i915_private_t,
  1991.                                 mm.retire_work.work);
  1992.         dev = dev_priv->dev;
  1993.  
  1994.         /* Come back later if the device is busy... */
  1995.         if (!mutex_trylock(&dev->struct_mutex)) {
  1996.         queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
  1997.         return;
  1998.         }
  1999.  
  2000.         i915_gem_retire_requests(dev);
  2001.  
  2002.         /* Send a periodic flush down the ring so we don't hold onto GEM
  2003.          * objects indefinitely.
  2004.          */
  2005.         idle = true;
  2006.         for_each_ring(ring, dev_priv, i) {
  2007.                 if (ring->gpu_caches_dirty)
  2008.                         i915_add_request(ring, NULL, NULL);
  2009.  
  2010.                 idle &= list_empty(&ring->request_list);
  2011.         }
  2012.  
  2013.    if (!dev_priv->mm.suspended && !idle)
  2014.        queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
  2015.         if (idle)
  2016.                 intel_mark_idle(dev);
  2017.  
  2018.         mutex_unlock(&dev->struct_mutex);
  2019. }
  2020.  
  2021. /**
  2022.  * Ensures that an object will eventually get non-busy by flushing any required
  2023.  * write domains, emitting any outstanding lazy request and retiring and
  2024.  * completed requests.
  2025.  */
  2026. static int
  2027. i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
  2028. {
  2029.         int ret;
  2030.  
  2031.         if (obj->active) {
  2032.                 ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
  2033.                 if (ret)
  2034.                         return ret;
  2035.  
  2036.                 i915_gem_retire_requests_ring(obj->ring);
  2037.         }
  2038.  
  2039.         return 0;
  2040. }
  2041.  
  2042. /**
  2043.  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
  2044.  * @DRM_IOCTL_ARGS: standard ioctl arguments
  2045.  *
  2046.  * Returns 0 if successful, else an error is returned with the remaining time in
  2047.  * the timeout parameter.
  2048.  *  -ETIME: object is still busy after timeout
  2049.  *  -ERESTARTSYS: signal interrupted the wait
  2050.  *  -ENONENT: object doesn't exist
  2051.  * Also possible, but rare:
  2052.  *  -EAGAIN: GPU wedged
  2053.  *  -ENOMEM: damn
  2054.  *  -ENODEV: Internal IRQ fail
  2055.  *  -E?: The add request failed
  2056.  *
  2057.  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
  2058.  * non-zero timeout parameter the wait ioctl will wait for the given number of
  2059.  * nanoseconds on an object becoming unbusy. Since the wait itself does so
  2060.  * without holding struct_mutex the object may become re-busied before this
  2061.  * function completes. A similar but shorter * race condition exists in the busy
  2062.  * ioctl
  2063.  */
  2064.  
  2065.  
  2066.  
  2067.  
  2068.  
  2069.  
  2070.  
  2071.  
  2072.  
  2073.  
  2074.  
  2075.  
  2076. /**
  2077.  * i915_gem_object_sync - sync an object to a ring.
  2078.  *
  2079.  * @obj: object which may be in use on another ring.
  2080.  * @to: ring we wish to use the object on. May be NULL.
  2081.  *
  2082.  * This code is meant to abstract object synchronization with the GPU.
  2083.  * Calling with NULL implies synchronizing the object with the CPU
  2084.  * rather than a particular GPU ring.
  2085.  *
  2086.  * Returns 0 if successful, else propagates up the lower layer error.
  2087.  */
  2088. int
  2089. i915_gem_object_sync(struct drm_i915_gem_object *obj,
  2090.                      struct intel_ring_buffer *to)
  2091. {
  2092.         struct intel_ring_buffer *from = obj->ring;
  2093.         u32 seqno;
  2094.         int ret, idx;
  2095.  
  2096.         if (from == NULL || to == from)
  2097.                 return 0;
  2098.  
  2099.         if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
  2100.                 return i915_gem_object_wait_rendering(obj, false);
  2101.  
  2102.         idx = intel_ring_sync_index(from, to);
  2103.  
  2104.         seqno = obj->last_read_seqno;
  2105.         if (seqno <= from->sync_seqno[idx])
  2106.                 return 0;
  2107.  
  2108.         ret = i915_gem_check_olr(obj->ring, seqno);
  2109.         if (ret)
  2110.                 return ret;
  2111.  
  2112.         ret = to->sync_to(to, from, seqno);
  2113.         if (!ret)
  2114.                 /* We use last_read_seqno because sync_to()
  2115.                  * might have just caused seqno wrap under
  2116.                  * the radar.
  2117.                  */
  2118.                 from->sync_seqno[idx] = obj->last_read_seqno;
  2119.  
  2120.         return ret;
  2121. }
  2122.  
  2123. static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
  2124. {
  2125.         u32 old_write_domain, old_read_domains;
  2126.  
  2127.         /* Act a barrier for all accesses through the GTT */
  2128.         mb();
  2129.  
  2130.         /* Force a pagefault for domain tracking on next user access */
  2131. //      i915_gem_release_mmap(obj);
  2132.  
  2133.         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
  2134.                 return;
  2135.  
  2136.         old_read_domains = obj->base.read_domains;
  2137.         old_write_domain = obj->base.write_domain;
  2138.  
  2139.         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
  2140.         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
  2141.  
  2142.         trace_i915_gem_object_change_domain(obj,
  2143.                                             old_read_domains,
  2144.                                             old_write_domain);
  2145. }
  2146.  
  2147. /**
  2148.  * Unbinds an object from the GTT aperture.
  2149.  */
  2150. int
  2151. i915_gem_object_unbind(struct drm_i915_gem_object *obj)
  2152. {
  2153.         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
  2154.         int ret = 0;
  2155.  
  2156.     if(obj == get_fb_obj())
  2157.         return 0;
  2158.  
  2159.         if (obj->gtt_space == NULL)
  2160.                 return 0;
  2161.  
  2162.         if (obj->pin_count)
  2163.                 return -EBUSY;
  2164.  
  2165.         BUG_ON(obj->pages == NULL);
  2166.  
  2167.         ret = i915_gem_object_finish_gpu(obj);
  2168.         if (ret)
  2169.                 return ret;
  2170.         /* Continue on if we fail due to EIO, the GPU is hung so we
  2171.          * should be safe and we need to cleanup or else we might
  2172.          * cause memory corruption through use-after-free.
  2173.          */
  2174.  
  2175.         i915_gem_object_finish_gtt(obj);
  2176.  
  2177.         /* release the fence reg _after_ flushing */
  2178.         ret = i915_gem_object_put_fence(obj);
  2179.         if (ret)
  2180.                 return ret;
  2181.  
  2182.         trace_i915_gem_object_unbind(obj);
  2183.  
  2184.         if (obj->has_global_gtt_mapping)
  2185.         i915_gem_gtt_unbind_object(obj);
  2186.         if (obj->has_aliasing_ppgtt_mapping) {
  2187.                 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
  2188.                 obj->has_aliasing_ppgtt_mapping = 0;
  2189.         }
  2190.         i915_gem_gtt_finish_object(obj);
  2191.  
  2192.         list_del(&obj->mm_list);
  2193.         list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
  2194.         /* Avoid an unnecessary call to unbind on rebind. */
  2195.         obj->map_and_fenceable = true;
  2196.  
  2197.         drm_mm_put_block(obj->gtt_space);
  2198.         obj->gtt_space = NULL;
  2199.         obj->gtt_offset = 0;
  2200.  
  2201.         return 0;
  2202. }
  2203.  
  2204. int i915_gpu_idle(struct drm_device *dev)
  2205. {
  2206.         drm_i915_private_t *dev_priv = dev->dev_private;
  2207.         struct intel_ring_buffer *ring;
  2208.         int ret, i;
  2209.  
  2210.         /* Flush everything onto the inactive list. */
  2211.         for_each_ring(ring, dev_priv, i) {
  2212.                 ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
  2213.                 if (ret)
  2214.                         return ret;
  2215.  
  2216.                 ret = intel_ring_idle(ring);
  2217.                 if (ret)
  2218.                         return ret;
  2219.         }
  2220.  
  2221.         return 0;
  2222. }
  2223.  
  2224. static void sandybridge_write_fence_reg(struct drm_device *dev, int reg,
  2225.                                         struct drm_i915_gem_object *obj)
  2226. {
  2227.         drm_i915_private_t *dev_priv = dev->dev_private;
  2228.         uint64_t val;
  2229.  
  2230.         if (obj) {
  2231.                 u32 size = obj->gtt_space->size;
  2232.  
  2233.                 val = (uint64_t)((obj->gtt_offset + size - 4096) &
  2234.                                  0xfffff000) << 32;
  2235.                 val |= obj->gtt_offset & 0xfffff000;
  2236.                 val |= (uint64_t)((obj->stride / 128) - 1) <<
  2237.                         SANDYBRIDGE_FENCE_PITCH_SHIFT;
  2238.  
  2239.                 if (obj->tiling_mode == I915_TILING_Y)
  2240.                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
  2241.                 val |= I965_FENCE_REG_VALID;
  2242.         } else
  2243.                 val = 0;
  2244.  
  2245.         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val);
  2246.         POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8);
  2247. }
  2248.  
  2249. static void i965_write_fence_reg(struct drm_device *dev, int reg,
  2250.                                  struct drm_i915_gem_object *obj)
  2251. {
  2252.         drm_i915_private_t *dev_priv = dev->dev_private;
  2253.         uint64_t val;
  2254.  
  2255.         if (obj) {
  2256.                 u32 size = obj->gtt_space->size;
  2257.  
  2258.                 val = (uint64_t)((obj->gtt_offset + size - 4096) &
  2259.                                  0xfffff000) << 32;
  2260.                 val |= obj->gtt_offset & 0xfffff000;
  2261.                 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
  2262.                 if (obj->tiling_mode == I915_TILING_Y)
  2263.                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
  2264.                 val |= I965_FENCE_REG_VALID;
  2265.         } else
  2266.                 val = 0;
  2267.  
  2268.         I915_WRITE64(FENCE_REG_965_0 + reg * 8, val);
  2269.         POSTING_READ(FENCE_REG_965_0 + reg * 8);
  2270. }
  2271.  
  2272. static void i915_write_fence_reg(struct drm_device *dev, int reg,
  2273.                                  struct drm_i915_gem_object *obj)
  2274. {
  2275.         drm_i915_private_t *dev_priv = dev->dev_private;
  2276.         u32 val;
  2277.  
  2278.         if (obj) {
  2279.                 u32 size = obj->gtt_space->size;
  2280.                 int pitch_val;
  2281.                 int tile_width;
  2282.  
  2283.                 WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
  2284.                      (size & -size) != size ||
  2285.                      (obj->gtt_offset & (size - 1)),
  2286.                      "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
  2287.                      obj->gtt_offset, obj->map_and_fenceable, size);
  2288.  
  2289.                 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
  2290.                         tile_width = 128;
  2291.                 else
  2292.                         tile_width = 512;
  2293.  
  2294.                 /* Note: pitch better be a power of two tile widths */
  2295.                 pitch_val = obj->stride / tile_width;
  2296.                 pitch_val = ffs(pitch_val) - 1;
  2297.  
  2298.                 val = obj->gtt_offset;
  2299.                 if (obj->tiling_mode == I915_TILING_Y)
  2300.                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
  2301.                 val |= I915_FENCE_SIZE_BITS(size);
  2302.                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
  2303.                 val |= I830_FENCE_REG_VALID;
  2304.         } else
  2305.                 val = 0;
  2306.  
  2307.         if (reg < 8)
  2308.                 reg = FENCE_REG_830_0 + reg * 4;
  2309.         else
  2310.                 reg = FENCE_REG_945_8 + (reg - 8) * 4;
  2311.  
  2312.         I915_WRITE(reg, val);
  2313.         POSTING_READ(reg);
  2314. }
  2315.  
  2316. static void i830_write_fence_reg(struct drm_device *dev, int reg,
  2317.                                 struct drm_i915_gem_object *obj)
  2318. {
  2319.         drm_i915_private_t *dev_priv = dev->dev_private;
  2320.         uint32_t val;
  2321.  
  2322.         if (obj) {
  2323.                 u32 size = obj->gtt_space->size;
  2324.                 uint32_t pitch_val;
  2325.  
  2326.                 WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
  2327.                      (size & -size) != size ||
  2328.                      (obj->gtt_offset & (size - 1)),
  2329.                      "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
  2330.                      obj->gtt_offset, size);
  2331.  
  2332.                 pitch_val = obj->stride / 128;
  2333.                 pitch_val = ffs(pitch_val) - 1;
  2334.  
  2335.                 val = obj->gtt_offset;
  2336.                 if (obj->tiling_mode == I915_TILING_Y)
  2337.                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
  2338.                 val |= I830_FENCE_SIZE_BITS(size);
  2339.                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
  2340.                 val |= I830_FENCE_REG_VALID;
  2341.         } else
  2342.                 val = 0;
  2343.  
  2344.         I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
  2345.         POSTING_READ(FENCE_REG_830_0 + reg * 4);
  2346. }
  2347.  
  2348. static void i915_gem_write_fence(struct drm_device *dev, int reg,
  2349.                                  struct drm_i915_gem_object *obj)
  2350. {
  2351.         switch (INTEL_INFO(dev)->gen) {
  2352.         case 7:
  2353.         case 6: sandybridge_write_fence_reg(dev, reg, obj); break;
  2354.         case 5:
  2355.         case 4: i965_write_fence_reg(dev, reg, obj); break;
  2356.         case 3: i915_write_fence_reg(dev, reg, obj); break;
  2357.         case 2: i830_write_fence_reg(dev, reg, obj); break;
  2358.         default: break;
  2359.         }
  2360. }
  2361.  
  2362. static inline int fence_number(struct drm_i915_private *dev_priv,
  2363.                                struct drm_i915_fence_reg *fence)
  2364. {
  2365.         return fence - dev_priv->fence_regs;
  2366. }
  2367.  
  2368. static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
  2369.                                          struct drm_i915_fence_reg *fence,
  2370.                                          bool enable)
  2371. {
  2372.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  2373.         int reg = fence_number(dev_priv, fence);
  2374.  
  2375.         i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
  2376.  
  2377.         if (enable) {
  2378.                 obj->fence_reg = reg;
  2379.                 fence->obj = obj;
  2380.                 list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
  2381.         } else {
  2382.                 obj->fence_reg = I915_FENCE_REG_NONE;
  2383.                 fence->obj = NULL;
  2384.                 list_del_init(&fence->lru_list);
  2385.         }
  2386. }
  2387.  
  2388. static int
  2389. i915_gem_object_flush_fence(struct drm_i915_gem_object *obj)
  2390. {
  2391.         if (obj->last_fenced_seqno) {
  2392.                 int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
  2393.                         if (ret)
  2394.                                 return ret;
  2395.  
  2396.                 obj->last_fenced_seqno = 0;
  2397.         }
  2398.  
  2399.         /* Ensure that all CPU reads are completed before installing a fence
  2400.          * and all writes before removing the fence.
  2401.          */
  2402.         if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
  2403.                 mb();
  2404.  
  2405.         obj->fenced_gpu_access = false;
  2406.         return 0;
  2407. }
  2408.  
  2409. int
  2410. i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
  2411. {
  2412.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  2413.         int ret;
  2414.  
  2415.         ret = i915_gem_object_flush_fence(obj);
  2416.         if (ret)
  2417.                 return ret;
  2418.  
  2419.         if (obj->fence_reg == I915_FENCE_REG_NONE)
  2420.                 return 0;
  2421.  
  2422.         i915_gem_object_update_fence(obj,
  2423.                                      &dev_priv->fence_regs[obj->fence_reg],
  2424.                                      false);
  2425.         i915_gem_object_fence_lost(obj);
  2426.  
  2427.         return 0;
  2428. }
  2429.  
  2430. static struct drm_i915_fence_reg *
  2431. i915_find_fence_reg(struct drm_device *dev)
  2432. {
  2433.         struct drm_i915_private *dev_priv = dev->dev_private;
  2434.         struct drm_i915_fence_reg *reg, *avail;
  2435.         int i;
  2436.  
  2437.         /* First try to find a free reg */
  2438.         avail = NULL;
  2439.         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
  2440.                 reg = &dev_priv->fence_regs[i];
  2441.                 if (!reg->obj)
  2442.                         return reg;
  2443.  
  2444.                 if (!reg->pin_count)
  2445.                         avail = reg;
  2446.         }
  2447.  
  2448.         if (avail == NULL)
  2449.                 return NULL;
  2450.  
  2451.         /* None available, try to steal one or wait for a user to finish */
  2452.         list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
  2453.                 if (reg->pin_count)
  2454.                         continue;
  2455.  
  2456.                 return reg;
  2457.         }
  2458.  
  2459.         return NULL;
  2460. }
  2461.  
  2462. /**
  2463.  * i915_gem_object_get_fence - set up fencing for an object
  2464.  * @obj: object to map through a fence reg
  2465.  *
  2466.  * When mapping objects through the GTT, userspace wants to be able to write
  2467.  * to them without having to worry about swizzling if the object is tiled.
  2468.  * This function walks the fence regs looking for a free one for @obj,
  2469.  * stealing one if it can't find any.
  2470.  *
  2471.  * It then sets up the reg based on the object's properties: address, pitch
  2472.  * and tiling format.
  2473.  *
  2474.  * For an untiled surface, this removes any existing fence.
  2475.  */
  2476. int
  2477. i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
  2478. {
  2479.         struct drm_device *dev = obj->base.dev;
  2480.         struct drm_i915_private *dev_priv = dev->dev_private;
  2481.         bool enable = obj->tiling_mode != I915_TILING_NONE;
  2482.         struct drm_i915_fence_reg *reg;
  2483.         int ret;
  2484.  
  2485.         /* Have we updated the tiling parameters upon the object and so
  2486.          * will need to serialise the write to the associated fence register?
  2487.          */
  2488.         if (obj->fence_dirty) {
  2489.                 ret = i915_gem_object_flush_fence(obj);
  2490.                 if (ret)
  2491.                         return ret;
  2492.         }
  2493.  
  2494.         /* Just update our place in the LRU if our fence is getting reused. */
  2495.         if (obj->fence_reg != I915_FENCE_REG_NONE) {
  2496.                 reg = &dev_priv->fence_regs[obj->fence_reg];
  2497.                 if (!obj->fence_dirty) {
  2498.                         list_move_tail(&reg->lru_list,
  2499.                                        &dev_priv->mm.fence_list);
  2500.                         return 0;
  2501.                 }
  2502.         } else if (enable) {
  2503.                 reg = i915_find_fence_reg(dev);
  2504.                 if (reg == NULL)
  2505.                         return -EDEADLK;
  2506.  
  2507.                 if (reg->obj) {
  2508.                         struct drm_i915_gem_object *old = reg->obj;
  2509.  
  2510.                         ret = i915_gem_object_flush_fence(old);
  2511.                         if (ret)
  2512.                                 return ret;
  2513.  
  2514.                         i915_gem_object_fence_lost(old);
  2515.                 }
  2516.         } else
  2517.                 return 0;
  2518.  
  2519.         i915_gem_object_update_fence(obj, reg, enable);
  2520.         obj->fence_dirty = false;
  2521.  
  2522.         return 0;
  2523. }
  2524.  
  2525. static bool i915_gem_valid_gtt_space(struct drm_device *dev,
  2526.                                      struct drm_mm_node *gtt_space,
  2527.                                      unsigned long cache_level)
  2528. {
  2529.         struct drm_mm_node *other;
  2530.  
  2531.         /* On non-LLC machines we have to be careful when putting differing
  2532.          * types of snoopable memory together to avoid the prefetcher
  2533.          * crossing memory domains and dieing.
  2534.          */
  2535.         if (HAS_LLC(dev))
  2536.                 return true;
  2537.  
  2538.         if (gtt_space == NULL)
  2539.                 return true;
  2540.  
  2541.         if (list_empty(&gtt_space->node_list))
  2542.                 return true;
  2543.  
  2544.         other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
  2545.         if (other->allocated && !other->hole_follows && other->color != cache_level)
  2546.                 return false;
  2547.  
  2548.         other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
  2549.         if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
  2550.                 return false;
  2551.  
  2552.         return true;
  2553. }
  2554.  
  2555. static void i915_gem_verify_gtt(struct drm_device *dev)
  2556. {
  2557. #if WATCH_GTT
  2558.         struct drm_i915_private *dev_priv = dev->dev_private;
  2559.         struct drm_i915_gem_object *obj;
  2560.         int err = 0;
  2561.  
  2562.         list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
  2563.                 if (obj->gtt_space == NULL) {
  2564.                         printk(KERN_ERR "object found on GTT list with no space reserved\n");
  2565.                         err++;
  2566.                         continue;
  2567.                 }
  2568.  
  2569.                 if (obj->cache_level != obj->gtt_space->color) {
  2570.                         printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
  2571.                                obj->gtt_space->start,
  2572.                                obj->gtt_space->start + obj->gtt_space->size,
  2573.                                obj->cache_level,
  2574.                                obj->gtt_space->color);
  2575.                         err++;
  2576.                         continue;
  2577.                 }
  2578.  
  2579.                 if (!i915_gem_valid_gtt_space(dev,
  2580.                                               obj->gtt_space,
  2581.                                               obj->cache_level)) {
  2582.                         printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
  2583.                                obj->gtt_space->start,
  2584.                                obj->gtt_space->start + obj->gtt_space->size,
  2585.                                obj->cache_level);
  2586.                         err++;
  2587.                         continue;
  2588.                 }
  2589.         }
  2590.  
  2591.         WARN_ON(err);
  2592. #endif
  2593. }
  2594.  
  2595. /**
  2596.  * Finds free space in the GTT aperture and binds the object there.
  2597.  */
  2598. static int
  2599. i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  2600.                             unsigned alignment,
  2601.                             bool map_and_fenceable,
  2602.                             bool nonblocking)
  2603. {
  2604.         struct drm_device *dev = obj->base.dev;
  2605.         drm_i915_private_t *dev_priv = dev->dev_private;
  2606.         struct drm_mm_node *node;
  2607.         u32 size, fence_size, fence_alignment, unfenced_alignment;
  2608.         bool mappable, fenceable;
  2609.         int ret;
  2610.  
  2611.         if (obj->madv != I915_MADV_WILLNEED) {
  2612.         DRM_ERROR("Attempting to bind a purgeable object\n");
  2613.                 return -EINVAL;
  2614.         }
  2615.  
  2616.         fence_size = i915_gem_get_gtt_size(dev,
  2617.                                            obj->base.size,
  2618.                                            obj->tiling_mode);
  2619.         fence_alignment = i915_gem_get_gtt_alignment(dev,
  2620.                                                      obj->base.size,
  2621.                                                      obj->tiling_mode);
  2622.         unfenced_alignment =
  2623.                 i915_gem_get_unfenced_gtt_alignment(dev,
  2624.                                                     obj->base.size,
  2625.                                                     obj->tiling_mode);
  2626.  
  2627.         if (alignment == 0)
  2628.                 alignment = map_and_fenceable ? fence_alignment :
  2629.                                                 unfenced_alignment;
  2630.         if (map_and_fenceable && alignment & (fence_alignment - 1)) {
  2631.                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
  2632.                 return -EINVAL;
  2633.         }
  2634.  
  2635.         size = map_and_fenceable ? fence_size : obj->base.size;
  2636.  
  2637.         /* If the object is bigger than the entire aperture, reject it early
  2638.          * before evicting everything in a vain attempt to find space.
  2639.          */
  2640.         if (obj->base.size >
  2641.             (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
  2642.                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
  2643.                 return -E2BIG;
  2644.         }
  2645.  
  2646.         ret = i915_gem_object_get_pages(obj);
  2647.         if (ret)
  2648.                 return ret;
  2649.  
  2650.         i915_gem_object_pin_pages(obj);
  2651.  
  2652.         node = kzalloc(sizeof(*node), GFP_KERNEL);
  2653.         if (node == NULL) {
  2654.                 i915_gem_object_unpin_pages(obj);
  2655.                 return -ENOMEM;
  2656.         }
  2657.  
  2658.  search_free:
  2659.         if (map_and_fenceable)
  2660.                 ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space, node,
  2661.                                                           size, alignment, obj->cache_level,
  2662.                                                           0, dev_priv->mm.gtt_mappable_end);
  2663.         else
  2664.                 ret = drm_mm_insert_node_generic(&dev_priv->mm.gtt_space, node,
  2665.                                                  size, alignment, obj->cache_level);
  2666.         if (ret) {
  2667.  
  2668.                 i915_gem_object_unpin_pages(obj);
  2669.                 kfree(node);
  2670.                         return ret;
  2671.         }
  2672.         if (WARN_ON(!i915_gem_valid_gtt_space(dev, node, obj->cache_level))) {
  2673.                 i915_gem_object_unpin_pages(obj);
  2674.                 drm_mm_put_block(node);
  2675.                 return -EINVAL;
  2676.         }
  2677.  
  2678.         ret = i915_gem_gtt_prepare_object(obj);
  2679.         if (ret) {
  2680.                 i915_gem_object_unpin_pages(obj);
  2681.                 drm_mm_put_block(node);
  2682.                         return ret;
  2683.         }
  2684.  
  2685.         list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list);
  2686.         list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
  2687.  
  2688.         obj->gtt_space = node;
  2689.         obj->gtt_offset = node->start;
  2690.  
  2691.         fenceable =
  2692.                 node->size == fence_size &&
  2693.                 (node->start & (fence_alignment - 1)) == 0;
  2694.  
  2695.         mappable =
  2696.                 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
  2697.  
  2698.         obj->map_and_fenceable = mappable && fenceable;
  2699.  
  2700.         i915_gem_object_unpin_pages(obj);
  2701.         trace_i915_gem_object_bind(obj, map_and_fenceable);
  2702.         i915_gem_verify_gtt(dev);
  2703.         return 0;
  2704. }
  2705.  
  2706. void
  2707. i915_gem_clflush_object(struct drm_i915_gem_object *obj)
  2708. {
  2709.         /* If we don't have a page list set up, then we're not pinned
  2710.          * to GPU, and we can ignore the cache flush because it'll happen
  2711.          * again at bind time.
  2712.          */
  2713.         if (obj->pages == NULL)
  2714.                 return;
  2715.  
  2716.         /* If the GPU is snooping the contents of the CPU cache,
  2717.          * we do not need to manually clear the CPU cache lines.  However,
  2718.          * the caches are only snooped when the render cache is
  2719.          * flushed/invalidated.  As we always have to emit invalidations
  2720.          * and flushes when moving into and out of the RENDER domain, correct
  2721.          * snooping behaviour occurs naturally as the result of our domain
  2722.          * tracking.
  2723.          */
  2724.         if (obj->cache_level != I915_CACHE_NONE)
  2725.                 return;
  2726. #if 0
  2727.      if(obj->mapped != NULL)
  2728.      {
  2729.         uint8_t *page_virtual;
  2730.         unsigned int i;
  2731.  
  2732.         page_virtual = obj->mapped;
  2733.         asm volatile("mfence");
  2734.         for (i = 0; i < obj->base.size; i += x86_clflush_size)
  2735.             clflush(page_virtual + i);
  2736.         asm volatile("mfence");
  2737.      }
  2738.      else
  2739.      {
  2740.         uint8_t *page_virtual;
  2741.         unsigned int i;
  2742.         page_virtual = AllocKernelSpace(obj->base.size);
  2743.         if(page_virtual != NULL)
  2744.         {
  2745.             dma_addr_t *src, *dst;
  2746.             u32 count;
  2747.  
  2748. #define page_tabs  0xFDC00000      /* really dirty hack */
  2749.  
  2750.             src =  obj->pages.page;
  2751.             dst =  &((dma_addr_t*)page_tabs)[(u32_t)page_virtual >> 12];
  2752.             count = obj->base.size/4096;
  2753.  
  2754.             while(count--)
  2755.             {
  2756.                 *dst++ = (0xFFFFF000 & *src++) | 0x001 ;
  2757.             };
  2758.  
  2759.             asm volatile("mfence");
  2760.             for (i = 0; i < obj->base.size; i += x86_clflush_size)
  2761.                 clflush(page_virtual + i);
  2762.             asm volatile("mfence");
  2763.             FreeKernelSpace(page_virtual);
  2764.         }
  2765.         else
  2766.         {
  2767.             asm volatile (
  2768.             "mfence         \n"
  2769.             "wbinvd         \n"                 /* this is really ugly  */
  2770.             "mfence");
  2771.         }
  2772.      }
  2773. #endif
  2774.  
  2775. }
  2776.  
  2777. /** Flushes the GTT write domain for the object if it's dirty. */
  2778. static void
  2779. i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
  2780. {
  2781.         uint32_t old_write_domain;
  2782.  
  2783.         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
  2784.                 return;
  2785.  
  2786.         /* No actual flushing is required for the GTT write domain.  Writes
  2787.          * to it immediately go to main memory as far as we know, so there's
  2788.          * no chipset flush.  It also doesn't land in render cache.
  2789.          *
  2790.          * However, we do have to enforce the order so that all writes through
  2791.          * the GTT land before any writes to the device, such as updates to
  2792.          * the GATT itself.
  2793.          */
  2794.         wmb();
  2795.  
  2796.         old_write_domain = obj->base.write_domain;
  2797.         obj->base.write_domain = 0;
  2798.  
  2799.         trace_i915_gem_object_change_domain(obj,
  2800.                                             obj->base.read_domains,
  2801.                                             old_write_domain);
  2802. }
  2803.  
  2804. /** Flushes the CPU write domain for the object if it's dirty. */
  2805. static void
  2806. i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
  2807. {
  2808.         uint32_t old_write_domain;
  2809.  
  2810.         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
  2811.                 return;
  2812.  
  2813.         i915_gem_clflush_object(obj);
  2814.         i915_gem_chipset_flush(obj->base.dev);
  2815.         old_write_domain = obj->base.write_domain;
  2816.         obj->base.write_domain = 0;
  2817.  
  2818.         trace_i915_gem_object_change_domain(obj,
  2819.                                             obj->base.read_domains,
  2820.                                             old_write_domain);
  2821. }
  2822.  
  2823. /**
  2824.  * Moves a single object to the GTT read, and possibly write domain.
  2825.  *
  2826.  * This function returns when the move is complete, including waiting on
  2827.  * flushes to occur.
  2828.  */
  2829. int
  2830. i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
  2831. {
  2832.         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
  2833.         uint32_t old_write_domain, old_read_domains;
  2834.         int ret;
  2835.  
  2836.         /* Not valid to be called on unbound objects. */
  2837.         if (obj->gtt_space == NULL)
  2838.                 return -EINVAL;
  2839.  
  2840.         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
  2841.                 return 0;
  2842.  
  2843.         ret = i915_gem_object_wait_rendering(obj, !write);
  2844.                 if (ret)
  2845.                         return ret;
  2846.  
  2847.         i915_gem_object_flush_cpu_write_domain(obj);
  2848.  
  2849.         old_write_domain = obj->base.write_domain;
  2850.         old_read_domains = obj->base.read_domains;
  2851.  
  2852.         /* It should now be out of any other write domains, and we can update
  2853.          * the domain values for our changes.
  2854.          */
  2855.         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
  2856.         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
  2857.         if (write) {
  2858.                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
  2859.                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
  2860.                 obj->dirty = 1;
  2861.         }
  2862.  
  2863.         trace_i915_gem_object_change_domain(obj,
  2864.                                             old_read_domains,
  2865.                                             old_write_domain);
  2866.  
  2867.         /* And bump the LRU for this access */
  2868.         if (i915_gem_object_is_inactive(obj))
  2869.                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
  2870.  
  2871.         return 0;
  2872. }
  2873.  
  2874. int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
  2875.                                     enum i915_cache_level cache_level)
  2876. {
  2877.         struct drm_device *dev = obj->base.dev;
  2878.         drm_i915_private_t *dev_priv = dev->dev_private;
  2879.         int ret;
  2880.  
  2881.         if (obj->cache_level == cache_level)
  2882.                 return 0;
  2883.  
  2884.         if (obj->pin_count) {
  2885.                 DRM_DEBUG("can not change the cache level of pinned objects\n");
  2886.                 return -EBUSY;
  2887.         }
  2888.  
  2889.         if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) {
  2890.                 ret = i915_gem_object_unbind(obj);
  2891.                 if (ret)
  2892.                         return ret;
  2893.         }
  2894.  
  2895.         if (obj->gtt_space) {
  2896.                 ret = i915_gem_object_finish_gpu(obj);
  2897.                 if (ret)
  2898.                         return ret;
  2899.  
  2900.                 i915_gem_object_finish_gtt(obj);
  2901.  
  2902.                 /* Before SandyBridge, you could not use tiling or fence
  2903.                  * registers with snooped memory, so relinquish any fences
  2904.                  * currently pointing to our region in the aperture.
  2905.                  */
  2906.                 if (INTEL_INFO(dev)->gen < 6) {
  2907.                         ret = i915_gem_object_put_fence(obj);
  2908.                         if (ret)
  2909.                                 return ret;
  2910.                 }
  2911.  
  2912.                 if (obj->has_global_gtt_mapping)
  2913.                         i915_gem_gtt_bind_object(obj, cache_level);
  2914.                 if (obj->has_aliasing_ppgtt_mapping)
  2915.                         i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
  2916.                                                obj, cache_level);
  2917.  
  2918.                 obj->gtt_space->color = cache_level;
  2919.         }
  2920.  
  2921.         if (cache_level == I915_CACHE_NONE) {
  2922.                 u32 old_read_domains, old_write_domain;
  2923.  
  2924.                 /* If we're coming from LLC cached, then we haven't
  2925.                  * actually been tracking whether the data is in the
  2926.                  * CPU cache or not, since we only allow one bit set
  2927.                  * in obj->write_domain and have been skipping the clflushes.
  2928.                  * Just set it to the CPU cache for now.
  2929.                  */
  2930.                 WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
  2931.                 WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
  2932.  
  2933.                 old_read_domains = obj->base.read_domains;
  2934.                 old_write_domain = obj->base.write_domain;
  2935.  
  2936.                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  2937.                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  2938.  
  2939.                 trace_i915_gem_object_change_domain(obj,
  2940.                                                     old_read_domains,
  2941.                                                     old_write_domain);
  2942.     }
  2943.  
  2944.         obj->cache_level = cache_level;
  2945.         i915_gem_verify_gtt(dev);
  2946.         return 0;
  2947. }
  2948.  
  2949. int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
  2950.                                struct drm_file *file)
  2951. {
  2952.         struct drm_i915_gem_caching *args = data;
  2953.         struct drm_i915_gem_object *obj;
  2954.         int ret;
  2955.  
  2956.         ret = i915_mutex_lock_interruptible(dev);
  2957.         if (ret)
  2958.                 return ret;
  2959.  
  2960.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  2961.         if (&obj->base == NULL) {
  2962.                 ret = -ENOENT;
  2963.                 goto unlock;
  2964.         }
  2965.  
  2966.         args->caching = obj->cache_level != I915_CACHE_NONE;
  2967.  
  2968.         drm_gem_object_unreference(&obj->base);
  2969. unlock:
  2970.         mutex_unlock(&dev->struct_mutex);
  2971.         return ret;
  2972. }
  2973.  
  2974. int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
  2975.                                struct drm_file *file)
  2976. {
  2977.         struct drm_i915_gem_caching *args = data;
  2978.         struct drm_i915_gem_object *obj;
  2979.         enum i915_cache_level level;
  2980.         int ret;
  2981.  
  2982.         switch (args->caching) {
  2983.         case I915_CACHING_NONE:
  2984.                 level = I915_CACHE_NONE;
  2985.                 break;
  2986.         case I915_CACHING_CACHED:
  2987.                 level = I915_CACHE_LLC;
  2988.                 break;
  2989.         default:
  2990.                 return -EINVAL;
  2991.         }
  2992.  
  2993.         ret = i915_mutex_lock_interruptible(dev);
  2994.         if (ret)
  2995.                 return ret;
  2996.  
  2997.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  2998.         if (&obj->base == NULL) {
  2999.                 ret = -ENOENT;
  3000.                 goto unlock;
  3001.         }
  3002.  
  3003.         ret = i915_gem_object_set_cache_level(obj, level);
  3004.  
  3005.         drm_gem_object_unreference(&obj->base);
  3006. unlock:
  3007.         mutex_unlock(&dev->struct_mutex);
  3008.         return ret;
  3009. }
  3010.  
  3011. /*
  3012.  * Prepare buffer for display plane (scanout, cursors, etc).
  3013.  * Can be called from an uninterruptible phase (modesetting) and allows
  3014.  * any flushes to be pipelined (for pageflips).
  3015.  */
  3016. int
  3017. i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
  3018.                                      u32 alignment,
  3019.                                      struct intel_ring_buffer *pipelined)
  3020. {
  3021.         u32 old_read_domains, old_write_domain;
  3022.         int ret;
  3023.  
  3024.         if (pipelined != obj->ring) {
  3025.                 ret = i915_gem_object_sync(obj, pipelined);
  3026.         if (ret)
  3027.                 return ret;
  3028.         }
  3029.  
  3030.         /* The display engine is not coherent with the LLC cache on gen6.  As
  3031.          * a result, we make sure that the pinning that is about to occur is
  3032.          * done with uncached PTEs. This is lowest common denominator for all
  3033.          * chipsets.
  3034.          *
  3035.          * However for gen6+, we could do better by using the GFDT bit instead
  3036.          * of uncaching, which would allow us to flush all the LLC-cached data
  3037.          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
  3038.          */
  3039.         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
  3040.         if (ret)
  3041.                 return ret;
  3042.  
  3043.         /* As the user may map the buffer once pinned in the display plane
  3044.          * (e.g. libkms for the bootup splash), we have to ensure that we
  3045.          * always use map_and_fenceable for all scanout buffers.
  3046.          */
  3047.         ret = i915_gem_object_pin(obj, alignment, true, false);
  3048.         if (ret)
  3049.                 return ret;
  3050.  
  3051.         i915_gem_object_flush_cpu_write_domain(obj);
  3052.  
  3053.         old_write_domain = obj->base.write_domain;
  3054.         old_read_domains = obj->base.read_domains;
  3055.  
  3056.         /* It should now be out of any other write domains, and we can update
  3057.          * the domain values for our changes.
  3058.          */
  3059.         obj->base.write_domain = 0;
  3060.         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
  3061.  
  3062.         trace_i915_gem_object_change_domain(obj,
  3063.                                             old_read_domains,
  3064.                                             old_write_domain);
  3065.  
  3066.         return 0;
  3067. }
  3068.  
  3069. int
  3070. i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
  3071. {
  3072.         int ret;
  3073.  
  3074.         if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
  3075.                 return 0;
  3076.  
  3077.         ret = i915_gem_object_wait_rendering(obj, false);
  3078.     if (ret)
  3079.         return ret;
  3080.  
  3081.         /* Ensure that we invalidate the GPU's caches and TLBs. */
  3082.         obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
  3083.         return 0;
  3084. }
  3085.  
  3086. /**
  3087.  * Moves a single object to the CPU read, and possibly write domain.
  3088.  *
  3089.  * This function returns when the move is complete, including waiting on
  3090.  * flushes to occur.
  3091.  */
  3092. int
  3093. i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
  3094. {
  3095.         uint32_t old_write_domain, old_read_domains;
  3096.         int ret;
  3097.  
  3098.         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
  3099.                 return 0;
  3100.  
  3101.         ret = i915_gem_object_wait_rendering(obj, !write);
  3102.         if (ret)
  3103.                 return ret;
  3104.  
  3105.         i915_gem_object_flush_gtt_write_domain(obj);
  3106.  
  3107.         old_write_domain = obj->base.write_domain;
  3108.         old_read_domains = obj->base.read_domains;
  3109.  
  3110.         /* Flush the CPU cache if it's still invalid. */
  3111.         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
  3112.                 i915_gem_clflush_object(obj);
  3113.  
  3114.                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
  3115.         }
  3116.  
  3117.         /* It should now be out of any other write domains, and we can update
  3118.          * the domain values for our changes.
  3119.          */
  3120.         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
  3121.  
  3122.         /* If we're writing through the CPU, then the GPU read domains will
  3123.          * need to be invalidated at next use.
  3124.          */
  3125.         if (write) {
  3126.                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  3127.                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  3128.         }
  3129.  
  3130.         trace_i915_gem_object_change_domain(obj,
  3131.                                             old_read_domains,
  3132.                                             old_write_domain);
  3133.  
  3134.         return 0;
  3135. }
  3136.  
  3137. /* Throttle our rendering by waiting until the ring has completed our requests
  3138.  * emitted over 20 msec ago.
  3139.  *
  3140.  * Note that if we were to use the current jiffies each time around the loop,
  3141.  * we wouldn't escape the function with any frames outstanding if the time to
  3142.  * render a frame was over 20ms.
  3143.  *
  3144.  * This should get us reasonable parallelism between CPU and GPU but also
  3145.  * relatively low latency when blocking on a particular request to finish.
  3146.  */
  3147. static int
  3148. i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
  3149. {
  3150.         struct drm_i915_private *dev_priv = dev->dev_private;
  3151.         struct drm_i915_file_private *file_priv = file->driver_priv;
  3152.         unsigned long recent_enough = GetTimerTicks() - msecs_to_jiffies(20);
  3153.         struct drm_i915_gem_request *request;
  3154.         struct intel_ring_buffer *ring = NULL;
  3155.         u32 seqno = 0;
  3156.         int ret;
  3157.  
  3158.         if (atomic_read(&dev_priv->mm.wedged))
  3159.                 return -EIO;
  3160.  
  3161.         spin_lock(&file_priv->mm.lock);
  3162.         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
  3163.                 if (time_after_eq(request->emitted_jiffies, recent_enough))
  3164.                         break;
  3165.  
  3166.                 ring = request->ring;
  3167.                 seqno = request->seqno;
  3168.         }
  3169.         spin_unlock(&file_priv->mm.lock);
  3170.  
  3171.         if (seqno == 0)
  3172.                 return 0;
  3173.  
  3174.         ret = __wait_seqno(ring, seqno, true, NULL);
  3175.         if (ret == 0)
  3176.                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
  3177.  
  3178.         return ret;
  3179. }
  3180.  
  3181. int
  3182. i915_gem_object_pin(struct drm_i915_gem_object *obj,
  3183.                     uint32_t alignment,
  3184.                     bool map_and_fenceable,
  3185.                     bool nonblocking)
  3186. {
  3187.         int ret;
  3188.  
  3189.         if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
  3190.                 return -EBUSY;
  3191.  
  3192.         if (obj->gtt_space != NULL) {
  3193.                 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
  3194.                     (map_and_fenceable && !obj->map_and_fenceable)) {
  3195.                         WARN(obj->pin_count,
  3196.                              "bo is already pinned with incorrect alignment:"
  3197.                              " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
  3198.                              " obj->map_and_fenceable=%d\n",
  3199.                              obj->gtt_offset, alignment,
  3200.                              map_and_fenceable,
  3201.                              obj->map_and_fenceable);
  3202.                         ret = i915_gem_object_unbind(obj);
  3203.                         if (ret)
  3204.                                 return ret;
  3205.                 }
  3206.         }
  3207.  
  3208.         if (obj->gtt_space == NULL) {
  3209.                 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  3210.  
  3211.                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
  3212.                                                   map_and_fenceable,
  3213.                                                   nonblocking);
  3214.                 if (ret)
  3215.                         return ret;
  3216.  
  3217.                 if (!dev_priv->mm.aliasing_ppgtt)
  3218.                         i915_gem_gtt_bind_object(obj, obj->cache_level);
  3219.         }
  3220.  
  3221.         if (!obj->has_global_gtt_mapping && map_and_fenceable)
  3222.                 i915_gem_gtt_bind_object(obj, obj->cache_level);
  3223.  
  3224.         obj->pin_count++;
  3225.         obj->pin_mappable |= map_and_fenceable;
  3226.  
  3227.         return 0;
  3228. }
  3229.  
  3230. void
  3231. i915_gem_object_unpin(struct drm_i915_gem_object *obj)
  3232. {
  3233.         BUG_ON(obj->pin_count == 0);
  3234.         BUG_ON(obj->gtt_space == NULL);
  3235.  
  3236.         if (--obj->pin_count == 0)
  3237.                 obj->pin_mappable = false;
  3238. }
  3239.  
  3240. int
  3241. i915_gem_pin_ioctl(struct drm_device *dev, void *data,
  3242.                    struct drm_file *file)
  3243. {
  3244.         struct drm_i915_gem_pin *args = data;
  3245.         struct drm_i915_gem_object *obj;
  3246.         int ret;
  3247.  
  3248.         ret = i915_mutex_lock_interruptible(dev);
  3249.         if (ret)
  3250.                 return ret;
  3251.  
  3252.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  3253.         if (&obj->base == NULL) {
  3254.                 ret = -ENOENT;
  3255.                 goto unlock;
  3256.         }
  3257.  
  3258.         if (obj->madv != I915_MADV_WILLNEED) {
  3259.                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
  3260.                 ret = -EINVAL;
  3261.                 goto out;
  3262.         }
  3263.  
  3264.         if (obj->pin_filp != NULL && obj->pin_filp != file) {
  3265.                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
  3266.                           args->handle);
  3267.                 ret = -EINVAL;
  3268.                 goto out;
  3269.         }
  3270.  
  3271.         if (obj->user_pin_count == 0) {
  3272.                 ret = i915_gem_object_pin(obj, args->alignment, true, false);
  3273.                 if (ret)
  3274.                         goto out;
  3275.         }
  3276.  
  3277.         obj->user_pin_count++;
  3278.         obj->pin_filp = file;
  3279.  
  3280.         /* XXX - flush the CPU caches for pinned objects
  3281.          * as the X server doesn't manage domains yet
  3282.          */
  3283.         i915_gem_object_flush_cpu_write_domain(obj);
  3284.         args->offset = obj->gtt_offset;
  3285. out:
  3286.         drm_gem_object_unreference(&obj->base);
  3287. unlock:
  3288.         mutex_unlock(&dev->struct_mutex);
  3289.         return ret;
  3290. }
  3291.  
  3292. #if 0
  3293.  
  3294. int
  3295. i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
  3296.                      struct drm_file *file)
  3297. {
  3298.         struct drm_i915_gem_pin *args = data;
  3299.         struct drm_i915_gem_object *obj;
  3300.         int ret;
  3301.  
  3302.         ret = i915_mutex_lock_interruptible(dev);
  3303.         if (ret)
  3304.                 return ret;
  3305.  
  3306.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  3307.         if (&obj->base == NULL) {
  3308.                 ret = -ENOENT;
  3309.                 goto unlock;
  3310.         }
  3311.  
  3312.         if (obj->pin_filp != file) {
  3313.                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
  3314.                           args->handle);
  3315.                 ret = -EINVAL;
  3316.                 goto out;
  3317.         }
  3318.         obj->user_pin_count--;
  3319.         if (obj->user_pin_count == 0) {
  3320.                 obj->pin_filp = NULL;
  3321.                 i915_gem_object_unpin(obj);
  3322.         }
  3323.  
  3324. out:
  3325.         drm_gem_object_unreference(&obj->base);
  3326. unlock:
  3327.         mutex_unlock(&dev->struct_mutex);
  3328.         return ret;
  3329. }
  3330.  
  3331. #endif
  3332.  
  3333. int
  3334. i915_gem_busy_ioctl(struct drm_device *dev, void *data,
  3335.                     struct drm_file *file)
  3336. {
  3337.         struct drm_i915_gem_busy *args = data;
  3338.         struct drm_i915_gem_object *obj;
  3339.         int ret;
  3340.  
  3341.         ret = i915_mutex_lock_interruptible(dev);
  3342.         if (ret)
  3343.                 return ret;
  3344.  
  3345.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  3346.         if (&obj->base == NULL) {
  3347.                 ret = -ENOENT;
  3348.                 goto unlock;
  3349.         }
  3350.  
  3351.         /* Count all active objects as busy, even if they are currently not used
  3352.          * by the gpu. Users of this interface expect objects to eventually
  3353.          * become non-busy without any further actions, therefore emit any
  3354.          * necessary flushes here.
  3355.          */
  3356.         ret = i915_gem_object_flush_active(obj);
  3357.  
  3358.         args->busy = obj->active;
  3359.         if (obj->ring) {
  3360.                 BUILD_BUG_ON(I915_NUM_RINGS > 16);
  3361.                 args->busy |= intel_ring_flag(obj->ring) << 16;
  3362.         }
  3363.  
  3364.         drm_gem_object_unreference(&obj->base);
  3365. unlock:
  3366.         mutex_unlock(&dev->struct_mutex);
  3367.         return ret;
  3368. }
  3369.  
  3370. int
  3371. i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
  3372.                         struct drm_file *file_priv)
  3373. {
  3374.         return i915_gem_ring_throttle(dev, file_priv);
  3375. }
  3376.  
  3377. #if 0
  3378.  
  3379. int
  3380. i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
  3381.                        struct drm_file *file_priv)
  3382. {
  3383.         struct drm_i915_gem_madvise *args = data;
  3384.         struct drm_i915_gem_object *obj;
  3385.         int ret;
  3386.  
  3387.         switch (args->madv) {
  3388.         case I915_MADV_DONTNEED:
  3389.         case I915_MADV_WILLNEED:
  3390.             break;
  3391.         default:
  3392.             return -EINVAL;
  3393.         }
  3394.  
  3395.         ret = i915_mutex_lock_interruptible(dev);
  3396.         if (ret)
  3397.                 return ret;
  3398.  
  3399.         obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
  3400.         if (&obj->base == NULL) {
  3401.                 ret = -ENOENT;
  3402.                 goto unlock;
  3403.         }
  3404.  
  3405.         if (obj->pin_count) {
  3406.                 ret = -EINVAL;
  3407.                 goto out;
  3408.         }
  3409.  
  3410.         if (obj->madv != __I915_MADV_PURGED)
  3411.                 obj->madv = args->madv;
  3412.  
  3413.         /* if the object is no longer attached, discard its backing storage */
  3414.         if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
  3415.                 i915_gem_object_truncate(obj);
  3416.  
  3417.         args->retained = obj->madv != __I915_MADV_PURGED;
  3418.  
  3419. out:
  3420.         drm_gem_object_unreference(&obj->base);
  3421. unlock:
  3422.         mutex_unlock(&dev->struct_mutex);
  3423.         return ret;
  3424. }
  3425. #endif
  3426.  
  3427. void i915_gem_object_init(struct drm_i915_gem_object *obj,
  3428.                           const struct drm_i915_gem_object_ops *ops)
  3429. {
  3430.         INIT_LIST_HEAD(&obj->mm_list);
  3431.         INIT_LIST_HEAD(&obj->gtt_list);
  3432.         INIT_LIST_HEAD(&obj->ring_list);
  3433.         INIT_LIST_HEAD(&obj->exec_list);
  3434.  
  3435.         obj->ops = ops;
  3436.  
  3437.         obj->fence_reg = I915_FENCE_REG_NONE;
  3438.         obj->madv = I915_MADV_WILLNEED;
  3439.         /* Avoid an unnecessary call to unbind on the first bind. */
  3440.         obj->map_and_fenceable = true;
  3441.  
  3442.         i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
  3443. }
  3444.  
  3445. static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
  3446.         .get_pages = i915_gem_object_get_pages_gtt,
  3447.         .put_pages = i915_gem_object_put_pages_gtt,
  3448. };
  3449.  
  3450. struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
  3451.                                                   size_t size)
  3452. {
  3453.         struct drm_i915_gem_object *obj;
  3454.         struct address_space *mapping;
  3455.         u32 mask;
  3456.  
  3457.         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
  3458.         if (obj == NULL)
  3459.                 return NULL;
  3460.  
  3461.         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
  3462.                 kfree(obj);
  3463.                 return NULL;
  3464.         }
  3465.  
  3466.  
  3467.         i915_gem_object_init(obj, &i915_gem_object_ops);
  3468.  
  3469.         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  3470.         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  3471.  
  3472.         if (HAS_LLC(dev)) {
  3473.                 /* On some devices, we can have the GPU use the LLC (the CPU
  3474.                  * cache) for about a 10% performance improvement
  3475.                  * compared to uncached.  Graphics requests other than
  3476.                  * display scanout are coherent with the CPU in
  3477.                  * accessing this cache.  This means in this mode we
  3478.                  * don't need to clflush on the CPU side, and on the
  3479.                  * GPU side we only need to flush internal caches to
  3480.                  * get data visible to the CPU.
  3481.                  *
  3482.                  * However, we maintain the display planes as UC, and so
  3483.                  * need to rebind when first used as such.
  3484.                  */
  3485.                 obj->cache_level = I915_CACHE_LLC;
  3486.         } else
  3487.                 obj->cache_level = I915_CACHE_NONE;
  3488.  
  3489.         return obj;
  3490. }
  3491.  
  3492. int i915_gem_init_object(struct drm_gem_object *obj)
  3493. {
  3494.         BUG();
  3495.  
  3496.         return 0;
  3497. }
  3498.  
  3499. void i915_gem_free_object(struct drm_gem_object *gem_obj)
  3500. {
  3501.         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
  3502.         struct drm_device *dev = obj->base.dev;
  3503.         drm_i915_private_t *dev_priv = dev->dev_private;
  3504.  
  3505.         trace_i915_gem_object_destroy(obj);
  3506.  
  3507. //   if (obj->phys_obj)
  3508. //       i915_gem_detach_phys_object(dev, obj);
  3509.  
  3510.         obj->pin_count = 0;
  3511.         if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
  3512.                 bool was_interruptible;
  3513.  
  3514.                 was_interruptible = dev_priv->mm.interruptible;
  3515.                 dev_priv->mm.interruptible = false;
  3516.  
  3517.                 WARN_ON(i915_gem_object_unbind(obj));
  3518.  
  3519.                 dev_priv->mm.interruptible = was_interruptible;
  3520.         }
  3521.  
  3522.         obj->pages_pin_count = 0;
  3523.         i915_gem_object_put_pages(obj);
  3524. //   i915_gem_object_free_mmap_offset(obj);
  3525.  
  3526.         BUG_ON(obj->pages);
  3527.  
  3528. //   if (obj->base.import_attach)
  3529. //       drm_prime_gem_destroy(&obj->base, NULL);
  3530.  
  3531.         drm_gem_object_release(&obj->base);
  3532.         i915_gem_info_remove_obj(dev_priv, obj->base.size);
  3533.  
  3534.         kfree(obj->bit_17);
  3535.         kfree(obj);
  3536. }
  3537.  
  3538. #if 0
  3539. int
  3540. i915_gem_idle(struct drm_device *dev)
  3541. {
  3542.         drm_i915_private_t *dev_priv = dev->dev_private;
  3543.         int ret;
  3544.  
  3545.         mutex_lock(&dev->struct_mutex);
  3546.  
  3547.         if (dev_priv->mm.suspended) {
  3548.                 mutex_unlock(&dev->struct_mutex);
  3549.                 return 0;
  3550.         }
  3551.  
  3552.         ret = i915_gpu_idle(dev);
  3553.         if (ret) {
  3554.                 mutex_unlock(&dev->struct_mutex);
  3555.                 return ret;
  3556.         }
  3557.         i915_gem_retire_requests(dev);
  3558.  
  3559.         i915_gem_reset_fences(dev);
  3560.  
  3561.         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
  3562.          * We need to replace this with a semaphore, or something.
  3563.          * And not confound mm.suspended!
  3564.          */
  3565.         dev_priv->mm.suspended = 1;
  3566.         del_timer_sync(&dev_priv->hangcheck_timer);
  3567.  
  3568.         i915_kernel_lost_context(dev);
  3569.         i915_gem_cleanup_ringbuffer(dev);
  3570.  
  3571.         mutex_unlock(&dev->struct_mutex);
  3572.  
  3573.         /* Cancel the retire work handler, which should be idle now. */
  3574.         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
  3575.  
  3576.         return 0;
  3577. }
  3578. #endif
  3579.  
  3580. void i915_gem_l3_remap(struct drm_device *dev)
  3581. {
  3582.         drm_i915_private_t *dev_priv = dev->dev_private;
  3583.         u32 misccpctl;
  3584.         int i;
  3585.  
  3586.         if (!IS_IVYBRIDGE(dev))
  3587.                 return;
  3588.  
  3589.         if (!dev_priv->l3_parity.remap_info)
  3590.                 return;
  3591.  
  3592.         misccpctl = I915_READ(GEN7_MISCCPCTL);
  3593.         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
  3594.         POSTING_READ(GEN7_MISCCPCTL);
  3595.  
  3596.         for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
  3597.                 u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
  3598.                 if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
  3599.                         DRM_DEBUG("0x%x was already programmed to %x\n",
  3600.                                   GEN7_L3LOG_BASE + i, remap);
  3601.                 if (remap && !dev_priv->l3_parity.remap_info[i/4])
  3602.                         DRM_DEBUG_DRIVER("Clearing remapped register\n");
  3603.                 I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
  3604.         }
  3605.  
  3606.         /* Make sure all the writes land before disabling dop clock gating */
  3607.         POSTING_READ(GEN7_L3LOG_BASE);
  3608.  
  3609.         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
  3610. }
  3611.  
  3612. void i915_gem_init_swizzling(struct drm_device *dev)
  3613. {
  3614.         drm_i915_private_t *dev_priv = dev->dev_private;
  3615.  
  3616.         if (INTEL_INFO(dev)->gen < 5 ||
  3617.             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
  3618.                 return;
  3619.  
  3620.         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
  3621.                                  DISP_TILE_SURFACE_SWIZZLING);
  3622.  
  3623.         if (IS_GEN5(dev))
  3624.                 return;
  3625.  
  3626.         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
  3627.         if (IS_GEN6(dev))
  3628.                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
  3629.         else
  3630.                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
  3631. }
  3632.  
  3633. static bool
  3634. intel_enable_blt(struct drm_device *dev)
  3635. {
  3636.         if (!HAS_BLT(dev))
  3637.                 return false;
  3638.  
  3639.         /* The blitter was dysfunctional on early prototypes */
  3640.         if (IS_GEN6(dev) && dev->pdev->revision < 8) {
  3641.                 DRM_INFO("BLT not supported on this pre-production hardware;"
  3642.                          " graphics performance will be degraded.\n");
  3643.                 return false;
  3644.         }
  3645.  
  3646.         return true;
  3647. }
  3648.  
  3649. int
  3650. i915_gem_init_hw(struct drm_device *dev)
  3651. {
  3652.         drm_i915_private_t *dev_priv = dev->dev_private;
  3653.         int ret;
  3654.  
  3655.         if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
  3656.                 return -EIO;
  3657.  
  3658.         if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))
  3659.                 I915_WRITE(0x9008, I915_READ(0x9008) | 0xf0000);
  3660.  
  3661.         i915_gem_l3_remap(dev);
  3662.  
  3663.         i915_gem_init_swizzling(dev);
  3664.  
  3665.         ret = intel_init_render_ring_buffer(dev);
  3666.         if (ret)
  3667.                 return ret;
  3668.  
  3669.     if (HAS_BSD(dev)) {
  3670.                 ret = intel_init_bsd_ring_buffer(dev);
  3671.                 if (ret)
  3672.                         goto cleanup_render_ring;
  3673.         }
  3674.  
  3675.         if (intel_enable_blt(dev)) {
  3676.                 ret = intel_init_blt_ring_buffer(dev);
  3677.                 if (ret)
  3678.                         goto cleanup_bsd_ring;
  3679.         }
  3680.  
  3681.         dev_priv->next_seqno = 1;
  3682.  
  3683.         /*
  3684.          * XXX: There was some w/a described somewhere suggesting loading
  3685.          * contexts before PPGTT.
  3686.          */
  3687.         i915_gem_context_init(dev);
  3688.         i915_gem_init_ppgtt(dev);
  3689.  
  3690.         return 0;
  3691.  
  3692. cleanup_bsd_ring:
  3693.         intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
  3694. cleanup_render_ring:
  3695.         intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
  3696.         return ret;
  3697. }
  3698.  
  3699. static bool
  3700. intel_enable_ppgtt(struct drm_device *dev)
  3701. {
  3702.         if (i915_enable_ppgtt >= 0)
  3703.                 return i915_enable_ppgtt;
  3704.  
  3705. #ifdef CONFIG_INTEL_IOMMU
  3706.         /* Disable ppgtt on SNB if VT-d is on. */
  3707.         if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
  3708.                 return false;
  3709. #endif
  3710.  
  3711.         return true;
  3712. }
  3713.  
  3714. #define LFB_SIZE 0xC00000
  3715.  
  3716. int i915_gem_init(struct drm_device *dev)
  3717. {
  3718.         struct drm_i915_private *dev_priv = dev->dev_private;
  3719.         unsigned long gtt_size, mappable_size;
  3720.         int ret;
  3721.  
  3722.         gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT;
  3723.         mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
  3724.  
  3725.         mutex_lock(&dev->struct_mutex);
  3726.         if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
  3727.                 /* PPGTT pdes are stolen from global gtt ptes, so shrink the
  3728.                  * aperture accordingly when using aliasing ppgtt. */
  3729.                 gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
  3730.  
  3731.         i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE);
  3732.  
  3733.                 ret = i915_gem_init_aliasing_ppgtt(dev);
  3734.                 if (ret) {
  3735.                         mutex_unlock(&dev->struct_mutex);
  3736.                         return ret;
  3737.                 }
  3738.         } else {
  3739.                 /* Let GEM Manage all of the aperture.
  3740.                  *
  3741.                  * However, leave one page at the end still bound to the scratch
  3742.                  * page.  There are a number of places where the hardware
  3743.                  * apparently prefetches past the end of the object, and we've
  3744.                  * seen multiple hangs with the GPU head pointer stuck in a
  3745.                  * batchbuffer bound at the last page of the aperture.  One page
  3746.                  * should be enough to keep any prefetching inside of the
  3747.                  * aperture.
  3748.                  */
  3749.         i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE);
  3750.         }
  3751.  
  3752.         ret = i915_gem_init_hw(dev);
  3753.         mutex_unlock(&dev->struct_mutex);
  3754.         if (ret) {
  3755.                 i915_gem_cleanup_aliasing_ppgtt(dev);
  3756.                 return ret;
  3757.         }
  3758.  
  3759.     return 0;
  3760. }
  3761.  
  3762. void
  3763. i915_gem_cleanup_ringbuffer(struct drm_device *dev)
  3764. {
  3765.         drm_i915_private_t *dev_priv = dev->dev_private;
  3766.         struct intel_ring_buffer *ring;
  3767.         int i;
  3768.  
  3769.         for_each_ring(ring, dev_priv, i)
  3770.                 intel_cleanup_ring_buffer(ring);
  3771. }
  3772.  
  3773. #if 0
  3774.  
  3775. int
  3776. i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
  3777.                        struct drm_file *file_priv)
  3778. {
  3779.         drm_i915_private_t *dev_priv = dev->dev_private;
  3780.         int ret;
  3781.  
  3782.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  3783.                 return 0;
  3784.  
  3785.         if (atomic_read(&dev_priv->mm.wedged)) {
  3786.                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
  3787.                 atomic_set(&dev_priv->mm.wedged, 0);
  3788.         }
  3789.  
  3790.         mutex_lock(&dev->struct_mutex);
  3791.         dev_priv->mm.suspended = 0;
  3792.  
  3793.         ret = i915_gem_init_hw(dev);
  3794.         if (ret != 0) {
  3795.                 mutex_unlock(&dev->struct_mutex);
  3796.                 return ret;
  3797.         }
  3798.  
  3799.         BUG_ON(!list_empty(&dev_priv->mm.active_list));
  3800.         mutex_unlock(&dev->struct_mutex);
  3801.  
  3802.         ret = drm_irq_install(dev);
  3803.         if (ret)
  3804.                 goto cleanup_ringbuffer;
  3805.  
  3806.         return 0;
  3807.  
  3808. cleanup_ringbuffer:
  3809.         mutex_lock(&dev->struct_mutex);
  3810.         i915_gem_cleanup_ringbuffer(dev);
  3811.         dev_priv->mm.suspended = 1;
  3812.         mutex_unlock(&dev->struct_mutex);
  3813.  
  3814.         return ret;
  3815. }
  3816.  
  3817. int
  3818. i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
  3819.                        struct drm_file *file_priv)
  3820. {
  3821.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  3822.                 return 0;
  3823.  
  3824.         drm_irq_uninstall(dev);
  3825.         return i915_gem_idle(dev);
  3826. }
  3827.  
  3828. void
  3829. i915_gem_lastclose(struct drm_device *dev)
  3830. {
  3831.         int ret;
  3832.  
  3833.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  3834.                 return;
  3835.  
  3836.         ret = i915_gem_idle(dev);
  3837.         if (ret)
  3838.                 DRM_ERROR("failed to idle hardware: %d\n", ret);
  3839. }
  3840. #endif
  3841.  
  3842. static void
  3843. init_ring_lists(struct intel_ring_buffer *ring)
  3844. {
  3845.     INIT_LIST_HEAD(&ring->active_list);
  3846.     INIT_LIST_HEAD(&ring->request_list);
  3847. }
  3848.  
  3849. void
  3850. i915_gem_load(struct drm_device *dev)
  3851. {
  3852.     int i;
  3853.     drm_i915_private_t *dev_priv = dev->dev_private;
  3854.  
  3855.     INIT_LIST_HEAD(&dev_priv->mm.active_list);
  3856.     INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
  3857.         INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
  3858.         INIT_LIST_HEAD(&dev_priv->mm.bound_list);
  3859.     INIT_LIST_HEAD(&dev_priv->mm.fence_list);
  3860.     for (i = 0; i < I915_NUM_RINGS; i++)
  3861.         init_ring_lists(&dev_priv->ring[i]);
  3862.         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
  3863.         INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
  3864.         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
  3865.                           i915_gem_retire_work_handler);
  3866.  
  3867.     /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
  3868.     if (IS_GEN3(dev)) {
  3869.                 I915_WRITE(MI_ARB_STATE,
  3870.                            _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
  3871.     }
  3872.  
  3873.     dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
  3874.  
  3875.     if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
  3876.         dev_priv->num_fence_regs = 16;
  3877.     else
  3878.         dev_priv->num_fence_regs = 8;
  3879.  
  3880.     /* Initialize fence registers to zero */
  3881.         i915_gem_reset_fences(dev);
  3882.  
  3883.     i915_gem_detect_bit_6_swizzle(dev);
  3884.  
  3885.     dev_priv->mm.interruptible = true;
  3886.  
  3887. //    dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
  3888. //    dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
  3889. //    register_shrinker(&dev_priv->mm.inactive_shrinker);
  3890. }
  3891.  
  3892.  
  3893.