Subversion Repositories Kolibri OS

Rev

Rev 3290 | Rev 3480 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright © 2008 Intel Corporation
  3.  *
  4.  * Permission is hereby granted, free of charge, to any person obtaining a
  5.  * copy of this software and associated documentation files (the "Software"),
  6.  * to deal in the Software without restriction, including without limitation
  7.  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8.  * and/or sell copies of the Software, and to permit persons to whom the
  9.  * Software is furnished to do so, subject to the following conditions:
  10.  *
  11.  * The above copyright notice and this permission notice (including the next
  12.  * paragraph) shall be included in all copies or substantial portions of the
  13.  * Software.
  14.  *
  15.  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16.  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17.  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18.  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19.  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20.  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21.  * IN THE SOFTWARE.
  22.  *
  23.  * Authors:
  24.  *    Eric Anholt <eric@anholt.net>
  25.  *
  26.  */
  27.  
  28. #include <drm/drmP.h>
  29. #include <drm/i915_drm.h>
  30. #include "i915_drv.h"
  31. #include "i915_trace.h"
  32. #include "intel_drv.h"
  33. #include <linux/shmem_fs.h>
  34. #include <linux/slab.h>
  35. //#include <linux/swap.h>
  36. #include <linux/pci.h>
  37.  
  38. extern int x86_clflush_size;
  39.  
  40. #define PROT_READ       0x1             /* page can be read */
  41. #define PROT_WRITE      0x2             /* page can be written */
  42. #define MAP_SHARED      0x01            /* Share changes */
  43.  
  44. #undef mb
  45. #undef rmb
  46. #undef wmb
  47. #define mb() asm volatile("mfence")
  48. #define rmb() asm volatile ("lfence")
  49. #define wmb() asm volatile ("sfence")
  50.  
  51. struct drm_i915_gem_object *get_fb_obj();
  52.  
  53. unsigned long vm_mmap(struct file *file, unsigned long addr,
  54.          unsigned long len, unsigned long prot,
  55.          unsigned long flag, unsigned long offset);
  56.  
  57. static inline void clflush(volatile void *__p)
  58. {
  59.     asm volatile("clflush %0" : "+m" (*(volatile char*)__p));
  60. }
  61.  
  62. #define MAX_ERRNO       4095
  63.  
  64. #define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
  65.  
  66.  
  67. #define I915_EXEC_CONSTANTS_MASK        (3<<6)
  68. #define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */
  69. #define I915_EXEC_CONSTANTS_ABSOLUTE    (1<<6)
  70. #define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */
  71.  
  72. static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  73. static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  74. static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  75.                                                     unsigned alignment,
  76.                                                     bool map_and_fenceable,
  77.                                                     bool nonblocking);
  78. static int i915_gem_phys_pwrite(struct drm_device *dev,
  79.                                 struct drm_i915_gem_object *obj,
  80.                                 struct drm_i915_gem_pwrite *args,
  81.                                 struct drm_file *file);
  82.  
  83. static void i915_gem_write_fence(struct drm_device *dev, int reg,
  84.                                  struct drm_i915_gem_object *obj);
  85. static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
  86.                                          struct drm_i915_fence_reg *fence,
  87.                                          bool enable);
  88.  
  89. static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
  90. static void i915_gem_shrink_all(struct drm_i915_private *dev_priv);
  91. static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
  92.  
  93. static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
  94. {
  95.         if (obj->tiling_mode)
  96.                 i915_gem_release_mmap(obj);
  97.  
  98.         /* As we do not have an associated fence register, we will force
  99.          * a tiling change if we ever need to acquire one.
  100.          */
  101.         obj->fence_dirty = false;
  102.         obj->fence_reg = I915_FENCE_REG_NONE;
  103. }
  104.  
  105. /* some bookkeeping */
  106. static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  107.                                   size_t size)
  108. {
  109.         dev_priv->mm.object_count++;
  110.         dev_priv->mm.object_memory += size;
  111. }
  112.  
  113. static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  114.                                      size_t size)
  115. {
  116.         dev_priv->mm.object_count--;
  117.         dev_priv->mm.object_memory -= size;
  118. }
  119.  
  120. static int
  121. i915_gem_wait_for_error(struct drm_device *dev)
  122. {
  123.         struct drm_i915_private *dev_priv = dev->dev_private;
  124.         struct completion *x = &dev_priv->error_completion;
  125.         unsigned long flags;
  126.         int ret;
  127.  
  128.         if (!atomic_read(&dev_priv->mm.wedged))
  129.                 return 0;
  130. #if 0
  131.         /*
  132.          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
  133.          * userspace. If it takes that long something really bad is going on and
  134.          * we should simply try to bail out and fail as gracefully as possible.
  135.          */
  136.         ret = wait_for_completion_interruptible_timeout(x, 10*HZ);
  137.         if (ret == 0) {
  138.                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
  139.                 return -EIO;
  140.         } else if (ret < 0) {
  141.                 return ret;
  142.         }
  143.  
  144.         if (atomic_read(&dev_priv->mm.wedged)) {
  145.                 /* GPU is hung, bump the completion count to account for
  146.                  * the token we just consumed so that we never hit zero and
  147.                  * end up waiting upon a subsequent completion event that
  148.                  * will never happen.
  149.                  */
  150.                 spin_lock_irqsave(&x->wait.lock, flags);
  151.                 x->done++;
  152.                 spin_unlock_irqrestore(&x->wait.lock, flags);
  153.         }
  154. #endif
  155.  
  156.         return 0;
  157. }
  158.  
  159. int i915_mutex_lock_interruptible(struct drm_device *dev)
  160. {
  161.         int ret;
  162.  
  163.         ret = i915_gem_wait_for_error(dev);
  164.         if (ret)
  165.                 return ret;
  166.  
  167.     mutex_lock(&dev->struct_mutex);
  168.  
  169.         WARN_ON(i915_verify_lists(dev));
  170.         return 0;
  171. }
  172.  
  173. static inline bool
  174. i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
  175. {
  176.         return obj->gtt_space && !obj->active;
  177. }
  178.  
  179.  
  180. #if 0
  181.  
  182. int
  183. i915_gem_init_ioctl(struct drm_device *dev, void *data,
  184.                     struct drm_file *file)
  185. {
  186.         struct drm_i915_gem_init *args = data;
  187.  
  188.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  189.                 return -ENODEV;
  190.  
  191.         if (args->gtt_start >= args->gtt_end ||
  192.             (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
  193.                 return -EINVAL;
  194.  
  195.         /* GEM with user mode setting was never supported on ilk and later. */
  196.         if (INTEL_INFO(dev)->gen >= 5)
  197.                 return -ENODEV;
  198.  
  199.         mutex_lock(&dev->struct_mutex);
  200.         i915_gem_init_global_gtt(dev, args->gtt_start,
  201.                                  args->gtt_end, args->gtt_end);
  202.         mutex_unlock(&dev->struct_mutex);
  203.  
  204.         return 0;
  205. }
  206. #endif
  207.  
  208. int
  209. i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  210.                             struct drm_file *file)
  211. {
  212.         struct drm_i915_private *dev_priv = dev->dev_private;
  213.         struct drm_i915_gem_get_aperture *args = data;
  214.         struct drm_i915_gem_object *obj;
  215.         size_t pinned;
  216.  
  217.         pinned = 0;
  218.         mutex_lock(&dev->struct_mutex);
  219.         list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
  220.                 if (obj->pin_count)
  221.                 pinned += obj->gtt_space->size;
  222.         mutex_unlock(&dev->struct_mutex);
  223.  
  224.         args->aper_size = dev_priv->mm.gtt_total;
  225.         args->aper_available_size = args->aper_size - pinned;
  226.  
  227.         return 0;
  228. }
  229.  
  230. static int
  231. i915_gem_create(struct drm_file *file,
  232.                 struct drm_device *dev,
  233.                 uint64_t size,
  234.                 uint32_t *handle_p)
  235. {
  236.         struct drm_i915_gem_object *obj;
  237.         int ret;
  238.         u32 handle;
  239.  
  240.         size = roundup(size, PAGE_SIZE);
  241.         if (size == 0)
  242.                 return -EINVAL;
  243.  
  244.         /* Allocate the new object */
  245.         obj = i915_gem_alloc_object(dev, size);
  246.         if (obj == NULL)
  247.                 return -ENOMEM;
  248.  
  249.         ret = drm_gem_handle_create(file, &obj->base, &handle);
  250.         if (ret) {
  251.                 drm_gem_object_release(&obj->base);
  252.                 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
  253.                 kfree(obj);
  254.                 return ret;
  255.         }
  256.  
  257.         /* drop reference from allocate - handle holds it now */
  258.         drm_gem_object_unreference(&obj->base);
  259.         trace_i915_gem_object_create(obj);
  260.  
  261.         *handle_p = handle;
  262.  
  263. //    printf("%s obj %p handle %d\n", __FUNCTION__, obj, handle);
  264.  
  265.         return 0;
  266. }
  267.  
  268. int
  269. i915_gem_dumb_create(struct drm_file *file,
  270.                      struct drm_device *dev,
  271.                      struct drm_mode_create_dumb *args)
  272. {
  273.         /* have to work out size/pitch and return them */
  274.         args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
  275.         args->size = args->pitch * args->height;
  276.         return i915_gem_create(file, dev,
  277.                                args->size, &args->handle);
  278. }
  279.  
  280. int i915_gem_dumb_destroy(struct drm_file *file,
  281.                           struct drm_device *dev,
  282.                           uint32_t handle)
  283. {
  284.         return drm_gem_handle_delete(file, handle);
  285. }
  286.  
  287. /**
  288.  * Creates a new mm object and returns a handle to it.
  289.  */
  290. int
  291. i915_gem_create_ioctl(struct drm_device *dev, void *data,
  292.                       struct drm_file *file)
  293. {
  294.         struct drm_i915_gem_create *args = data;
  295.  
  296.         return i915_gem_create(file, dev,
  297.                                args->size, &args->handle);
  298. }
  299.  
  300. static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
  301. {
  302.         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
  303.  
  304.         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
  305.                 obj->tiling_mode != I915_TILING_NONE;
  306. }
  307. #if 0
  308.  
  309. static inline int
  310. __copy_to_user_swizzled(char __user *cpu_vaddr,
  311.                         const char *gpu_vaddr, int gpu_offset,
  312.                 int length)
  313. {
  314.         int ret, cpu_offset = 0;
  315.  
  316.         while (length > 0) {
  317.                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
  318.                 int this_length = min(cacheline_end - gpu_offset, length);
  319.                 int swizzled_gpu_offset = gpu_offset ^ 64;
  320.  
  321.                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
  322.                                      gpu_vaddr + swizzled_gpu_offset,
  323.                                      this_length);
  324.                 if (ret)
  325.                         return ret + length;
  326.  
  327.                 cpu_offset += this_length;
  328.                 gpu_offset += this_length;
  329.                 length -= this_length;
  330.         }
  331.  
  332.         return 0;
  333. }
  334.  
  335. static inline int
  336. __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
  337.                           const char __user *cpu_vaddr,
  338.                           int length)
  339. {
  340.         int ret, cpu_offset = 0;
  341.  
  342.         while (length > 0) {
  343.                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
  344.                 int this_length = min(cacheline_end - gpu_offset, length);
  345.                 int swizzled_gpu_offset = gpu_offset ^ 64;
  346.  
  347.                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
  348.                                cpu_vaddr + cpu_offset,
  349.                                this_length);
  350.                 if (ret)
  351.                         return ret + length;
  352.  
  353.                 cpu_offset += this_length;
  354.                 gpu_offset += this_length;
  355.                 length -= this_length;
  356.         }
  357.  
  358.         return 0;
  359. }
  360.  
  361. /* Per-page copy function for the shmem pread fastpath.
  362.  * Flushes invalid cachelines before reading the target if
  363.  * needs_clflush is set. */
  364. static int
  365. shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
  366.                  char __user *user_data,
  367.                  bool page_do_bit17_swizzling, bool needs_clflush)
  368. {
  369.                 char *vaddr;
  370.                 int ret;
  371.  
  372.         if (unlikely(page_do_bit17_swizzling))
  373.                 return -EINVAL;
  374.  
  375.                 vaddr = kmap_atomic(page);
  376.         if (needs_clflush)
  377.                 drm_clflush_virt_range(vaddr + shmem_page_offset,
  378.                                        page_length);
  379.                 ret = __copy_to_user_inatomic(user_data,
  380.                                       vaddr + shmem_page_offset,
  381.                                               page_length);
  382.                 kunmap_atomic(vaddr);
  383.  
  384.         return ret ? -EFAULT : 0;
  385. }
  386.  
  387. static void
  388. shmem_clflush_swizzled_range(char *addr, unsigned long length,
  389.                              bool swizzled)
  390. {
  391.         if (unlikely(swizzled)) {
  392.                 unsigned long start = (unsigned long) addr;
  393.                 unsigned long end = (unsigned long) addr + length;
  394.  
  395.                 /* For swizzling simply ensure that we always flush both
  396.                  * channels. Lame, but simple and it works. Swizzled
  397.                  * pwrite/pread is far from a hotpath - current userspace
  398.                  * doesn't use it at all. */
  399.                 start = round_down(start, 128);
  400.                 end = round_up(end, 128);
  401.  
  402.                 drm_clflush_virt_range((void *)start, end - start);
  403.         } else {
  404.                 drm_clflush_virt_range(addr, length);
  405.         }
  406.  
  407. }
  408.  
  409. /* Only difference to the fast-path function is that this can handle bit17
  410.  * and uses non-atomic copy and kmap functions. */
  411. static int
  412. shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
  413.                  char __user *user_data,
  414.                  bool page_do_bit17_swizzling, bool needs_clflush)
  415. {
  416.         char *vaddr;
  417.         int ret;
  418.  
  419.         vaddr = kmap(page);
  420.         if (needs_clflush)
  421.                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  422.                                              page_length,
  423.                                              page_do_bit17_swizzling);
  424.  
  425.         if (page_do_bit17_swizzling)
  426.                 ret = __copy_to_user_swizzled(user_data,
  427.                                               vaddr, shmem_page_offset,
  428.                                               page_length);
  429.         else
  430.                 ret = __copy_to_user(user_data,
  431.                                      vaddr + shmem_page_offset,
  432.                                      page_length);
  433.         kunmap(page);
  434.  
  435.         return ret ? - EFAULT : 0;
  436. }
  437.  
  438. static int
  439. i915_gem_shmem_pread(struct drm_device *dev,
  440.                           struct drm_i915_gem_object *obj,
  441.                           struct drm_i915_gem_pread *args,
  442.                           struct drm_file *file)
  443. {
  444.         char __user *user_data;
  445.         ssize_t remain;
  446.         loff_t offset;
  447.         int shmem_page_offset, page_length, ret = 0;
  448.         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
  449.         int hit_slowpath = 0;
  450.         int prefaulted = 0;
  451.         int needs_clflush = 0;
  452.         struct scatterlist *sg;
  453.         int i;
  454.  
  455.         user_data = (char __user *) (uintptr_t) args->data_ptr;
  456.         remain = args->size;
  457.  
  458.         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
  459.  
  460.         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
  461.                 /* If we're not in the cpu read domain, set ourself into the gtt
  462.                  * read domain and manually flush cachelines (if required). This
  463.                  * optimizes for the case when the gpu will dirty the data
  464.                  * anyway again before the next pread happens. */
  465.                 if (obj->cache_level == I915_CACHE_NONE)
  466.                         needs_clflush = 1;
  467.                 if (obj->gtt_space) {
  468.                         ret = i915_gem_object_set_to_gtt_domain(obj, false);
  469.                         if (ret)
  470.                                 return ret;
  471.                 }
  472.         }
  473.  
  474.         ret = i915_gem_object_get_pages(obj);
  475.         if (ret)
  476.                 return ret;
  477.  
  478.         i915_gem_object_pin_pages(obj);
  479.  
  480.         offset = args->offset;
  481.  
  482.         for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
  483.                 struct page *page;
  484.  
  485.                 if (i < offset >> PAGE_SHIFT)
  486.                         continue;
  487.  
  488.                 if (remain <= 0)
  489.                         break;
  490.  
  491.                 /* Operation in this page
  492.                  *
  493.                  * shmem_page_offset = offset within page in shmem file
  494.                  * page_length = bytes to copy for this page
  495.                  */
  496.                 shmem_page_offset = offset_in_page(offset);
  497.                 page_length = remain;
  498.                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
  499.                         page_length = PAGE_SIZE - shmem_page_offset;
  500.  
  501.                 page = sg_page(sg);
  502.                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
  503.                         (page_to_phys(page) & (1 << 17)) != 0;
  504.  
  505.                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
  506.                                        user_data, page_do_bit17_swizzling,
  507.                                        needs_clflush);
  508.                 if (ret == 0)
  509.                         goto next_page;
  510.  
  511.                 hit_slowpath = 1;
  512.                 mutex_unlock(&dev->struct_mutex);
  513.  
  514.                 if (!prefaulted) {
  515.                         ret = fault_in_multipages_writeable(user_data, remain);
  516.                         /* Userspace is tricking us, but we've already clobbered
  517.                          * its pages with the prefault and promised to write the
  518.                          * data up to the first fault. Hence ignore any errors
  519.                          * and just continue. */
  520.                         (void)ret;
  521.                         prefaulted = 1;
  522.                 }
  523.  
  524.                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
  525.                                        user_data, page_do_bit17_swizzling,
  526.                                        needs_clflush);
  527.  
  528.                 mutex_lock(&dev->struct_mutex);
  529.  
  530. next_page:
  531.                 mark_page_accessed(page);
  532.  
  533.                 if (ret)
  534.                         goto out;
  535.  
  536.                 remain -= page_length;
  537.                 user_data += page_length;
  538.                 offset += page_length;
  539.         }
  540.  
  541. out:
  542.         i915_gem_object_unpin_pages(obj);
  543.  
  544.         if (hit_slowpath) {
  545.                 /* Fixup: Kill any reinstated backing storage pages */
  546.                 if (obj->madv == __I915_MADV_PURGED)
  547.                         i915_gem_object_truncate(obj);
  548.         }
  549.  
  550.         return ret;
  551. }
  552.  
  553. /**
  554.  * Reads data from the object referenced by handle.
  555.  *
  556.  * On error, the contents of *data are undefined.
  557.  */
  558. int
  559. i915_gem_pread_ioctl(struct drm_device *dev, void *data,
  560.                      struct drm_file *file)
  561. {
  562.         struct drm_i915_gem_pread *args = data;
  563.         struct drm_i915_gem_object *obj;
  564.         int ret = 0;
  565.  
  566.         if (args->size == 0)
  567.                 return 0;
  568.  
  569.         if (!access_ok(VERIFY_WRITE,
  570.                        (char __user *)(uintptr_t)args->data_ptr,
  571.                        args->size))
  572.                 return -EFAULT;
  573.  
  574.         ret = i915_mutex_lock_interruptible(dev);
  575.         if (ret)
  576.                 return ret;
  577.  
  578.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  579.         if (&obj->base == NULL) {
  580.                 ret = -ENOENT;
  581.                 goto unlock;
  582.         }
  583.  
  584.         /* Bounds check source.  */
  585.         if (args->offset > obj->base.size ||
  586.             args->size > obj->base.size - args->offset) {
  587.                 ret = -EINVAL;
  588.                 goto out;
  589.         }
  590.  
  591.         /* prime objects have no backing filp to GEM pread/pwrite
  592.          * pages from.
  593.          */
  594.         if (!obj->base.filp) {
  595.                 ret = -EINVAL;
  596.                 goto out;
  597.         }
  598.  
  599.         trace_i915_gem_object_pread(obj, args->offset, args->size);
  600.  
  601.         ret = i915_gem_shmem_pread(dev, obj, args, file);
  602.  
  603. out:
  604.         drm_gem_object_unreference(&obj->base);
  605. unlock:
  606.         mutex_unlock(&dev->struct_mutex);
  607.         return ret;
  608. }
  609.  
  610. /* This is the fast write path which cannot handle
  611.  * page faults in the source data
  612.  */
  613.  
  614. static inline int
  615. fast_user_write(struct io_mapping *mapping,
  616.                 loff_t page_base, int page_offset,
  617.                 char __user *user_data,
  618.                 int length)
  619. {
  620.         void __iomem *vaddr_atomic;
  621.         void *vaddr;
  622.         unsigned long unwritten;
  623.  
  624.         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
  625.         /* We can use the cpu mem copy function because this is X86. */
  626.         vaddr = (void __force*)vaddr_atomic + page_offset;
  627.         unwritten = __copy_from_user_inatomic_nocache(vaddr,
  628.                                                       user_data, length);
  629.         io_mapping_unmap_atomic(vaddr_atomic);
  630.         return unwritten;
  631. }
  632. #endif
  633.  
  634. #define offset_in_page(p)       ((unsigned long)(p) & ~PAGE_MASK)
  635. /**
  636.  * This is the fast pwrite path, where we copy the data directly from the
  637.  * user into the GTT, uncached.
  638.  */
  639. static int
  640. i915_gem_gtt_pwrite_fast(struct drm_device *dev,
  641.                          struct drm_i915_gem_object *obj,
  642.                          struct drm_i915_gem_pwrite *args,
  643.                          struct drm_file *file)
  644. {
  645.         drm_i915_private_t *dev_priv = dev->dev_private;
  646.         ssize_t remain;
  647.         loff_t offset, page_base;
  648.         char __user *user_data;
  649.         int page_offset, page_length, ret;
  650.     char *vaddr;
  651.  
  652.         ret = i915_gem_object_pin(obj, 0, true, true);
  653.         if (ret)
  654.                 goto out;
  655.  
  656.         ret = i915_gem_object_set_to_gtt_domain(obj, true);
  657.         if (ret)
  658.                 goto out_unpin;
  659.  
  660.         ret = i915_gem_object_put_fence(obj);
  661.         if (ret)
  662.                 goto out_unpin;
  663.  
  664.     vaddr = AllocKernelSpace(4096);
  665.     if(vaddr == NULL)
  666.     {
  667.         ret = -ENOSPC;
  668.         goto out_unpin;
  669.     };
  670.  
  671.         user_data = (char __user *) (uintptr_t) args->data_ptr;
  672.         remain = args->size;
  673.  
  674.         offset = obj->gtt_offset + args->offset;
  675.  
  676.         while (remain > 0) {
  677.                 /* Operation in this page
  678.                  *
  679.                  * page_base = page offset within aperture
  680.                  * page_offset = offset within page
  681.                  * page_length = bytes to copy for this page
  682.                  */
  683.                 page_base = offset & PAGE_MASK;
  684.                 page_offset = offset_in_page(offset);
  685.                 page_length = remain;
  686.                 if ((page_offset + remain) > PAGE_SIZE)
  687.                         page_length = PAGE_SIZE - page_offset;
  688.  
  689.         MapPage(vaddr, page_base, PG_SW|PG_NOCACHE);
  690.  
  691.         memcpy(vaddr+page_offset, user_data, page_length);
  692.  
  693.                 remain -= page_length;
  694.                 user_data += page_length;
  695.                 offset += page_length;
  696.         }
  697.  
  698.     FreeKernelSpace(vaddr);
  699.  
  700. out_unpin:
  701.         i915_gem_object_unpin(obj);
  702. out:
  703.     printf("% s ret = %d\n", __FUNCTION__, ret);
  704.  
  705.         return ret;
  706. }
  707.  
  708. /* Per-page copy function for the shmem pwrite fastpath.
  709.  * Flushes invalid cachelines before writing to the target if
  710.  * needs_clflush_before is set and flushes out any written cachelines after
  711.  * writing if needs_clflush is set. */
  712. static int
  713. shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
  714.                   char __user *user_data,
  715.                   bool page_do_bit17_swizzling,
  716.                   bool needs_clflush_before,
  717.                   bool needs_clflush_after)
  718. {
  719.         char *vaddr;
  720.         int ret = 0;
  721.  
  722.         if (unlikely(page_do_bit17_swizzling))
  723.                 return -EINVAL;
  724.  
  725.         vaddr = (char *)MapIoMem((addr_t)page, 4096, PG_SW);
  726.         if (needs_clflush_before)
  727.                 drm_clflush_virt_range(vaddr + shmem_page_offset,
  728.                                        page_length);
  729.         memcpy(vaddr + shmem_page_offset,
  730.                                                 user_data,
  731.                                                 page_length);
  732.         if (needs_clflush_after)
  733.                 drm_clflush_virt_range(vaddr + shmem_page_offset,
  734.                                        page_length);
  735.         FreeKernelSpace(vaddr);
  736.  
  737.         return ret ? -EFAULT : 0;
  738. }
  739. #if 0
  740.  
  741. /* Only difference to the fast-path function is that this can handle bit17
  742.  * and uses non-atomic copy and kmap functions. */
  743. static int
  744. shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
  745.                   char __user *user_data,
  746.                   bool page_do_bit17_swizzling,
  747.                   bool needs_clflush_before,
  748.                   bool needs_clflush_after)
  749. {
  750.         char *vaddr;
  751.         int ret;
  752.  
  753.         vaddr = kmap(page);
  754.         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
  755.                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  756.                                              page_length,
  757.                                              page_do_bit17_swizzling);
  758.         if (page_do_bit17_swizzling)
  759.                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
  760.                                                 user_data,
  761.                                                 page_length);
  762.         else
  763.                 ret = __copy_from_user(vaddr + shmem_page_offset,
  764.                                        user_data,
  765.                                        page_length);
  766.         if (needs_clflush_after)
  767.                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  768.                                              page_length,
  769.                                              page_do_bit17_swizzling);
  770.         kunmap(page);
  771.  
  772.         return ret ? -EFAULT : 0;
  773. }
  774. #endif
  775.  
  776.  
  777. static int
  778. i915_gem_shmem_pwrite(struct drm_device *dev,
  779.                       struct drm_i915_gem_object *obj,
  780.                       struct drm_i915_gem_pwrite *args,
  781.                       struct drm_file *file)
  782. {
  783.         ssize_t remain;
  784.         loff_t offset;
  785.         char __user *user_data;
  786.         int shmem_page_offset, page_length, ret = 0;
  787.         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
  788.         int hit_slowpath = 0;
  789.         int needs_clflush_after = 0;
  790.         int needs_clflush_before = 0;
  791.         int i;
  792.         struct scatterlist *sg;
  793.  
  794.         user_data = (char __user *) (uintptr_t) args->data_ptr;
  795.         remain = args->size;
  796.  
  797.         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
  798.  
  799.         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  800.                 /* If we're not in the cpu write domain, set ourself into the gtt
  801.                  * write domain and manually flush cachelines (if required). This
  802.                  * optimizes for the case when the gpu will use the data
  803.                  * right away and we therefore have to clflush anyway. */
  804.                 if (obj->cache_level == I915_CACHE_NONE)
  805.                         needs_clflush_after = 1;
  806.                 if (obj->gtt_space) {
  807.                         ret = i915_gem_object_set_to_gtt_domain(obj, true);
  808.                         if (ret)
  809.                                 return ret;
  810.                 }
  811.         }
  812.         /* Same trick applies for invalidate partially written cachelines before
  813.          * writing.  */
  814.         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
  815.             && obj->cache_level == I915_CACHE_NONE)
  816.                 needs_clflush_before = 1;
  817.  
  818.         ret = i915_gem_object_get_pages(obj);
  819.         if (ret)
  820.                 return ret;
  821.  
  822.         i915_gem_object_pin_pages(obj);
  823.  
  824.         offset = args->offset;
  825.         obj->dirty = 1;
  826.  
  827.         for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
  828.                 struct page *page;
  829.                 int partial_cacheline_write;
  830.  
  831.                 if (i < offset >> PAGE_SHIFT)
  832.                         continue;
  833.  
  834.                 if (remain <= 0)
  835.                         break;
  836.  
  837.                 /* Operation in this page
  838.                  *
  839.                  * shmem_page_offset = offset within page in shmem file
  840.                  * page_length = bytes to copy for this page
  841.                  */
  842.                 shmem_page_offset = offset_in_page(offset);
  843.  
  844.                 page_length = remain;
  845.                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
  846.                         page_length = PAGE_SIZE - shmem_page_offset;
  847.  
  848.                 /* If we don't overwrite a cacheline completely we need to be
  849.                  * careful to have up-to-date data by first clflushing. Don't
  850.                  * overcomplicate things and flush the entire patch. */
  851.                 partial_cacheline_write = needs_clflush_before &&
  852.                         ((shmem_page_offset | page_length)
  853.                                 & (x86_clflush_size - 1));
  854.  
  855.                 page = sg_page(sg);
  856.                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
  857.                         (page_to_phys(page) & (1 << 17)) != 0;
  858.  
  859.                 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
  860.                                         user_data, page_do_bit17_swizzling,
  861.                                         partial_cacheline_write,
  862.                                         needs_clflush_after);
  863.                 if (ret == 0)
  864.                         goto next_page;
  865.  
  866.                 hit_slowpath = 1;
  867.                 mutex_unlock(&dev->struct_mutex);
  868.                 dbgprintf("%s need shmem_pwrite_slow\n",__FUNCTION__);
  869.  
  870. //              ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
  871. //                                      user_data, page_do_bit17_swizzling,
  872. //                                      partial_cacheline_write,
  873. //                                      needs_clflush_after);
  874.  
  875.                 mutex_lock(&dev->struct_mutex);
  876.  
  877. next_page:
  878.  
  879.                 if (ret)
  880.                         goto out;
  881.  
  882.                 remain -= page_length;
  883.                 user_data += page_length;
  884.                 offset += page_length;
  885.         }
  886.  
  887. out:
  888.         i915_gem_object_unpin_pages(obj);
  889.  
  890.         if (hit_slowpath) {
  891.                 /* Fixup: Kill any reinstated backing storage pages */
  892.                 if (obj->madv == __I915_MADV_PURGED)
  893.                         i915_gem_object_truncate(obj);
  894.                 /* and flush dirty cachelines in case the object isn't in the cpu write
  895.                  * domain anymore. */
  896.                 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  897.                         i915_gem_clflush_object(obj);
  898.                         i915_gem_chipset_flush(dev);
  899.                 }
  900.         }
  901.  
  902.         if (needs_clflush_after)
  903.                 i915_gem_chipset_flush(dev);
  904.  
  905.         return ret;
  906. }
  907.  
  908. /**
  909.  * Writes data to the object referenced by handle.
  910.  *
  911.  * On error, the contents of the buffer that were to be modified are undefined.
  912.  */
  913. int
  914. i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
  915.                       struct drm_file *file)
  916. {
  917.         struct drm_i915_gem_pwrite *args = data;
  918.         struct drm_i915_gem_object *obj;
  919.         int ret;
  920.  
  921.         if (args->size == 0)
  922.                 return 0;
  923.  
  924.         ret = i915_mutex_lock_interruptible(dev);
  925.         if (ret)
  926.                 return ret;
  927.  
  928.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  929.         if (&obj->base == NULL) {
  930.                 ret = -ENOENT;
  931.                 goto unlock;
  932.         }
  933.  
  934.         /* Bounds check destination. */
  935.         if (args->offset > obj->base.size ||
  936.             args->size > obj->base.size - args->offset) {
  937.                 ret = -EINVAL;
  938.                 goto out;
  939.         }
  940.  
  941.         /* prime objects have no backing filp to GEM pread/pwrite
  942.          * pages from.
  943.          */
  944.         if (!obj->base.filp) {
  945.                 ret = -EINVAL;
  946.                 goto out;
  947.         }
  948.  
  949.         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
  950.  
  951.         ret = -EFAULT;
  952.         /* We can only do the GTT pwrite on untiled buffers, as otherwise
  953.          * it would end up going through the fenced access, and we'll get
  954.          * different detiling behavior between reading and writing.
  955.          * pread/pwrite currently are reading and writing from the CPU
  956.          * perspective, requiring manual detiling by the client.
  957.          */
  958. //   if (obj->phys_obj) {
  959. //       ret = i915_gem_phys_pwrite(dev, obj, args, file);
  960. //       goto out;
  961. //   }
  962.  
  963.         if (obj->cache_level == I915_CACHE_NONE &&
  964.             obj->tiling_mode == I915_TILING_NONE &&
  965.             obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  966.                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
  967.                 /* Note that the gtt paths might fail with non-page-backed user
  968.                  * pointers (e.g. gtt mappings when moving data between
  969.                  * textures). Fallback to the shmem path in that case. */
  970.         }
  971.  
  972.         if (ret == -EFAULT || ret == -ENOSPC)
  973.        ret = i915_gem_shmem_pwrite(dev, obj, args, file);
  974.  
  975. out:
  976.         drm_gem_object_unreference(&obj->base);
  977. unlock:
  978.         mutex_unlock(&dev->struct_mutex);
  979.         return ret;
  980. }
  981.  
  982. int
  983. i915_gem_check_wedge(struct drm_i915_private *dev_priv,
  984.                      bool interruptible)
  985. {
  986.         if (atomic_read(&dev_priv->mm.wedged)) {
  987.                 struct completion *x = &dev_priv->error_completion;
  988.                 bool recovery_complete;
  989.                 unsigned long flags;
  990.  
  991.                 /* Give the error handler a chance to run. */
  992.                 spin_lock_irqsave(&x->wait.lock, flags);
  993.                 recovery_complete = x->done > 0;
  994.                 spin_unlock_irqrestore(&x->wait.lock, flags);
  995.  
  996.                 /* Non-interruptible callers can't handle -EAGAIN, hence return
  997.                  * -EIO unconditionally for these. */
  998.                 if (!interruptible)
  999.                         return -EIO;
  1000.  
  1001.                 /* Recovery complete, but still wedged means reset failure. */
  1002.                 if (recovery_complete)
  1003.                         return -EIO;
  1004.  
  1005.                 return -EAGAIN;
  1006.         }
  1007.  
  1008.         return 0;
  1009. }
  1010.  
  1011. /*
  1012.  * Compare seqno against outstanding lazy request. Emit a request if they are
  1013.  * equal.
  1014.  */
  1015. static int
  1016. i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
  1017. {
  1018.         int ret;
  1019.  
  1020.         BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
  1021.  
  1022.         ret = 0;
  1023.         if (seqno == ring->outstanding_lazy_request)
  1024.                 ret = i915_add_request(ring, NULL, NULL);
  1025.  
  1026.         return ret;
  1027. }
  1028.  
  1029. /**
  1030.  * __wait_seqno - wait until execution of seqno has finished
  1031.  * @ring: the ring expected to report seqno
  1032.  * @seqno: duh!
  1033.  * @interruptible: do an interruptible wait (normally yes)
  1034.  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
  1035.  *
  1036.  * Returns 0 if the seqno was found within the alloted time. Else returns the
  1037.  * errno with remaining time filled in timeout argument.
  1038.  */
  1039. static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
  1040.                         bool interruptible, struct timespec *timeout)
  1041. {
  1042.         drm_i915_private_t *dev_priv = ring->dev->dev_private;
  1043.         struct timespec before, now, wait_time={1,0};
  1044.         unsigned long timeout_jiffies;
  1045.         long end;
  1046.         bool wait_forever = true;
  1047.         int ret;
  1048.  
  1049.         if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
  1050.                 return 0;
  1051.  
  1052.         trace_i915_gem_request_wait_begin(ring, seqno);
  1053.  
  1054.         if (timeout != NULL) {
  1055.                 wait_time = *timeout;
  1056.                 wait_forever = false;
  1057.         }
  1058.  
  1059.         timeout_jiffies = timespec_to_jiffies(&wait_time);
  1060.  
  1061.         if (WARN_ON(!ring->irq_get(ring)))
  1062.                 return -ENODEV;
  1063.  
  1064.     /* Record current time in case interrupted by signal, or wedged * */
  1065.         getrawmonotonic(&before);
  1066.  
  1067. #define EXIT_COND \
  1068.         (i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \
  1069.         atomic_read(&dev_priv->mm.wedged))
  1070.         do {
  1071.                 if (interruptible)
  1072.                         end = wait_event_interruptible_timeout(ring->irq_queue,
  1073.                                                                EXIT_COND,
  1074.                                                                timeout_jiffies);
  1075.                 else
  1076.                         end = wait_event_timeout(ring->irq_queue, EXIT_COND,
  1077.                                                  timeout_jiffies);
  1078.  
  1079.                 ret = i915_gem_check_wedge(dev_priv, interruptible);
  1080.                 if (ret)
  1081.                         end = ret;
  1082.         } while (end == 0 && wait_forever);
  1083.  
  1084.         getrawmonotonic(&now);
  1085.  
  1086.         ring->irq_put(ring);
  1087.         trace_i915_gem_request_wait_end(ring, seqno);
  1088. #undef EXIT_COND
  1089.  
  1090.         if (timeout) {
  1091. //       struct timespec sleep_time = timespec_sub(now, before);
  1092. //       *timeout = timespec_sub(*timeout, sleep_time);
  1093.         }
  1094.  
  1095.         switch (end) {
  1096.         case -EIO:
  1097.         case -EAGAIN: /* Wedged */
  1098.         case -ERESTARTSYS: /* Signal */
  1099.                 return (int)end;
  1100.         case 0: /* Timeout */
  1101.                 if (timeout)
  1102.                         set_normalized_timespec(timeout, 0, 0);
  1103.                 return -ETIME;
  1104.         default: /* Completed */
  1105.                 WARN_ON(end < 0); /* We're not aware of other errors */
  1106.                 return 0;
  1107.         }
  1108. }
  1109.  
  1110. /**
  1111.  * Waits for a sequence number to be signaled, and cleans up the
  1112.  * request and object lists appropriately for that event.
  1113.  */
  1114. int
  1115. i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
  1116. {
  1117.         struct drm_device *dev = ring->dev;
  1118.         struct drm_i915_private *dev_priv = dev->dev_private;
  1119.         bool interruptible = dev_priv->mm.interruptible;
  1120.         int ret;
  1121.  
  1122.         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
  1123.         BUG_ON(seqno == 0);
  1124.  
  1125.         ret = i915_gem_check_wedge(dev_priv, interruptible);
  1126.         if (ret)
  1127.                 return ret;
  1128.  
  1129.         ret = i915_gem_check_olr(ring, seqno);
  1130.         if (ret)
  1131.                 return ret;
  1132.  
  1133.         return __wait_seqno(ring, seqno, interruptible, NULL);
  1134. }
  1135.  
  1136. /**
  1137.  * Ensures that all rendering to the object has completed and the object is
  1138.  * safe to unbind from the GTT or access from the CPU.
  1139.  */
  1140. static __must_check int
  1141. i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
  1142.                                bool readonly)
  1143. {
  1144.         struct intel_ring_buffer *ring = obj->ring;
  1145.         u32 seqno;
  1146.         int ret;
  1147.  
  1148.         seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
  1149.         if (seqno == 0)
  1150.                 return 0;
  1151.  
  1152.         ret = i915_wait_seqno(ring, seqno);
  1153.        if (ret)
  1154.            return ret;
  1155.  
  1156.         i915_gem_retire_requests_ring(ring);
  1157.  
  1158.         /* Manually manage the write flush as we may have not yet
  1159.          * retired the buffer.
  1160.          */
  1161.         if (obj->last_write_seqno &&
  1162.             i915_seqno_passed(seqno, obj->last_write_seqno)) {
  1163.                 obj->last_write_seqno = 0;
  1164.                 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
  1165.         }
  1166.  
  1167.         return 0;
  1168. }
  1169.  
  1170. /* A nonblocking variant of the above wait. This is a highly dangerous routine
  1171.  * as the object state may change during this call.
  1172.  */
  1173. static __must_check int
  1174. i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
  1175.                                             bool readonly)
  1176. {
  1177.         struct drm_device *dev = obj->base.dev;
  1178.         struct drm_i915_private *dev_priv = dev->dev_private;
  1179.         struct intel_ring_buffer *ring = obj->ring;
  1180.         u32 seqno;
  1181.         int ret;
  1182.  
  1183.         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
  1184.         BUG_ON(!dev_priv->mm.interruptible);
  1185.  
  1186.         seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
  1187.         if (seqno == 0)
  1188.                 return 0;
  1189.  
  1190.         ret = i915_gem_check_wedge(dev_priv, true);
  1191.         if (ret)
  1192.                 return ret;
  1193.  
  1194.         ret = i915_gem_check_olr(ring, seqno);
  1195.         if (ret)
  1196.                 return ret;
  1197.  
  1198.         mutex_unlock(&dev->struct_mutex);
  1199.         ret = __wait_seqno(ring, seqno, true, NULL);
  1200.         mutex_lock(&dev->struct_mutex);
  1201.  
  1202.         i915_gem_retire_requests_ring(ring);
  1203.  
  1204.         /* Manually manage the write flush as we may have not yet
  1205.          * retired the buffer.
  1206.          */
  1207.         if (obj->last_write_seqno &&
  1208.             i915_seqno_passed(seqno, obj->last_write_seqno)) {
  1209.                 obj->last_write_seqno = 0;
  1210.                 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
  1211.         }
  1212.  
  1213.         return ret;
  1214. }
  1215.  
  1216. /**
  1217.  * Called when user space prepares to use an object with the CPU, either
  1218.  * through the mmap ioctl's mapping or a GTT mapping.
  1219.  */
  1220. int
  1221. i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
  1222.                           struct drm_file *file)
  1223. {
  1224.         struct drm_i915_gem_set_domain *args = data;
  1225.         struct drm_i915_gem_object *obj;
  1226.         uint32_t read_domains = args->read_domains;
  1227.         uint32_t write_domain = args->write_domain;
  1228.         int ret;
  1229.  
  1230.         /* Only handle setting domains to types used by the CPU. */
  1231.         if (write_domain & I915_GEM_GPU_DOMAINS)
  1232.                 return -EINVAL;
  1233.  
  1234.         if (read_domains & I915_GEM_GPU_DOMAINS)
  1235.                 return -EINVAL;
  1236.  
  1237.         /* Having something in the write domain implies it's in the read
  1238.          * domain, and only that read domain.  Enforce that in the request.
  1239.          */
  1240.         if (write_domain != 0 && read_domains != write_domain)
  1241.                 return -EINVAL;
  1242.  
  1243.         ret = i915_mutex_lock_interruptible(dev);
  1244.         if (ret)
  1245.                 return ret;
  1246.  
  1247.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  1248.         if (&obj->base == NULL) {
  1249.                 ret = -ENOENT;
  1250.                 goto unlock;
  1251.         }
  1252.  
  1253.         /* Try to flush the object off the GPU without holding the lock.
  1254.          * We will repeat the flush holding the lock in the normal manner
  1255.          * to catch cases where we are gazumped.
  1256.          */
  1257.         ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
  1258.         if (ret)
  1259.                 goto unref;
  1260.  
  1261.         if (read_domains & I915_GEM_DOMAIN_GTT) {
  1262.                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
  1263.  
  1264.                 /* Silently promote "you're not bound, there was nothing to do"
  1265.                  * to success, since the client was just asking us to
  1266.                  * make sure everything was done.
  1267.                  */
  1268.                 if (ret == -EINVAL)
  1269.                         ret = 0;
  1270.         } else {
  1271.                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
  1272.         }
  1273.  
  1274. unref:
  1275.         drm_gem_object_unreference(&obj->base);
  1276. unlock:
  1277.         mutex_unlock(&dev->struct_mutex);
  1278.         return ret;
  1279. }
  1280.  
  1281.  
  1282.  
  1283.  
  1284.  
  1285.  
  1286. /**
  1287.  * Maps the contents of an object, returning the address it is mapped
  1288.  * into.
  1289.  *
  1290.  * While the mapping holds a reference on the contents of the object, it doesn't
  1291.  * imply a ref on the object itself.
  1292.  */
  1293. int
  1294. i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
  1295.                     struct drm_file *file)
  1296. {
  1297.         struct drm_i915_gem_mmap *args = data;
  1298.         struct drm_gem_object *obj;
  1299.         unsigned long addr = 0;
  1300.  
  1301.         obj = drm_gem_object_lookup(dev, file, args->handle);
  1302.         if (obj == NULL)
  1303.                 return -ENOENT;
  1304.        
  1305.     //dbgprintf("%s offset %lld size %lld\n",
  1306. //                __FUNCTION__, args->offset, args->size);
  1307.         /* prime objects have no backing filp to GEM mmap
  1308.          * pages from.
  1309.          */
  1310.         if (!obj->filp) {
  1311.                 drm_gem_object_unreference_unlocked(obj);
  1312.                 return -EINVAL;
  1313.         }
  1314.  
  1315.     addr = vm_mmap(obj->filp, 0, args->size,
  1316.               PROT_READ | PROT_WRITE, MAP_SHARED,
  1317.               args->offset);
  1318.         drm_gem_object_unreference_unlocked(obj);
  1319.     if (IS_ERR((void *)addr))
  1320.         return addr;
  1321.  
  1322.         args->addr_ptr = (uint64_t) addr;
  1323.  
  1324.     return 0;
  1325. }
  1326.  
  1327.  
  1328.  
  1329.  
  1330.  
  1331.  
  1332.  
  1333.  
  1334.  
  1335.  
  1336.  
  1337.  
  1338.  
  1339. /**
  1340.  * i915_gem_release_mmap - remove physical page mappings
  1341.  * @obj: obj in question
  1342.  *
  1343.  * Preserve the reservation of the mmapping with the DRM core code, but
  1344.  * relinquish ownership of the pages back to the system.
  1345.  *
  1346.  * It is vital that we remove the page mapping if we have mapped a tiled
  1347.  * object through the GTT and then lose the fence register due to
  1348.  * resource pressure. Similarly if the object has been moved out of the
  1349.  * aperture, than pages mapped into userspace must be revoked. Removing the
  1350.  * mapping will then trigger a page fault on the next user access, allowing
  1351.  * fixup by i915_gem_fault().
  1352.  */
  1353. void
  1354. i915_gem_release_mmap(struct drm_i915_gem_object *obj)
  1355. {
  1356.         if (!obj->fault_mappable)
  1357.                 return;
  1358.  
  1359.         if (obj->base.dev->dev_mapping)
  1360. //              unmap_mapping_range(obj->base.dev->dev_mapping,
  1361. //                                  (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
  1362. //                                  obj->base.size, 1);
  1363.  
  1364.         obj->fault_mappable = false;
  1365. }
  1366.  
  1367. static uint32_t
  1368. i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
  1369. {
  1370.         uint32_t gtt_size;
  1371.  
  1372.         if (INTEL_INFO(dev)->gen >= 4 ||
  1373.             tiling_mode == I915_TILING_NONE)
  1374.                 return size;
  1375.  
  1376.         /* Previous chips need a power-of-two fence region when tiling */
  1377.         if (INTEL_INFO(dev)->gen == 3)
  1378.                 gtt_size = 1024*1024;
  1379.         else
  1380.                 gtt_size = 512*1024;
  1381.  
  1382.         while (gtt_size < size)
  1383.                 gtt_size <<= 1;
  1384.  
  1385.         return gtt_size;
  1386. }
  1387.  
  1388. /**
  1389.  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
  1390.  * @obj: object to check
  1391.  *
  1392.  * Return the required GTT alignment for an object, taking into account
  1393.  * potential fence register mapping.
  1394.  */
  1395. static uint32_t
  1396. i915_gem_get_gtt_alignment(struct drm_device *dev,
  1397.                            uint32_t size,
  1398.                            int tiling_mode)
  1399. {
  1400.         /*
  1401.          * Minimum alignment is 4k (GTT page size), but might be greater
  1402.          * if a fence register is needed for the object.
  1403.          */
  1404.         if (INTEL_INFO(dev)->gen >= 4 ||
  1405.             tiling_mode == I915_TILING_NONE)
  1406.                 return 4096;
  1407.  
  1408.         /*
  1409.          * Previous chips need to be aligned to the size of the smallest
  1410.          * fence register that can contain the object.
  1411.          */
  1412.         return i915_gem_get_gtt_size(dev, size, tiling_mode);
  1413. }
  1414.  
  1415. /**
  1416.  * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
  1417.  *                                       unfenced object
  1418.  * @dev: the device
  1419.  * @size: size of the object
  1420.  * @tiling_mode: tiling mode of the object
  1421.  *
  1422.  * Return the required GTT alignment for an object, only taking into account
  1423.  * unfenced tiled surface requirements.
  1424.  */
  1425. uint32_t
  1426. i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
  1427.                                     uint32_t size,
  1428.                                     int tiling_mode)
  1429. {
  1430.         /*
  1431.          * Minimum alignment is 4k (GTT page size) for sane hw.
  1432.          */
  1433.         if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
  1434.             tiling_mode == I915_TILING_NONE)
  1435.                 return 4096;
  1436.  
  1437.         /* Previous hardware however needs to be aligned to a power-of-two
  1438.          * tile height. The simplest method for determining this is to reuse
  1439.          * the power-of-tile object size.
  1440.          */
  1441.         return i915_gem_get_gtt_size(dev, size, tiling_mode);
  1442. }
  1443.  
  1444. /* Immediately discard the backing storage */
  1445. static void
  1446. i915_gem_object_truncate(struct drm_i915_gem_object *obj)
  1447. {
  1448. //      struct inode *inode;
  1449.  
  1450. //      i915_gem_object_free_mmap_offset(obj);
  1451.  
  1452.         if (obj->base.filp == NULL)
  1453.                 return;
  1454.  
  1455.         /* Our goal here is to return as much of the memory as
  1456.          * is possible back to the system as we are called from OOM.
  1457.          * To do this we must instruct the shmfs to drop all of its
  1458.          * backing pages, *now*.
  1459.          */
  1460. //      inode = obj->base.filp->f_path.dentry->d_inode;
  1461. //      shmem_truncate_range(inode, 0, (loff_t)-1);
  1462.  
  1463.         obj->madv = __I915_MADV_PURGED;
  1464. }
  1465.  
  1466. static inline int
  1467. i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
  1468. {
  1469.         return obj->madv == I915_MADV_DONTNEED;
  1470. }
  1471.  
  1472. static void
  1473. i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
  1474. {
  1475.         int page_count = obj->base.size / PAGE_SIZE;
  1476.         struct scatterlist *sg;
  1477.         int ret, i;
  1478.  
  1479.         BUG_ON(obj->madv == __I915_MADV_PURGED);
  1480.  
  1481.         ret = i915_gem_object_set_to_cpu_domain(obj, true);
  1482.         if (ret) {
  1483.                 /* In the event of a disaster, abandon all caches and
  1484.                  * hope for the best.
  1485.                  */
  1486.                 WARN_ON(ret != -EIO);
  1487.                 i915_gem_clflush_object(obj);
  1488.                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  1489.         }
  1490.  
  1491.         if (obj->madv == I915_MADV_DONTNEED)
  1492.                 obj->dirty = 0;
  1493.  
  1494.         for_each_sg(obj->pages->sgl, sg, page_count, i) {
  1495.                 struct page *page = sg_page(sg);
  1496.  
  1497.         page_cache_release(page);
  1498.         }
  1499.     //DRM_DEBUG_KMS("%s release %d pages\n", __FUNCTION__, page_count);
  1500.  
  1501.         obj->dirty = 0;
  1502.  
  1503.         sg_free_table(obj->pages);
  1504.         kfree(obj->pages);
  1505. }
  1506.  
  1507. static int
  1508. i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
  1509. {
  1510.         const struct drm_i915_gem_object_ops *ops = obj->ops;
  1511.  
  1512.         if (obj->pages == NULL)
  1513.                 return 0;
  1514.  
  1515.         BUG_ON(obj->gtt_space);
  1516.  
  1517.         if (obj->pages_pin_count)
  1518.                 return -EBUSY;
  1519.  
  1520.         /* ->put_pages might need to allocate memory for the bit17 swizzle
  1521.          * array, hence protect them from being reaped by removing them from gtt
  1522.          * lists early. */
  1523.         list_del(&obj->gtt_list);
  1524.  
  1525.         ops->put_pages(obj);
  1526.         obj->pages = NULL;
  1527.  
  1528.         if (i915_gem_object_is_purgeable(obj))
  1529.                 i915_gem_object_truncate(obj);
  1530.  
  1531.         return 0;
  1532. }
  1533.  
  1534.  
  1535.  
  1536.  
  1537.  
  1538.  
  1539.  
  1540.  
  1541. static int
  1542. i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
  1543. {
  1544.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  1545.     int page_count, i;
  1546.         struct address_space *mapping;
  1547.         struct sg_table *st;
  1548.         struct scatterlist *sg;
  1549.         struct page *page;
  1550.         gfp_t gfp;
  1551.  
  1552.         /* Assert that the object is not currently in any GPU domain. As it
  1553.          * wasn't in the GTT, there shouldn't be any way it could have been in
  1554.          * a GPU cache
  1555.          */
  1556.         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
  1557.         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
  1558.  
  1559.         st = kmalloc(sizeof(*st), GFP_KERNEL);
  1560.         if (st == NULL)
  1561.                 return -ENOMEM;
  1562.  
  1563.         page_count = obj->base.size / PAGE_SIZE;
  1564.         if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
  1565.                 sg_free_table(st);
  1566.                 kfree(st);
  1567.                 return -ENOMEM;
  1568.         }
  1569.  
  1570.         /* Get the list of pages out of our struct file.  They'll be pinned
  1571.          * at this point until we release them.
  1572.          *
  1573.          * Fail silently without starting the shrinker
  1574.          */
  1575.         for_each_sg(st->sgl, sg, page_count, i) {
  1576.                 page = shmem_read_mapping_page_gfp(obj->base.filp, i, gfp);
  1577.                 if (IS_ERR(page)) {
  1578.             dbgprintf("%s invalid page %p\n", __FUNCTION__, page);
  1579.                         goto err_pages;
  1580.  
  1581.                 }
  1582.                 sg_set_page(sg, page, PAGE_SIZE, 0);
  1583.         }
  1584.  
  1585.         obj->pages = st;
  1586.  
  1587. //    DRM_DEBUG_KMS("%s alloc %d pages\n", __FUNCTION__, page_count);
  1588.  
  1589.         return 0;
  1590.  
  1591. err_pages:
  1592.         for_each_sg(st->sgl, sg, i, page_count)
  1593.                 page_cache_release(sg_page(sg));
  1594.         sg_free_table(st);
  1595.         kfree(st);
  1596.         return PTR_ERR(page);
  1597. }
  1598.  
  1599. /* Ensure that the associated pages are gathered from the backing storage
  1600.  * and pinned into our object. i915_gem_object_get_pages() may be called
  1601.  * multiple times before they are released by a single call to
  1602.  * i915_gem_object_put_pages() - once the pages are no longer referenced
  1603.  * either as a result of memory pressure (reaping pages under the shrinker)
  1604.  * or as the object is itself released.
  1605.  */
  1606. int
  1607. i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
  1608. {
  1609.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  1610.         const struct drm_i915_gem_object_ops *ops = obj->ops;
  1611.         int ret;
  1612.  
  1613.         if (obj->pages)
  1614.                 return 0;
  1615.  
  1616.         BUG_ON(obj->pages_pin_count);
  1617.  
  1618.         ret = ops->get_pages(obj);
  1619.         if (ret)
  1620.                 return ret;
  1621.  
  1622.         list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
  1623.     return 0;
  1624. }
  1625.  
  1626. void
  1627. i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
  1628.                                struct intel_ring_buffer *ring)
  1629. {
  1630.         struct drm_device *dev = obj->base.dev;
  1631.         struct drm_i915_private *dev_priv = dev->dev_private;
  1632.         u32 seqno = intel_ring_get_seqno(ring);
  1633.  
  1634.         BUG_ON(ring == NULL);
  1635.         obj->ring = ring;
  1636.  
  1637.         /* Add a reference if we're newly entering the active list. */
  1638.         if (!obj->active) {
  1639.                 drm_gem_object_reference(&obj->base);
  1640.                 obj->active = 1;
  1641.         }
  1642.  
  1643.         /* Move from whatever list we were on to the tail of execution. */
  1644.         list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
  1645.         list_move_tail(&obj->ring_list, &ring->active_list);
  1646.  
  1647.         obj->last_read_seqno = seqno;
  1648.  
  1649.         if (obj->fenced_gpu_access) {
  1650.                 obj->last_fenced_seqno = seqno;
  1651.  
  1652.                 /* Bump MRU to take account of the delayed flush */
  1653.                 if (obj->fence_reg != I915_FENCE_REG_NONE) {
  1654.                 struct drm_i915_fence_reg *reg;
  1655.  
  1656.                 reg = &dev_priv->fence_regs[obj->fence_reg];
  1657.                         list_move_tail(&reg->lru_list,
  1658.                                        &dev_priv->mm.fence_list);
  1659.                 }
  1660.         }
  1661. }
  1662.  
  1663. static void
  1664. i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
  1665. {
  1666.         struct drm_device *dev = obj->base.dev;
  1667.         struct drm_i915_private *dev_priv = dev->dev_private;
  1668.  
  1669.         BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
  1670.         BUG_ON(!obj->active);
  1671.  
  1672.         if (obj->pin_count) /* are we a framebuffer? */
  1673.                 intel_mark_fb_idle(obj);
  1674.  
  1675.                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
  1676.  
  1677.         list_del_init(&obj->ring_list);
  1678.         obj->ring = NULL;
  1679.  
  1680.         obj->last_read_seqno = 0;
  1681.         obj->last_write_seqno = 0;
  1682.         obj->base.write_domain = 0;
  1683.  
  1684.         obj->last_fenced_seqno = 0;
  1685.         obj->fenced_gpu_access = false;
  1686.  
  1687.         obj->active = 0;
  1688.         drm_gem_object_unreference(&obj->base);
  1689.  
  1690.         WARN_ON(i915_verify_lists(dev));
  1691. }
  1692.  
  1693. static int
  1694. i915_gem_handle_seqno_wrap(struct drm_device *dev)
  1695. {
  1696.         struct drm_i915_private *dev_priv = dev->dev_private;
  1697.         struct intel_ring_buffer *ring;
  1698.         int ret, i, j;
  1699.  
  1700.         /* The hardware uses various monotonic 32-bit counters, if we
  1701.          * detect that they will wraparound we need to idle the GPU
  1702.          * and reset those counters.
  1703.          */
  1704.         ret = 0;
  1705.         for_each_ring(ring, dev_priv, i) {
  1706.                 for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
  1707.                         ret |= ring->sync_seqno[j] != 0;
  1708.         }
  1709.         if (ret == 0)
  1710.                 return ret;
  1711.  
  1712.         ret = i915_gpu_idle(dev);
  1713.         if (ret)
  1714.                 return ret;
  1715.  
  1716.         i915_gem_retire_requests(dev);
  1717.         for_each_ring(ring, dev_priv, i) {
  1718.                 for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
  1719.                         ring->sync_seqno[j] = 0;
  1720.         }
  1721.  
  1722.         return 0;
  1723. }
  1724.  
  1725. int
  1726. i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
  1727. {
  1728.         struct drm_i915_private *dev_priv = dev->dev_private;
  1729.  
  1730.         /* reserve 0 for non-seqno */
  1731.         if (dev_priv->next_seqno == 0) {
  1732.                 int ret = i915_gem_handle_seqno_wrap(dev);
  1733.                 if (ret)
  1734.                         return ret;
  1735.  
  1736.                 dev_priv->next_seqno = 1;
  1737.         }
  1738.  
  1739.         *seqno = dev_priv->next_seqno++;
  1740.         return 0;
  1741. }
  1742.  
  1743. int
  1744. i915_add_request(struct intel_ring_buffer *ring,
  1745.                  struct drm_file *file,
  1746.                  u32 *out_seqno)
  1747. {
  1748.         drm_i915_private_t *dev_priv = ring->dev->dev_private;
  1749.         struct drm_i915_gem_request *request;
  1750.         u32 request_ring_position;
  1751.         int was_empty;
  1752.         int ret;
  1753.  
  1754.         /*
  1755.          * Emit any outstanding flushes - execbuf can fail to emit the flush
  1756.          * after having emitted the batchbuffer command. Hence we need to fix
  1757.          * things up similar to emitting the lazy request. The difference here
  1758.          * is that the flush _must_ happen before the next request, no matter
  1759.          * what.
  1760.          */
  1761.         ret = intel_ring_flush_all_caches(ring);
  1762.         if (ret)
  1763.                 return ret;
  1764.  
  1765.         request = kmalloc(sizeof(*request), GFP_KERNEL);
  1766.         if (request == NULL)
  1767.                 return -ENOMEM;
  1768.  
  1769.  
  1770.         /* Record the position of the start of the request so that
  1771.          * should we detect the updated seqno part-way through the
  1772.          * GPU processing the request, we never over-estimate the
  1773.          * position of the head.
  1774.          */
  1775.         request_ring_position = intel_ring_get_tail(ring);
  1776.  
  1777.         ret = ring->add_request(ring);
  1778.         if (ret) {
  1779.                 kfree(request);
  1780.             return ret;
  1781.         }
  1782.  
  1783.         request->seqno = intel_ring_get_seqno(ring);
  1784.         request->ring = ring;
  1785.         request->tail = request_ring_position;
  1786.     request->emitted_jiffies = GetTimerTicks();
  1787.         was_empty = list_empty(&ring->request_list);
  1788.         list_add_tail(&request->list, &ring->request_list);
  1789.         request->file_priv = NULL;
  1790.  
  1791.         if (file) {
  1792.                 struct drm_i915_file_private *file_priv = file->driver_priv;
  1793.  
  1794.                 spin_lock(&file_priv->mm.lock);
  1795.                 request->file_priv = file_priv;
  1796.                 list_add_tail(&request->client_list,
  1797.                               &file_priv->mm.request_list);
  1798.                 spin_unlock(&file_priv->mm.lock);
  1799.         }
  1800.  
  1801.         trace_i915_gem_request_add(ring, request->seqno);
  1802.         ring->outstanding_lazy_request = 0;
  1803.  
  1804.         if (!dev_priv->mm.suspended) {
  1805.                 if (i915_enable_hangcheck) {
  1806. //                      mod_timer(&dev_priv->hangcheck_timer,
  1807. //                                jiffies +
  1808. //                                msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
  1809.                 }
  1810.                 if (was_empty) {
  1811.            queue_delayed_work(dev_priv->wq,
  1812.                       &dev_priv->mm.retire_work, HZ);
  1813.                         intel_mark_busy(dev_priv->dev);
  1814.                 }
  1815.         }
  1816.  
  1817.         if (out_seqno)
  1818.                 *out_seqno = request->seqno;
  1819.         return 0;
  1820. }
  1821.  
  1822. static inline void
  1823. i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
  1824. {
  1825.         struct drm_i915_file_private *file_priv = request->file_priv;
  1826.  
  1827.         if (!file_priv)
  1828.                 return;
  1829.  
  1830.         spin_lock(&file_priv->mm.lock);
  1831.         if (request->file_priv) {
  1832.                 list_del(&request->client_list);
  1833.                 request->file_priv = NULL;
  1834.         }
  1835.         spin_unlock(&file_priv->mm.lock);
  1836. }
  1837.  
  1838. static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
  1839.                                       struct intel_ring_buffer *ring)
  1840. {
  1841.         while (!list_empty(&ring->request_list)) {
  1842.                 struct drm_i915_gem_request *request;
  1843.  
  1844.                 request = list_first_entry(&ring->request_list,
  1845.                                            struct drm_i915_gem_request,
  1846.                                            list);
  1847.  
  1848.                 list_del(&request->list);
  1849.                 i915_gem_request_remove_from_client(request);
  1850.                 kfree(request);
  1851.         }
  1852.  
  1853.         while (!list_empty(&ring->active_list)) {
  1854.                 struct drm_i915_gem_object *obj;
  1855.  
  1856.                 obj = list_first_entry(&ring->active_list,
  1857.                                        struct drm_i915_gem_object,
  1858.                                        ring_list);
  1859.  
  1860.                 i915_gem_object_move_to_inactive(obj);
  1861.         }
  1862. }
  1863.  
  1864. static void i915_gem_reset_fences(struct drm_device *dev)
  1865. {
  1866.         struct drm_i915_private *dev_priv = dev->dev_private;
  1867.         int i;
  1868.  
  1869.         for (i = 0; i < dev_priv->num_fence_regs; i++) {
  1870.                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
  1871.  
  1872.                 i915_gem_write_fence(dev, i, NULL);
  1873.  
  1874.                 if (reg->obj)
  1875.                         i915_gem_object_fence_lost(reg->obj);
  1876.  
  1877.                 reg->pin_count = 0;
  1878.                 reg->obj = NULL;
  1879.                 INIT_LIST_HEAD(&reg->lru_list);
  1880.         }
  1881.  
  1882.         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
  1883. }
  1884.  
  1885. void i915_gem_reset(struct drm_device *dev)
  1886. {
  1887.         struct drm_i915_private *dev_priv = dev->dev_private;
  1888.         struct drm_i915_gem_object *obj;
  1889.         struct intel_ring_buffer *ring;
  1890.         int i;
  1891.  
  1892.         for_each_ring(ring, dev_priv, i)
  1893.                 i915_gem_reset_ring_lists(dev_priv, ring);
  1894.  
  1895.         /* Move everything out of the GPU domains to ensure we do any
  1896.          * necessary invalidation upon reuse.
  1897.          */
  1898.         list_for_each_entry(obj,
  1899.                             &dev_priv->mm.inactive_list,
  1900.                             mm_list)
  1901.         {
  1902.                 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
  1903.         }
  1904.  
  1905.         /* The fence registers are invalidated so clear them out */
  1906.         i915_gem_reset_fences(dev);
  1907. }
  1908.  
  1909. /**
  1910.  * This function clears the request list as sequence numbers are passed.
  1911.  */
  1912. void
  1913. i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
  1914. {
  1915.         uint32_t seqno;
  1916.  
  1917.         if (list_empty(&ring->request_list))
  1918.                 return;
  1919.  
  1920.         WARN_ON(i915_verify_lists(ring->dev));
  1921.  
  1922.         seqno = ring->get_seqno(ring, true);
  1923.  
  1924.         while (!list_empty(&ring->request_list)) {
  1925.                 struct drm_i915_gem_request *request;
  1926.  
  1927.                 request = list_first_entry(&ring->request_list,
  1928.                                            struct drm_i915_gem_request,
  1929.                                            list);
  1930.  
  1931.                 if (!i915_seqno_passed(seqno, request->seqno))
  1932.                         break;
  1933.  
  1934.                 trace_i915_gem_request_retire(ring, request->seqno);
  1935.                 /* We know the GPU must have read the request to have
  1936.                  * sent us the seqno + interrupt, so use the position
  1937.                  * of tail of the request to update the last known position
  1938.                  * of the GPU head.
  1939.                  */
  1940.                 ring->last_retired_head = request->tail;
  1941.  
  1942.                 list_del(&request->list);
  1943.                 i915_gem_request_remove_from_client(request);
  1944.                 kfree(request);
  1945.         }
  1946.  
  1947.         /* Move any buffers on the active list that are no longer referenced
  1948.          * by the ringbuffer to the flushing/inactive lists as appropriate.
  1949.          */
  1950.         while (!list_empty(&ring->active_list)) {
  1951.                 struct drm_i915_gem_object *obj;
  1952.  
  1953.                 obj = list_first_entry(&ring->active_list,
  1954.                                       struct drm_i915_gem_object,
  1955.                                       ring_list);
  1956.  
  1957.                 if (!i915_seqno_passed(seqno, obj->last_read_seqno))
  1958.                         break;
  1959.  
  1960.                         i915_gem_object_move_to_inactive(obj);
  1961.         }
  1962.  
  1963.         if (unlikely(ring->trace_irq_seqno &&
  1964.                      i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
  1965.                 ring->irq_put(ring);
  1966.                 ring->trace_irq_seqno = 0;
  1967.         }
  1968.  
  1969.         WARN_ON(i915_verify_lists(ring->dev));
  1970. }
  1971.  
  1972. void
  1973. i915_gem_retire_requests(struct drm_device *dev)
  1974. {
  1975.         drm_i915_private_t *dev_priv = dev->dev_private;
  1976.         struct intel_ring_buffer *ring;
  1977.         int i;
  1978.  
  1979.         for_each_ring(ring, dev_priv, i)
  1980.                 i915_gem_retire_requests_ring(ring);
  1981. }
  1982.  
  1983. static void
  1984. i915_gem_retire_work_handler(struct work_struct *work)
  1985. {
  1986.         drm_i915_private_t *dev_priv;
  1987.         struct drm_device *dev;
  1988.         struct intel_ring_buffer *ring;
  1989.         bool idle;
  1990.         int i;
  1991.  
  1992.         dev_priv = container_of(work, drm_i915_private_t,
  1993.                                 mm.retire_work.work);
  1994.         dev = dev_priv->dev;
  1995.  
  1996.         /* Come back later if the device is busy... */
  1997.         if (!mutex_trylock(&dev->struct_mutex)) {
  1998.         queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
  1999.         return;
  2000.         }
  2001.  
  2002.         i915_gem_retire_requests(dev);
  2003.  
  2004.         /* Send a periodic flush down the ring so we don't hold onto GEM
  2005.          * objects indefinitely.
  2006.          */
  2007.         idle = true;
  2008.         for_each_ring(ring, dev_priv, i) {
  2009.                 if (ring->gpu_caches_dirty)
  2010.                         i915_add_request(ring, NULL, NULL);
  2011.  
  2012.                 idle &= list_empty(&ring->request_list);
  2013.         }
  2014.  
  2015.    if (!dev_priv->mm.suspended && !idle)
  2016.        queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
  2017.         if (idle)
  2018.                 intel_mark_idle(dev);
  2019.  
  2020.         mutex_unlock(&dev->struct_mutex);
  2021. }
  2022.  
  2023. /**
  2024.  * Ensures that an object will eventually get non-busy by flushing any required
  2025.  * write domains, emitting any outstanding lazy request and retiring and
  2026.  * completed requests.
  2027.  */
  2028. static int
  2029. i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
  2030. {
  2031.         int ret;
  2032.  
  2033.         if (obj->active) {
  2034.                 ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
  2035.                 if (ret)
  2036.                         return ret;
  2037.  
  2038.                 i915_gem_retire_requests_ring(obj->ring);
  2039.         }
  2040.  
  2041.         return 0;
  2042. }
  2043.  
  2044. /**
  2045.  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
  2046.  * @DRM_IOCTL_ARGS: standard ioctl arguments
  2047.  *
  2048.  * Returns 0 if successful, else an error is returned with the remaining time in
  2049.  * the timeout parameter.
  2050.  *  -ETIME: object is still busy after timeout
  2051.  *  -ERESTARTSYS: signal interrupted the wait
  2052.  *  -ENONENT: object doesn't exist
  2053.  * Also possible, but rare:
  2054.  *  -EAGAIN: GPU wedged
  2055.  *  -ENOMEM: damn
  2056.  *  -ENODEV: Internal IRQ fail
  2057.  *  -E?: The add request failed
  2058.  *
  2059.  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
  2060.  * non-zero timeout parameter the wait ioctl will wait for the given number of
  2061.  * nanoseconds on an object becoming unbusy. Since the wait itself does so
  2062.  * without holding struct_mutex the object may become re-busied before this
  2063.  * function completes. A similar but shorter * race condition exists in the busy
  2064.  * ioctl
  2065.  */
  2066.  
  2067.  
  2068.  
  2069.  
  2070.  
  2071.  
  2072.  
  2073.  
  2074.  
  2075.  
  2076.  
  2077.  
  2078. /**
  2079.  * i915_gem_object_sync - sync an object to a ring.
  2080.  *
  2081.  * @obj: object which may be in use on another ring.
  2082.  * @to: ring we wish to use the object on. May be NULL.
  2083.  *
  2084.  * This code is meant to abstract object synchronization with the GPU.
  2085.  * Calling with NULL implies synchronizing the object with the CPU
  2086.  * rather than a particular GPU ring.
  2087.  *
  2088.  * Returns 0 if successful, else propagates up the lower layer error.
  2089.  */
  2090. int
  2091. i915_gem_object_sync(struct drm_i915_gem_object *obj,
  2092.                      struct intel_ring_buffer *to)
  2093. {
  2094.         struct intel_ring_buffer *from = obj->ring;
  2095.         u32 seqno;
  2096.         int ret, idx;
  2097.  
  2098.         if (from == NULL || to == from)
  2099.                 return 0;
  2100.  
  2101.         if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
  2102.                 return i915_gem_object_wait_rendering(obj, false);
  2103.  
  2104.         idx = intel_ring_sync_index(from, to);
  2105.  
  2106.         seqno = obj->last_read_seqno;
  2107.         if (seqno <= from->sync_seqno[idx])
  2108.                 return 0;
  2109.  
  2110.         ret = i915_gem_check_olr(obj->ring, seqno);
  2111.         if (ret)
  2112.                 return ret;
  2113.  
  2114.         ret = to->sync_to(to, from, seqno);
  2115.         if (!ret)
  2116.                 /* We use last_read_seqno because sync_to()
  2117.                  * might have just caused seqno wrap under
  2118.                  * the radar.
  2119.                  */
  2120.                 from->sync_seqno[idx] = obj->last_read_seqno;
  2121.  
  2122.         return ret;
  2123. }
  2124.  
  2125. static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
  2126. {
  2127.         u32 old_write_domain, old_read_domains;
  2128.  
  2129.         /* Act a barrier for all accesses through the GTT */
  2130.         mb();
  2131.  
  2132.         /* Force a pagefault for domain tracking on next user access */
  2133. //      i915_gem_release_mmap(obj);
  2134.  
  2135.         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
  2136.                 return;
  2137.  
  2138.         old_read_domains = obj->base.read_domains;
  2139.         old_write_domain = obj->base.write_domain;
  2140.  
  2141.         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
  2142.         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
  2143.  
  2144.         trace_i915_gem_object_change_domain(obj,
  2145.                                             old_read_domains,
  2146.                                             old_write_domain);
  2147. }
  2148.  
  2149. /**
  2150.  * Unbinds an object from the GTT aperture.
  2151.  */
  2152. int
  2153. i915_gem_object_unbind(struct drm_i915_gem_object *obj)
  2154. {
  2155.         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
  2156.         int ret = 0;
  2157.  
  2158.     if(obj == get_fb_obj())
  2159.         return 0;
  2160.  
  2161.         if (obj->gtt_space == NULL)
  2162.                 return 0;
  2163.  
  2164.         if (obj->pin_count)
  2165.                 return -EBUSY;
  2166.  
  2167.         BUG_ON(obj->pages == NULL);
  2168.  
  2169.         ret = i915_gem_object_finish_gpu(obj);
  2170.         if (ret)
  2171.                 return ret;
  2172.         /* Continue on if we fail due to EIO, the GPU is hung so we
  2173.          * should be safe and we need to cleanup or else we might
  2174.          * cause memory corruption through use-after-free.
  2175.          */
  2176.  
  2177.         i915_gem_object_finish_gtt(obj);
  2178.  
  2179.         /* release the fence reg _after_ flushing */
  2180.         ret = i915_gem_object_put_fence(obj);
  2181.         if (ret)
  2182.                 return ret;
  2183.  
  2184.         trace_i915_gem_object_unbind(obj);
  2185.  
  2186.         if (obj->has_global_gtt_mapping)
  2187.         i915_gem_gtt_unbind_object(obj);
  2188.         if (obj->has_aliasing_ppgtt_mapping) {
  2189.                 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
  2190.                 obj->has_aliasing_ppgtt_mapping = 0;
  2191.         }
  2192.         i915_gem_gtt_finish_object(obj);
  2193.  
  2194.         list_del(&obj->mm_list);
  2195.         list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
  2196.         /* Avoid an unnecessary call to unbind on rebind. */
  2197.         obj->map_and_fenceable = true;
  2198.  
  2199.         drm_mm_put_block(obj->gtt_space);
  2200.         obj->gtt_space = NULL;
  2201.         obj->gtt_offset = 0;
  2202.  
  2203.         return 0;
  2204. }
  2205.  
  2206. int i915_gpu_idle(struct drm_device *dev)
  2207. {
  2208.         drm_i915_private_t *dev_priv = dev->dev_private;
  2209.         struct intel_ring_buffer *ring;
  2210.         int ret, i;
  2211.  
  2212.         /* Flush everything onto the inactive list. */
  2213.         for_each_ring(ring, dev_priv, i) {
  2214.                 ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
  2215.                 if (ret)
  2216.                         return ret;
  2217.  
  2218.                 ret = intel_ring_idle(ring);
  2219.                 if (ret)
  2220.                         return ret;
  2221.         }
  2222.  
  2223.         return 0;
  2224. }
  2225.  
  2226. static void sandybridge_write_fence_reg(struct drm_device *dev, int reg,
  2227.                                         struct drm_i915_gem_object *obj)
  2228. {
  2229.         drm_i915_private_t *dev_priv = dev->dev_private;
  2230.         uint64_t val;
  2231.  
  2232.         if (obj) {
  2233.                 u32 size = obj->gtt_space->size;
  2234.  
  2235.                 val = (uint64_t)((obj->gtt_offset + size - 4096) &
  2236.                                  0xfffff000) << 32;
  2237.                 val |= obj->gtt_offset & 0xfffff000;
  2238.                 val |= (uint64_t)((obj->stride / 128) - 1) <<
  2239.                         SANDYBRIDGE_FENCE_PITCH_SHIFT;
  2240.  
  2241.                 if (obj->tiling_mode == I915_TILING_Y)
  2242.                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
  2243.                 val |= I965_FENCE_REG_VALID;
  2244.         } else
  2245.                 val = 0;
  2246.  
  2247.         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val);
  2248.         POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8);
  2249. }
  2250.  
  2251. static void i965_write_fence_reg(struct drm_device *dev, int reg,
  2252.                                  struct drm_i915_gem_object *obj)
  2253. {
  2254.         drm_i915_private_t *dev_priv = dev->dev_private;
  2255.         uint64_t val;
  2256.  
  2257.         if (obj) {
  2258.                 u32 size = obj->gtt_space->size;
  2259.  
  2260.                 val = (uint64_t)((obj->gtt_offset + size - 4096) &
  2261.                                  0xfffff000) << 32;
  2262.                 val |= obj->gtt_offset & 0xfffff000;
  2263.                 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
  2264.                 if (obj->tiling_mode == I915_TILING_Y)
  2265.                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
  2266.                 val |= I965_FENCE_REG_VALID;
  2267.         } else
  2268.                 val = 0;
  2269.  
  2270.         I915_WRITE64(FENCE_REG_965_0 + reg * 8, val);
  2271.         POSTING_READ(FENCE_REG_965_0 + reg * 8);
  2272. }
  2273.  
  2274. static void i915_write_fence_reg(struct drm_device *dev, int reg,
  2275.                                  struct drm_i915_gem_object *obj)
  2276. {
  2277.         drm_i915_private_t *dev_priv = dev->dev_private;
  2278.         u32 val;
  2279.  
  2280.         if (obj) {
  2281.                 u32 size = obj->gtt_space->size;
  2282.                 int pitch_val;
  2283.                 int tile_width;
  2284.  
  2285.                 WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
  2286.                      (size & -size) != size ||
  2287.                      (obj->gtt_offset & (size - 1)),
  2288.                      "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
  2289.                      obj->gtt_offset, obj->map_and_fenceable, size);
  2290.  
  2291.                 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
  2292.                         tile_width = 128;
  2293.                 else
  2294.                         tile_width = 512;
  2295.  
  2296.                 /* Note: pitch better be a power of two tile widths */
  2297.                 pitch_val = obj->stride / tile_width;
  2298.                 pitch_val = ffs(pitch_val) - 1;
  2299.  
  2300.                 val = obj->gtt_offset;
  2301.                 if (obj->tiling_mode == I915_TILING_Y)
  2302.                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
  2303.                 val |= I915_FENCE_SIZE_BITS(size);
  2304.                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
  2305.                 val |= I830_FENCE_REG_VALID;
  2306.         } else
  2307.                 val = 0;
  2308.  
  2309.         if (reg < 8)
  2310.                 reg = FENCE_REG_830_0 + reg * 4;
  2311.         else
  2312.                 reg = FENCE_REG_945_8 + (reg - 8) * 4;
  2313.  
  2314.         I915_WRITE(reg, val);
  2315.         POSTING_READ(reg);
  2316. }
  2317.  
  2318. static void i830_write_fence_reg(struct drm_device *dev, int reg,
  2319.                                 struct drm_i915_gem_object *obj)
  2320. {
  2321.         drm_i915_private_t *dev_priv = dev->dev_private;
  2322.         uint32_t val;
  2323.  
  2324.         if (obj) {
  2325.                 u32 size = obj->gtt_space->size;
  2326.                 uint32_t pitch_val;
  2327.  
  2328.                 WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
  2329.                      (size & -size) != size ||
  2330.                      (obj->gtt_offset & (size - 1)),
  2331.                      "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
  2332.                      obj->gtt_offset, size);
  2333.  
  2334.                 pitch_val = obj->stride / 128;
  2335.                 pitch_val = ffs(pitch_val) - 1;
  2336.  
  2337.                 val = obj->gtt_offset;
  2338.                 if (obj->tiling_mode == I915_TILING_Y)
  2339.                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
  2340.                 val |= I830_FENCE_SIZE_BITS(size);
  2341.                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
  2342.                 val |= I830_FENCE_REG_VALID;
  2343.         } else
  2344.                 val = 0;
  2345.  
  2346.         I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
  2347.         POSTING_READ(FENCE_REG_830_0 + reg * 4);
  2348. }
  2349.  
  2350. static void i915_gem_write_fence(struct drm_device *dev, int reg,
  2351.                                  struct drm_i915_gem_object *obj)
  2352. {
  2353.         switch (INTEL_INFO(dev)->gen) {
  2354.         case 7:
  2355.         case 6: sandybridge_write_fence_reg(dev, reg, obj); break;
  2356.         case 5:
  2357.         case 4: i965_write_fence_reg(dev, reg, obj); break;
  2358.         case 3: i915_write_fence_reg(dev, reg, obj); break;
  2359.         case 2: i830_write_fence_reg(dev, reg, obj); break;
  2360.         default: break;
  2361.         }
  2362. }
  2363.  
  2364. static inline int fence_number(struct drm_i915_private *dev_priv,
  2365.                                struct drm_i915_fence_reg *fence)
  2366. {
  2367.         return fence - dev_priv->fence_regs;
  2368. }
  2369.  
  2370. static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
  2371.                                          struct drm_i915_fence_reg *fence,
  2372.                                          bool enable)
  2373. {
  2374.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  2375.         int reg = fence_number(dev_priv, fence);
  2376.  
  2377.         i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
  2378.  
  2379.         if (enable) {
  2380.                 obj->fence_reg = reg;
  2381.                 fence->obj = obj;
  2382.                 list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
  2383.         } else {
  2384.                 obj->fence_reg = I915_FENCE_REG_NONE;
  2385.                 fence->obj = NULL;
  2386.                 list_del_init(&fence->lru_list);
  2387.         }
  2388. }
  2389.  
  2390. static int
  2391. i915_gem_object_flush_fence(struct drm_i915_gem_object *obj)
  2392. {
  2393.         if (obj->last_fenced_seqno) {
  2394.                 int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
  2395.                         if (ret)
  2396.                                 return ret;
  2397.  
  2398.                 obj->last_fenced_seqno = 0;
  2399.         }
  2400.  
  2401.         /* Ensure that all CPU reads are completed before installing a fence
  2402.          * and all writes before removing the fence.
  2403.          */
  2404.         if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
  2405.                 mb();
  2406.  
  2407.         obj->fenced_gpu_access = false;
  2408.         return 0;
  2409. }
  2410.  
  2411. int
  2412. i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
  2413. {
  2414.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  2415.         int ret;
  2416.  
  2417.         ret = i915_gem_object_flush_fence(obj);
  2418.         if (ret)
  2419.                 return ret;
  2420.  
  2421.         if (obj->fence_reg == I915_FENCE_REG_NONE)
  2422.                 return 0;
  2423.  
  2424.         i915_gem_object_update_fence(obj,
  2425.                                      &dev_priv->fence_regs[obj->fence_reg],
  2426.                                      false);
  2427.         i915_gem_object_fence_lost(obj);
  2428.  
  2429.         return 0;
  2430. }
  2431.  
  2432. static struct drm_i915_fence_reg *
  2433. i915_find_fence_reg(struct drm_device *dev)
  2434. {
  2435.         struct drm_i915_private *dev_priv = dev->dev_private;
  2436.         struct drm_i915_fence_reg *reg, *avail;
  2437.         int i;
  2438.  
  2439.         /* First try to find a free reg */
  2440.         avail = NULL;
  2441.         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
  2442.                 reg = &dev_priv->fence_regs[i];
  2443.                 if (!reg->obj)
  2444.                         return reg;
  2445.  
  2446.                 if (!reg->pin_count)
  2447.                         avail = reg;
  2448.         }
  2449.  
  2450.         if (avail == NULL)
  2451.                 return NULL;
  2452.  
  2453.         /* None available, try to steal one or wait for a user to finish */
  2454.         list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
  2455.                 if (reg->pin_count)
  2456.                         continue;
  2457.  
  2458.                 return reg;
  2459.         }
  2460.  
  2461.         return NULL;
  2462. }
  2463.  
  2464. /**
  2465.  * i915_gem_object_get_fence - set up fencing for an object
  2466.  * @obj: object to map through a fence reg
  2467.  *
  2468.  * When mapping objects through the GTT, userspace wants to be able to write
  2469.  * to them without having to worry about swizzling if the object is tiled.
  2470.  * This function walks the fence regs looking for a free one for @obj,
  2471.  * stealing one if it can't find any.
  2472.  *
  2473.  * It then sets up the reg based on the object's properties: address, pitch
  2474.  * and tiling format.
  2475.  *
  2476.  * For an untiled surface, this removes any existing fence.
  2477.  */
  2478. int
  2479. i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
  2480. {
  2481.         struct drm_device *dev = obj->base.dev;
  2482.         struct drm_i915_private *dev_priv = dev->dev_private;
  2483.         bool enable = obj->tiling_mode != I915_TILING_NONE;
  2484.         struct drm_i915_fence_reg *reg;
  2485.         int ret;
  2486.  
  2487.         /* Have we updated the tiling parameters upon the object and so
  2488.          * will need to serialise the write to the associated fence register?
  2489.          */
  2490.         if (obj->fence_dirty) {
  2491.                 ret = i915_gem_object_flush_fence(obj);
  2492.                 if (ret)
  2493.                         return ret;
  2494.         }
  2495.  
  2496.         /* Just update our place in the LRU if our fence is getting reused. */
  2497.         if (obj->fence_reg != I915_FENCE_REG_NONE) {
  2498.                 reg = &dev_priv->fence_regs[obj->fence_reg];
  2499.                 if (!obj->fence_dirty) {
  2500.                         list_move_tail(&reg->lru_list,
  2501.                                        &dev_priv->mm.fence_list);
  2502.                         return 0;
  2503.                 }
  2504.         } else if (enable) {
  2505.                 reg = i915_find_fence_reg(dev);
  2506.                 if (reg == NULL)
  2507.                         return -EDEADLK;
  2508.  
  2509.                 if (reg->obj) {
  2510.                         struct drm_i915_gem_object *old = reg->obj;
  2511.  
  2512.                         ret = i915_gem_object_flush_fence(old);
  2513.                         if (ret)
  2514.                                 return ret;
  2515.  
  2516.                         i915_gem_object_fence_lost(old);
  2517.                 }
  2518.         } else
  2519.                 return 0;
  2520.  
  2521.         i915_gem_object_update_fence(obj, reg, enable);
  2522.         obj->fence_dirty = false;
  2523.  
  2524.         return 0;
  2525. }
  2526.  
  2527. static bool i915_gem_valid_gtt_space(struct drm_device *dev,
  2528.                                      struct drm_mm_node *gtt_space,
  2529.                                      unsigned long cache_level)
  2530. {
  2531.         struct drm_mm_node *other;
  2532.  
  2533.         /* On non-LLC machines we have to be careful when putting differing
  2534.          * types of snoopable memory together to avoid the prefetcher
  2535.          * crossing memory domains and dieing.
  2536.          */
  2537.         if (HAS_LLC(dev))
  2538.                 return true;
  2539.  
  2540.         if (gtt_space == NULL)
  2541.                 return true;
  2542.  
  2543.         if (list_empty(&gtt_space->node_list))
  2544.                 return true;
  2545.  
  2546.         other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
  2547.         if (other->allocated && !other->hole_follows && other->color != cache_level)
  2548.                 return false;
  2549.  
  2550.         other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
  2551.         if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
  2552.                 return false;
  2553.  
  2554.         return true;
  2555. }
  2556.  
  2557. static void i915_gem_verify_gtt(struct drm_device *dev)
  2558. {
  2559. #if WATCH_GTT
  2560.         struct drm_i915_private *dev_priv = dev->dev_private;
  2561.         struct drm_i915_gem_object *obj;
  2562.         int err = 0;
  2563.  
  2564.         list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
  2565.                 if (obj->gtt_space == NULL) {
  2566.                         printk(KERN_ERR "object found on GTT list with no space reserved\n");
  2567.                         err++;
  2568.                         continue;
  2569.                 }
  2570.  
  2571.                 if (obj->cache_level != obj->gtt_space->color) {
  2572.                         printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
  2573.                                obj->gtt_space->start,
  2574.                                obj->gtt_space->start + obj->gtt_space->size,
  2575.                                obj->cache_level,
  2576.                                obj->gtt_space->color);
  2577.                         err++;
  2578.                         continue;
  2579.                 }
  2580.  
  2581.                 if (!i915_gem_valid_gtt_space(dev,
  2582.                                               obj->gtt_space,
  2583.                                               obj->cache_level)) {
  2584.                         printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
  2585.                                obj->gtt_space->start,
  2586.                                obj->gtt_space->start + obj->gtt_space->size,
  2587.                                obj->cache_level);
  2588.                         err++;
  2589.                         continue;
  2590.                 }
  2591.         }
  2592.  
  2593.         WARN_ON(err);
  2594. #endif
  2595. }
  2596.  
  2597. /**
  2598.  * Finds free space in the GTT aperture and binds the object there.
  2599.  */
  2600. static int
  2601. i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  2602.                             unsigned alignment,
  2603.                             bool map_and_fenceable,
  2604.                             bool nonblocking)
  2605. {
  2606.         struct drm_device *dev = obj->base.dev;
  2607.         drm_i915_private_t *dev_priv = dev->dev_private;
  2608.         struct drm_mm_node *node;
  2609.         u32 size, fence_size, fence_alignment, unfenced_alignment;
  2610.         bool mappable, fenceable;
  2611.         int ret;
  2612.  
  2613.         if (obj->madv != I915_MADV_WILLNEED) {
  2614.         DRM_ERROR("Attempting to bind a purgeable object\n");
  2615.                 return -EINVAL;
  2616.         }
  2617.  
  2618.         fence_size = i915_gem_get_gtt_size(dev,
  2619.                                            obj->base.size,
  2620.                                            obj->tiling_mode);
  2621.         fence_alignment = i915_gem_get_gtt_alignment(dev,
  2622.                                                      obj->base.size,
  2623.                                                      obj->tiling_mode);
  2624.         unfenced_alignment =
  2625.                 i915_gem_get_unfenced_gtt_alignment(dev,
  2626.                                                     obj->base.size,
  2627.                                                     obj->tiling_mode);
  2628.  
  2629.         if (alignment == 0)
  2630.                 alignment = map_and_fenceable ? fence_alignment :
  2631.                                                 unfenced_alignment;
  2632.         if (map_and_fenceable && alignment & (fence_alignment - 1)) {
  2633.                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
  2634.                 return -EINVAL;
  2635.         }
  2636.  
  2637.         size = map_and_fenceable ? fence_size : obj->base.size;
  2638.  
  2639.         /* If the object is bigger than the entire aperture, reject it early
  2640.          * before evicting everything in a vain attempt to find space.
  2641.          */
  2642.         if (obj->base.size >
  2643.             (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
  2644.                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
  2645.                 return -E2BIG;
  2646.         }
  2647.  
  2648.         ret = i915_gem_object_get_pages(obj);
  2649.         if (ret)
  2650.                 return ret;
  2651.  
  2652.         i915_gem_object_pin_pages(obj);
  2653.  
  2654.         node = kzalloc(sizeof(*node), GFP_KERNEL);
  2655.         if (node == NULL) {
  2656.                 i915_gem_object_unpin_pages(obj);
  2657.                 return -ENOMEM;
  2658.         }
  2659.  
  2660.  search_free:
  2661.         if (map_and_fenceable)
  2662.                 ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space, node,
  2663.                                                           size, alignment, obj->cache_level,
  2664.                                                           0, dev_priv->mm.gtt_mappable_end);
  2665.         else
  2666.                 ret = drm_mm_insert_node_generic(&dev_priv->mm.gtt_space, node,
  2667.                                                  size, alignment, obj->cache_level);
  2668.         if (ret) {
  2669.  
  2670.                 i915_gem_object_unpin_pages(obj);
  2671.                 kfree(node);
  2672.                         return ret;
  2673.         }
  2674.         if (WARN_ON(!i915_gem_valid_gtt_space(dev, node, obj->cache_level))) {
  2675.                 i915_gem_object_unpin_pages(obj);
  2676.                 drm_mm_put_block(node);
  2677.                 return -EINVAL;
  2678.         }
  2679.  
  2680.         ret = i915_gem_gtt_prepare_object(obj);
  2681.         if (ret) {
  2682.                 i915_gem_object_unpin_pages(obj);
  2683.                 drm_mm_put_block(node);
  2684.                         return ret;
  2685.         }
  2686.  
  2687.         list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list);
  2688.         list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
  2689.  
  2690.         obj->gtt_space = node;
  2691.         obj->gtt_offset = node->start;
  2692.  
  2693.         fenceable =
  2694.                 node->size == fence_size &&
  2695.                 (node->start & (fence_alignment - 1)) == 0;
  2696.  
  2697.         mappable =
  2698.                 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
  2699.  
  2700.         obj->map_and_fenceable = mappable && fenceable;
  2701.  
  2702.         i915_gem_object_unpin_pages(obj);
  2703.         trace_i915_gem_object_bind(obj, map_and_fenceable);
  2704.         i915_gem_verify_gtt(dev);
  2705.         return 0;
  2706. }
  2707.  
  2708. void
  2709. i915_gem_clflush_object(struct drm_i915_gem_object *obj)
  2710. {
  2711.         /* If we don't have a page list set up, then we're not pinned
  2712.          * to GPU, and we can ignore the cache flush because it'll happen
  2713.          * again at bind time.
  2714.          */
  2715.         if (obj->pages == NULL)
  2716.                 return;
  2717.  
  2718.         /* If the GPU is snooping the contents of the CPU cache,
  2719.          * we do not need to manually clear the CPU cache lines.  However,
  2720.          * the caches are only snooped when the render cache is
  2721.          * flushed/invalidated.  As we always have to emit invalidations
  2722.          * and flushes when moving into and out of the RENDER domain, correct
  2723.          * snooping behaviour occurs naturally as the result of our domain
  2724.          * tracking.
  2725.          */
  2726.         if (obj->cache_level != I915_CACHE_NONE)
  2727.                 return;
  2728. #if 0
  2729.      if(obj->mapped != NULL)
  2730.      {
  2731.         uint8_t *page_virtual;
  2732.         unsigned int i;
  2733.  
  2734.         page_virtual = obj->mapped;
  2735.         asm volatile("mfence");
  2736.         for (i = 0; i < obj->base.size; i += x86_clflush_size)
  2737.             clflush(page_virtual + i);
  2738.         asm volatile("mfence");
  2739.      }
  2740.      else
  2741.      {
  2742.         uint8_t *page_virtual;
  2743.         unsigned int i;
  2744.         page_virtual = AllocKernelSpace(obj->base.size);
  2745.         if(page_virtual != NULL)
  2746.         {
  2747.             dma_addr_t *src, *dst;
  2748.             u32 count;
  2749.  
  2750. #define page_tabs  0xFDC00000      /* really dirty hack */
  2751.  
  2752.             src =  obj->pages.page;
  2753.             dst =  &((dma_addr_t*)page_tabs)[(u32_t)page_virtual >> 12];
  2754.             count = obj->base.size/4096;
  2755.  
  2756.             while(count--)
  2757.             {
  2758.                 *dst++ = (0xFFFFF000 & *src++) | 0x001 ;
  2759.             };
  2760.  
  2761.             asm volatile("mfence");
  2762.             for (i = 0; i < obj->base.size; i += x86_clflush_size)
  2763.                 clflush(page_virtual + i);
  2764.             asm volatile("mfence");
  2765.             FreeKernelSpace(page_virtual);
  2766.         }
  2767.         else
  2768.         {
  2769.             asm volatile (
  2770.             "mfence         \n"
  2771.             "wbinvd         \n"                 /* this is really ugly  */
  2772.             "mfence");
  2773.         }
  2774.      }
  2775. #endif
  2776.  
  2777. }
  2778.  
  2779. /** Flushes the GTT write domain for the object if it's dirty. */
  2780. static void
  2781. i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
  2782. {
  2783.         uint32_t old_write_domain;
  2784.  
  2785.         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
  2786.                 return;
  2787.  
  2788.         /* No actual flushing is required for the GTT write domain.  Writes
  2789.          * to it immediately go to main memory as far as we know, so there's
  2790.          * no chipset flush.  It also doesn't land in render cache.
  2791.          *
  2792.          * However, we do have to enforce the order so that all writes through
  2793.          * the GTT land before any writes to the device, such as updates to
  2794.          * the GATT itself.
  2795.          */
  2796.         wmb();
  2797.  
  2798.         old_write_domain = obj->base.write_domain;
  2799.         obj->base.write_domain = 0;
  2800.  
  2801.         trace_i915_gem_object_change_domain(obj,
  2802.                                             obj->base.read_domains,
  2803.                                             old_write_domain);
  2804. }
  2805.  
  2806. /** Flushes the CPU write domain for the object if it's dirty. */
  2807. static void
  2808. i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
  2809. {
  2810.         uint32_t old_write_domain;
  2811.  
  2812.         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
  2813.                 return;
  2814.  
  2815.         i915_gem_clflush_object(obj);
  2816.         i915_gem_chipset_flush(obj->base.dev);
  2817.         old_write_domain = obj->base.write_domain;
  2818.         obj->base.write_domain = 0;
  2819.  
  2820.         trace_i915_gem_object_change_domain(obj,
  2821.                                             obj->base.read_domains,
  2822.                                             old_write_domain);
  2823. }
  2824.  
  2825. /**
  2826.  * Moves a single object to the GTT read, and possibly write domain.
  2827.  *
  2828.  * This function returns when the move is complete, including waiting on
  2829.  * flushes to occur.
  2830.  */
  2831. int
  2832. i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
  2833. {
  2834.         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
  2835.         uint32_t old_write_domain, old_read_domains;
  2836.         int ret;
  2837.  
  2838.         /* Not valid to be called on unbound objects. */
  2839.         if (obj->gtt_space == NULL)
  2840.                 return -EINVAL;
  2841.  
  2842.         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
  2843.                 return 0;
  2844.  
  2845.         ret = i915_gem_object_wait_rendering(obj, !write);
  2846.                 if (ret)
  2847.                         return ret;
  2848.  
  2849.         i915_gem_object_flush_cpu_write_domain(obj);
  2850.  
  2851.         old_write_domain = obj->base.write_domain;
  2852.         old_read_domains = obj->base.read_domains;
  2853.  
  2854.         /* It should now be out of any other write domains, and we can update
  2855.          * the domain values for our changes.
  2856.          */
  2857.         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
  2858.         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
  2859.         if (write) {
  2860.                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
  2861.                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
  2862.                 obj->dirty = 1;
  2863.         }
  2864.  
  2865.         trace_i915_gem_object_change_domain(obj,
  2866.                                             old_read_domains,
  2867.                                             old_write_domain);
  2868.  
  2869.         /* And bump the LRU for this access */
  2870.         if (i915_gem_object_is_inactive(obj))
  2871.                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
  2872.  
  2873.         return 0;
  2874. }
  2875.  
  2876. int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
  2877.                                     enum i915_cache_level cache_level)
  2878. {
  2879.         struct drm_device *dev = obj->base.dev;
  2880.         drm_i915_private_t *dev_priv = dev->dev_private;
  2881.         int ret;
  2882.  
  2883.         if (obj->cache_level == cache_level)
  2884.                 return 0;
  2885.  
  2886.         if (obj->pin_count) {
  2887.                 DRM_DEBUG("can not change the cache level of pinned objects\n");
  2888.                 return -EBUSY;
  2889.         }
  2890.  
  2891.         if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) {
  2892.                 ret = i915_gem_object_unbind(obj);
  2893.                 if (ret)
  2894.                         return ret;
  2895.         }
  2896.  
  2897.         if (obj->gtt_space) {
  2898.                 ret = i915_gem_object_finish_gpu(obj);
  2899.                 if (ret)
  2900.                         return ret;
  2901.  
  2902.                 i915_gem_object_finish_gtt(obj);
  2903.  
  2904.                 /* Before SandyBridge, you could not use tiling or fence
  2905.                  * registers with snooped memory, so relinquish any fences
  2906.                  * currently pointing to our region in the aperture.
  2907.                  */
  2908.                 if (INTEL_INFO(dev)->gen < 6) {
  2909.                         ret = i915_gem_object_put_fence(obj);
  2910.                         if (ret)
  2911.                                 return ret;
  2912.                 }
  2913.  
  2914.                 if (obj->has_global_gtt_mapping)
  2915.                         i915_gem_gtt_bind_object(obj, cache_level);
  2916.                 if (obj->has_aliasing_ppgtt_mapping)
  2917.                         i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
  2918.                                                obj, cache_level);
  2919.  
  2920.                 obj->gtt_space->color = cache_level;
  2921.         }
  2922.  
  2923.         if (cache_level == I915_CACHE_NONE) {
  2924.                 u32 old_read_domains, old_write_domain;
  2925.  
  2926.                 /* If we're coming from LLC cached, then we haven't
  2927.                  * actually been tracking whether the data is in the
  2928.                  * CPU cache or not, since we only allow one bit set
  2929.                  * in obj->write_domain and have been skipping the clflushes.
  2930.                  * Just set it to the CPU cache for now.
  2931.                  */
  2932.                 WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
  2933.                 WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
  2934.  
  2935.                 old_read_domains = obj->base.read_domains;
  2936.                 old_write_domain = obj->base.write_domain;
  2937.  
  2938.                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  2939.                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  2940.  
  2941.                 trace_i915_gem_object_change_domain(obj,
  2942.                                                     old_read_domains,
  2943.                                                     old_write_domain);
  2944.     }
  2945.  
  2946.         obj->cache_level = cache_level;
  2947.         i915_gem_verify_gtt(dev);
  2948.         return 0;
  2949. }
  2950.  
  2951. int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
  2952.                                struct drm_file *file)
  2953. {
  2954.         struct drm_i915_gem_caching *args = data;
  2955.         struct drm_i915_gem_object *obj;
  2956.         int ret;
  2957.  
  2958.         ret = i915_mutex_lock_interruptible(dev);
  2959.         if (ret)
  2960.                 return ret;
  2961.  
  2962.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  2963.         if (&obj->base == NULL) {
  2964.                 ret = -ENOENT;
  2965.                 goto unlock;
  2966.         }
  2967.  
  2968.         args->caching = obj->cache_level != I915_CACHE_NONE;
  2969.  
  2970.         drm_gem_object_unreference(&obj->base);
  2971. unlock:
  2972.         mutex_unlock(&dev->struct_mutex);
  2973.         return ret;
  2974. }
  2975.  
  2976. int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
  2977.                                struct drm_file *file)
  2978. {
  2979.         struct drm_i915_gem_caching *args = data;
  2980.         struct drm_i915_gem_object *obj;
  2981.         enum i915_cache_level level;
  2982.         int ret;
  2983.  
  2984.         switch (args->caching) {
  2985.         case I915_CACHING_NONE:
  2986.                 level = I915_CACHE_NONE;
  2987.                 break;
  2988.         case I915_CACHING_CACHED:
  2989.                 level = I915_CACHE_LLC;
  2990.                 break;
  2991.         default:
  2992.                 return -EINVAL;
  2993.         }
  2994.  
  2995.         ret = i915_mutex_lock_interruptible(dev);
  2996.         if (ret)
  2997.                 return ret;
  2998.  
  2999.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  3000.         if (&obj->base == NULL) {
  3001.                 ret = -ENOENT;
  3002.                 goto unlock;
  3003.         }
  3004.  
  3005.         ret = i915_gem_object_set_cache_level(obj, level);
  3006.  
  3007.         drm_gem_object_unreference(&obj->base);
  3008. unlock:
  3009.         mutex_unlock(&dev->struct_mutex);
  3010.         return ret;
  3011. }
  3012.  
  3013. /*
  3014.  * Prepare buffer for display plane (scanout, cursors, etc).
  3015.  * Can be called from an uninterruptible phase (modesetting) and allows
  3016.  * any flushes to be pipelined (for pageflips).
  3017.  */
  3018. int
  3019. i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
  3020.                                      u32 alignment,
  3021.                                      struct intel_ring_buffer *pipelined)
  3022. {
  3023.         u32 old_read_domains, old_write_domain;
  3024.         int ret;
  3025.  
  3026.         if (pipelined != obj->ring) {
  3027.                 ret = i915_gem_object_sync(obj, pipelined);
  3028.         if (ret)
  3029.                 return ret;
  3030.         }
  3031.  
  3032.         /* The display engine is not coherent with the LLC cache on gen6.  As
  3033.          * a result, we make sure that the pinning that is about to occur is
  3034.          * done with uncached PTEs. This is lowest common denominator for all
  3035.          * chipsets.
  3036.          *
  3037.          * However for gen6+, we could do better by using the GFDT bit instead
  3038.          * of uncaching, which would allow us to flush all the LLC-cached data
  3039.          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
  3040.          */
  3041.         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
  3042.         if (ret)
  3043.                 return ret;
  3044.  
  3045.         /* As the user may map the buffer once pinned in the display plane
  3046.          * (e.g. libkms for the bootup splash), we have to ensure that we
  3047.          * always use map_and_fenceable for all scanout buffers.
  3048.          */
  3049.         ret = i915_gem_object_pin(obj, alignment, true, false);
  3050.         if (ret)
  3051.                 return ret;
  3052.  
  3053.         i915_gem_object_flush_cpu_write_domain(obj);
  3054.  
  3055.         old_write_domain = obj->base.write_domain;
  3056.         old_read_domains = obj->base.read_domains;
  3057.  
  3058.         /* It should now be out of any other write domains, and we can update
  3059.          * the domain values for our changes.
  3060.          */
  3061.         obj->base.write_domain = 0;
  3062.         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
  3063.  
  3064.         trace_i915_gem_object_change_domain(obj,
  3065.                                             old_read_domains,
  3066.                                             old_write_domain);
  3067.  
  3068.         return 0;
  3069. }
  3070.  
  3071. int
  3072. i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
  3073. {
  3074.         int ret;
  3075.  
  3076.         if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
  3077.                 return 0;
  3078.  
  3079.         ret = i915_gem_object_wait_rendering(obj, false);
  3080.     if (ret)
  3081.         return ret;
  3082.  
  3083.         /* Ensure that we invalidate the GPU's caches and TLBs. */
  3084.         obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
  3085.         return 0;
  3086. }
  3087.  
  3088. /**
  3089.  * Moves a single object to the CPU read, and possibly write domain.
  3090.  *
  3091.  * This function returns when the move is complete, including waiting on
  3092.  * flushes to occur.
  3093.  */
  3094. int
  3095. i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
  3096. {
  3097.         uint32_t old_write_domain, old_read_domains;
  3098.         int ret;
  3099.  
  3100.         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
  3101.                 return 0;
  3102.  
  3103.         ret = i915_gem_object_wait_rendering(obj, !write);
  3104.         if (ret)
  3105.                 return ret;
  3106.  
  3107.         i915_gem_object_flush_gtt_write_domain(obj);
  3108.  
  3109.         old_write_domain = obj->base.write_domain;
  3110.         old_read_domains = obj->base.read_domains;
  3111.  
  3112.         /* Flush the CPU cache if it's still invalid. */
  3113.         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
  3114.                 i915_gem_clflush_object(obj);
  3115.  
  3116.                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
  3117.         }
  3118.  
  3119.         /* It should now be out of any other write domains, and we can update
  3120.          * the domain values for our changes.
  3121.          */
  3122.         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
  3123.  
  3124.         /* If we're writing through the CPU, then the GPU read domains will
  3125.          * need to be invalidated at next use.
  3126.          */
  3127.         if (write) {
  3128.                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  3129.                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  3130.         }
  3131.  
  3132.         trace_i915_gem_object_change_domain(obj,
  3133.                                             old_read_domains,
  3134.                                             old_write_domain);
  3135.  
  3136.         return 0;
  3137. }
  3138.  
  3139. /* Throttle our rendering by waiting until the ring has completed our requests
  3140.  * emitted over 20 msec ago.
  3141.  *
  3142.  * Note that if we were to use the current jiffies each time around the loop,
  3143.  * we wouldn't escape the function with any frames outstanding if the time to
  3144.  * render a frame was over 20ms.
  3145.  *
  3146.  * This should get us reasonable parallelism between CPU and GPU but also
  3147.  * relatively low latency when blocking on a particular request to finish.
  3148.  */
  3149. static int
  3150. i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
  3151. {
  3152.         struct drm_i915_private *dev_priv = dev->dev_private;
  3153.         struct drm_i915_file_private *file_priv = file->driver_priv;
  3154.         unsigned long recent_enough = GetTimerTicks() - msecs_to_jiffies(20);
  3155.         struct drm_i915_gem_request *request;
  3156.         struct intel_ring_buffer *ring = NULL;
  3157.         u32 seqno = 0;
  3158.         int ret;
  3159.  
  3160.         if (atomic_read(&dev_priv->mm.wedged))
  3161.                 return -EIO;
  3162.  
  3163.         spin_lock(&file_priv->mm.lock);
  3164.         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
  3165.                 if (time_after_eq(request->emitted_jiffies, recent_enough))
  3166.                         break;
  3167.  
  3168.                 ring = request->ring;
  3169.                 seqno = request->seqno;
  3170.         }
  3171.         spin_unlock(&file_priv->mm.lock);
  3172.  
  3173.         if (seqno == 0)
  3174.                 return 0;
  3175.  
  3176.         ret = __wait_seqno(ring, seqno, true, NULL);
  3177.         if (ret == 0)
  3178.                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
  3179.  
  3180.         return ret;
  3181. }
  3182.  
  3183. int
  3184. i915_gem_object_pin(struct drm_i915_gem_object *obj,
  3185.                     uint32_t alignment,
  3186.                     bool map_and_fenceable,
  3187.                     bool nonblocking)
  3188. {
  3189.         int ret;
  3190.  
  3191.         if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
  3192.                 return -EBUSY;
  3193.  
  3194.         if (obj->gtt_space != NULL) {
  3195.                 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
  3196.                     (map_and_fenceable && !obj->map_and_fenceable)) {
  3197.                         WARN(obj->pin_count,
  3198.                              "bo is already pinned with incorrect alignment:"
  3199.                              " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
  3200.                              " obj->map_and_fenceable=%d\n",
  3201.                              obj->gtt_offset, alignment,
  3202.                              map_and_fenceable,
  3203.                              obj->map_and_fenceable);
  3204.                         ret = i915_gem_object_unbind(obj);
  3205.                         if (ret)
  3206.                                 return ret;
  3207.                 }
  3208.         }
  3209.  
  3210.         if (obj->gtt_space == NULL) {
  3211.                 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  3212.  
  3213.                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
  3214.                                                   map_and_fenceable,
  3215.                                                   nonblocking);
  3216.                 if (ret)
  3217.                         return ret;
  3218.  
  3219.                 if (!dev_priv->mm.aliasing_ppgtt)
  3220.                         i915_gem_gtt_bind_object(obj, obj->cache_level);
  3221.         }
  3222.  
  3223.         if (!obj->has_global_gtt_mapping && map_and_fenceable)
  3224.                 i915_gem_gtt_bind_object(obj, obj->cache_level);
  3225.  
  3226.         obj->pin_count++;
  3227.         obj->pin_mappable |= map_and_fenceable;
  3228.  
  3229.         return 0;
  3230. }
  3231.  
  3232. void
  3233. i915_gem_object_unpin(struct drm_i915_gem_object *obj)
  3234. {
  3235.         BUG_ON(obj->pin_count == 0);
  3236.         BUG_ON(obj->gtt_space == NULL);
  3237.  
  3238.         if (--obj->pin_count == 0)
  3239.                 obj->pin_mappable = false;
  3240. }
  3241.  
  3242. int
  3243. i915_gem_pin_ioctl(struct drm_device *dev, void *data,
  3244.                    struct drm_file *file)
  3245. {
  3246.         struct drm_i915_gem_pin *args = data;
  3247.         struct drm_i915_gem_object *obj;
  3248.         int ret;
  3249.  
  3250.         ret = i915_mutex_lock_interruptible(dev);
  3251.         if (ret)
  3252.                 return ret;
  3253.  
  3254.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  3255.         if (&obj->base == NULL) {
  3256.                 ret = -ENOENT;
  3257.                 goto unlock;
  3258.         }
  3259.  
  3260.         if (obj->madv != I915_MADV_WILLNEED) {
  3261.                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
  3262.                 ret = -EINVAL;
  3263.                 goto out;
  3264.         }
  3265.  
  3266.         if (obj->pin_filp != NULL && obj->pin_filp != file) {
  3267.                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
  3268.                           args->handle);
  3269.                 ret = -EINVAL;
  3270.                 goto out;
  3271.         }
  3272.  
  3273.         if (obj->user_pin_count == 0) {
  3274.                 ret = i915_gem_object_pin(obj, args->alignment, true, false);
  3275.                 if (ret)
  3276.                         goto out;
  3277.         }
  3278.  
  3279.         obj->user_pin_count++;
  3280.         obj->pin_filp = file;
  3281.  
  3282.         /* XXX - flush the CPU caches for pinned objects
  3283.          * as the X server doesn't manage domains yet
  3284.          */
  3285.         i915_gem_object_flush_cpu_write_domain(obj);
  3286.         args->offset = obj->gtt_offset;
  3287. out:
  3288.         drm_gem_object_unreference(&obj->base);
  3289. unlock:
  3290.         mutex_unlock(&dev->struct_mutex);
  3291.         return ret;
  3292. }
  3293.  
  3294. #if 0
  3295.  
  3296. int
  3297. i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
  3298.                      struct drm_file *file)
  3299. {
  3300.         struct drm_i915_gem_pin *args = data;
  3301.         struct drm_i915_gem_object *obj;
  3302.         int ret;
  3303.  
  3304.         ret = i915_mutex_lock_interruptible(dev);
  3305.         if (ret)
  3306.                 return ret;
  3307.  
  3308.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  3309.         if (&obj->base == NULL) {
  3310.                 ret = -ENOENT;
  3311.                 goto unlock;
  3312.         }
  3313.  
  3314.         if (obj->pin_filp != file) {
  3315.                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
  3316.                           args->handle);
  3317.                 ret = -EINVAL;
  3318.                 goto out;
  3319.         }
  3320.         obj->user_pin_count--;
  3321.         if (obj->user_pin_count == 0) {
  3322.                 obj->pin_filp = NULL;
  3323.                 i915_gem_object_unpin(obj);
  3324.         }
  3325.  
  3326. out:
  3327.         drm_gem_object_unreference(&obj->base);
  3328. unlock:
  3329.         mutex_unlock(&dev->struct_mutex);
  3330.         return ret;
  3331. }
  3332.  
  3333. #endif
  3334.  
  3335. int
  3336. i915_gem_busy_ioctl(struct drm_device *dev, void *data,
  3337.                     struct drm_file *file)
  3338. {
  3339.         struct drm_i915_gem_busy *args = data;
  3340.         struct drm_i915_gem_object *obj;
  3341.         int ret;
  3342.  
  3343.         ret = i915_mutex_lock_interruptible(dev);
  3344.         if (ret)
  3345.                 return ret;
  3346.  
  3347.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  3348.         if (&obj->base == NULL) {
  3349.                 ret = -ENOENT;
  3350.                 goto unlock;
  3351.         }
  3352.  
  3353.         /* Count all active objects as busy, even if they are currently not used
  3354.          * by the gpu. Users of this interface expect objects to eventually
  3355.          * become non-busy without any further actions, therefore emit any
  3356.          * necessary flushes here.
  3357.          */
  3358.         ret = i915_gem_object_flush_active(obj);
  3359.  
  3360.         args->busy = obj->active;
  3361.         if (obj->ring) {
  3362.                 BUILD_BUG_ON(I915_NUM_RINGS > 16);
  3363.                 args->busy |= intel_ring_flag(obj->ring) << 16;
  3364.         }
  3365.  
  3366.         drm_gem_object_unreference(&obj->base);
  3367. unlock:
  3368.         mutex_unlock(&dev->struct_mutex);
  3369.         return ret;
  3370. }
  3371.  
  3372. int
  3373. i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
  3374.                         struct drm_file *file_priv)
  3375. {
  3376.         return i915_gem_ring_throttle(dev, file_priv);
  3377. }
  3378.  
  3379. #if 0
  3380.  
  3381. int
  3382. i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
  3383.                        struct drm_file *file_priv)
  3384. {
  3385.         struct drm_i915_gem_madvise *args = data;
  3386.         struct drm_i915_gem_object *obj;
  3387.         int ret;
  3388.  
  3389.         switch (args->madv) {
  3390.         case I915_MADV_DONTNEED:
  3391.         case I915_MADV_WILLNEED:
  3392.             break;
  3393.         default:
  3394.             return -EINVAL;
  3395.         }
  3396.  
  3397.         ret = i915_mutex_lock_interruptible(dev);
  3398.         if (ret)
  3399.                 return ret;
  3400.  
  3401.         obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
  3402.         if (&obj->base == NULL) {
  3403.                 ret = -ENOENT;
  3404.                 goto unlock;
  3405.         }
  3406.  
  3407.         if (obj->pin_count) {
  3408.                 ret = -EINVAL;
  3409.                 goto out;
  3410.         }
  3411.  
  3412.         if (obj->madv != __I915_MADV_PURGED)
  3413.                 obj->madv = args->madv;
  3414.  
  3415.         /* if the object is no longer attached, discard its backing storage */
  3416.         if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
  3417.                 i915_gem_object_truncate(obj);
  3418.  
  3419.         args->retained = obj->madv != __I915_MADV_PURGED;
  3420.  
  3421. out:
  3422.         drm_gem_object_unreference(&obj->base);
  3423. unlock:
  3424.         mutex_unlock(&dev->struct_mutex);
  3425.         return ret;
  3426. }
  3427. #endif
  3428.  
  3429. void i915_gem_object_init(struct drm_i915_gem_object *obj,
  3430.                           const struct drm_i915_gem_object_ops *ops)
  3431. {
  3432.         INIT_LIST_HEAD(&obj->mm_list);
  3433.         INIT_LIST_HEAD(&obj->gtt_list);
  3434.         INIT_LIST_HEAD(&obj->ring_list);
  3435.         INIT_LIST_HEAD(&obj->exec_list);
  3436.  
  3437.         obj->ops = ops;
  3438.  
  3439.         obj->fence_reg = I915_FENCE_REG_NONE;
  3440.         obj->madv = I915_MADV_WILLNEED;
  3441.         /* Avoid an unnecessary call to unbind on the first bind. */
  3442.         obj->map_and_fenceable = true;
  3443.  
  3444.         i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
  3445. }
  3446.  
  3447. static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
  3448.         .get_pages = i915_gem_object_get_pages_gtt,
  3449.         .put_pages = i915_gem_object_put_pages_gtt,
  3450. };
  3451.  
  3452. struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
  3453.                                                   size_t size)
  3454. {
  3455.         struct drm_i915_gem_object *obj;
  3456.         struct address_space *mapping;
  3457.         u32 mask;
  3458.  
  3459.         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
  3460.         if (obj == NULL)
  3461.                 return NULL;
  3462.  
  3463.         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
  3464.                 kfree(obj);
  3465.                 return NULL;
  3466.         }
  3467.  
  3468.  
  3469.         i915_gem_object_init(obj, &i915_gem_object_ops);
  3470.  
  3471.         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  3472.         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  3473.  
  3474.         if (HAS_LLC(dev)) {
  3475.                 /* On some devices, we can have the GPU use the LLC (the CPU
  3476.                  * cache) for about a 10% performance improvement
  3477.                  * compared to uncached.  Graphics requests other than
  3478.                  * display scanout are coherent with the CPU in
  3479.                  * accessing this cache.  This means in this mode we
  3480.                  * don't need to clflush on the CPU side, and on the
  3481.                  * GPU side we only need to flush internal caches to
  3482.                  * get data visible to the CPU.
  3483.                  *
  3484.                  * However, we maintain the display planes as UC, and so
  3485.                  * need to rebind when first used as such.
  3486.                  */
  3487.                 obj->cache_level = I915_CACHE_LLC;
  3488.         } else
  3489.                 obj->cache_level = I915_CACHE_NONE;
  3490.  
  3491.         return obj;
  3492. }
  3493.  
  3494. int i915_gem_init_object(struct drm_gem_object *obj)
  3495. {
  3496.         BUG();
  3497.  
  3498.         return 0;
  3499. }
  3500.  
  3501. void i915_gem_free_object(struct drm_gem_object *gem_obj)
  3502. {
  3503.         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
  3504.         struct drm_device *dev = obj->base.dev;
  3505.         drm_i915_private_t *dev_priv = dev->dev_private;
  3506.  
  3507.         trace_i915_gem_object_destroy(obj);
  3508.  
  3509. //   if (obj->phys_obj)
  3510. //       i915_gem_detach_phys_object(dev, obj);
  3511. //    printf("%s obj %p\n", __FUNCTION__, obj);
  3512.  
  3513.         obj->pin_count = 0;
  3514.         if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
  3515.                 bool was_interruptible;
  3516.  
  3517.                 was_interruptible = dev_priv->mm.interruptible;
  3518.                 dev_priv->mm.interruptible = false;
  3519.  
  3520.                 WARN_ON(i915_gem_object_unbind(obj));
  3521.  
  3522.                 dev_priv->mm.interruptible = was_interruptible;
  3523.         }
  3524.  
  3525.         obj->pages_pin_count = 0;
  3526.         i915_gem_object_put_pages(obj);
  3527. //   i915_gem_object_free_mmap_offset(obj);
  3528.  
  3529.         BUG_ON(obj->pages);
  3530.  
  3531. //   if (obj->base.import_attach)
  3532. //       drm_prime_gem_destroy(&obj->base, NULL);
  3533.  
  3534.     if(obj->base.filp != NULL)
  3535.     {
  3536. //        printf("filp %p\n", obj->base.filp);
  3537.         shmem_file_delete(obj->base.filp);
  3538.     }
  3539.  
  3540.         drm_gem_object_release(&obj->base);
  3541.         i915_gem_info_remove_obj(dev_priv, obj->base.size);
  3542.  
  3543.         kfree(obj->bit_17);
  3544.         kfree(obj);
  3545. }
  3546.  
  3547. #if 0
  3548. int
  3549. i915_gem_idle(struct drm_device *dev)
  3550. {
  3551.         drm_i915_private_t *dev_priv = dev->dev_private;
  3552.         int ret;
  3553.  
  3554.         mutex_lock(&dev->struct_mutex);
  3555.  
  3556.         if (dev_priv->mm.suspended) {
  3557.                 mutex_unlock(&dev->struct_mutex);
  3558.                 return 0;
  3559.         }
  3560.  
  3561.         ret = i915_gpu_idle(dev);
  3562.         if (ret) {
  3563.                 mutex_unlock(&dev->struct_mutex);
  3564.                 return ret;
  3565.         }
  3566.         i915_gem_retire_requests(dev);
  3567.  
  3568.         i915_gem_reset_fences(dev);
  3569.  
  3570.         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
  3571.          * We need to replace this with a semaphore, or something.
  3572.          * And not confound mm.suspended!
  3573.          */
  3574.         dev_priv->mm.suspended = 1;
  3575.         del_timer_sync(&dev_priv->hangcheck_timer);
  3576.  
  3577.         i915_kernel_lost_context(dev);
  3578.         i915_gem_cleanup_ringbuffer(dev);
  3579.  
  3580.         mutex_unlock(&dev->struct_mutex);
  3581.  
  3582.         /* Cancel the retire work handler, which should be idle now. */
  3583.         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
  3584.  
  3585.         return 0;
  3586. }
  3587. #endif
  3588.  
  3589. void i915_gem_l3_remap(struct drm_device *dev)
  3590. {
  3591.         drm_i915_private_t *dev_priv = dev->dev_private;
  3592.         u32 misccpctl;
  3593.         int i;
  3594.  
  3595.         if (!IS_IVYBRIDGE(dev))
  3596.                 return;
  3597.  
  3598.         if (!dev_priv->l3_parity.remap_info)
  3599.                 return;
  3600.  
  3601.         misccpctl = I915_READ(GEN7_MISCCPCTL);
  3602.         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
  3603.         POSTING_READ(GEN7_MISCCPCTL);
  3604.  
  3605.         for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
  3606.                 u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
  3607.                 if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
  3608.                         DRM_DEBUG("0x%x was already programmed to %x\n",
  3609.                                   GEN7_L3LOG_BASE + i, remap);
  3610.                 if (remap && !dev_priv->l3_parity.remap_info[i/4])
  3611.                         DRM_DEBUG_DRIVER("Clearing remapped register\n");
  3612.                 I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
  3613.         }
  3614.  
  3615.         /* Make sure all the writes land before disabling dop clock gating */
  3616.         POSTING_READ(GEN7_L3LOG_BASE);
  3617.  
  3618.         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
  3619. }
  3620.  
  3621. void i915_gem_init_swizzling(struct drm_device *dev)
  3622. {
  3623.         drm_i915_private_t *dev_priv = dev->dev_private;
  3624.  
  3625.         if (INTEL_INFO(dev)->gen < 5 ||
  3626.             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
  3627.                 return;
  3628.  
  3629.         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
  3630.                                  DISP_TILE_SURFACE_SWIZZLING);
  3631.  
  3632.         if (IS_GEN5(dev))
  3633.                 return;
  3634.  
  3635.         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
  3636.         if (IS_GEN6(dev))
  3637.                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
  3638.         else
  3639.                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
  3640. }
  3641.  
  3642. static bool
  3643. intel_enable_blt(struct drm_device *dev)
  3644. {
  3645.         if (!HAS_BLT(dev))
  3646.                 return false;
  3647.  
  3648.         /* The blitter was dysfunctional on early prototypes */
  3649.         if (IS_GEN6(dev) && dev->pdev->revision < 8) {
  3650.                 DRM_INFO("BLT not supported on this pre-production hardware;"
  3651.                          " graphics performance will be degraded.\n");
  3652.                 return false;
  3653.         }
  3654.  
  3655.         return true;
  3656. }
  3657.  
  3658. int
  3659. i915_gem_init_hw(struct drm_device *dev)
  3660. {
  3661.         drm_i915_private_t *dev_priv = dev->dev_private;
  3662.         int ret;
  3663.  
  3664.         if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
  3665.                 return -EIO;
  3666.  
  3667.         if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))
  3668.                 I915_WRITE(0x9008, I915_READ(0x9008) | 0xf0000);
  3669.  
  3670.         i915_gem_l3_remap(dev);
  3671.  
  3672.         i915_gem_init_swizzling(dev);
  3673.  
  3674.         ret = intel_init_render_ring_buffer(dev);
  3675.         if (ret)
  3676.                 return ret;
  3677.  
  3678.     if (HAS_BSD(dev)) {
  3679.                 ret = intel_init_bsd_ring_buffer(dev);
  3680.                 if (ret)
  3681.                         goto cleanup_render_ring;
  3682.         }
  3683.  
  3684.         if (intel_enable_blt(dev)) {
  3685.                 ret = intel_init_blt_ring_buffer(dev);
  3686.                 if (ret)
  3687.                         goto cleanup_bsd_ring;
  3688.         }
  3689.  
  3690.         dev_priv->next_seqno = 1;
  3691.  
  3692.         /*
  3693.          * XXX: There was some w/a described somewhere suggesting loading
  3694.          * contexts before PPGTT.
  3695.          */
  3696.         i915_gem_context_init(dev);
  3697.         i915_gem_init_ppgtt(dev);
  3698.  
  3699.         return 0;
  3700.  
  3701. cleanup_bsd_ring:
  3702.         intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
  3703. cleanup_render_ring:
  3704.         intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
  3705.         return ret;
  3706. }
  3707.  
  3708. static bool
  3709. intel_enable_ppgtt(struct drm_device *dev)
  3710. {
  3711.         if (i915_enable_ppgtt >= 0)
  3712.                 return i915_enable_ppgtt;
  3713.  
  3714. #ifdef CONFIG_INTEL_IOMMU
  3715.         /* Disable ppgtt on SNB if VT-d is on. */
  3716.         if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
  3717.                 return false;
  3718. #endif
  3719.  
  3720.         return true;
  3721. }
  3722.  
  3723. #define LFB_SIZE 0xC00000
  3724.  
  3725. int i915_gem_init(struct drm_device *dev)
  3726. {
  3727.         struct drm_i915_private *dev_priv = dev->dev_private;
  3728.         unsigned long gtt_size, mappable_size;
  3729.         int ret;
  3730.  
  3731.         gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT;
  3732.         mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
  3733.  
  3734.         mutex_lock(&dev->struct_mutex);
  3735.         if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
  3736.                 /* PPGTT pdes are stolen from global gtt ptes, so shrink the
  3737.                  * aperture accordingly when using aliasing ppgtt. */
  3738.                 gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
  3739.  
  3740.         i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE);
  3741.  
  3742.                 ret = i915_gem_init_aliasing_ppgtt(dev);
  3743.                 if (ret) {
  3744.                         mutex_unlock(&dev->struct_mutex);
  3745.                         return ret;
  3746.                 }
  3747.         } else {
  3748.                 /* Let GEM Manage all of the aperture.
  3749.                  *
  3750.                  * However, leave one page at the end still bound to the scratch
  3751.                  * page.  There are a number of places where the hardware
  3752.                  * apparently prefetches past the end of the object, and we've
  3753.                  * seen multiple hangs with the GPU head pointer stuck in a
  3754.                  * batchbuffer bound at the last page of the aperture.  One page
  3755.                  * should be enough to keep any prefetching inside of the
  3756.                  * aperture.
  3757.                  */
  3758.         i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE);
  3759.         }
  3760.  
  3761.         ret = i915_gem_init_hw(dev);
  3762.         mutex_unlock(&dev->struct_mutex);
  3763.         if (ret) {
  3764.                 i915_gem_cleanup_aliasing_ppgtt(dev);
  3765.                 return ret;
  3766.         }
  3767.  
  3768.     return 0;
  3769. }
  3770.  
  3771. void
  3772. i915_gem_cleanup_ringbuffer(struct drm_device *dev)
  3773. {
  3774.         drm_i915_private_t *dev_priv = dev->dev_private;
  3775.         struct intel_ring_buffer *ring;
  3776.         int i;
  3777.  
  3778.         for_each_ring(ring, dev_priv, i)
  3779.                 intel_cleanup_ring_buffer(ring);
  3780. }
  3781.  
  3782. #if 0
  3783.  
  3784. int
  3785. i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
  3786.                        struct drm_file *file_priv)
  3787. {
  3788.         drm_i915_private_t *dev_priv = dev->dev_private;
  3789.         int ret;
  3790.  
  3791.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  3792.                 return 0;
  3793.  
  3794.         if (atomic_read(&dev_priv->mm.wedged)) {
  3795.                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
  3796.                 atomic_set(&dev_priv->mm.wedged, 0);
  3797.         }
  3798.  
  3799.         mutex_lock(&dev->struct_mutex);
  3800.         dev_priv->mm.suspended = 0;
  3801.  
  3802.         ret = i915_gem_init_hw(dev);
  3803.         if (ret != 0) {
  3804.                 mutex_unlock(&dev->struct_mutex);
  3805.                 return ret;
  3806.         }
  3807.  
  3808.         BUG_ON(!list_empty(&dev_priv->mm.active_list));
  3809.         mutex_unlock(&dev->struct_mutex);
  3810.  
  3811.         ret = drm_irq_install(dev);
  3812.         if (ret)
  3813.                 goto cleanup_ringbuffer;
  3814.  
  3815.         return 0;
  3816.  
  3817. cleanup_ringbuffer:
  3818.         mutex_lock(&dev->struct_mutex);
  3819.         i915_gem_cleanup_ringbuffer(dev);
  3820.         dev_priv->mm.suspended = 1;
  3821.         mutex_unlock(&dev->struct_mutex);
  3822.  
  3823.         return ret;
  3824. }
  3825.  
  3826. int
  3827. i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
  3828.                        struct drm_file *file_priv)
  3829. {
  3830.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  3831.                 return 0;
  3832.  
  3833.         drm_irq_uninstall(dev);
  3834.         return i915_gem_idle(dev);
  3835. }
  3836.  
  3837. void
  3838. i915_gem_lastclose(struct drm_device *dev)
  3839. {
  3840.         int ret;
  3841.  
  3842.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  3843.                 return;
  3844.  
  3845.         ret = i915_gem_idle(dev);
  3846.         if (ret)
  3847.                 DRM_ERROR("failed to idle hardware: %d\n", ret);
  3848. }
  3849. #endif
  3850.  
  3851. static void
  3852. init_ring_lists(struct intel_ring_buffer *ring)
  3853. {
  3854.     INIT_LIST_HEAD(&ring->active_list);
  3855.     INIT_LIST_HEAD(&ring->request_list);
  3856. }
  3857.  
  3858. void
  3859. i915_gem_load(struct drm_device *dev)
  3860. {
  3861.     int i;
  3862.     drm_i915_private_t *dev_priv = dev->dev_private;
  3863.  
  3864.     INIT_LIST_HEAD(&dev_priv->mm.active_list);
  3865.     INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
  3866.         INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
  3867.         INIT_LIST_HEAD(&dev_priv->mm.bound_list);
  3868.     INIT_LIST_HEAD(&dev_priv->mm.fence_list);
  3869.     for (i = 0; i < I915_NUM_RINGS; i++)
  3870.         init_ring_lists(&dev_priv->ring[i]);
  3871.         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
  3872.         INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
  3873.         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
  3874.                           i915_gem_retire_work_handler);
  3875.  
  3876.     /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
  3877.     if (IS_GEN3(dev)) {
  3878.                 I915_WRITE(MI_ARB_STATE,
  3879.                            _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
  3880.     }
  3881.  
  3882.     dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
  3883.  
  3884.     if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
  3885.         dev_priv->num_fence_regs = 16;
  3886.     else
  3887.         dev_priv->num_fence_regs = 8;
  3888.  
  3889.     /* Initialize fence registers to zero */
  3890.         i915_gem_reset_fences(dev);
  3891.  
  3892.     i915_gem_detect_bit_6_swizzle(dev);
  3893.  
  3894.     dev_priv->mm.interruptible = true;
  3895.  
  3896. //    dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
  3897. //    dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
  3898. //    register_shrinker(&dev_priv->mm.inactive_shrinker);
  3899. }
  3900.  
  3901.  
  3902.