Subversion Repositories Kolibri OS

Rev

Rev 3266 | Rev 3298 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright © 2008 Intel Corporation
  3.  *
  4.  * Permission is hereby granted, free of charge, to any person obtaining a
  5.  * copy of this software and associated documentation files (the "Software"),
  6.  * to deal in the Software without restriction, including without limitation
  7.  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8.  * and/or sell copies of the Software, and to permit persons to whom the
  9.  * Software is furnished to do so, subject to the following conditions:
  10.  *
  11.  * The above copyright notice and this permission notice (including the next
  12.  * paragraph) shall be included in all copies or substantial portions of the
  13.  * Software.
  14.  *
  15.  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16.  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17.  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18.  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19.  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20.  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21.  * IN THE SOFTWARE.
  22.  *
  23.  * Authors:
  24.  *    Eric Anholt <eric@anholt.net>
  25.  *
  26.  */
  27.  
  28. #include <drm/drmP.h>
  29. #include <drm/i915_drm.h>
  30. #include "i915_drv.h"
  31. #include "i915_trace.h"
  32. #include "intel_drv.h"
  33. #include <linux/shmem_fs.h>
  34. #include <linux/slab.h>
  35. //#include <linux/swap.h>
  36. #include <linux/pci.h>
  37.  
  38. extern int x86_clflush_size;
  39.  
  40. #define PROT_READ       0x1             /* page can be read */
  41. #define PROT_WRITE      0x2             /* page can be written */
  42. #define MAP_SHARED      0x01            /* Share changes */
  43.  
  44. #undef mb
  45. #undef rmb
  46. #undef wmb
  47. #define mb() asm volatile("mfence")
  48. #define rmb() asm volatile ("lfence")
  49. #define wmb() asm volatile ("sfence")
  50.  
  51. struct drm_i915_gem_object *get_fb_obj();
  52.  
  53. unsigned long vm_mmap(struct file *file, unsigned long addr,
  54.          unsigned long len, unsigned long prot,
  55.          unsigned long flag, unsigned long offset);
  56.  
  57. static inline void clflush(volatile void *__p)
  58. {
  59.     asm volatile("clflush %0" : "+m" (*(volatile char*)__p));
  60. }
  61.  
  62. #define MAX_ERRNO       4095
  63.  
  64. #define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
  65.  
  66.  
  67. #define I915_EXEC_CONSTANTS_MASK        (3<<6)
  68. #define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */
  69. #define I915_EXEC_CONSTANTS_ABSOLUTE    (1<<6)
  70. #define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */
  71.  
  72. static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  73. static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  74. static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  75.                                                     unsigned alignment,
  76.                                                     bool map_and_fenceable,
  77.                                                     bool nonblocking);
  78. static int i915_gem_phys_pwrite(struct drm_device *dev,
  79.                                 struct drm_i915_gem_object *obj,
  80.                                 struct drm_i915_gem_pwrite *args,
  81.                                 struct drm_file *file);
  82.  
  83. static void i915_gem_write_fence(struct drm_device *dev, int reg,
  84.                                  struct drm_i915_gem_object *obj);
  85. static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
  86.                                          struct drm_i915_fence_reg *fence,
  87.                                          bool enable);
  88.  
  89. static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
  90. static void i915_gem_shrink_all(struct drm_i915_private *dev_priv);
  91. static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
  92.  
  93. static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
  94. {
  95.         if (obj->tiling_mode)
  96.                 i915_gem_release_mmap(obj);
  97.  
  98.         /* As we do not have an associated fence register, we will force
  99.          * a tiling change if we ever need to acquire one.
  100.          */
  101.         obj->fence_dirty = false;
  102.         obj->fence_reg = I915_FENCE_REG_NONE;
  103. }
  104.  
  105. /* some bookkeeping */
  106. static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  107.                                   size_t size)
  108. {
  109.         dev_priv->mm.object_count++;
  110.         dev_priv->mm.object_memory += size;
  111. }
  112.  
  113. static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  114.                                      size_t size)
  115. {
  116.         dev_priv->mm.object_count--;
  117.         dev_priv->mm.object_memory -= size;
  118. }
  119.  
  120. static int
  121. i915_gem_wait_for_error(struct drm_device *dev)
  122. {
  123.         struct drm_i915_private *dev_priv = dev->dev_private;
  124.         struct completion *x = &dev_priv->error_completion;
  125.         unsigned long flags;
  126.         int ret;
  127.  
  128.         if (!atomic_read(&dev_priv->mm.wedged))
  129.                 return 0;
  130. #if 0
  131.         /*
  132.          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
  133.          * userspace. If it takes that long something really bad is going on and
  134.          * we should simply try to bail out and fail as gracefully as possible.
  135.          */
  136.         ret = wait_for_completion_interruptible_timeout(x, 10*HZ);
  137.         if (ret == 0) {
  138.                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
  139.                 return -EIO;
  140.         } else if (ret < 0) {
  141.                 return ret;
  142.         }
  143.  
  144.         if (atomic_read(&dev_priv->mm.wedged)) {
  145.                 /* GPU is hung, bump the completion count to account for
  146.                  * the token we just consumed so that we never hit zero and
  147.                  * end up waiting upon a subsequent completion event that
  148.                  * will never happen.
  149.                  */
  150.                 spin_lock_irqsave(&x->wait.lock, flags);
  151.                 x->done++;
  152.                 spin_unlock_irqrestore(&x->wait.lock, flags);
  153.         }
  154. #endif
  155.  
  156.         return 0;
  157. }
  158.  
  159. int i915_mutex_lock_interruptible(struct drm_device *dev)
  160. {
  161.         int ret;
  162.  
  163.         ret = i915_gem_wait_for_error(dev);
  164.         if (ret)
  165.                 return ret;
  166.  
  167.     mutex_lock(&dev->struct_mutex);
  168.  
  169.         WARN_ON(i915_verify_lists(dev));
  170.         return 0;
  171. }
  172.  
  173. static inline bool
  174. i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
  175. {
  176.         return obj->gtt_space && !obj->active;
  177. }
  178.  
  179.  
  180. #if 0
  181.  
  182. int
  183. i915_gem_init_ioctl(struct drm_device *dev, void *data,
  184.                     struct drm_file *file)
  185. {
  186.         struct drm_i915_gem_init *args = data;
  187.  
  188.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  189.                 return -ENODEV;
  190.  
  191.         if (args->gtt_start >= args->gtt_end ||
  192.             (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
  193.                 return -EINVAL;
  194.  
  195.         /* GEM with user mode setting was never supported on ilk and later. */
  196.         if (INTEL_INFO(dev)->gen >= 5)
  197.                 return -ENODEV;
  198.  
  199.         mutex_lock(&dev->struct_mutex);
  200.         i915_gem_init_global_gtt(dev, args->gtt_start,
  201.                                  args->gtt_end, args->gtt_end);
  202.         mutex_unlock(&dev->struct_mutex);
  203.  
  204.         return 0;
  205. }
  206. #endif
  207.  
  208. int
  209. i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  210.                             struct drm_file *file)
  211. {
  212.         struct drm_i915_private *dev_priv = dev->dev_private;
  213.         struct drm_i915_gem_get_aperture *args = data;
  214.         struct drm_i915_gem_object *obj;
  215.         size_t pinned;
  216.  
  217.         pinned = 0;
  218.         mutex_lock(&dev->struct_mutex);
  219.         list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
  220.                 if (obj->pin_count)
  221.                 pinned += obj->gtt_space->size;
  222.         mutex_unlock(&dev->struct_mutex);
  223.  
  224.         args->aper_size = dev_priv->mm.gtt_total;
  225.         args->aper_available_size = args->aper_size - pinned;
  226.  
  227.         return 0;
  228. }
  229.  
  230. static int
  231. i915_gem_create(struct drm_file *file,
  232.                 struct drm_device *dev,
  233.                 uint64_t size,
  234.                 uint32_t *handle_p)
  235. {
  236.         struct drm_i915_gem_object *obj;
  237.         int ret;
  238.         u32 handle;
  239.  
  240.     ENTER();
  241.  
  242.         size = roundup(size, PAGE_SIZE);
  243.         if (size == 0)
  244.                 return -EINVAL;
  245.  
  246.         /* Allocate the new object */
  247.         obj = i915_gem_alloc_object(dev, size);
  248.         if (obj == NULL)
  249.                 return -ENOMEM;
  250.  
  251.         ret = drm_gem_handle_create(file, &obj->base, &handle);
  252.         if (ret) {
  253.                 drm_gem_object_release(&obj->base);
  254.                 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
  255.                 kfree(obj);
  256.                 return ret;
  257.         }
  258.  
  259.         /* drop reference from allocate - handle holds it now */
  260.         drm_gem_object_unreference(&obj->base);
  261.         trace_i915_gem_object_create(obj);
  262.  
  263.         *handle_p = handle;
  264.  
  265.     printf("%s obj %p handle %d\n", __FUNCTION__, obj, handle);
  266.  
  267.         return 0;
  268. }
  269.  
  270. int
  271. i915_gem_dumb_create(struct drm_file *file,
  272.                      struct drm_device *dev,
  273.                      struct drm_mode_create_dumb *args)
  274. {
  275.         /* have to work out size/pitch and return them */
  276.         args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
  277.         args->size = args->pitch * args->height;
  278.         return i915_gem_create(file, dev,
  279.                                args->size, &args->handle);
  280. }
  281.  
  282. int i915_gem_dumb_destroy(struct drm_file *file,
  283.                           struct drm_device *dev,
  284.                           uint32_t handle)
  285. {
  286.         return drm_gem_handle_delete(file, handle);
  287. }
  288.  
  289. /**
  290.  * Creates a new mm object and returns a handle to it.
  291.  */
  292. int
  293. i915_gem_create_ioctl(struct drm_device *dev, void *data,
  294.                       struct drm_file *file)
  295. {
  296.         struct drm_i915_gem_create *args = data;
  297.  
  298.         return i915_gem_create(file, dev,
  299.                                args->size, &args->handle);
  300. }
  301.  
  302. static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
  303. {
  304.         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
  305.  
  306.         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
  307.                 obj->tiling_mode != I915_TILING_NONE;
  308. }
  309. #if 0
  310.  
  311. static inline int
  312. __copy_to_user_swizzled(char __user *cpu_vaddr,
  313.                         const char *gpu_vaddr, int gpu_offset,
  314.                 int length)
  315. {
  316.         int ret, cpu_offset = 0;
  317.  
  318.         while (length > 0) {
  319.                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
  320.                 int this_length = min(cacheline_end - gpu_offset, length);
  321.                 int swizzled_gpu_offset = gpu_offset ^ 64;
  322.  
  323.                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
  324.                                      gpu_vaddr + swizzled_gpu_offset,
  325.                                      this_length);
  326.                 if (ret)
  327.                         return ret + length;
  328.  
  329.                 cpu_offset += this_length;
  330.                 gpu_offset += this_length;
  331.                 length -= this_length;
  332.         }
  333.  
  334.         return 0;
  335. }
  336.  
  337. static inline int
  338. __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
  339.                           const char __user *cpu_vaddr,
  340.                           int length)
  341. {
  342.         int ret, cpu_offset = 0;
  343.  
  344.         while (length > 0) {
  345.                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
  346.                 int this_length = min(cacheline_end - gpu_offset, length);
  347.                 int swizzled_gpu_offset = gpu_offset ^ 64;
  348.  
  349.                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
  350.                                cpu_vaddr + cpu_offset,
  351.                                this_length);
  352.                 if (ret)
  353.                         return ret + length;
  354.  
  355.                 cpu_offset += this_length;
  356.                 gpu_offset += this_length;
  357.                 length -= this_length;
  358.         }
  359.  
  360.         return 0;
  361. }
  362.  
  363. /* Per-page copy function for the shmem pread fastpath.
  364.  * Flushes invalid cachelines before reading the target if
  365.  * needs_clflush is set. */
  366. static int
  367. shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
  368.                  char __user *user_data,
  369.                  bool page_do_bit17_swizzling, bool needs_clflush)
  370. {
  371.                 char *vaddr;
  372.                 int ret;
  373.  
  374.         if (unlikely(page_do_bit17_swizzling))
  375.                 return -EINVAL;
  376.  
  377.                 vaddr = kmap_atomic(page);
  378.         if (needs_clflush)
  379.                 drm_clflush_virt_range(vaddr + shmem_page_offset,
  380.                                        page_length);
  381.                 ret = __copy_to_user_inatomic(user_data,
  382.                                       vaddr + shmem_page_offset,
  383.                                               page_length);
  384.                 kunmap_atomic(vaddr);
  385.  
  386.         return ret ? -EFAULT : 0;
  387. }
  388.  
  389. static void
  390. shmem_clflush_swizzled_range(char *addr, unsigned long length,
  391.                              bool swizzled)
  392. {
  393.         if (unlikely(swizzled)) {
  394.                 unsigned long start = (unsigned long) addr;
  395.                 unsigned long end = (unsigned long) addr + length;
  396.  
  397.                 /* For swizzling simply ensure that we always flush both
  398.                  * channels. Lame, but simple and it works. Swizzled
  399.                  * pwrite/pread is far from a hotpath - current userspace
  400.                  * doesn't use it at all. */
  401.                 start = round_down(start, 128);
  402.                 end = round_up(end, 128);
  403.  
  404.                 drm_clflush_virt_range((void *)start, end - start);
  405.         } else {
  406.                 drm_clflush_virt_range(addr, length);
  407.         }
  408.  
  409. }
  410.  
  411. /* Only difference to the fast-path function is that this can handle bit17
  412.  * and uses non-atomic copy and kmap functions. */
  413. static int
  414. shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
  415.                  char __user *user_data,
  416.                  bool page_do_bit17_swizzling, bool needs_clflush)
  417. {
  418.         char *vaddr;
  419.         int ret;
  420.  
  421.         vaddr = kmap(page);
  422.         if (needs_clflush)
  423.                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  424.                                              page_length,
  425.                                              page_do_bit17_swizzling);
  426.  
  427.         if (page_do_bit17_swizzling)
  428.                 ret = __copy_to_user_swizzled(user_data,
  429.                                               vaddr, shmem_page_offset,
  430.                                               page_length);
  431.         else
  432.                 ret = __copy_to_user(user_data,
  433.                                      vaddr + shmem_page_offset,
  434.                                      page_length);
  435.         kunmap(page);
  436.  
  437.         return ret ? - EFAULT : 0;
  438. }
  439.  
  440. static int
  441. i915_gem_shmem_pread(struct drm_device *dev,
  442.                           struct drm_i915_gem_object *obj,
  443.                           struct drm_i915_gem_pread *args,
  444.                           struct drm_file *file)
  445. {
  446.         char __user *user_data;
  447.         ssize_t remain;
  448.         loff_t offset;
  449.         int shmem_page_offset, page_length, ret = 0;
  450.         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
  451.         int hit_slowpath = 0;
  452.         int prefaulted = 0;
  453.         int needs_clflush = 0;
  454.         struct scatterlist *sg;
  455.         int i;
  456.  
  457.         user_data = (char __user *) (uintptr_t) args->data_ptr;
  458.         remain = args->size;
  459.  
  460.         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
  461.  
  462.         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
  463.                 /* If we're not in the cpu read domain, set ourself into the gtt
  464.                  * read domain and manually flush cachelines (if required). This
  465.                  * optimizes for the case when the gpu will dirty the data
  466.                  * anyway again before the next pread happens. */
  467.                 if (obj->cache_level == I915_CACHE_NONE)
  468.                         needs_clflush = 1;
  469.                 if (obj->gtt_space) {
  470.                         ret = i915_gem_object_set_to_gtt_domain(obj, false);
  471.                         if (ret)
  472.                                 return ret;
  473.                 }
  474.         }
  475.  
  476.         ret = i915_gem_object_get_pages(obj);
  477.         if (ret)
  478.                 return ret;
  479.  
  480.         i915_gem_object_pin_pages(obj);
  481.  
  482.         offset = args->offset;
  483.  
  484.         for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
  485.                 struct page *page;
  486.  
  487.                 if (i < offset >> PAGE_SHIFT)
  488.                         continue;
  489.  
  490.                 if (remain <= 0)
  491.                         break;
  492.  
  493.                 /* Operation in this page
  494.                  *
  495.                  * shmem_page_offset = offset within page in shmem file
  496.                  * page_length = bytes to copy for this page
  497.                  */
  498.                 shmem_page_offset = offset_in_page(offset);
  499.                 page_length = remain;
  500.                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
  501.                         page_length = PAGE_SIZE - shmem_page_offset;
  502.  
  503.                 page = sg_page(sg);
  504.                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
  505.                         (page_to_phys(page) & (1 << 17)) != 0;
  506.  
  507.                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
  508.                                        user_data, page_do_bit17_swizzling,
  509.                                        needs_clflush);
  510.                 if (ret == 0)
  511.                         goto next_page;
  512.  
  513.                 hit_slowpath = 1;
  514.                 mutex_unlock(&dev->struct_mutex);
  515.  
  516.                 if (!prefaulted) {
  517.                         ret = fault_in_multipages_writeable(user_data, remain);
  518.                         /* Userspace is tricking us, but we've already clobbered
  519.                          * its pages with the prefault and promised to write the
  520.                          * data up to the first fault. Hence ignore any errors
  521.                          * and just continue. */
  522.                         (void)ret;
  523.                         prefaulted = 1;
  524.                 }
  525.  
  526.                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
  527.                                        user_data, page_do_bit17_swizzling,
  528.                                        needs_clflush);
  529.  
  530.                 mutex_lock(&dev->struct_mutex);
  531.  
  532. next_page:
  533.                 mark_page_accessed(page);
  534.  
  535.                 if (ret)
  536.                         goto out;
  537.  
  538.                 remain -= page_length;
  539.                 user_data += page_length;
  540.                 offset += page_length;
  541.         }
  542.  
  543. out:
  544.         i915_gem_object_unpin_pages(obj);
  545.  
  546.         if (hit_slowpath) {
  547.                 /* Fixup: Kill any reinstated backing storage pages */
  548.                 if (obj->madv == __I915_MADV_PURGED)
  549.                         i915_gem_object_truncate(obj);
  550.         }
  551.  
  552.         return ret;
  553. }
  554.  
  555. /**
  556.  * Reads data from the object referenced by handle.
  557.  *
  558.  * On error, the contents of *data are undefined.
  559.  */
  560. int
  561. i915_gem_pread_ioctl(struct drm_device *dev, void *data,
  562.                      struct drm_file *file)
  563. {
  564.         struct drm_i915_gem_pread *args = data;
  565.         struct drm_i915_gem_object *obj;
  566.         int ret = 0;
  567.  
  568.         if (args->size == 0)
  569.                 return 0;
  570.  
  571.         if (!access_ok(VERIFY_WRITE,
  572.                        (char __user *)(uintptr_t)args->data_ptr,
  573.                        args->size))
  574.                 return -EFAULT;
  575.  
  576.         ret = i915_mutex_lock_interruptible(dev);
  577.         if (ret)
  578.                 return ret;
  579.  
  580.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  581.         if (&obj->base == NULL) {
  582.                 ret = -ENOENT;
  583.                 goto unlock;
  584.         }
  585.  
  586.         /* Bounds check source.  */
  587.         if (args->offset > obj->base.size ||
  588.             args->size > obj->base.size - args->offset) {
  589.                 ret = -EINVAL;
  590.                 goto out;
  591.         }
  592.  
  593.         /* prime objects have no backing filp to GEM pread/pwrite
  594.          * pages from.
  595.          */
  596.         if (!obj->base.filp) {
  597.                 ret = -EINVAL;
  598.                 goto out;
  599.         }
  600.  
  601.         trace_i915_gem_object_pread(obj, args->offset, args->size);
  602.  
  603.         ret = i915_gem_shmem_pread(dev, obj, args, file);
  604.  
  605. out:
  606.         drm_gem_object_unreference(&obj->base);
  607. unlock:
  608.         mutex_unlock(&dev->struct_mutex);
  609.         return ret;
  610. }
  611.  
  612. /* This is the fast write path which cannot handle
  613.  * page faults in the source data
  614.  */
  615.  
  616. static inline int
  617. fast_user_write(struct io_mapping *mapping,
  618.                 loff_t page_base, int page_offset,
  619.                 char __user *user_data,
  620.                 int length)
  621. {
  622.         void __iomem *vaddr_atomic;
  623.         void *vaddr;
  624.         unsigned long unwritten;
  625.  
  626.         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
  627.         /* We can use the cpu mem copy function because this is X86. */
  628.         vaddr = (void __force*)vaddr_atomic + page_offset;
  629.         unwritten = __copy_from_user_inatomic_nocache(vaddr,
  630.                                                       user_data, length);
  631.         io_mapping_unmap_atomic(vaddr_atomic);
  632.         return unwritten;
  633. }
  634. #endif
  635.  
  636. #define offset_in_page(p)       ((unsigned long)(p) & ~PAGE_MASK)
  637. /**
  638.  * This is the fast pwrite path, where we copy the data directly from the
  639.  * user into the GTT, uncached.
  640.  */
  641. static int
  642. i915_gem_gtt_pwrite_fast(struct drm_device *dev,
  643.                          struct drm_i915_gem_object *obj,
  644.                          struct drm_i915_gem_pwrite *args,
  645.                          struct drm_file *file)
  646. {
  647.         drm_i915_private_t *dev_priv = dev->dev_private;
  648.         ssize_t remain;
  649.         loff_t offset, page_base;
  650.         char __user *user_data;
  651.         int page_offset, page_length, ret;
  652.     char *vaddr;
  653.  
  654.         ret = i915_gem_object_pin(obj, 0, true, true);
  655.         if (ret)
  656.                 goto out;
  657.  
  658.         ret = i915_gem_object_set_to_gtt_domain(obj, true);
  659.         if (ret)
  660.                 goto out_unpin;
  661.  
  662.         ret = i915_gem_object_put_fence(obj);
  663.         if (ret)
  664.                 goto out_unpin;
  665.  
  666.     vaddr = AllocKernelSpace(4096);
  667.     if(vaddr == NULL)
  668.     {
  669.         ret = -ENOSPC;
  670.         goto out_unpin;
  671.     };
  672.  
  673.         user_data = (char __user *) (uintptr_t) args->data_ptr;
  674.         remain = args->size;
  675.  
  676.         offset = obj->gtt_offset + args->offset;
  677.  
  678.         while (remain > 0) {
  679.                 /* Operation in this page
  680.                  *
  681.                  * page_base = page offset within aperture
  682.                  * page_offset = offset within page
  683.                  * page_length = bytes to copy for this page
  684.                  */
  685.                 page_base = offset & PAGE_MASK;
  686.                 page_offset = offset_in_page(offset);
  687.                 page_length = remain;
  688.                 if ((page_offset + remain) > PAGE_SIZE)
  689.                         page_length = PAGE_SIZE - page_offset;
  690.  
  691.         MapPage(vaddr, page_base, PG_SW|PG_NOCACHE);
  692.  
  693.         memcpy(vaddr+page_offset, user_data, page_length);
  694.  
  695.                 remain -= page_length;
  696.                 user_data += page_length;
  697.                 offset += page_length;
  698.         }
  699.  
  700.     FreeKernelSpace(vaddr);
  701.  
  702. out_unpin:
  703.         i915_gem_object_unpin(obj);
  704. out:
  705.     printf("% s ret = %d\n", __FUNCTION__, ret);
  706.  
  707.         return ret;
  708. }
  709.  
  710. /* Per-page copy function for the shmem pwrite fastpath.
  711.  * Flushes invalid cachelines before writing to the target if
  712.  * needs_clflush_before is set and flushes out any written cachelines after
  713.  * writing if needs_clflush is set. */
  714. static int
  715. shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
  716.                   char __user *user_data,
  717.                   bool page_do_bit17_swizzling,
  718.                   bool needs_clflush_before,
  719.                   bool needs_clflush_after)
  720. {
  721.         char *vaddr;
  722.         int ret = 0;
  723.  
  724.         if (unlikely(page_do_bit17_swizzling))
  725.                 return -EINVAL;
  726.  
  727.         vaddr = (char *)MapIoMem((addr_t)page, 4096, PG_SW);
  728.         if (needs_clflush_before)
  729.                 drm_clflush_virt_range(vaddr + shmem_page_offset,
  730.                                        page_length);
  731.         memcpy(vaddr + shmem_page_offset,
  732.                                                 user_data,
  733.                                                 page_length);
  734.         if (needs_clflush_after)
  735.                 drm_clflush_virt_range(vaddr + shmem_page_offset,
  736.                                        page_length);
  737.         FreeKernelSpace(vaddr);
  738.  
  739.         return ret ? -EFAULT : 0;
  740. }
  741. #if 0
  742.  
  743. /* Only difference to the fast-path function is that this can handle bit17
  744.  * and uses non-atomic copy and kmap functions. */
  745. static int
  746. shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
  747.                   char __user *user_data,
  748.                   bool page_do_bit17_swizzling,
  749.                   bool needs_clflush_before,
  750.                   bool needs_clflush_after)
  751. {
  752.         char *vaddr;
  753.         int ret;
  754.  
  755.         vaddr = kmap(page);
  756.         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
  757.                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  758.                                              page_length,
  759.                                              page_do_bit17_swizzling);
  760.         if (page_do_bit17_swizzling)
  761.                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
  762.                                                 user_data,
  763.                                                 page_length);
  764.         else
  765.                 ret = __copy_from_user(vaddr + shmem_page_offset,
  766.                                        user_data,
  767.                                        page_length);
  768.         if (needs_clflush_after)
  769.                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
  770.                                              page_length,
  771.                                              page_do_bit17_swizzling);
  772.         kunmap(page);
  773.  
  774.         return ret ? -EFAULT : 0;
  775. }
  776. #endif
  777.  
  778.  
  779. static int
  780. i915_gem_shmem_pwrite(struct drm_device *dev,
  781.                       struct drm_i915_gem_object *obj,
  782.                       struct drm_i915_gem_pwrite *args,
  783.                       struct drm_file *file)
  784. {
  785.         ssize_t remain;
  786.         loff_t offset;
  787.         char __user *user_data;
  788.         int shmem_page_offset, page_length, ret = 0;
  789.         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
  790.         int hit_slowpath = 0;
  791.         int needs_clflush_after = 0;
  792.         int needs_clflush_before = 0;
  793.         int i;
  794.         struct scatterlist *sg;
  795.  
  796.         user_data = (char __user *) (uintptr_t) args->data_ptr;
  797.         remain = args->size;
  798.  
  799.         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
  800.  
  801.         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  802.                 /* If we're not in the cpu write domain, set ourself into the gtt
  803.                  * write domain and manually flush cachelines (if required). This
  804.                  * optimizes for the case when the gpu will use the data
  805.                  * right away and we therefore have to clflush anyway. */
  806.                 if (obj->cache_level == I915_CACHE_NONE)
  807.                         needs_clflush_after = 1;
  808.                 if (obj->gtt_space) {
  809.                         ret = i915_gem_object_set_to_gtt_domain(obj, true);
  810.                         if (ret)
  811.                                 return ret;
  812.                 }
  813.         }
  814.         /* Same trick applies for invalidate partially written cachelines before
  815.          * writing.  */
  816.         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
  817.             && obj->cache_level == I915_CACHE_NONE)
  818.                 needs_clflush_before = 1;
  819.  
  820.         ret = i915_gem_object_get_pages(obj);
  821.         if (ret)
  822.                 return ret;
  823.  
  824.         i915_gem_object_pin_pages(obj);
  825.  
  826.         offset = args->offset;
  827.         obj->dirty = 1;
  828.  
  829.         for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
  830.                 struct page *page;
  831.                 int partial_cacheline_write;
  832.  
  833.                 if (i < offset >> PAGE_SHIFT)
  834.                         continue;
  835.  
  836.                 if (remain <= 0)
  837.                         break;
  838.  
  839.                 /* Operation in this page
  840.                  *
  841.                  * shmem_page_offset = offset within page in shmem file
  842.                  * page_length = bytes to copy for this page
  843.                  */
  844.                 shmem_page_offset = offset_in_page(offset);
  845.  
  846.                 page_length = remain;
  847.                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
  848.                         page_length = PAGE_SIZE - shmem_page_offset;
  849.  
  850.                 /* If we don't overwrite a cacheline completely we need to be
  851.                  * careful to have up-to-date data by first clflushing. Don't
  852.                  * overcomplicate things and flush the entire patch. */
  853.                 partial_cacheline_write = needs_clflush_before &&
  854.                         ((shmem_page_offset | page_length)
  855.                                 & (x86_clflush_size - 1));
  856.  
  857.                 page = sg_page(sg);
  858.                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
  859.                         (page_to_phys(page) & (1 << 17)) != 0;
  860.  
  861.                 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
  862.                                         user_data, page_do_bit17_swizzling,
  863.                                         partial_cacheline_write,
  864.                                         needs_clflush_after);
  865.                 if (ret == 0)
  866.                         goto next_page;
  867.  
  868.                 hit_slowpath = 1;
  869.                 mutex_unlock(&dev->struct_mutex);
  870.                 dbgprintf("%s need shmem_pwrite_slow\n",__FUNCTION__);
  871.  
  872. //              ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
  873. //                                      user_data, page_do_bit17_swizzling,
  874. //                                      partial_cacheline_write,
  875. //                                      needs_clflush_after);
  876.  
  877.                 mutex_lock(&dev->struct_mutex);
  878.  
  879. next_page:
  880.  
  881.                 if (ret)
  882.                         goto out;
  883.  
  884.                 remain -= page_length;
  885.                 user_data += page_length;
  886.                 offset += page_length;
  887.         }
  888.  
  889. out:
  890.         i915_gem_object_unpin_pages(obj);
  891.  
  892.         if (hit_slowpath) {
  893.                 /* Fixup: Kill any reinstated backing storage pages */
  894.                 if (obj->madv == __I915_MADV_PURGED)
  895.                         i915_gem_object_truncate(obj);
  896.                 /* and flush dirty cachelines in case the object isn't in the cpu write
  897.                  * domain anymore. */
  898.                 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  899.                         i915_gem_clflush_object(obj);
  900.                         i915_gem_chipset_flush(dev);
  901.                 }
  902.         }
  903.  
  904.         if (needs_clflush_after)
  905.                 i915_gem_chipset_flush(dev);
  906.  
  907.         return ret;
  908. }
  909.  
  910. /**
  911.  * Writes data to the object referenced by handle.
  912.  *
  913.  * On error, the contents of the buffer that were to be modified are undefined.
  914.  */
  915. int
  916. i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
  917.                       struct drm_file *file)
  918. {
  919.         struct drm_i915_gem_pwrite *args = data;
  920.         struct drm_i915_gem_object *obj;
  921.         int ret;
  922.  
  923.         if (args->size == 0)
  924.                 return 0;
  925.  
  926.         ret = i915_mutex_lock_interruptible(dev);
  927.         if (ret)
  928.                 return ret;
  929.  
  930.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  931.         if (&obj->base == NULL) {
  932.                 ret = -ENOENT;
  933.                 goto unlock;
  934.         }
  935.  
  936.         /* Bounds check destination. */
  937.         if (args->offset > obj->base.size ||
  938.             args->size > obj->base.size - args->offset) {
  939.                 ret = -EINVAL;
  940.                 goto out;
  941.         }
  942.  
  943.         /* prime objects have no backing filp to GEM pread/pwrite
  944.          * pages from.
  945.          */
  946.         if (!obj->base.filp) {
  947.                 ret = -EINVAL;
  948.                 goto out;
  949.         }
  950.  
  951.         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
  952.  
  953.         ret = -EFAULT;
  954.         /* We can only do the GTT pwrite on untiled buffers, as otherwise
  955.          * it would end up going through the fenced access, and we'll get
  956.          * different detiling behavior between reading and writing.
  957.          * pread/pwrite currently are reading and writing from the CPU
  958.          * perspective, requiring manual detiling by the client.
  959.          */
  960. //   if (obj->phys_obj) {
  961. //       ret = i915_gem_phys_pwrite(dev, obj, args, file);
  962. //       goto out;
  963. //   }
  964.  
  965.         if (obj->cache_level == I915_CACHE_NONE &&
  966.             obj->tiling_mode == I915_TILING_NONE &&
  967.             obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
  968.                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
  969.                 /* Note that the gtt paths might fail with non-page-backed user
  970.                  * pointers (e.g. gtt mappings when moving data between
  971.                  * textures). Fallback to the shmem path in that case. */
  972.         }
  973.  
  974.         if (ret == -EFAULT || ret == -ENOSPC)
  975.        ret = i915_gem_shmem_pwrite(dev, obj, args, file);
  976.  
  977. out:
  978.         drm_gem_object_unreference(&obj->base);
  979. unlock:
  980.         mutex_unlock(&dev->struct_mutex);
  981.         return ret;
  982. }
  983.  
  984. int
  985. i915_gem_check_wedge(struct drm_i915_private *dev_priv,
  986.                      bool interruptible)
  987. {
  988.         if (atomic_read(&dev_priv->mm.wedged)) {
  989.                 struct completion *x = &dev_priv->error_completion;
  990.                 bool recovery_complete;
  991.                 unsigned long flags;
  992.  
  993.                 /* Give the error handler a chance to run. */
  994.                 spin_lock_irqsave(&x->wait.lock, flags);
  995.                 recovery_complete = x->done > 0;
  996.                 spin_unlock_irqrestore(&x->wait.lock, flags);
  997.  
  998.                 /* Non-interruptible callers can't handle -EAGAIN, hence return
  999.                  * -EIO unconditionally for these. */
  1000.                 if (!interruptible)
  1001.                         return -EIO;
  1002.  
  1003.                 /* Recovery complete, but still wedged means reset failure. */
  1004.                 if (recovery_complete)
  1005.                         return -EIO;
  1006.  
  1007.                 return -EAGAIN;
  1008.         }
  1009.  
  1010.         return 0;
  1011. }
  1012.  
  1013. /*
  1014.  * Compare seqno against outstanding lazy request. Emit a request if they are
  1015.  * equal.
  1016.  */
  1017. static int
  1018. i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
  1019. {
  1020.         int ret;
  1021.  
  1022.         BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
  1023.  
  1024.         ret = 0;
  1025.         if (seqno == ring->outstanding_lazy_request)
  1026.                 ret = i915_add_request(ring, NULL, NULL);
  1027.  
  1028.         return ret;
  1029. }
  1030.  
  1031. /**
  1032.  * __wait_seqno - wait until execution of seqno has finished
  1033.  * @ring: the ring expected to report seqno
  1034.  * @seqno: duh!
  1035.  * @interruptible: do an interruptible wait (normally yes)
  1036.  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
  1037.  *
  1038.  * Returns 0 if the seqno was found within the alloted time. Else returns the
  1039.  * errno with remaining time filled in timeout argument.
  1040.  */
  1041. static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
  1042.                         bool interruptible, struct timespec *timeout)
  1043. {
  1044.         drm_i915_private_t *dev_priv = ring->dev->dev_private;
  1045.         struct timespec before, now, wait_time={1,0};
  1046.         unsigned long timeout_jiffies;
  1047.         long end;
  1048.         bool wait_forever = true;
  1049.         int ret;
  1050.  
  1051.         if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
  1052.                 return 0;
  1053.  
  1054.         trace_i915_gem_request_wait_begin(ring, seqno);
  1055.  
  1056.         if (timeout != NULL) {
  1057.                 wait_time = *timeout;
  1058.                 wait_forever = false;
  1059.         }
  1060.  
  1061.         timeout_jiffies = timespec_to_jiffies(&wait_time);
  1062.  
  1063.         if (WARN_ON(!ring->irq_get(ring)))
  1064.                 return -ENODEV;
  1065.  
  1066.     /* Record current time in case interrupted by signal, or wedged * */
  1067.         getrawmonotonic(&before);
  1068.  
  1069. #define EXIT_COND \
  1070.         (i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \
  1071.         atomic_read(&dev_priv->mm.wedged))
  1072.         do {
  1073.                 if (interruptible)
  1074.                         end = wait_event_interruptible_timeout(ring->irq_queue,
  1075.                                                                EXIT_COND,
  1076.                                                                timeout_jiffies);
  1077.                 else
  1078.                         end = wait_event_timeout(ring->irq_queue, EXIT_COND,
  1079.                                                  timeout_jiffies);
  1080.  
  1081.                 ret = i915_gem_check_wedge(dev_priv, interruptible);
  1082.                 if (ret)
  1083.                         end = ret;
  1084.         } while (end == 0 && wait_forever);
  1085.  
  1086.         getrawmonotonic(&now);
  1087.  
  1088.         ring->irq_put(ring);
  1089.         trace_i915_gem_request_wait_end(ring, seqno);
  1090. #undef EXIT_COND
  1091.  
  1092.         if (timeout) {
  1093. //       struct timespec sleep_time = timespec_sub(now, before);
  1094. //       *timeout = timespec_sub(*timeout, sleep_time);
  1095.         }
  1096.  
  1097.         switch (end) {
  1098.         case -EIO:
  1099.         case -EAGAIN: /* Wedged */
  1100.         case -ERESTARTSYS: /* Signal */
  1101.                 return (int)end;
  1102.         case 0: /* Timeout */
  1103.                 if (timeout)
  1104.                         set_normalized_timespec(timeout, 0, 0);
  1105.                 return -ETIME;
  1106.         default: /* Completed */
  1107.                 WARN_ON(end < 0); /* We're not aware of other errors */
  1108.                 return 0;
  1109.         }
  1110. }
  1111.  
  1112. /**
  1113.  * Waits for a sequence number to be signaled, and cleans up the
  1114.  * request and object lists appropriately for that event.
  1115.  */
  1116. int
  1117. i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
  1118. {
  1119.         struct drm_device *dev = ring->dev;
  1120.         struct drm_i915_private *dev_priv = dev->dev_private;
  1121.         bool interruptible = dev_priv->mm.interruptible;
  1122.         int ret;
  1123.  
  1124.         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
  1125.         BUG_ON(seqno == 0);
  1126.  
  1127.         ret = i915_gem_check_wedge(dev_priv, interruptible);
  1128.         if (ret)
  1129.                 return ret;
  1130.  
  1131.         ret = i915_gem_check_olr(ring, seqno);
  1132.         if (ret)
  1133.                 return ret;
  1134.  
  1135.         return __wait_seqno(ring, seqno, interruptible, NULL);
  1136. }
  1137.  
  1138. /**
  1139.  * Ensures that all rendering to the object has completed and the object is
  1140.  * safe to unbind from the GTT or access from the CPU.
  1141.  */
  1142. static __must_check int
  1143. i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
  1144.                                bool readonly)
  1145. {
  1146.         struct intel_ring_buffer *ring = obj->ring;
  1147.         u32 seqno;
  1148.         int ret;
  1149.  
  1150.         seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
  1151.         if (seqno == 0)
  1152.                 return 0;
  1153.  
  1154.         ret = i915_wait_seqno(ring, seqno);
  1155.        if (ret)
  1156.            return ret;
  1157.  
  1158.         i915_gem_retire_requests_ring(ring);
  1159.  
  1160.         /* Manually manage the write flush as we may have not yet
  1161.          * retired the buffer.
  1162.          */
  1163.         if (obj->last_write_seqno &&
  1164.             i915_seqno_passed(seqno, obj->last_write_seqno)) {
  1165.                 obj->last_write_seqno = 0;
  1166.                 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
  1167.         }
  1168.  
  1169.         return 0;
  1170. }
  1171.  
  1172. /* A nonblocking variant of the above wait. This is a highly dangerous routine
  1173.  * as the object state may change during this call.
  1174.  */
  1175. static __must_check int
  1176. i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
  1177.                                             bool readonly)
  1178. {
  1179.         struct drm_device *dev = obj->base.dev;
  1180.         struct drm_i915_private *dev_priv = dev->dev_private;
  1181.         struct intel_ring_buffer *ring = obj->ring;
  1182.         u32 seqno;
  1183.         int ret;
  1184.  
  1185.         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
  1186.         BUG_ON(!dev_priv->mm.interruptible);
  1187.  
  1188.         seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
  1189.         if (seqno == 0)
  1190.                 return 0;
  1191.  
  1192.         ret = i915_gem_check_wedge(dev_priv, true);
  1193.         if (ret)
  1194.                 return ret;
  1195.  
  1196.         ret = i915_gem_check_olr(ring, seqno);
  1197.         if (ret)
  1198.                 return ret;
  1199.  
  1200.         mutex_unlock(&dev->struct_mutex);
  1201.         ret = __wait_seqno(ring, seqno, true, NULL);
  1202.         mutex_lock(&dev->struct_mutex);
  1203.  
  1204.         i915_gem_retire_requests_ring(ring);
  1205.  
  1206.         /* Manually manage the write flush as we may have not yet
  1207.          * retired the buffer.
  1208.          */
  1209.         if (obj->last_write_seqno &&
  1210.             i915_seqno_passed(seqno, obj->last_write_seqno)) {
  1211.                 obj->last_write_seqno = 0;
  1212.                 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
  1213.         }
  1214.  
  1215.         return ret;
  1216. }
  1217.  
  1218. /**
  1219.  * Called when user space prepares to use an object with the CPU, either
  1220.  * through the mmap ioctl's mapping or a GTT mapping.
  1221.  */
  1222. int
  1223. i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
  1224.                           struct drm_file *file)
  1225. {
  1226.         struct drm_i915_gem_set_domain *args = data;
  1227.         struct drm_i915_gem_object *obj;
  1228.         uint32_t read_domains = args->read_domains;
  1229.         uint32_t write_domain = args->write_domain;
  1230.         int ret;
  1231.  
  1232.         /* Only handle setting domains to types used by the CPU. */
  1233.         if (write_domain & I915_GEM_GPU_DOMAINS)
  1234.                 return -EINVAL;
  1235.  
  1236.         if (read_domains & I915_GEM_GPU_DOMAINS)
  1237.                 return -EINVAL;
  1238.  
  1239.         /* Having something in the write domain implies it's in the read
  1240.          * domain, and only that read domain.  Enforce that in the request.
  1241.          */
  1242.         if (write_domain != 0 && read_domains != write_domain)
  1243.                 return -EINVAL;
  1244.  
  1245.         ret = i915_mutex_lock_interruptible(dev);
  1246.         if (ret)
  1247.                 return ret;
  1248.  
  1249.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  1250.         if (&obj->base == NULL) {
  1251.                 ret = -ENOENT;
  1252.                 goto unlock;
  1253.         }
  1254.  
  1255.         /* Try to flush the object off the GPU without holding the lock.
  1256.          * We will repeat the flush holding the lock in the normal manner
  1257.          * to catch cases where we are gazumped.
  1258.          */
  1259.         ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
  1260.         if (ret)
  1261.                 goto unref;
  1262.  
  1263.         if (read_domains & I915_GEM_DOMAIN_GTT) {
  1264.                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
  1265.  
  1266.                 /* Silently promote "you're not bound, there was nothing to do"
  1267.                  * to success, since the client was just asking us to
  1268.                  * make sure everything was done.
  1269.                  */
  1270.                 if (ret == -EINVAL)
  1271.                         ret = 0;
  1272.         } else {
  1273.                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
  1274.         }
  1275.  
  1276. unref:
  1277.         drm_gem_object_unreference(&obj->base);
  1278. unlock:
  1279.         mutex_unlock(&dev->struct_mutex);
  1280.         return ret;
  1281. }
  1282.  
  1283.  
  1284.  
  1285.  
  1286.  
  1287.  
  1288. /**
  1289.  * Maps the contents of an object, returning the address it is mapped
  1290.  * into.
  1291.  *
  1292.  * While the mapping holds a reference on the contents of the object, it doesn't
  1293.  * imply a ref on the object itself.
  1294.  */
  1295. int
  1296. i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
  1297.                     struct drm_file *file)
  1298. {
  1299.         struct drm_i915_gem_mmap *args = data;
  1300.         struct drm_gem_object *obj;
  1301.         unsigned long addr = 0;
  1302.  
  1303.         obj = drm_gem_object_lookup(dev, file, args->handle);
  1304.         if (obj == NULL)
  1305.                 return -ENOENT;
  1306.        
  1307.     dbgprintf("%s offset %lld size %lld\n",
  1308.                 __FUNCTION__, args->offset, args->size);
  1309.         /* prime objects have no backing filp to GEM mmap
  1310.          * pages from.
  1311.          */
  1312.         if (!obj->filp) {
  1313.                 drm_gem_object_unreference_unlocked(obj);
  1314.                 return -EINVAL;
  1315.         }
  1316.  
  1317.     addr = vm_mmap(obj->filp, 0, args->size,
  1318.               PROT_READ | PROT_WRITE, MAP_SHARED,
  1319.               args->offset);
  1320.         drm_gem_object_unreference_unlocked(obj);
  1321.     if (IS_ERR((void *)addr))
  1322.         return addr;
  1323.  
  1324.         args->addr_ptr = (uint64_t) addr;
  1325.  
  1326.     return 0;
  1327. }
  1328.  
  1329.  
  1330.  
  1331.  
  1332.  
  1333.  
  1334.  
  1335.  
  1336.  
  1337.  
  1338.  
  1339.  
  1340.  
  1341. /**
  1342.  * i915_gem_release_mmap - remove physical page mappings
  1343.  * @obj: obj in question
  1344.  *
  1345.  * Preserve the reservation of the mmapping with the DRM core code, but
  1346.  * relinquish ownership of the pages back to the system.
  1347.  *
  1348.  * It is vital that we remove the page mapping if we have mapped a tiled
  1349.  * object through the GTT and then lose the fence register due to
  1350.  * resource pressure. Similarly if the object has been moved out of the
  1351.  * aperture, than pages mapped into userspace must be revoked. Removing the
  1352.  * mapping will then trigger a page fault on the next user access, allowing
  1353.  * fixup by i915_gem_fault().
  1354.  */
  1355. void
  1356. i915_gem_release_mmap(struct drm_i915_gem_object *obj)
  1357. {
  1358.         if (!obj->fault_mappable)
  1359.                 return;
  1360.  
  1361.         if (obj->base.dev->dev_mapping)
  1362. //              unmap_mapping_range(obj->base.dev->dev_mapping,
  1363. //                                  (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
  1364. //                                  obj->base.size, 1);
  1365.  
  1366.         obj->fault_mappable = false;
  1367. }
  1368.  
  1369. static uint32_t
  1370. i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
  1371. {
  1372.         uint32_t gtt_size;
  1373.  
  1374.         if (INTEL_INFO(dev)->gen >= 4 ||
  1375.             tiling_mode == I915_TILING_NONE)
  1376.                 return size;
  1377.  
  1378.         /* Previous chips need a power-of-two fence region when tiling */
  1379.         if (INTEL_INFO(dev)->gen == 3)
  1380.                 gtt_size = 1024*1024;
  1381.         else
  1382.                 gtt_size = 512*1024;
  1383.  
  1384.         while (gtt_size < size)
  1385.                 gtt_size <<= 1;
  1386.  
  1387.         return gtt_size;
  1388. }
  1389.  
  1390. /**
  1391.  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
  1392.  * @obj: object to check
  1393.  *
  1394.  * Return the required GTT alignment for an object, taking into account
  1395.  * potential fence register mapping.
  1396.  */
  1397. static uint32_t
  1398. i915_gem_get_gtt_alignment(struct drm_device *dev,
  1399.                            uint32_t size,
  1400.                            int tiling_mode)
  1401. {
  1402.         /*
  1403.          * Minimum alignment is 4k (GTT page size), but might be greater
  1404.          * if a fence register is needed for the object.
  1405.          */
  1406.         if (INTEL_INFO(dev)->gen >= 4 ||
  1407.             tiling_mode == I915_TILING_NONE)
  1408.                 return 4096;
  1409.  
  1410.         /*
  1411.          * Previous chips need to be aligned to the size of the smallest
  1412.          * fence register that can contain the object.
  1413.          */
  1414.         return i915_gem_get_gtt_size(dev, size, tiling_mode);
  1415. }
  1416.  
  1417. /**
  1418.  * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
  1419.  *                                       unfenced object
  1420.  * @dev: the device
  1421.  * @size: size of the object
  1422.  * @tiling_mode: tiling mode of the object
  1423.  *
  1424.  * Return the required GTT alignment for an object, only taking into account
  1425.  * unfenced tiled surface requirements.
  1426.  */
  1427. uint32_t
  1428. i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
  1429.                                     uint32_t size,
  1430.                                     int tiling_mode)
  1431. {
  1432.         /*
  1433.          * Minimum alignment is 4k (GTT page size) for sane hw.
  1434.          */
  1435.         if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
  1436.             tiling_mode == I915_TILING_NONE)
  1437.                 return 4096;
  1438.  
  1439.         /* Previous hardware however needs to be aligned to a power-of-two
  1440.          * tile height. The simplest method for determining this is to reuse
  1441.          * the power-of-tile object size.
  1442.          */
  1443.         return i915_gem_get_gtt_size(dev, size, tiling_mode);
  1444. }
  1445.  
  1446. /* Immediately discard the backing storage */
  1447. static void
  1448. i915_gem_object_truncate(struct drm_i915_gem_object *obj)
  1449. {
  1450. //      struct inode *inode;
  1451.  
  1452. //      i915_gem_object_free_mmap_offset(obj);
  1453.  
  1454.         if (obj->base.filp == NULL)
  1455.                 return;
  1456.  
  1457.         /* Our goal here is to return as much of the memory as
  1458.          * is possible back to the system as we are called from OOM.
  1459.          * To do this we must instruct the shmfs to drop all of its
  1460.          * backing pages, *now*.
  1461.          */
  1462. //      inode = obj->base.filp->f_path.dentry->d_inode;
  1463. //      shmem_truncate_range(inode, 0, (loff_t)-1);
  1464.  
  1465.         obj->madv = __I915_MADV_PURGED;
  1466. }
  1467.  
  1468. static inline int
  1469. i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
  1470. {
  1471.         return obj->madv == I915_MADV_DONTNEED;
  1472. }
  1473.  
  1474. static void
  1475. i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
  1476. {
  1477.         int page_count = obj->base.size / PAGE_SIZE;
  1478.         struct scatterlist *sg;
  1479.         int ret, i;
  1480.  
  1481.         BUG_ON(obj->madv == __I915_MADV_PURGED);
  1482.  
  1483.         ret = i915_gem_object_set_to_cpu_domain(obj, true);
  1484.         if (ret) {
  1485.                 /* In the event of a disaster, abandon all caches and
  1486.                  * hope for the best.
  1487.                  */
  1488.                 WARN_ON(ret != -EIO);
  1489.                 i915_gem_clflush_object(obj);
  1490.                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  1491.         }
  1492.  
  1493.         if (obj->madv == I915_MADV_DONTNEED)
  1494.                 obj->dirty = 0;
  1495.  
  1496.         for_each_sg(obj->pages->sgl, sg, page_count, i) {
  1497.                 struct page *page = sg_page(sg);
  1498.  
  1499.         page_cache_release(page);
  1500.         }
  1501.     //DRM_DEBUG_KMS("%s release %d pages\n", __FUNCTION__, page_count);
  1502.     printf("%s release %d pages\n", __FUNCTION__, page_count);
  1503.  
  1504.         obj->dirty = 0;
  1505.  
  1506.         sg_free_table(obj->pages);
  1507.         kfree(obj->pages);
  1508. }
  1509.  
  1510. static int
  1511. i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
  1512. {
  1513.         const struct drm_i915_gem_object_ops *ops = obj->ops;
  1514.  
  1515.         if (obj->pages == NULL)
  1516.                 return 0;
  1517.  
  1518.         BUG_ON(obj->gtt_space);
  1519.  
  1520.         if (obj->pages_pin_count)
  1521.                 return -EBUSY;
  1522.  
  1523.         /* ->put_pages might need to allocate memory for the bit17 swizzle
  1524.          * array, hence protect them from being reaped by removing them from gtt
  1525.          * lists early. */
  1526.         list_del(&obj->gtt_list);
  1527.  
  1528.         ops->put_pages(obj);
  1529.         obj->pages = NULL;
  1530.  
  1531.         if (i915_gem_object_is_purgeable(obj))
  1532.                 i915_gem_object_truncate(obj);
  1533.  
  1534.         return 0;
  1535. }
  1536.  
  1537.  
  1538.  
  1539.  
  1540.  
  1541.  
  1542.  
  1543.  
  1544. static int
  1545. i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
  1546. {
  1547.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  1548.     int page_count, i;
  1549.         struct address_space *mapping;
  1550.         struct sg_table *st;
  1551.         struct scatterlist *sg;
  1552.         struct page *page;
  1553.         gfp_t gfp;
  1554.  
  1555.         /* Assert that the object is not currently in any GPU domain. As it
  1556.          * wasn't in the GTT, there shouldn't be any way it could have been in
  1557.          * a GPU cache
  1558.          */
  1559.         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
  1560.         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
  1561.  
  1562.         st = kmalloc(sizeof(*st), GFP_KERNEL);
  1563.         if (st == NULL)
  1564.                 return -ENOMEM;
  1565.  
  1566.         page_count = obj->base.size / PAGE_SIZE;
  1567.         if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
  1568.                 sg_free_table(st);
  1569.                 kfree(st);
  1570.                 return -ENOMEM;
  1571.         }
  1572.  
  1573.         /* Get the list of pages out of our struct file.  They'll be pinned
  1574.          * at this point until we release them.
  1575.          *
  1576.          * Fail silently without starting the shrinker
  1577.          */
  1578.         for_each_sg(st->sgl, sg, page_count, i) {
  1579.                 page = shmem_read_mapping_page_gfp(obj->base.filp, i, gfp);
  1580.                 if (IS_ERR(page)) {
  1581.             dbgprintf("%s invalid page %p\n", __FUNCTION__, page);
  1582.                         goto err_pages;
  1583.  
  1584.                 }
  1585.                 sg_set_page(sg, page, PAGE_SIZE, 0);
  1586.         }
  1587.  
  1588.         obj->pages = st;
  1589.  
  1590.     DRM_DEBUG_KMS("%s alloc %d pages\n", __FUNCTION__, page_count);
  1591.  
  1592.         return 0;
  1593.  
  1594. err_pages:
  1595.         for_each_sg(st->sgl, sg, i, page_count)
  1596.                 page_cache_release(sg_page(sg));
  1597.         sg_free_table(st);
  1598.         kfree(st);
  1599.         return PTR_ERR(page);
  1600. }
  1601.  
  1602. /* Ensure that the associated pages are gathered from the backing storage
  1603.  * and pinned into our object. i915_gem_object_get_pages() may be called
  1604.  * multiple times before they are released by a single call to
  1605.  * i915_gem_object_put_pages() - once the pages are no longer referenced
  1606.  * either as a result of memory pressure (reaping pages under the shrinker)
  1607.  * or as the object is itself released.
  1608.  */
  1609. int
  1610. i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
  1611. {
  1612.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  1613.         const struct drm_i915_gem_object_ops *ops = obj->ops;
  1614.         int ret;
  1615.  
  1616.         if (obj->pages)
  1617.                 return 0;
  1618.  
  1619.         BUG_ON(obj->pages_pin_count);
  1620.  
  1621.         ret = ops->get_pages(obj);
  1622.         if (ret)
  1623.                 return ret;
  1624.  
  1625.         list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
  1626.     return 0;
  1627. }
  1628.  
  1629. void
  1630. i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
  1631.                                struct intel_ring_buffer *ring)
  1632. {
  1633.         struct drm_device *dev = obj->base.dev;
  1634.         struct drm_i915_private *dev_priv = dev->dev_private;
  1635.         u32 seqno = intel_ring_get_seqno(ring);
  1636.  
  1637.         BUG_ON(ring == NULL);
  1638.         obj->ring = ring;
  1639.  
  1640.         /* Add a reference if we're newly entering the active list. */
  1641.         if (!obj->active) {
  1642.                 drm_gem_object_reference(&obj->base);
  1643.                 obj->active = 1;
  1644.         }
  1645.  
  1646.         /* Move from whatever list we were on to the tail of execution. */
  1647.         list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
  1648.         list_move_tail(&obj->ring_list, &ring->active_list);
  1649.  
  1650.         obj->last_read_seqno = seqno;
  1651.  
  1652.         if (obj->fenced_gpu_access) {
  1653.                 obj->last_fenced_seqno = seqno;
  1654.  
  1655.                 /* Bump MRU to take account of the delayed flush */
  1656.                 if (obj->fence_reg != I915_FENCE_REG_NONE) {
  1657.                 struct drm_i915_fence_reg *reg;
  1658.  
  1659.                 reg = &dev_priv->fence_regs[obj->fence_reg];
  1660.                         list_move_tail(&reg->lru_list,
  1661.                                        &dev_priv->mm.fence_list);
  1662.                 }
  1663.         }
  1664. }
  1665.  
  1666. static void
  1667. i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
  1668. {
  1669.         struct drm_device *dev = obj->base.dev;
  1670.         struct drm_i915_private *dev_priv = dev->dev_private;
  1671.  
  1672.         BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
  1673.         BUG_ON(!obj->active);
  1674.  
  1675.         if (obj->pin_count) /* are we a framebuffer? */
  1676.                 intel_mark_fb_idle(obj);
  1677.  
  1678.                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
  1679.  
  1680.         list_del_init(&obj->ring_list);
  1681.         obj->ring = NULL;
  1682.  
  1683.         obj->last_read_seqno = 0;
  1684.         obj->last_write_seqno = 0;
  1685.         obj->base.write_domain = 0;
  1686.  
  1687.         obj->last_fenced_seqno = 0;
  1688.         obj->fenced_gpu_access = false;
  1689.  
  1690.         obj->active = 0;
  1691.         drm_gem_object_unreference(&obj->base);
  1692.  
  1693.         WARN_ON(i915_verify_lists(dev));
  1694. }
  1695.  
  1696. static int
  1697. i915_gem_handle_seqno_wrap(struct drm_device *dev)
  1698. {
  1699.         struct drm_i915_private *dev_priv = dev->dev_private;
  1700.         struct intel_ring_buffer *ring;
  1701.         int ret, i, j;
  1702.  
  1703.         /* The hardware uses various monotonic 32-bit counters, if we
  1704.          * detect that they will wraparound we need to idle the GPU
  1705.          * and reset those counters.
  1706.          */
  1707.         ret = 0;
  1708.         for_each_ring(ring, dev_priv, i) {
  1709.                 for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
  1710.                         ret |= ring->sync_seqno[j] != 0;
  1711.         }
  1712.         if (ret == 0)
  1713.                 return ret;
  1714.  
  1715.         ret = i915_gpu_idle(dev);
  1716.         if (ret)
  1717.                 return ret;
  1718.  
  1719.         i915_gem_retire_requests(dev);
  1720.         for_each_ring(ring, dev_priv, i) {
  1721.                 for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
  1722.                         ring->sync_seqno[j] = 0;
  1723.         }
  1724.  
  1725.         return 0;
  1726. }
  1727.  
  1728. int
  1729. i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
  1730. {
  1731.         struct drm_i915_private *dev_priv = dev->dev_private;
  1732.  
  1733.         /* reserve 0 for non-seqno */
  1734.         if (dev_priv->next_seqno == 0) {
  1735.                 int ret = i915_gem_handle_seqno_wrap(dev);
  1736.                 if (ret)
  1737.                         return ret;
  1738.  
  1739.                 dev_priv->next_seqno = 1;
  1740.         }
  1741.  
  1742.         *seqno = dev_priv->next_seqno++;
  1743.         return 0;
  1744. }
  1745.  
  1746. int
  1747. i915_add_request(struct intel_ring_buffer *ring,
  1748.                  struct drm_file *file,
  1749.                  u32 *out_seqno)
  1750. {
  1751.         drm_i915_private_t *dev_priv = ring->dev->dev_private;
  1752.         struct drm_i915_gem_request *request;
  1753.         u32 request_ring_position;
  1754.         int was_empty;
  1755.         int ret;
  1756.  
  1757.         /*
  1758.          * Emit any outstanding flushes - execbuf can fail to emit the flush
  1759.          * after having emitted the batchbuffer command. Hence we need to fix
  1760.          * things up similar to emitting the lazy request. The difference here
  1761.          * is that the flush _must_ happen before the next request, no matter
  1762.          * what.
  1763.          */
  1764.         ret = intel_ring_flush_all_caches(ring);
  1765.         if (ret)
  1766.                 return ret;
  1767.  
  1768.         request = kmalloc(sizeof(*request), GFP_KERNEL);
  1769.         if (request == NULL)
  1770.                 return -ENOMEM;
  1771.  
  1772.  
  1773.         /* Record the position of the start of the request so that
  1774.          * should we detect the updated seqno part-way through the
  1775.          * GPU processing the request, we never over-estimate the
  1776.          * position of the head.
  1777.          */
  1778.         request_ring_position = intel_ring_get_tail(ring);
  1779.  
  1780.         ret = ring->add_request(ring);
  1781.         if (ret) {
  1782.                 kfree(request);
  1783.             return ret;
  1784.         }
  1785.  
  1786.         request->seqno = intel_ring_get_seqno(ring);
  1787.         request->ring = ring;
  1788.         request->tail = request_ring_position;
  1789.     request->emitted_jiffies = GetTimerTicks();
  1790.         was_empty = list_empty(&ring->request_list);
  1791.         list_add_tail(&request->list, &ring->request_list);
  1792.         request->file_priv = NULL;
  1793.  
  1794.         if (file) {
  1795.                 struct drm_i915_file_private *file_priv = file->driver_priv;
  1796.  
  1797.                 spin_lock(&file_priv->mm.lock);
  1798.                 request->file_priv = file_priv;
  1799.                 list_add_tail(&request->client_list,
  1800.                               &file_priv->mm.request_list);
  1801.                 spin_unlock(&file_priv->mm.lock);
  1802.         }
  1803.  
  1804.         trace_i915_gem_request_add(ring, request->seqno);
  1805.         ring->outstanding_lazy_request = 0;
  1806.  
  1807.         if (!dev_priv->mm.suspended) {
  1808.                 if (i915_enable_hangcheck) {
  1809. //                      mod_timer(&dev_priv->hangcheck_timer,
  1810. //                                jiffies +
  1811. //                                msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
  1812.                 }
  1813.                 if (was_empty) {
  1814.            queue_delayed_work(dev_priv->wq,
  1815.                       &dev_priv->mm.retire_work, HZ);
  1816.                         intel_mark_busy(dev_priv->dev);
  1817.                 }
  1818.         }
  1819.  
  1820.         if (out_seqno)
  1821.                 *out_seqno = request->seqno;
  1822.         return 0;
  1823. }
  1824.  
  1825. static inline void
  1826. i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
  1827. {
  1828.         struct drm_i915_file_private *file_priv = request->file_priv;
  1829.  
  1830.         if (!file_priv)
  1831.                 return;
  1832.  
  1833.         spin_lock(&file_priv->mm.lock);
  1834.         if (request->file_priv) {
  1835.                 list_del(&request->client_list);
  1836.                 request->file_priv = NULL;
  1837.         }
  1838.         spin_unlock(&file_priv->mm.lock);
  1839. }
  1840.  
  1841. static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
  1842.                                       struct intel_ring_buffer *ring)
  1843. {
  1844.         while (!list_empty(&ring->request_list)) {
  1845.                 struct drm_i915_gem_request *request;
  1846.  
  1847.                 request = list_first_entry(&ring->request_list,
  1848.                                            struct drm_i915_gem_request,
  1849.                                            list);
  1850.  
  1851.                 list_del(&request->list);
  1852.                 i915_gem_request_remove_from_client(request);
  1853.                 kfree(request);
  1854.         }
  1855.  
  1856.         while (!list_empty(&ring->active_list)) {
  1857.                 struct drm_i915_gem_object *obj;
  1858.  
  1859.                 obj = list_first_entry(&ring->active_list,
  1860.                                        struct drm_i915_gem_object,
  1861.                                        ring_list);
  1862.  
  1863.                 i915_gem_object_move_to_inactive(obj);
  1864.         }
  1865. }
  1866.  
  1867. static void i915_gem_reset_fences(struct drm_device *dev)
  1868. {
  1869.         struct drm_i915_private *dev_priv = dev->dev_private;
  1870.         int i;
  1871.  
  1872.         for (i = 0; i < dev_priv->num_fence_regs; i++) {
  1873.                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
  1874.  
  1875.                 i915_gem_write_fence(dev, i, NULL);
  1876.  
  1877.                 if (reg->obj)
  1878.                         i915_gem_object_fence_lost(reg->obj);
  1879.  
  1880.                 reg->pin_count = 0;
  1881.                 reg->obj = NULL;
  1882.                 INIT_LIST_HEAD(&reg->lru_list);
  1883.         }
  1884.  
  1885.         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
  1886. }
  1887.  
  1888. void i915_gem_reset(struct drm_device *dev)
  1889. {
  1890.         struct drm_i915_private *dev_priv = dev->dev_private;
  1891.         struct drm_i915_gem_object *obj;
  1892.         struct intel_ring_buffer *ring;
  1893.         int i;
  1894.  
  1895.         for_each_ring(ring, dev_priv, i)
  1896.                 i915_gem_reset_ring_lists(dev_priv, ring);
  1897.  
  1898.         /* Move everything out of the GPU domains to ensure we do any
  1899.          * necessary invalidation upon reuse.
  1900.          */
  1901.         list_for_each_entry(obj,
  1902.                             &dev_priv->mm.inactive_list,
  1903.                             mm_list)
  1904.         {
  1905.                 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
  1906.         }
  1907.  
  1908.         /* The fence registers are invalidated so clear them out */
  1909.         i915_gem_reset_fences(dev);
  1910. }
  1911.  
  1912. /**
  1913.  * This function clears the request list as sequence numbers are passed.
  1914.  */
  1915. void
  1916. i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
  1917. {
  1918.         uint32_t seqno;
  1919.  
  1920.         if (list_empty(&ring->request_list))
  1921.                 return;
  1922.  
  1923.         WARN_ON(i915_verify_lists(ring->dev));
  1924.  
  1925.         seqno = ring->get_seqno(ring, true);
  1926.  
  1927.         while (!list_empty(&ring->request_list)) {
  1928.                 struct drm_i915_gem_request *request;
  1929.  
  1930.                 request = list_first_entry(&ring->request_list,
  1931.                                            struct drm_i915_gem_request,
  1932.                                            list);
  1933.  
  1934.                 if (!i915_seqno_passed(seqno, request->seqno))
  1935.                         break;
  1936.  
  1937.                 trace_i915_gem_request_retire(ring, request->seqno);
  1938.                 /* We know the GPU must have read the request to have
  1939.                  * sent us the seqno + interrupt, so use the position
  1940.                  * of tail of the request to update the last known position
  1941.                  * of the GPU head.
  1942.                  */
  1943.                 ring->last_retired_head = request->tail;
  1944.  
  1945.                 list_del(&request->list);
  1946.                 i915_gem_request_remove_from_client(request);
  1947.                 kfree(request);
  1948.         }
  1949.  
  1950.         /* Move any buffers on the active list that are no longer referenced
  1951.          * by the ringbuffer to the flushing/inactive lists as appropriate.
  1952.          */
  1953.         while (!list_empty(&ring->active_list)) {
  1954.                 struct drm_i915_gem_object *obj;
  1955.  
  1956.                 obj = list_first_entry(&ring->active_list,
  1957.                                       struct drm_i915_gem_object,
  1958.                                       ring_list);
  1959.  
  1960.                 if (!i915_seqno_passed(seqno, obj->last_read_seqno))
  1961.                         break;
  1962.  
  1963.                         i915_gem_object_move_to_inactive(obj);
  1964.         }
  1965.  
  1966.         if (unlikely(ring->trace_irq_seqno &&
  1967.                      i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
  1968.                 ring->irq_put(ring);
  1969.                 ring->trace_irq_seqno = 0;
  1970.         }
  1971.  
  1972.         WARN_ON(i915_verify_lists(ring->dev));
  1973. }
  1974.  
  1975. void
  1976. i915_gem_retire_requests(struct drm_device *dev)
  1977. {
  1978.         drm_i915_private_t *dev_priv = dev->dev_private;
  1979.         struct intel_ring_buffer *ring;
  1980.         int i;
  1981.  
  1982.         for_each_ring(ring, dev_priv, i)
  1983.                 i915_gem_retire_requests_ring(ring);
  1984. }
  1985.  
  1986. static void
  1987. i915_gem_retire_work_handler(struct work_struct *work)
  1988. {
  1989.         drm_i915_private_t *dev_priv;
  1990.         struct drm_device *dev;
  1991.         struct intel_ring_buffer *ring;
  1992.         bool idle;
  1993.         int i;
  1994.  
  1995.         dev_priv = container_of(work, drm_i915_private_t,
  1996.                                 mm.retire_work.work);
  1997.         dev = dev_priv->dev;
  1998.  
  1999.         /* Come back later if the device is busy... */
  2000.         if (!mutex_trylock(&dev->struct_mutex)) {
  2001.         queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
  2002.         return;
  2003.         }
  2004.  
  2005.         i915_gem_retire_requests(dev);
  2006.  
  2007.         /* Send a periodic flush down the ring so we don't hold onto GEM
  2008.          * objects indefinitely.
  2009.          */
  2010.         idle = true;
  2011.         for_each_ring(ring, dev_priv, i) {
  2012.                 if (ring->gpu_caches_dirty)
  2013.                         i915_add_request(ring, NULL, NULL);
  2014.  
  2015.                 idle &= list_empty(&ring->request_list);
  2016.         }
  2017.  
  2018.    if (!dev_priv->mm.suspended && !idle)
  2019.        queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
  2020.         if (idle)
  2021.                 intel_mark_idle(dev);
  2022.  
  2023.         mutex_unlock(&dev->struct_mutex);
  2024. }
  2025.  
  2026. /**
  2027.  * Ensures that an object will eventually get non-busy by flushing any required
  2028.  * write domains, emitting any outstanding lazy request and retiring and
  2029.  * completed requests.
  2030.  */
  2031. static int
  2032. i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
  2033. {
  2034.         int ret;
  2035.  
  2036.         if (obj->active) {
  2037.                 ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
  2038.                 if (ret)
  2039.                         return ret;
  2040.  
  2041.                 i915_gem_retire_requests_ring(obj->ring);
  2042.         }
  2043.  
  2044.         return 0;
  2045. }
  2046.  
  2047. /**
  2048.  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
  2049.  * @DRM_IOCTL_ARGS: standard ioctl arguments
  2050.  *
  2051.  * Returns 0 if successful, else an error is returned with the remaining time in
  2052.  * the timeout parameter.
  2053.  *  -ETIME: object is still busy after timeout
  2054.  *  -ERESTARTSYS: signal interrupted the wait
  2055.  *  -ENONENT: object doesn't exist
  2056.  * Also possible, but rare:
  2057.  *  -EAGAIN: GPU wedged
  2058.  *  -ENOMEM: damn
  2059.  *  -ENODEV: Internal IRQ fail
  2060.  *  -E?: The add request failed
  2061.  *
  2062.  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
  2063.  * non-zero timeout parameter the wait ioctl will wait for the given number of
  2064.  * nanoseconds on an object becoming unbusy. Since the wait itself does so
  2065.  * without holding struct_mutex the object may become re-busied before this
  2066.  * function completes. A similar but shorter * race condition exists in the busy
  2067.  * ioctl
  2068.  */
  2069.  
  2070.  
  2071.  
  2072.  
  2073.  
  2074.  
  2075.  
  2076.  
  2077.  
  2078.  
  2079.  
  2080.  
  2081. /**
  2082.  * i915_gem_object_sync - sync an object to a ring.
  2083.  *
  2084.  * @obj: object which may be in use on another ring.
  2085.  * @to: ring we wish to use the object on. May be NULL.
  2086.  *
  2087.  * This code is meant to abstract object synchronization with the GPU.
  2088.  * Calling with NULL implies synchronizing the object with the CPU
  2089.  * rather than a particular GPU ring.
  2090.  *
  2091.  * Returns 0 if successful, else propagates up the lower layer error.
  2092.  */
  2093. int
  2094. i915_gem_object_sync(struct drm_i915_gem_object *obj,
  2095.                      struct intel_ring_buffer *to)
  2096. {
  2097.         struct intel_ring_buffer *from = obj->ring;
  2098.         u32 seqno;
  2099.         int ret, idx;
  2100.  
  2101.         if (from == NULL || to == from)
  2102.                 return 0;
  2103.  
  2104.         if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
  2105.                 return i915_gem_object_wait_rendering(obj, false);
  2106.  
  2107.         idx = intel_ring_sync_index(from, to);
  2108.  
  2109.         seqno = obj->last_read_seqno;
  2110.         if (seqno <= from->sync_seqno[idx])
  2111.                 return 0;
  2112.  
  2113.         ret = i915_gem_check_olr(obj->ring, seqno);
  2114.         if (ret)
  2115.                 return ret;
  2116.  
  2117.         ret = to->sync_to(to, from, seqno);
  2118.         if (!ret)
  2119.                 /* We use last_read_seqno because sync_to()
  2120.                  * might have just caused seqno wrap under
  2121.                  * the radar.
  2122.                  */
  2123.                 from->sync_seqno[idx] = obj->last_read_seqno;
  2124.  
  2125.         return ret;
  2126. }
  2127.  
  2128. static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
  2129. {
  2130.         u32 old_write_domain, old_read_domains;
  2131.  
  2132.         /* Act a barrier for all accesses through the GTT */
  2133.         mb();
  2134.  
  2135.         /* Force a pagefault for domain tracking on next user access */
  2136. //      i915_gem_release_mmap(obj);
  2137.  
  2138.         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
  2139.                 return;
  2140.  
  2141.         old_read_domains = obj->base.read_domains;
  2142.         old_write_domain = obj->base.write_domain;
  2143.  
  2144.         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
  2145.         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
  2146.  
  2147.         trace_i915_gem_object_change_domain(obj,
  2148.                                             old_read_domains,
  2149.                                             old_write_domain);
  2150. }
  2151.  
  2152. /**
  2153.  * Unbinds an object from the GTT aperture.
  2154.  */
  2155. int
  2156. i915_gem_object_unbind(struct drm_i915_gem_object *obj)
  2157. {
  2158.         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
  2159.         int ret = 0;
  2160.  
  2161.     if(obj == get_fb_obj())
  2162.         return 0;
  2163.  
  2164.         if (obj->gtt_space == NULL)
  2165.                 return 0;
  2166.  
  2167.         if (obj->pin_count)
  2168.                 return -EBUSY;
  2169.  
  2170.         BUG_ON(obj->pages == NULL);
  2171.  
  2172.         ret = i915_gem_object_finish_gpu(obj);
  2173.         if (ret)
  2174.                 return ret;
  2175.         /* Continue on if we fail due to EIO, the GPU is hung so we
  2176.          * should be safe and we need to cleanup or else we might
  2177.          * cause memory corruption through use-after-free.
  2178.          */
  2179.  
  2180.         i915_gem_object_finish_gtt(obj);
  2181.  
  2182.         /* release the fence reg _after_ flushing */
  2183.         ret = i915_gem_object_put_fence(obj);
  2184.         if (ret)
  2185.                 return ret;
  2186.  
  2187.         trace_i915_gem_object_unbind(obj);
  2188.  
  2189.         if (obj->has_global_gtt_mapping)
  2190.         i915_gem_gtt_unbind_object(obj);
  2191.         if (obj->has_aliasing_ppgtt_mapping) {
  2192.                 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
  2193.                 obj->has_aliasing_ppgtt_mapping = 0;
  2194.         }
  2195.         i915_gem_gtt_finish_object(obj);
  2196.  
  2197.         list_del(&obj->mm_list);
  2198.         list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
  2199.         /* Avoid an unnecessary call to unbind on rebind. */
  2200.         obj->map_and_fenceable = true;
  2201.  
  2202.         drm_mm_put_block(obj->gtt_space);
  2203.         obj->gtt_space = NULL;
  2204.         obj->gtt_offset = 0;
  2205.  
  2206.         return 0;
  2207. }
  2208.  
  2209. int i915_gpu_idle(struct drm_device *dev)
  2210. {
  2211.         drm_i915_private_t *dev_priv = dev->dev_private;
  2212.         struct intel_ring_buffer *ring;
  2213.         int ret, i;
  2214.  
  2215.         /* Flush everything onto the inactive list. */
  2216.         for_each_ring(ring, dev_priv, i) {
  2217.                 ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
  2218.                 if (ret)
  2219.                         return ret;
  2220.  
  2221.                 ret = intel_ring_idle(ring);
  2222.                 if (ret)
  2223.                         return ret;
  2224.         }
  2225.  
  2226.         return 0;
  2227. }
  2228.  
  2229. static void sandybridge_write_fence_reg(struct drm_device *dev, int reg,
  2230.                                         struct drm_i915_gem_object *obj)
  2231. {
  2232.         drm_i915_private_t *dev_priv = dev->dev_private;
  2233.         uint64_t val;
  2234.  
  2235.         if (obj) {
  2236.                 u32 size = obj->gtt_space->size;
  2237.  
  2238.                 val = (uint64_t)((obj->gtt_offset + size - 4096) &
  2239.                                  0xfffff000) << 32;
  2240.                 val |= obj->gtt_offset & 0xfffff000;
  2241.                 val |= (uint64_t)((obj->stride / 128) - 1) <<
  2242.                         SANDYBRIDGE_FENCE_PITCH_SHIFT;
  2243.  
  2244.                 if (obj->tiling_mode == I915_TILING_Y)
  2245.                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
  2246.                 val |= I965_FENCE_REG_VALID;
  2247.         } else
  2248.                 val = 0;
  2249.  
  2250.         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val);
  2251.         POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8);
  2252. }
  2253.  
  2254. static void i965_write_fence_reg(struct drm_device *dev, int reg,
  2255.                                  struct drm_i915_gem_object *obj)
  2256. {
  2257.         drm_i915_private_t *dev_priv = dev->dev_private;
  2258.         uint64_t val;
  2259.  
  2260.         if (obj) {
  2261.                 u32 size = obj->gtt_space->size;
  2262.  
  2263.                 val = (uint64_t)((obj->gtt_offset + size - 4096) &
  2264.                                  0xfffff000) << 32;
  2265.                 val |= obj->gtt_offset & 0xfffff000;
  2266.                 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
  2267.                 if (obj->tiling_mode == I915_TILING_Y)
  2268.                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
  2269.                 val |= I965_FENCE_REG_VALID;
  2270.         } else
  2271.                 val = 0;
  2272.  
  2273.         I915_WRITE64(FENCE_REG_965_0 + reg * 8, val);
  2274.         POSTING_READ(FENCE_REG_965_0 + reg * 8);
  2275. }
  2276.  
  2277. static void i915_write_fence_reg(struct drm_device *dev, int reg,
  2278.                                  struct drm_i915_gem_object *obj)
  2279. {
  2280.         drm_i915_private_t *dev_priv = dev->dev_private;
  2281.         u32 val;
  2282.  
  2283.         if (obj) {
  2284.                 u32 size = obj->gtt_space->size;
  2285.                 int pitch_val;
  2286.                 int tile_width;
  2287.  
  2288.                 WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
  2289.                      (size & -size) != size ||
  2290.                      (obj->gtt_offset & (size - 1)),
  2291.                      "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
  2292.                      obj->gtt_offset, obj->map_and_fenceable, size);
  2293.  
  2294.                 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
  2295.                         tile_width = 128;
  2296.                 else
  2297.                         tile_width = 512;
  2298.  
  2299.                 /* Note: pitch better be a power of two tile widths */
  2300.                 pitch_val = obj->stride / tile_width;
  2301.                 pitch_val = ffs(pitch_val) - 1;
  2302.  
  2303.                 val = obj->gtt_offset;
  2304.                 if (obj->tiling_mode == I915_TILING_Y)
  2305.                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
  2306.                 val |= I915_FENCE_SIZE_BITS(size);
  2307.                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
  2308.                 val |= I830_FENCE_REG_VALID;
  2309.         } else
  2310.                 val = 0;
  2311.  
  2312.         if (reg < 8)
  2313.                 reg = FENCE_REG_830_0 + reg * 4;
  2314.         else
  2315.                 reg = FENCE_REG_945_8 + (reg - 8) * 4;
  2316.  
  2317.         I915_WRITE(reg, val);
  2318.         POSTING_READ(reg);
  2319. }
  2320.  
  2321. static void i830_write_fence_reg(struct drm_device *dev, int reg,
  2322.                                 struct drm_i915_gem_object *obj)
  2323. {
  2324.         drm_i915_private_t *dev_priv = dev->dev_private;
  2325.         uint32_t val;
  2326.  
  2327.         if (obj) {
  2328.                 u32 size = obj->gtt_space->size;
  2329.                 uint32_t pitch_val;
  2330.  
  2331.                 WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
  2332.                      (size & -size) != size ||
  2333.                      (obj->gtt_offset & (size - 1)),
  2334.                      "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
  2335.                      obj->gtt_offset, size);
  2336.  
  2337.                 pitch_val = obj->stride / 128;
  2338.                 pitch_val = ffs(pitch_val) - 1;
  2339.  
  2340.                 val = obj->gtt_offset;
  2341.                 if (obj->tiling_mode == I915_TILING_Y)
  2342.                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
  2343.                 val |= I830_FENCE_SIZE_BITS(size);
  2344.                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
  2345.                 val |= I830_FENCE_REG_VALID;
  2346.         } else
  2347.                 val = 0;
  2348.  
  2349.         I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
  2350.         POSTING_READ(FENCE_REG_830_0 + reg * 4);
  2351. }
  2352.  
  2353. static void i915_gem_write_fence(struct drm_device *dev, int reg,
  2354.                                  struct drm_i915_gem_object *obj)
  2355. {
  2356.         switch (INTEL_INFO(dev)->gen) {
  2357.         case 7:
  2358.         case 6: sandybridge_write_fence_reg(dev, reg, obj); break;
  2359.         case 5:
  2360.         case 4: i965_write_fence_reg(dev, reg, obj); break;
  2361.         case 3: i915_write_fence_reg(dev, reg, obj); break;
  2362.         case 2: i830_write_fence_reg(dev, reg, obj); break;
  2363.         default: break;
  2364.         }
  2365. }
  2366.  
  2367. static inline int fence_number(struct drm_i915_private *dev_priv,
  2368.                                struct drm_i915_fence_reg *fence)
  2369. {
  2370.         return fence - dev_priv->fence_regs;
  2371. }
  2372.  
  2373. static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
  2374.                                          struct drm_i915_fence_reg *fence,
  2375.                                          bool enable)
  2376. {
  2377.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  2378.         int reg = fence_number(dev_priv, fence);
  2379.  
  2380.         i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
  2381.  
  2382.         if (enable) {
  2383.                 obj->fence_reg = reg;
  2384.                 fence->obj = obj;
  2385.                 list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
  2386.         } else {
  2387.                 obj->fence_reg = I915_FENCE_REG_NONE;
  2388.                 fence->obj = NULL;
  2389.                 list_del_init(&fence->lru_list);
  2390.         }
  2391. }
  2392.  
  2393. static int
  2394. i915_gem_object_flush_fence(struct drm_i915_gem_object *obj)
  2395. {
  2396.         if (obj->last_fenced_seqno) {
  2397.                 int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
  2398.                         if (ret)
  2399.                                 return ret;
  2400.  
  2401.                 obj->last_fenced_seqno = 0;
  2402.         }
  2403.  
  2404.         /* Ensure that all CPU reads are completed before installing a fence
  2405.          * and all writes before removing the fence.
  2406.          */
  2407.         if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
  2408.                 mb();
  2409.  
  2410.         obj->fenced_gpu_access = false;
  2411.         return 0;
  2412. }
  2413.  
  2414. int
  2415. i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
  2416. {
  2417.         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  2418.         int ret;
  2419.  
  2420.         ret = i915_gem_object_flush_fence(obj);
  2421.         if (ret)
  2422.                 return ret;
  2423.  
  2424.         if (obj->fence_reg == I915_FENCE_REG_NONE)
  2425.                 return 0;
  2426.  
  2427.         i915_gem_object_update_fence(obj,
  2428.                                      &dev_priv->fence_regs[obj->fence_reg],
  2429.                                      false);
  2430.         i915_gem_object_fence_lost(obj);
  2431.  
  2432.         return 0;
  2433. }
  2434.  
  2435. static struct drm_i915_fence_reg *
  2436. i915_find_fence_reg(struct drm_device *dev)
  2437. {
  2438.         struct drm_i915_private *dev_priv = dev->dev_private;
  2439.         struct drm_i915_fence_reg *reg, *avail;
  2440.         int i;
  2441.  
  2442.         /* First try to find a free reg */
  2443.         avail = NULL;
  2444.         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
  2445.                 reg = &dev_priv->fence_regs[i];
  2446.                 if (!reg->obj)
  2447.                         return reg;
  2448.  
  2449.                 if (!reg->pin_count)
  2450.                         avail = reg;
  2451.         }
  2452.  
  2453.         if (avail == NULL)
  2454.                 return NULL;
  2455.  
  2456.         /* None available, try to steal one or wait for a user to finish */
  2457.         list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
  2458.                 if (reg->pin_count)
  2459.                         continue;
  2460.  
  2461.                 return reg;
  2462.         }
  2463.  
  2464.         return NULL;
  2465. }
  2466.  
  2467. /**
  2468.  * i915_gem_object_get_fence - set up fencing for an object
  2469.  * @obj: object to map through a fence reg
  2470.  *
  2471.  * When mapping objects through the GTT, userspace wants to be able to write
  2472.  * to them without having to worry about swizzling if the object is tiled.
  2473.  * This function walks the fence regs looking for a free one for @obj,
  2474.  * stealing one if it can't find any.
  2475.  *
  2476.  * It then sets up the reg based on the object's properties: address, pitch
  2477.  * and tiling format.
  2478.  *
  2479.  * For an untiled surface, this removes any existing fence.
  2480.  */
  2481. int
  2482. i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
  2483. {
  2484.         struct drm_device *dev = obj->base.dev;
  2485.         struct drm_i915_private *dev_priv = dev->dev_private;
  2486.         bool enable = obj->tiling_mode != I915_TILING_NONE;
  2487.         struct drm_i915_fence_reg *reg;
  2488.         int ret;
  2489.  
  2490.         /* Have we updated the tiling parameters upon the object and so
  2491.          * will need to serialise the write to the associated fence register?
  2492.          */
  2493.         if (obj->fence_dirty) {
  2494.                 ret = i915_gem_object_flush_fence(obj);
  2495.                 if (ret)
  2496.                         return ret;
  2497.         }
  2498.  
  2499.         /* Just update our place in the LRU if our fence is getting reused. */
  2500.         if (obj->fence_reg != I915_FENCE_REG_NONE) {
  2501.                 reg = &dev_priv->fence_regs[obj->fence_reg];
  2502.                 if (!obj->fence_dirty) {
  2503.                         list_move_tail(&reg->lru_list,
  2504.                                        &dev_priv->mm.fence_list);
  2505.                         return 0;
  2506.                 }
  2507.         } else if (enable) {
  2508.                 reg = i915_find_fence_reg(dev);
  2509.                 if (reg == NULL)
  2510.                         return -EDEADLK;
  2511.  
  2512.                 if (reg->obj) {
  2513.                         struct drm_i915_gem_object *old = reg->obj;
  2514.  
  2515.                         ret = i915_gem_object_flush_fence(old);
  2516.                         if (ret)
  2517.                                 return ret;
  2518.  
  2519.                         i915_gem_object_fence_lost(old);
  2520.                 }
  2521.         } else
  2522.                 return 0;
  2523.  
  2524.         i915_gem_object_update_fence(obj, reg, enable);
  2525.         obj->fence_dirty = false;
  2526.  
  2527.         return 0;
  2528. }
  2529.  
  2530. static bool i915_gem_valid_gtt_space(struct drm_device *dev,
  2531.                                      struct drm_mm_node *gtt_space,
  2532.                                      unsigned long cache_level)
  2533. {
  2534.         struct drm_mm_node *other;
  2535.  
  2536.         /* On non-LLC machines we have to be careful when putting differing
  2537.          * types of snoopable memory together to avoid the prefetcher
  2538.          * crossing memory domains and dieing.
  2539.          */
  2540.         if (HAS_LLC(dev))
  2541.                 return true;
  2542.  
  2543.         if (gtt_space == NULL)
  2544.                 return true;
  2545.  
  2546.         if (list_empty(&gtt_space->node_list))
  2547.                 return true;
  2548.  
  2549.         other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
  2550.         if (other->allocated && !other->hole_follows && other->color != cache_level)
  2551.                 return false;
  2552.  
  2553.         other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
  2554.         if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
  2555.                 return false;
  2556.  
  2557.         return true;
  2558. }
  2559.  
  2560. static void i915_gem_verify_gtt(struct drm_device *dev)
  2561. {
  2562. #if WATCH_GTT
  2563.         struct drm_i915_private *dev_priv = dev->dev_private;
  2564.         struct drm_i915_gem_object *obj;
  2565.         int err = 0;
  2566.  
  2567.         list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
  2568.                 if (obj->gtt_space == NULL) {
  2569.                         printk(KERN_ERR "object found on GTT list with no space reserved\n");
  2570.                         err++;
  2571.                         continue;
  2572.                 }
  2573.  
  2574.                 if (obj->cache_level != obj->gtt_space->color) {
  2575.                         printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
  2576.                                obj->gtt_space->start,
  2577.                                obj->gtt_space->start + obj->gtt_space->size,
  2578.                                obj->cache_level,
  2579.                                obj->gtt_space->color);
  2580.                         err++;
  2581.                         continue;
  2582.                 }
  2583.  
  2584.                 if (!i915_gem_valid_gtt_space(dev,
  2585.                                               obj->gtt_space,
  2586.                                               obj->cache_level)) {
  2587.                         printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
  2588.                                obj->gtt_space->start,
  2589.                                obj->gtt_space->start + obj->gtt_space->size,
  2590.                                obj->cache_level);
  2591.                         err++;
  2592.                         continue;
  2593.                 }
  2594.         }
  2595.  
  2596.         WARN_ON(err);
  2597. #endif
  2598. }
  2599.  
  2600. /**
  2601.  * Finds free space in the GTT aperture and binds the object there.
  2602.  */
  2603. static int
  2604. i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  2605.                             unsigned alignment,
  2606.                             bool map_and_fenceable,
  2607.                             bool nonblocking)
  2608. {
  2609.         struct drm_device *dev = obj->base.dev;
  2610.         drm_i915_private_t *dev_priv = dev->dev_private;
  2611.         struct drm_mm_node *node;
  2612.         u32 size, fence_size, fence_alignment, unfenced_alignment;
  2613.         bool mappable, fenceable;
  2614.         int ret;
  2615.  
  2616.         if (obj->madv != I915_MADV_WILLNEED) {
  2617.         DRM_ERROR("Attempting to bind a purgeable object\n");
  2618.                 return -EINVAL;
  2619.         }
  2620.  
  2621.         fence_size = i915_gem_get_gtt_size(dev,
  2622.                                            obj->base.size,
  2623.                                            obj->tiling_mode);
  2624.         fence_alignment = i915_gem_get_gtt_alignment(dev,
  2625.                                                      obj->base.size,
  2626.                                                      obj->tiling_mode);
  2627.         unfenced_alignment =
  2628.                 i915_gem_get_unfenced_gtt_alignment(dev,
  2629.                                                     obj->base.size,
  2630.                                                     obj->tiling_mode);
  2631.  
  2632.         if (alignment == 0)
  2633.                 alignment = map_and_fenceable ? fence_alignment :
  2634.                                                 unfenced_alignment;
  2635.         if (map_and_fenceable && alignment & (fence_alignment - 1)) {
  2636.                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
  2637.                 return -EINVAL;
  2638.         }
  2639.  
  2640.         size = map_and_fenceable ? fence_size : obj->base.size;
  2641.  
  2642.         /* If the object is bigger than the entire aperture, reject it early
  2643.          * before evicting everything in a vain attempt to find space.
  2644.          */
  2645.         if (obj->base.size >
  2646.             (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
  2647.                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
  2648.                 return -E2BIG;
  2649.         }
  2650.  
  2651.         ret = i915_gem_object_get_pages(obj);
  2652.         if (ret)
  2653.                 return ret;
  2654.  
  2655.         i915_gem_object_pin_pages(obj);
  2656.  
  2657.         node = kzalloc(sizeof(*node), GFP_KERNEL);
  2658.         if (node == NULL) {
  2659.                 i915_gem_object_unpin_pages(obj);
  2660.                 return -ENOMEM;
  2661.         }
  2662.  
  2663.  search_free:
  2664.         if (map_and_fenceable)
  2665.                 ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space, node,
  2666.                                                           size, alignment, obj->cache_level,
  2667.                                                           0, dev_priv->mm.gtt_mappable_end);
  2668.         else
  2669.                 ret = drm_mm_insert_node_generic(&dev_priv->mm.gtt_space, node,
  2670.                                                  size, alignment, obj->cache_level);
  2671.         if (ret) {
  2672.  
  2673.                 i915_gem_object_unpin_pages(obj);
  2674.                 kfree(node);
  2675.                         return ret;
  2676.         }
  2677.         if (WARN_ON(!i915_gem_valid_gtt_space(dev, node, obj->cache_level))) {
  2678.                 i915_gem_object_unpin_pages(obj);
  2679.                 drm_mm_put_block(node);
  2680.                 return -EINVAL;
  2681.         }
  2682.  
  2683.         ret = i915_gem_gtt_prepare_object(obj);
  2684.         if (ret) {
  2685.                 i915_gem_object_unpin_pages(obj);
  2686.                 drm_mm_put_block(node);
  2687.                         return ret;
  2688.         }
  2689.  
  2690.         list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list);
  2691.         list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
  2692.  
  2693.         obj->gtt_space = node;
  2694.         obj->gtt_offset = node->start;
  2695.  
  2696.         fenceable =
  2697.                 node->size == fence_size &&
  2698.                 (node->start & (fence_alignment - 1)) == 0;
  2699.  
  2700.         mappable =
  2701.                 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
  2702.  
  2703.         obj->map_and_fenceable = mappable && fenceable;
  2704.  
  2705.         i915_gem_object_unpin_pages(obj);
  2706.         trace_i915_gem_object_bind(obj, map_and_fenceable);
  2707.         i915_gem_verify_gtt(dev);
  2708.         return 0;
  2709. }
  2710.  
  2711. void
  2712. i915_gem_clflush_object(struct drm_i915_gem_object *obj)
  2713. {
  2714.         /* If we don't have a page list set up, then we're not pinned
  2715.          * to GPU, and we can ignore the cache flush because it'll happen
  2716.          * again at bind time.
  2717.          */
  2718.         if (obj->pages == NULL)
  2719.                 return;
  2720.  
  2721.         /* If the GPU is snooping the contents of the CPU cache,
  2722.          * we do not need to manually clear the CPU cache lines.  However,
  2723.          * the caches are only snooped when the render cache is
  2724.          * flushed/invalidated.  As we always have to emit invalidations
  2725.          * and flushes when moving into and out of the RENDER domain, correct
  2726.          * snooping behaviour occurs naturally as the result of our domain
  2727.          * tracking.
  2728.          */
  2729.         if (obj->cache_level != I915_CACHE_NONE)
  2730.                 return;
  2731. #if 0
  2732.      if(obj->mapped != NULL)
  2733.      {
  2734.         uint8_t *page_virtual;
  2735.         unsigned int i;
  2736.  
  2737.         page_virtual = obj->mapped;
  2738.         asm volatile("mfence");
  2739.         for (i = 0; i < obj->base.size; i += x86_clflush_size)
  2740.             clflush(page_virtual + i);
  2741.         asm volatile("mfence");
  2742.      }
  2743.      else
  2744.      {
  2745.         uint8_t *page_virtual;
  2746.         unsigned int i;
  2747.         page_virtual = AllocKernelSpace(obj->base.size);
  2748.         if(page_virtual != NULL)
  2749.         {
  2750.             dma_addr_t *src, *dst;
  2751.             u32 count;
  2752.  
  2753. #define page_tabs  0xFDC00000      /* really dirty hack */
  2754.  
  2755.             src =  obj->pages.page;
  2756.             dst =  &((dma_addr_t*)page_tabs)[(u32_t)page_virtual >> 12];
  2757.             count = obj->base.size/4096;
  2758.  
  2759.             while(count--)
  2760.             {
  2761.                 *dst++ = (0xFFFFF000 & *src++) | 0x001 ;
  2762.             };
  2763.  
  2764.             asm volatile("mfence");
  2765.             for (i = 0; i < obj->base.size; i += x86_clflush_size)
  2766.                 clflush(page_virtual + i);
  2767.             asm volatile("mfence");
  2768.             FreeKernelSpace(page_virtual);
  2769.         }
  2770.         else
  2771.         {
  2772.             asm volatile (
  2773.             "mfence         \n"
  2774.             "wbinvd         \n"                 /* this is really ugly  */
  2775.             "mfence");
  2776.         }
  2777.      }
  2778. #endif
  2779.  
  2780. }
  2781.  
  2782. /** Flushes the GTT write domain for the object if it's dirty. */
  2783. static void
  2784. i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
  2785. {
  2786.         uint32_t old_write_domain;
  2787.  
  2788.         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
  2789.                 return;
  2790.  
  2791.         /* No actual flushing is required for the GTT write domain.  Writes
  2792.          * to it immediately go to main memory as far as we know, so there's
  2793.          * no chipset flush.  It also doesn't land in render cache.
  2794.          *
  2795.          * However, we do have to enforce the order so that all writes through
  2796.          * the GTT land before any writes to the device, such as updates to
  2797.          * the GATT itself.
  2798.          */
  2799.         wmb();
  2800.  
  2801.         old_write_domain = obj->base.write_domain;
  2802.         obj->base.write_domain = 0;
  2803.  
  2804.         trace_i915_gem_object_change_domain(obj,
  2805.                                             obj->base.read_domains,
  2806.                                             old_write_domain);
  2807. }
  2808.  
  2809. /** Flushes the CPU write domain for the object if it's dirty. */
  2810. static void
  2811. i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
  2812. {
  2813.         uint32_t old_write_domain;
  2814.  
  2815.         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
  2816.                 return;
  2817.  
  2818.         i915_gem_clflush_object(obj);
  2819.         i915_gem_chipset_flush(obj->base.dev);
  2820.         old_write_domain = obj->base.write_domain;
  2821.         obj->base.write_domain = 0;
  2822.  
  2823.         trace_i915_gem_object_change_domain(obj,
  2824.                                             obj->base.read_domains,
  2825.                                             old_write_domain);
  2826. }
  2827.  
  2828. /**
  2829.  * Moves a single object to the GTT read, and possibly write domain.
  2830.  *
  2831.  * This function returns when the move is complete, including waiting on
  2832.  * flushes to occur.
  2833.  */
  2834. int
  2835. i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
  2836. {
  2837.         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
  2838.         uint32_t old_write_domain, old_read_domains;
  2839.         int ret;
  2840.  
  2841.         /* Not valid to be called on unbound objects. */
  2842.         if (obj->gtt_space == NULL)
  2843.                 return -EINVAL;
  2844.  
  2845.         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
  2846.                 return 0;
  2847.  
  2848.         ret = i915_gem_object_wait_rendering(obj, !write);
  2849.                 if (ret)
  2850.                         return ret;
  2851.  
  2852.         i915_gem_object_flush_cpu_write_domain(obj);
  2853.  
  2854.         old_write_domain = obj->base.write_domain;
  2855.         old_read_domains = obj->base.read_domains;
  2856.  
  2857.         /* It should now be out of any other write domains, and we can update
  2858.          * the domain values for our changes.
  2859.          */
  2860.         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
  2861.         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
  2862.         if (write) {
  2863.                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
  2864.                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
  2865.                 obj->dirty = 1;
  2866.         }
  2867.  
  2868.         trace_i915_gem_object_change_domain(obj,
  2869.                                             old_read_domains,
  2870.                                             old_write_domain);
  2871.  
  2872.         /* And bump the LRU for this access */
  2873.         if (i915_gem_object_is_inactive(obj))
  2874.                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
  2875.  
  2876.         return 0;
  2877. }
  2878.  
  2879. int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
  2880.                                     enum i915_cache_level cache_level)
  2881. {
  2882.         struct drm_device *dev = obj->base.dev;
  2883.         drm_i915_private_t *dev_priv = dev->dev_private;
  2884.         int ret;
  2885.  
  2886.         if (obj->cache_level == cache_level)
  2887.                 return 0;
  2888.  
  2889.         if (obj->pin_count) {
  2890.                 DRM_DEBUG("can not change the cache level of pinned objects\n");
  2891.                 return -EBUSY;
  2892.         }
  2893.  
  2894.         if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) {
  2895.                 ret = i915_gem_object_unbind(obj);
  2896.                 if (ret)
  2897.                         return ret;
  2898.         }
  2899.  
  2900.         if (obj->gtt_space) {
  2901.                 ret = i915_gem_object_finish_gpu(obj);
  2902.                 if (ret)
  2903.                         return ret;
  2904.  
  2905.                 i915_gem_object_finish_gtt(obj);
  2906.  
  2907.                 /* Before SandyBridge, you could not use tiling or fence
  2908.                  * registers with snooped memory, so relinquish any fences
  2909.                  * currently pointing to our region in the aperture.
  2910.                  */
  2911.                 if (INTEL_INFO(dev)->gen < 6) {
  2912.                         ret = i915_gem_object_put_fence(obj);
  2913.                         if (ret)
  2914.                                 return ret;
  2915.                 }
  2916.  
  2917.                 if (obj->has_global_gtt_mapping)
  2918.                         i915_gem_gtt_bind_object(obj, cache_level);
  2919.                 if (obj->has_aliasing_ppgtt_mapping)
  2920.                         i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
  2921.                                                obj, cache_level);
  2922.  
  2923.                 obj->gtt_space->color = cache_level;
  2924.         }
  2925.  
  2926.         if (cache_level == I915_CACHE_NONE) {
  2927.                 u32 old_read_domains, old_write_domain;
  2928.  
  2929.                 /* If we're coming from LLC cached, then we haven't
  2930.                  * actually been tracking whether the data is in the
  2931.                  * CPU cache or not, since we only allow one bit set
  2932.                  * in obj->write_domain and have been skipping the clflushes.
  2933.                  * Just set it to the CPU cache for now.
  2934.                  */
  2935.                 WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
  2936.                 WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
  2937.  
  2938.                 old_read_domains = obj->base.read_domains;
  2939.                 old_write_domain = obj->base.write_domain;
  2940.  
  2941.                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  2942.                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  2943.  
  2944.                 trace_i915_gem_object_change_domain(obj,
  2945.                                                     old_read_domains,
  2946.                                                     old_write_domain);
  2947.     }
  2948.  
  2949.         obj->cache_level = cache_level;
  2950.         i915_gem_verify_gtt(dev);
  2951.         return 0;
  2952. }
  2953.  
  2954. int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
  2955.                                struct drm_file *file)
  2956. {
  2957.         struct drm_i915_gem_caching *args = data;
  2958.         struct drm_i915_gem_object *obj;
  2959.         int ret;
  2960.  
  2961.         ret = i915_mutex_lock_interruptible(dev);
  2962.         if (ret)
  2963.                 return ret;
  2964.  
  2965.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  2966.         if (&obj->base == NULL) {
  2967.                 ret = -ENOENT;
  2968.                 goto unlock;
  2969.         }
  2970.  
  2971.         args->caching = obj->cache_level != I915_CACHE_NONE;
  2972.  
  2973.         drm_gem_object_unreference(&obj->base);
  2974. unlock:
  2975.         mutex_unlock(&dev->struct_mutex);
  2976.         return ret;
  2977. }
  2978.  
  2979. int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
  2980.                                struct drm_file *file)
  2981. {
  2982.         struct drm_i915_gem_caching *args = data;
  2983.         struct drm_i915_gem_object *obj;
  2984.         enum i915_cache_level level;
  2985.         int ret;
  2986.  
  2987.         switch (args->caching) {
  2988.         case I915_CACHING_NONE:
  2989.                 level = I915_CACHE_NONE;
  2990.                 break;
  2991.         case I915_CACHING_CACHED:
  2992.                 level = I915_CACHE_LLC;
  2993.                 break;
  2994.         default:
  2995.                 return -EINVAL;
  2996.         }
  2997.  
  2998.         ret = i915_mutex_lock_interruptible(dev);
  2999.         if (ret)
  3000.                 return ret;
  3001.  
  3002.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  3003.         if (&obj->base == NULL) {
  3004.                 ret = -ENOENT;
  3005.                 goto unlock;
  3006.         }
  3007.  
  3008.         ret = i915_gem_object_set_cache_level(obj, level);
  3009.  
  3010.         drm_gem_object_unreference(&obj->base);
  3011. unlock:
  3012.         mutex_unlock(&dev->struct_mutex);
  3013.         return ret;
  3014. }
  3015.  
  3016. /*
  3017.  * Prepare buffer for display plane (scanout, cursors, etc).
  3018.  * Can be called from an uninterruptible phase (modesetting) and allows
  3019.  * any flushes to be pipelined (for pageflips).
  3020.  */
  3021. int
  3022. i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
  3023.                                      u32 alignment,
  3024.                                      struct intel_ring_buffer *pipelined)
  3025. {
  3026.         u32 old_read_domains, old_write_domain;
  3027.         int ret;
  3028.  
  3029.         if (pipelined != obj->ring) {
  3030.                 ret = i915_gem_object_sync(obj, pipelined);
  3031.         if (ret)
  3032.                 return ret;
  3033.         }
  3034.  
  3035.         /* The display engine is not coherent with the LLC cache on gen6.  As
  3036.          * a result, we make sure that the pinning that is about to occur is
  3037.          * done with uncached PTEs. This is lowest common denominator for all
  3038.          * chipsets.
  3039.          *
  3040.          * However for gen6+, we could do better by using the GFDT bit instead
  3041.          * of uncaching, which would allow us to flush all the LLC-cached data
  3042.          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
  3043.          */
  3044.         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
  3045.         if (ret)
  3046.                 return ret;
  3047.  
  3048.         /* As the user may map the buffer once pinned in the display plane
  3049.          * (e.g. libkms for the bootup splash), we have to ensure that we
  3050.          * always use map_and_fenceable for all scanout buffers.
  3051.          */
  3052.         ret = i915_gem_object_pin(obj, alignment, true, false);
  3053.         if (ret)
  3054.                 return ret;
  3055.  
  3056.         i915_gem_object_flush_cpu_write_domain(obj);
  3057.  
  3058.         old_write_domain = obj->base.write_domain;
  3059.         old_read_domains = obj->base.read_domains;
  3060.  
  3061.         /* It should now be out of any other write domains, and we can update
  3062.          * the domain values for our changes.
  3063.          */
  3064.         obj->base.write_domain = 0;
  3065.         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
  3066.  
  3067.         trace_i915_gem_object_change_domain(obj,
  3068.                                             old_read_domains,
  3069.                                             old_write_domain);
  3070.  
  3071.         return 0;
  3072. }
  3073.  
  3074. int
  3075. i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
  3076. {
  3077.         int ret;
  3078.  
  3079.         if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
  3080.                 return 0;
  3081.  
  3082.         ret = i915_gem_object_wait_rendering(obj, false);
  3083.     if (ret)
  3084.         return ret;
  3085.  
  3086.         /* Ensure that we invalidate the GPU's caches and TLBs. */
  3087.         obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
  3088.         return 0;
  3089. }
  3090.  
  3091. /**
  3092.  * Moves a single object to the CPU read, and possibly write domain.
  3093.  *
  3094.  * This function returns when the move is complete, including waiting on
  3095.  * flushes to occur.
  3096.  */
  3097. int
  3098. i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
  3099. {
  3100.         uint32_t old_write_domain, old_read_domains;
  3101.         int ret;
  3102.  
  3103.         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
  3104.                 return 0;
  3105.  
  3106.         ret = i915_gem_object_wait_rendering(obj, !write);
  3107.         if (ret)
  3108.                 return ret;
  3109.  
  3110.         i915_gem_object_flush_gtt_write_domain(obj);
  3111.  
  3112.         old_write_domain = obj->base.write_domain;
  3113.         old_read_domains = obj->base.read_domains;
  3114.  
  3115.         /* Flush the CPU cache if it's still invalid. */
  3116.         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
  3117.                 i915_gem_clflush_object(obj);
  3118.  
  3119.                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
  3120.         }
  3121.  
  3122.         /* It should now be out of any other write domains, and we can update
  3123.          * the domain values for our changes.
  3124.          */
  3125.         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
  3126.  
  3127.         /* If we're writing through the CPU, then the GPU read domains will
  3128.          * need to be invalidated at next use.
  3129.          */
  3130.         if (write) {
  3131.                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  3132.                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  3133.         }
  3134.  
  3135.         trace_i915_gem_object_change_domain(obj,
  3136.                                             old_read_domains,
  3137.                                             old_write_domain);
  3138.  
  3139.         return 0;
  3140. }
  3141.  
  3142. /* Throttle our rendering by waiting until the ring has completed our requests
  3143.  * emitted over 20 msec ago.
  3144.  *
  3145.  * Note that if we were to use the current jiffies each time around the loop,
  3146.  * we wouldn't escape the function with any frames outstanding if the time to
  3147.  * render a frame was over 20ms.
  3148.  *
  3149.  * This should get us reasonable parallelism between CPU and GPU but also
  3150.  * relatively low latency when blocking on a particular request to finish.
  3151.  */
  3152. static int
  3153. i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
  3154. {
  3155.         struct drm_i915_private *dev_priv = dev->dev_private;
  3156.         struct drm_i915_file_private *file_priv = file->driver_priv;
  3157.         unsigned long recent_enough = GetTimerTicks() - msecs_to_jiffies(20);
  3158.         struct drm_i915_gem_request *request;
  3159.         struct intel_ring_buffer *ring = NULL;
  3160.         u32 seqno = 0;
  3161.         int ret;
  3162.  
  3163.         if (atomic_read(&dev_priv->mm.wedged))
  3164.                 return -EIO;
  3165.  
  3166.         spin_lock(&file_priv->mm.lock);
  3167.         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
  3168.                 if (time_after_eq(request->emitted_jiffies, recent_enough))
  3169.                         break;
  3170.  
  3171.                 ring = request->ring;
  3172.                 seqno = request->seqno;
  3173.         }
  3174.         spin_unlock(&file_priv->mm.lock);
  3175.  
  3176.         if (seqno == 0)
  3177.                 return 0;
  3178.  
  3179.         ret = __wait_seqno(ring, seqno, true, NULL);
  3180.         if (ret == 0)
  3181.                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
  3182.  
  3183.         return ret;
  3184. }
  3185.  
  3186. int
  3187. i915_gem_object_pin(struct drm_i915_gem_object *obj,
  3188.                     uint32_t alignment,
  3189.                     bool map_and_fenceable,
  3190.                     bool nonblocking)
  3191. {
  3192.         int ret;
  3193.  
  3194.         if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
  3195.                 return -EBUSY;
  3196.  
  3197.         if (obj->gtt_space != NULL) {
  3198.                 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
  3199.                     (map_and_fenceable && !obj->map_and_fenceable)) {
  3200.                         WARN(obj->pin_count,
  3201.                              "bo is already pinned with incorrect alignment:"
  3202.                              " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
  3203.                              " obj->map_and_fenceable=%d\n",
  3204.                              obj->gtt_offset, alignment,
  3205.                              map_and_fenceable,
  3206.                              obj->map_and_fenceable);
  3207.                         ret = i915_gem_object_unbind(obj);
  3208.                         if (ret)
  3209.                                 return ret;
  3210.                 }
  3211.         }
  3212.  
  3213.         if (obj->gtt_space == NULL) {
  3214.                 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  3215.  
  3216.                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
  3217.                                                   map_and_fenceable,
  3218.                                                   nonblocking);
  3219.                 if (ret)
  3220.                         return ret;
  3221.  
  3222.                 if (!dev_priv->mm.aliasing_ppgtt)
  3223.                         i915_gem_gtt_bind_object(obj, obj->cache_level);
  3224.         }
  3225.  
  3226.         if (!obj->has_global_gtt_mapping && map_and_fenceable)
  3227.                 i915_gem_gtt_bind_object(obj, obj->cache_level);
  3228.  
  3229.         obj->pin_count++;
  3230.         obj->pin_mappable |= map_and_fenceable;
  3231.  
  3232.         return 0;
  3233. }
  3234.  
  3235. void
  3236. i915_gem_object_unpin(struct drm_i915_gem_object *obj)
  3237. {
  3238.         BUG_ON(obj->pin_count == 0);
  3239.         BUG_ON(obj->gtt_space == NULL);
  3240.  
  3241.         if (--obj->pin_count == 0)
  3242.                 obj->pin_mappable = false;
  3243. }
  3244.  
  3245. int
  3246. i915_gem_pin_ioctl(struct drm_device *dev, void *data,
  3247.                    struct drm_file *file)
  3248. {
  3249.         struct drm_i915_gem_pin *args = data;
  3250.         struct drm_i915_gem_object *obj;
  3251.         int ret;
  3252.  
  3253.         ret = i915_mutex_lock_interruptible(dev);
  3254.         if (ret)
  3255.                 return ret;
  3256.  
  3257.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  3258.         if (&obj->base == NULL) {
  3259.                 ret = -ENOENT;
  3260.                 goto unlock;
  3261.         }
  3262.  
  3263.         if (obj->madv != I915_MADV_WILLNEED) {
  3264.                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
  3265.                 ret = -EINVAL;
  3266.                 goto out;
  3267.         }
  3268.  
  3269.         if (obj->pin_filp != NULL && obj->pin_filp != file) {
  3270.                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
  3271.                           args->handle);
  3272.                 ret = -EINVAL;
  3273.                 goto out;
  3274.         }
  3275.  
  3276.         if (obj->user_pin_count == 0) {
  3277.                 ret = i915_gem_object_pin(obj, args->alignment, true, false);
  3278.                 if (ret)
  3279.                         goto out;
  3280.         }
  3281.  
  3282.         obj->user_pin_count++;
  3283.         obj->pin_filp = file;
  3284.  
  3285.         /* XXX - flush the CPU caches for pinned objects
  3286.          * as the X server doesn't manage domains yet
  3287.          */
  3288.         i915_gem_object_flush_cpu_write_domain(obj);
  3289.         args->offset = obj->gtt_offset;
  3290. out:
  3291.         drm_gem_object_unreference(&obj->base);
  3292. unlock:
  3293.         mutex_unlock(&dev->struct_mutex);
  3294.         return ret;
  3295. }
  3296.  
  3297. #if 0
  3298.  
  3299. int
  3300. i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
  3301.                      struct drm_file *file)
  3302. {
  3303.         struct drm_i915_gem_pin *args = data;
  3304.         struct drm_i915_gem_object *obj;
  3305.         int ret;
  3306.  
  3307.         ret = i915_mutex_lock_interruptible(dev);
  3308.         if (ret)
  3309.                 return ret;
  3310.  
  3311.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  3312.         if (&obj->base == NULL) {
  3313.                 ret = -ENOENT;
  3314.                 goto unlock;
  3315.         }
  3316.  
  3317.         if (obj->pin_filp != file) {
  3318.                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
  3319.                           args->handle);
  3320.                 ret = -EINVAL;
  3321.                 goto out;
  3322.         }
  3323.         obj->user_pin_count--;
  3324.         if (obj->user_pin_count == 0) {
  3325.                 obj->pin_filp = NULL;
  3326.                 i915_gem_object_unpin(obj);
  3327.         }
  3328.  
  3329. out:
  3330.         drm_gem_object_unreference(&obj->base);
  3331. unlock:
  3332.         mutex_unlock(&dev->struct_mutex);
  3333.         return ret;
  3334. }
  3335.  
  3336. #endif
  3337.  
  3338. int
  3339. i915_gem_busy_ioctl(struct drm_device *dev, void *data,
  3340.                     struct drm_file *file)
  3341. {
  3342.         struct drm_i915_gem_busy *args = data;
  3343.         struct drm_i915_gem_object *obj;
  3344.         int ret;
  3345.  
  3346.         ret = i915_mutex_lock_interruptible(dev);
  3347.         if (ret)
  3348.                 return ret;
  3349.  
  3350.         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
  3351.         if (&obj->base == NULL) {
  3352.                 ret = -ENOENT;
  3353.                 goto unlock;
  3354.         }
  3355.  
  3356.         /* Count all active objects as busy, even if they are currently not used
  3357.          * by the gpu. Users of this interface expect objects to eventually
  3358.          * become non-busy without any further actions, therefore emit any
  3359.          * necessary flushes here.
  3360.          */
  3361.         ret = i915_gem_object_flush_active(obj);
  3362.  
  3363.         args->busy = obj->active;
  3364.         if (obj->ring) {
  3365.                 BUILD_BUG_ON(I915_NUM_RINGS > 16);
  3366.                 args->busy |= intel_ring_flag(obj->ring) << 16;
  3367.         }
  3368.  
  3369.         drm_gem_object_unreference(&obj->base);
  3370. unlock:
  3371.         mutex_unlock(&dev->struct_mutex);
  3372.         return ret;
  3373. }
  3374.  
  3375. int
  3376. i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
  3377.                         struct drm_file *file_priv)
  3378. {
  3379.         return i915_gem_ring_throttle(dev, file_priv);
  3380. }
  3381.  
  3382. #if 0
  3383.  
  3384. int
  3385. i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
  3386.                        struct drm_file *file_priv)
  3387. {
  3388.         struct drm_i915_gem_madvise *args = data;
  3389.         struct drm_i915_gem_object *obj;
  3390.         int ret;
  3391.  
  3392.         switch (args->madv) {
  3393.         case I915_MADV_DONTNEED:
  3394.         case I915_MADV_WILLNEED:
  3395.             break;
  3396.         default:
  3397.             return -EINVAL;
  3398.         }
  3399.  
  3400.         ret = i915_mutex_lock_interruptible(dev);
  3401.         if (ret)
  3402.                 return ret;
  3403.  
  3404.         obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
  3405.         if (&obj->base == NULL) {
  3406.                 ret = -ENOENT;
  3407.                 goto unlock;
  3408.         }
  3409.  
  3410.         if (obj->pin_count) {
  3411.                 ret = -EINVAL;
  3412.                 goto out;
  3413.         }
  3414.  
  3415.         if (obj->madv != __I915_MADV_PURGED)
  3416.                 obj->madv = args->madv;
  3417.  
  3418.         /* if the object is no longer attached, discard its backing storage */
  3419.         if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
  3420.                 i915_gem_object_truncate(obj);
  3421.  
  3422.         args->retained = obj->madv != __I915_MADV_PURGED;
  3423.  
  3424. out:
  3425.         drm_gem_object_unreference(&obj->base);
  3426. unlock:
  3427.         mutex_unlock(&dev->struct_mutex);
  3428.         return ret;
  3429. }
  3430. #endif
  3431.  
  3432. void i915_gem_object_init(struct drm_i915_gem_object *obj,
  3433.                           const struct drm_i915_gem_object_ops *ops)
  3434. {
  3435.         INIT_LIST_HEAD(&obj->mm_list);
  3436.         INIT_LIST_HEAD(&obj->gtt_list);
  3437.         INIT_LIST_HEAD(&obj->ring_list);
  3438.         INIT_LIST_HEAD(&obj->exec_list);
  3439.  
  3440.         obj->ops = ops;
  3441.  
  3442.         obj->fence_reg = I915_FENCE_REG_NONE;
  3443.         obj->madv = I915_MADV_WILLNEED;
  3444.         /* Avoid an unnecessary call to unbind on the first bind. */
  3445.         obj->map_and_fenceable = true;
  3446.  
  3447.         i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
  3448. }
  3449.  
  3450. static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
  3451.         .get_pages = i915_gem_object_get_pages_gtt,
  3452.         .put_pages = i915_gem_object_put_pages_gtt,
  3453. };
  3454.  
  3455. struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
  3456.                                                   size_t size)
  3457. {
  3458.         struct drm_i915_gem_object *obj;
  3459.         struct address_space *mapping;
  3460.         u32 mask;
  3461.  
  3462.         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
  3463.         if (obj == NULL)
  3464.                 return NULL;
  3465.  
  3466.         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
  3467.                 kfree(obj);
  3468.                 return NULL;
  3469.         }
  3470.  
  3471.  
  3472.         i915_gem_object_init(obj, &i915_gem_object_ops);
  3473.  
  3474.         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  3475.         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  3476.  
  3477.         if (HAS_LLC(dev)) {
  3478.                 /* On some devices, we can have the GPU use the LLC (the CPU
  3479.                  * cache) for about a 10% performance improvement
  3480.                  * compared to uncached.  Graphics requests other than
  3481.                  * display scanout are coherent with the CPU in
  3482.                  * accessing this cache.  This means in this mode we
  3483.                  * don't need to clflush on the CPU side, and on the
  3484.                  * GPU side we only need to flush internal caches to
  3485.                  * get data visible to the CPU.
  3486.                  *
  3487.                  * However, we maintain the display planes as UC, and so
  3488.                  * need to rebind when first used as such.
  3489.                  */
  3490.                 obj->cache_level = I915_CACHE_LLC;
  3491.         } else
  3492.                 obj->cache_level = I915_CACHE_NONE;
  3493.  
  3494.         return obj;
  3495. }
  3496.  
  3497. int i915_gem_init_object(struct drm_gem_object *obj)
  3498. {
  3499.         BUG();
  3500.  
  3501.         return 0;
  3502. }
  3503.  
  3504. void i915_gem_free_object(struct drm_gem_object *gem_obj)
  3505. {
  3506.         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
  3507.         struct drm_device *dev = obj->base.dev;
  3508.         drm_i915_private_t *dev_priv = dev->dev_private;
  3509.  
  3510.         trace_i915_gem_object_destroy(obj);
  3511.  
  3512. //   if (obj->phys_obj)
  3513. //       i915_gem_detach_phys_object(dev, obj);
  3514.     printf("%s obj %p\n", __FUNCTION__, obj);
  3515.  
  3516.         obj->pin_count = 0;
  3517.         if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
  3518.                 bool was_interruptible;
  3519.  
  3520.                 was_interruptible = dev_priv->mm.interruptible;
  3521.                 dev_priv->mm.interruptible = false;
  3522.  
  3523.                 WARN_ON(i915_gem_object_unbind(obj));
  3524.  
  3525.                 dev_priv->mm.interruptible = was_interruptible;
  3526.         }
  3527.  
  3528.         obj->pages_pin_count = 0;
  3529.         i915_gem_object_put_pages(obj);
  3530. //   i915_gem_object_free_mmap_offset(obj);
  3531.  
  3532.         BUG_ON(obj->pages);
  3533.  
  3534. //   if (obj->base.import_attach)
  3535. //       drm_prime_gem_destroy(&obj->base, NULL);
  3536.  
  3537.     if(obj->base.filp != NULL)
  3538.     {
  3539.         printf("filp %p\n", obj->base.filp);
  3540.         shmem_file_delete(obj->base.filp);
  3541.     }
  3542.  
  3543.         drm_gem_object_release(&obj->base);
  3544.         i915_gem_info_remove_obj(dev_priv, obj->base.size);
  3545.  
  3546.         kfree(obj->bit_17);
  3547.         kfree(obj);
  3548. }
  3549.  
  3550. #if 0
  3551. int
  3552. i915_gem_idle(struct drm_device *dev)
  3553. {
  3554.         drm_i915_private_t *dev_priv = dev->dev_private;
  3555.         int ret;
  3556.  
  3557.         mutex_lock(&dev->struct_mutex);
  3558.  
  3559.         if (dev_priv->mm.suspended) {
  3560.                 mutex_unlock(&dev->struct_mutex);
  3561.                 return 0;
  3562.         }
  3563.  
  3564.         ret = i915_gpu_idle(dev);
  3565.         if (ret) {
  3566.                 mutex_unlock(&dev->struct_mutex);
  3567.                 return ret;
  3568.         }
  3569.         i915_gem_retire_requests(dev);
  3570.  
  3571.         i915_gem_reset_fences(dev);
  3572.  
  3573.         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
  3574.          * We need to replace this with a semaphore, or something.
  3575.          * And not confound mm.suspended!
  3576.          */
  3577.         dev_priv->mm.suspended = 1;
  3578.         del_timer_sync(&dev_priv->hangcheck_timer);
  3579.  
  3580.         i915_kernel_lost_context(dev);
  3581.         i915_gem_cleanup_ringbuffer(dev);
  3582.  
  3583.         mutex_unlock(&dev->struct_mutex);
  3584.  
  3585.         /* Cancel the retire work handler, which should be idle now. */
  3586.         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
  3587.  
  3588.         return 0;
  3589. }
  3590. #endif
  3591.  
  3592. void i915_gem_l3_remap(struct drm_device *dev)
  3593. {
  3594.         drm_i915_private_t *dev_priv = dev->dev_private;
  3595.         u32 misccpctl;
  3596.         int i;
  3597.  
  3598.         if (!IS_IVYBRIDGE(dev))
  3599.                 return;
  3600.  
  3601.         if (!dev_priv->l3_parity.remap_info)
  3602.                 return;
  3603.  
  3604.         misccpctl = I915_READ(GEN7_MISCCPCTL);
  3605.         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
  3606.         POSTING_READ(GEN7_MISCCPCTL);
  3607.  
  3608.         for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
  3609.                 u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
  3610.                 if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
  3611.                         DRM_DEBUG("0x%x was already programmed to %x\n",
  3612.                                   GEN7_L3LOG_BASE + i, remap);
  3613.                 if (remap && !dev_priv->l3_parity.remap_info[i/4])
  3614.                         DRM_DEBUG_DRIVER("Clearing remapped register\n");
  3615.                 I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
  3616.         }
  3617.  
  3618.         /* Make sure all the writes land before disabling dop clock gating */
  3619.         POSTING_READ(GEN7_L3LOG_BASE);
  3620.  
  3621.         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
  3622. }
  3623.  
  3624. void i915_gem_init_swizzling(struct drm_device *dev)
  3625. {
  3626.         drm_i915_private_t *dev_priv = dev->dev_private;
  3627.  
  3628.         if (INTEL_INFO(dev)->gen < 5 ||
  3629.             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
  3630.                 return;
  3631.  
  3632.         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
  3633.                                  DISP_TILE_SURFACE_SWIZZLING);
  3634.  
  3635.         if (IS_GEN5(dev))
  3636.                 return;
  3637.  
  3638.         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
  3639.         if (IS_GEN6(dev))
  3640.                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
  3641.         else
  3642.                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
  3643. }
  3644.  
  3645. static bool
  3646. intel_enable_blt(struct drm_device *dev)
  3647. {
  3648.         if (!HAS_BLT(dev))
  3649.                 return false;
  3650.  
  3651.         /* The blitter was dysfunctional on early prototypes */
  3652.         if (IS_GEN6(dev) && dev->pdev->revision < 8) {
  3653.                 DRM_INFO("BLT not supported on this pre-production hardware;"
  3654.                          " graphics performance will be degraded.\n");
  3655.                 return false;
  3656.         }
  3657.  
  3658.         return true;
  3659. }
  3660.  
  3661. int
  3662. i915_gem_init_hw(struct drm_device *dev)
  3663. {
  3664.         drm_i915_private_t *dev_priv = dev->dev_private;
  3665.         int ret;
  3666.  
  3667.         if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
  3668.                 return -EIO;
  3669.  
  3670.         if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))
  3671.                 I915_WRITE(0x9008, I915_READ(0x9008) | 0xf0000);
  3672.  
  3673.         i915_gem_l3_remap(dev);
  3674.  
  3675.         i915_gem_init_swizzling(dev);
  3676.  
  3677.         ret = intel_init_render_ring_buffer(dev);
  3678.         if (ret)
  3679.                 return ret;
  3680.  
  3681.     if (HAS_BSD(dev)) {
  3682.                 ret = intel_init_bsd_ring_buffer(dev);
  3683.                 if (ret)
  3684.                         goto cleanup_render_ring;
  3685.         }
  3686.  
  3687.         if (intel_enable_blt(dev)) {
  3688.                 ret = intel_init_blt_ring_buffer(dev);
  3689.                 if (ret)
  3690.                         goto cleanup_bsd_ring;
  3691.         }
  3692.  
  3693.         dev_priv->next_seqno = 1;
  3694.  
  3695.         /*
  3696.          * XXX: There was some w/a described somewhere suggesting loading
  3697.          * contexts before PPGTT.
  3698.          */
  3699.         i915_gem_context_init(dev);
  3700.         i915_gem_init_ppgtt(dev);
  3701.  
  3702.         return 0;
  3703.  
  3704. cleanup_bsd_ring:
  3705.         intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
  3706. cleanup_render_ring:
  3707.         intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
  3708.         return ret;
  3709. }
  3710.  
  3711. static bool
  3712. intel_enable_ppgtt(struct drm_device *dev)
  3713. {
  3714.         if (i915_enable_ppgtt >= 0)
  3715.                 return i915_enable_ppgtt;
  3716.  
  3717. #ifdef CONFIG_INTEL_IOMMU
  3718.         /* Disable ppgtt on SNB if VT-d is on. */
  3719.         if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
  3720.                 return false;
  3721. #endif
  3722.  
  3723.         return true;
  3724. }
  3725.  
  3726. #define LFB_SIZE 0xC00000
  3727.  
  3728. int i915_gem_init(struct drm_device *dev)
  3729. {
  3730.         struct drm_i915_private *dev_priv = dev->dev_private;
  3731.         unsigned long gtt_size, mappable_size;
  3732.         int ret;
  3733.  
  3734.         gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT;
  3735.         mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
  3736.  
  3737.         mutex_lock(&dev->struct_mutex);
  3738.         if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
  3739.                 /* PPGTT pdes are stolen from global gtt ptes, so shrink the
  3740.                  * aperture accordingly when using aliasing ppgtt. */
  3741.                 gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
  3742.  
  3743.         i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE);
  3744.  
  3745.                 ret = i915_gem_init_aliasing_ppgtt(dev);
  3746.                 if (ret) {
  3747.                         mutex_unlock(&dev->struct_mutex);
  3748.                         return ret;
  3749.                 }
  3750.         } else {
  3751.                 /* Let GEM Manage all of the aperture.
  3752.                  *
  3753.                  * However, leave one page at the end still bound to the scratch
  3754.                  * page.  There are a number of places where the hardware
  3755.                  * apparently prefetches past the end of the object, and we've
  3756.                  * seen multiple hangs with the GPU head pointer stuck in a
  3757.                  * batchbuffer bound at the last page of the aperture.  One page
  3758.                  * should be enough to keep any prefetching inside of the
  3759.                  * aperture.
  3760.                  */
  3761.         i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE);
  3762.         }
  3763.  
  3764.         ret = i915_gem_init_hw(dev);
  3765.         mutex_unlock(&dev->struct_mutex);
  3766.         if (ret) {
  3767.                 i915_gem_cleanup_aliasing_ppgtt(dev);
  3768.                 return ret;
  3769.         }
  3770.  
  3771.     return 0;
  3772. }
  3773.  
  3774. void
  3775. i915_gem_cleanup_ringbuffer(struct drm_device *dev)
  3776. {
  3777.         drm_i915_private_t *dev_priv = dev->dev_private;
  3778.         struct intel_ring_buffer *ring;
  3779.         int i;
  3780.  
  3781.         for_each_ring(ring, dev_priv, i)
  3782.                 intel_cleanup_ring_buffer(ring);
  3783. }
  3784.  
  3785. #if 0
  3786.  
  3787. int
  3788. i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
  3789.                        struct drm_file *file_priv)
  3790. {
  3791.         drm_i915_private_t *dev_priv = dev->dev_private;
  3792.         int ret;
  3793.  
  3794.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  3795.                 return 0;
  3796.  
  3797.         if (atomic_read(&dev_priv->mm.wedged)) {
  3798.                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
  3799.                 atomic_set(&dev_priv->mm.wedged, 0);
  3800.         }
  3801.  
  3802.         mutex_lock(&dev->struct_mutex);
  3803.         dev_priv->mm.suspended = 0;
  3804.  
  3805.         ret = i915_gem_init_hw(dev);
  3806.         if (ret != 0) {
  3807.                 mutex_unlock(&dev->struct_mutex);
  3808.                 return ret;
  3809.         }
  3810.  
  3811.         BUG_ON(!list_empty(&dev_priv->mm.active_list));
  3812.         mutex_unlock(&dev->struct_mutex);
  3813.  
  3814.         ret = drm_irq_install(dev);
  3815.         if (ret)
  3816.                 goto cleanup_ringbuffer;
  3817.  
  3818.         return 0;
  3819.  
  3820. cleanup_ringbuffer:
  3821.         mutex_lock(&dev->struct_mutex);
  3822.         i915_gem_cleanup_ringbuffer(dev);
  3823.         dev_priv->mm.suspended = 1;
  3824.         mutex_unlock(&dev->struct_mutex);
  3825.  
  3826.         return ret;
  3827. }
  3828.  
  3829. int
  3830. i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
  3831.                        struct drm_file *file_priv)
  3832. {
  3833.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  3834.                 return 0;
  3835.  
  3836.         drm_irq_uninstall(dev);
  3837.         return i915_gem_idle(dev);
  3838. }
  3839.  
  3840. void
  3841. i915_gem_lastclose(struct drm_device *dev)
  3842. {
  3843.         int ret;
  3844.  
  3845.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  3846.                 return;
  3847.  
  3848.         ret = i915_gem_idle(dev);
  3849.         if (ret)
  3850.                 DRM_ERROR("failed to idle hardware: %d\n", ret);
  3851. }
  3852. #endif
  3853.  
  3854. static void
  3855. init_ring_lists(struct intel_ring_buffer *ring)
  3856. {
  3857.     INIT_LIST_HEAD(&ring->active_list);
  3858.     INIT_LIST_HEAD(&ring->request_list);
  3859. }
  3860.  
  3861. void
  3862. i915_gem_load(struct drm_device *dev)
  3863. {
  3864.     int i;
  3865.     drm_i915_private_t *dev_priv = dev->dev_private;
  3866.  
  3867.     INIT_LIST_HEAD(&dev_priv->mm.active_list);
  3868.     INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
  3869.         INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
  3870.         INIT_LIST_HEAD(&dev_priv->mm.bound_list);
  3871.     INIT_LIST_HEAD(&dev_priv->mm.fence_list);
  3872.     for (i = 0; i < I915_NUM_RINGS; i++)
  3873.         init_ring_lists(&dev_priv->ring[i]);
  3874.         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
  3875.         INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
  3876.         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
  3877.                           i915_gem_retire_work_handler);
  3878.  
  3879.     /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
  3880.     if (IS_GEN3(dev)) {
  3881.                 I915_WRITE(MI_ARB_STATE,
  3882.                            _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
  3883.     }
  3884.  
  3885.     dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
  3886.  
  3887.     if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
  3888.         dev_priv->num_fence_regs = 16;
  3889.     else
  3890.         dev_priv->num_fence_regs = 8;
  3891.  
  3892.     /* Initialize fence registers to zero */
  3893.         i915_gem_reset_fences(dev);
  3894.  
  3895.     i915_gem_detect_bit_6_swizzle(dev);
  3896.  
  3897.     dev_priv->mm.interruptible = true;
  3898.  
  3899. //    dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
  3900. //    dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
  3901. //    register_shrinker(&dev_priv->mm.inactive_shrinker);
  3902. }
  3903.  
  3904.  
  3905.