Subversion Repositories Kolibri OS

Rev

Rev 2352 | Rev 3031 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright © 2008 Intel Corporation
  3.  *
  4.  * Permission is hereby granted, free of charge, to any person obtaining a
  5.  * copy of this software and associated documentation files (the "Software"),
  6.  * to deal in the Software without restriction, including without limitation
  7.  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8.  * and/or sell copies of the Software, and to permit persons to whom the
  9.  * Software is furnished to do so, subject to the following conditions:
  10.  *
  11.  * The above copyright notice and this permission notice (including the next
  12.  * paragraph) shall be included in all copies or substantial portions of the
  13.  * Software.
  14.  *
  15.  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16.  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17.  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18.  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19.  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20.  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21.  * IN THE SOFTWARE.
  22.  *
  23.  * Authors:
  24.  *    Eric Anholt <eric@anholt.net>
  25.  *
  26.  */
  27.  
  28. #include "drmP.h"
  29. #include "drm.h"
  30. #include "i915_drm.h"
  31. #include "i915_drv.h"
  32. #include "i915_trace.h"
  33. #include "intel_drv.h"
  34. //#include <linux/shmem_fs.h>
  35. #include <linux/slab.h>
  36. //#include <linux/swap.h>
  37. #include <linux/pci.h>
  38.  
  39. extern int x86_clflush_size;
  40.  
  41. #undef mb
  42. #undef rmb
  43. #undef wmb
  44. #define mb() asm volatile("mfence")
  45. #define rmb() asm volatile ("lfence")
  46. #define wmb() asm volatile ("sfence")
  47.  
  48. static inline void clflush(volatile void *__p)
  49. {
  50.     asm volatile("clflush %0" : "+m" (*(volatile char*)__p));
  51. }
  52.  
  53. #define MAX_ERRNO       4095
  54.  
  55. #define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
  56.  
  57. static inline long IS_ERR(const void *ptr)
  58. {
  59.     return IS_ERR_VALUE((unsigned long)ptr);
  60. }
  61.  
  62. static inline void *ERR_PTR(long error)
  63. {
  64.     return (void *) error;
  65. }
  66.  
  67. static inline long PTR_ERR(const void *ptr)
  68. {
  69.     return (long) ptr;
  70. }
  71.  
  72. void
  73. drm_gem_object_free(struct kref *kref)
  74. {
  75.     struct drm_gem_object *obj = (struct drm_gem_object *) kref;
  76.     struct drm_device *dev = obj->dev;
  77.  
  78.     BUG_ON(!mutex_is_locked(&dev->struct_mutex));
  79.  
  80.     i915_gem_free_object(obj);
  81. }
  82.  
  83. /**
  84.  * Initialize an already allocated GEM object of the specified size with
  85.  * shmfs backing store.
  86.  */
  87. int drm_gem_object_init(struct drm_device *dev,
  88.             struct drm_gem_object *obj, size_t size)
  89. {
  90.     BUG_ON((size & (PAGE_SIZE - 1)) != 0);
  91.  
  92.     obj->dev = dev;
  93.     kref_init(&obj->refcount);
  94.     atomic_set(&obj->handle_count, 0);
  95.     obj->size = size;
  96.  
  97.     return 0;
  98. }
  99.  
  100. void
  101. drm_gem_object_release(struct drm_gem_object *obj)
  102. { }
  103.  
  104.  
  105. #define I915_EXEC_CONSTANTS_MASK        (3<<6)
  106. #define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */
  107. #define I915_EXEC_CONSTANTS_ABSOLUTE    (1<<6)
  108. #define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */
  109.  
  110. static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
  111. static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  112. static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  113. static __must_check int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,
  114.                                                           bool write);
  115. static __must_check int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
  116.                                                                   uint64_t offset,
  117.                                                                   uint64_t size);
  118. static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj);
  119. static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  120.                                                     unsigned alignment,
  121.                                                     bool map_and_fenceable);
  122. static void i915_gem_clear_fence_reg(struct drm_device *dev,
  123.                                      struct drm_i915_fence_reg *reg);
  124. static int i915_gem_phys_pwrite(struct drm_device *dev,
  125.                                 struct drm_i915_gem_object *obj,
  126.                                 struct drm_i915_gem_pwrite *args,
  127.                                 struct drm_file *file);
  128. static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj);
  129.  
  130. //static int i915_gem_inactive_shrink(struct shrinker *shrinker,
  131. //                   struct shrink_control *sc);
  132.  
  133. /* some bookkeeping */
  134. static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  135.                                   size_t size)
  136. {
  137.         dev_priv->mm.object_count++;
  138.         dev_priv->mm.object_memory += size;
  139. }
  140.  
  141. static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  142.                                      size_t size)
  143. {
  144.         dev_priv->mm.object_count--;
  145.         dev_priv->mm.object_memory -= size;
  146. }
  147.  
  148. #if 0
  149.  
  150. static int
  151. i915_gem_wait_for_error(struct drm_device *dev)
  152. {
  153.         struct drm_i915_private *dev_priv = dev->dev_private;
  154.         struct completion *x = &dev_priv->error_completion;
  155.         unsigned long flags;
  156.         int ret;
  157.  
  158.         if (!atomic_read(&dev_priv->mm.wedged))
  159.                 return 0;
  160.  
  161.         ret = wait_for_completion_interruptible(x);
  162.         if (ret)
  163.                 return ret;
  164.  
  165.         if (atomic_read(&dev_priv->mm.wedged)) {
  166.                 /* GPU is hung, bump the completion count to account for
  167.                  * the token we just consumed so that we never hit zero and
  168.                  * end up waiting upon a subsequent completion event that
  169.                  * will never happen.
  170.                  */
  171.                 spin_lock_irqsave(&x->wait.lock, flags);
  172.                 x->done++;
  173.                 spin_unlock_irqrestore(&x->wait.lock, flags);
  174.         }
  175.         return 0;
  176. }
  177.  
  178. int i915_mutex_lock_interruptible(struct drm_device *dev)
  179. {
  180.         int ret;
  181.  
  182.         ret = i915_gem_wait_for_error(dev);
  183.         if (ret)
  184.                 return ret;
  185.  
  186.         ret = mutex_lock_interruptible(&dev->struct_mutex);
  187.         if (ret)
  188.                 return ret;
  189.  
  190.         WARN_ON(i915_verify_lists(dev));
  191.         return 0;
  192. }
  193. #endif
  194.  
  195. static inline bool
  196. i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
  197. {
  198.         return obj->gtt_space && !obj->active && obj->pin_count == 0;
  199. }
  200.  
  201. void i915_gem_do_init(struct drm_device *dev,
  202.                       unsigned long start,
  203.                       unsigned long mappable_end,
  204.                       unsigned long end)
  205. {
  206.         drm_i915_private_t *dev_priv = dev->dev_private;
  207.  
  208.         drm_mm_init(&dev_priv->mm.gtt_space, start, end - start);
  209.  
  210.         dev_priv->mm.gtt_start = start;
  211.         dev_priv->mm.gtt_mappable_end = mappable_end;
  212.         dev_priv->mm.gtt_end = end;
  213.         dev_priv->mm.gtt_total = end - start;
  214.         dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
  215.  
  216.         /* Take over this portion of the GTT */
  217.         intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
  218. }
  219.  
  220. #if 0
  221.  
  222. int
  223. i915_gem_init_ioctl(struct drm_device *dev, void *data,
  224.                     struct drm_file *file)
  225. {
  226.         struct drm_i915_gem_init *args = data;
  227.  
  228.         if (args->gtt_start >= args->gtt_end ||
  229.             (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
  230.                 return -EINVAL;
  231.  
  232.         mutex_lock(&dev->struct_mutex);
  233.         i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
  234.         mutex_unlock(&dev->struct_mutex);
  235.  
  236.         return 0;
  237. }
  238. #endif
  239.  
  240. int
  241. i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  242.                             struct drm_file *file)
  243. {
  244.         struct drm_i915_private *dev_priv = dev->dev_private;
  245.         struct drm_i915_gem_get_aperture *args = data;
  246.         struct drm_i915_gem_object *obj;
  247.         size_t pinned;
  248.  
  249.  
  250.         pinned = 0;
  251.         mutex_lock(&dev->struct_mutex);
  252.         list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
  253.                 pinned += obj->gtt_space->size;
  254.         mutex_unlock(&dev->struct_mutex);
  255.  
  256.         args->aper_size = dev_priv->mm.gtt_total;
  257.         args->aper_available_size = args->aper_size - pinned;
  258.  
  259.         return 0;
  260. }
  261.  
  262. #if 0
  263.  
  264. int i915_gem_create(struct drm_file *file,
  265.                 struct drm_device *dev,
  266.                 uint64_t size,
  267.                 uint32_t *handle_p)
  268. {
  269.         struct drm_i915_gem_object *obj;
  270.         int ret;
  271.         u32 handle;
  272.  
  273.         size = roundup(size, PAGE_SIZE);
  274.         if (size == 0)
  275.                 return -EINVAL;
  276.  
  277.         /* Allocate the new object */
  278.         obj = i915_gem_alloc_object(dev, size);
  279.         if (obj == NULL)
  280.                 return -ENOMEM;
  281.  
  282.         ret = drm_gem_handle_create(file, &obj->base, &handle);
  283.         if (ret) {
  284.                 drm_gem_object_release(&obj->base);
  285.                 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
  286.                 kfree(obj);
  287.                 return ret;
  288.         }
  289.  
  290.         /* drop reference from allocate - handle holds it now */
  291.         drm_gem_object_unreference(&obj->base);
  292.         trace_i915_gem_object_create(obj);
  293.  
  294.         *handle_p = handle;
  295.         return 0;
  296. }
  297.  
  298. int
  299. i915_gem_dumb_create(struct drm_file *file,
  300.                      struct drm_device *dev,
  301.                      struct drm_mode_create_dumb *args)
  302. {
  303.         /* have to work out size/pitch and return them */
  304.         args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
  305.         args->size = args->pitch * args->height;
  306.         return i915_gem_create(file, dev,
  307.                                args->size, &args->handle);
  308. }
  309.  
  310. int i915_gem_dumb_destroy(struct drm_file *file,
  311.                           struct drm_device *dev,
  312.                           uint32_t handle)
  313. {
  314.         return drm_gem_handle_delete(file, handle);
  315. }
  316.  
  317. /**
  318.  * Creates a new mm object and returns a handle to it.
  319.  */
  320. int
  321. i915_gem_create_ioctl(struct drm_device *dev, void *data,
  322.                       struct drm_file *file)
  323. {
  324.         struct drm_i915_gem_create *args = data;
  325.         return i915_gem_create(file, dev,
  326.                                args->size, &args->handle);
  327. }
  328.  
  329. static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
  330. {
  331.         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
  332.  
  333.         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
  334.                 obj->tiling_mode != I915_TILING_NONE;
  335. }
  336.  
  337. static inline void
  338. slow_shmem_copy(struct page *dst_page,
  339.                 int dst_offset,
  340.                 struct page *src_page,
  341.                 int src_offset,
  342.                 int length)
  343. {
  344.         char *dst_vaddr, *src_vaddr;
  345.  
  346.         dst_vaddr = kmap(dst_page);
  347.         src_vaddr = kmap(src_page);
  348.  
  349.         memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
  350.  
  351.         kunmap(src_page);
  352.         kunmap(dst_page);
  353. }
  354.  
  355. static inline void
  356. slow_shmem_bit17_copy(struct page *gpu_page,
  357.                       int gpu_offset,
  358.                       struct page *cpu_page,
  359.                       int cpu_offset,
  360.                       int length,
  361.                       int is_read)
  362. {
  363.         char *gpu_vaddr, *cpu_vaddr;
  364.  
  365.         /* Use the unswizzled path if this page isn't affected. */
  366.         if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
  367.                 if (is_read)
  368.                         return slow_shmem_copy(cpu_page, cpu_offset,
  369.                                                gpu_page, gpu_offset, length);
  370.                 else
  371.                         return slow_shmem_copy(gpu_page, gpu_offset,
  372.                                                cpu_page, cpu_offset, length);
  373.         }
  374.  
  375.         gpu_vaddr = kmap(gpu_page);
  376.         cpu_vaddr = kmap(cpu_page);
  377.  
  378.         /* Copy the data, XORing A6 with A17 (1). The user already knows he's
  379.          * XORing with the other bits (A9 for Y, A9 and A10 for X)
  380.          */
  381.         while (length > 0) {
  382.                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
  383.                 int this_length = min(cacheline_end - gpu_offset, length);
  384.                 int swizzled_gpu_offset = gpu_offset ^ 64;
  385.  
  386.                 if (is_read) {
  387.                         memcpy(cpu_vaddr + cpu_offset,
  388.                                gpu_vaddr + swizzled_gpu_offset,
  389.                                this_length);
  390.                 } else {
  391.                         memcpy(gpu_vaddr + swizzled_gpu_offset,
  392.                                cpu_vaddr + cpu_offset,
  393.                                this_length);
  394.                 }
  395.                 cpu_offset += this_length;
  396.                 gpu_offset += this_length;
  397.                 length -= this_length;
  398.         }
  399.  
  400.         kunmap(cpu_page);
  401.         kunmap(gpu_page);
  402. }
  403.  
  404. /**
  405.  * This is the fast shmem pread path, which attempts to copy_from_user directly
  406.  * from the backing pages of the object to the user's address space.  On a
  407.  * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
  408.  */
  409. static int
  410. i915_gem_shmem_pread_fast(struct drm_device *dev,
  411.                           struct drm_i915_gem_object *obj,
  412.                           struct drm_i915_gem_pread *args,
  413.                           struct drm_file *file)
  414. {
  415.         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
  416.         ssize_t remain;
  417.         loff_t offset;
  418.         char __user *user_data;
  419.         int page_offset, page_length;
  420.  
  421.         user_data = (char __user *) (uintptr_t) args->data_ptr;
  422.         remain = args->size;
  423.  
  424.         offset = args->offset;
  425.  
  426.         while (remain > 0) {
  427.                 struct page *page;
  428.                 char *vaddr;
  429.                 int ret;
  430.  
  431.                 /* Operation in this page
  432.                  *
  433.                  * page_offset = offset within page
  434.                  * page_length = bytes to copy for this page
  435.                  */
  436.                 page_offset = offset_in_page(offset);
  437.                 page_length = remain;
  438.                 if ((page_offset + remain) > PAGE_SIZE)
  439.                         page_length = PAGE_SIZE - page_offset;
  440.  
  441.                 page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
  442.                 if (IS_ERR(page))
  443.                         return PTR_ERR(page);
  444.  
  445.                 vaddr = kmap_atomic(page);
  446.                 ret = __copy_to_user_inatomic(user_data,
  447.                                               vaddr + page_offset,
  448.                                               page_length);
  449.                 kunmap_atomic(vaddr);
  450.  
  451.                 mark_page_accessed(page);
  452.                 page_cache_release(page);
  453.                 if (ret)
  454.                         return -EFAULT;
  455.  
  456.                 remain -= page_length;
  457.                 user_data += page_length;
  458.                 offset += page_length;
  459.         }
  460.  
  461.         return 0;
  462. }
  463.  
  464. /**
  465.  * This is the fallback shmem pread path, which allocates temporary storage
  466.  * in kernel space to copy_to_user into outside of the struct_mutex, so we
  467.  * can copy out of the object's backing pages while holding the struct mutex
  468.  * and not take page faults.
  469.  */
  470. static int
  471. i915_gem_shmem_pread_slow(struct drm_device *dev,
  472.                           struct drm_i915_gem_object *obj,
  473.                           struct drm_i915_gem_pread *args,
  474.                           struct drm_file *file)
  475. {
  476.         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
  477.         struct mm_struct *mm = current->mm;
  478.         struct page **user_pages;
  479.         ssize_t remain;
  480.         loff_t offset, pinned_pages, i;
  481.         loff_t first_data_page, last_data_page, num_pages;
  482.         int shmem_page_offset;
  483.         int data_page_index, data_page_offset;
  484.         int page_length;
  485.         int ret;
  486.         uint64_t data_ptr = args->data_ptr;
  487.         int do_bit17_swizzling;
  488.  
  489.         remain = args->size;
  490.  
  491.         /* Pin the user pages containing the data.  We can't fault while
  492.          * holding the struct mutex, yet we want to hold it while
  493.          * dereferencing the user data.
  494.          */
  495.         first_data_page = data_ptr / PAGE_SIZE;
  496.         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
  497.         num_pages = last_data_page - first_data_page + 1;
  498.  
  499.         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
  500.         if (user_pages == NULL)
  501.                 return -ENOMEM;
  502.  
  503.         mutex_unlock(&dev->struct_mutex);
  504.         down_read(&mm->mmap_sem);
  505.         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
  506.                                       num_pages, 1, 0, user_pages, NULL);
  507.         up_read(&mm->mmap_sem);
  508.         mutex_lock(&dev->struct_mutex);
  509.         if (pinned_pages < num_pages) {
  510.                 ret = -EFAULT;
  511.                 goto out;
  512.         }
  513.  
  514.         ret = i915_gem_object_set_cpu_read_domain_range(obj,
  515.                                                         args->offset,
  516.                                                         args->size);
  517.         if (ret)
  518.                 goto out;
  519.  
  520.         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
  521.  
  522.         offset = args->offset;
  523.  
  524.         while (remain > 0) {
  525.                 struct page *page;
  526.  
  527.                 /* Operation in this page
  528.                  *
  529.                  * shmem_page_offset = offset within page in shmem file
  530.                  * data_page_index = page number in get_user_pages return
  531.                  * data_page_offset = offset with data_page_index page.
  532.                  * page_length = bytes to copy for this page
  533.                  */
  534.                 shmem_page_offset = offset_in_page(offset);
  535.                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
  536.                 data_page_offset = offset_in_page(data_ptr);
  537.  
  538.                 page_length = remain;
  539.                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
  540.                         page_length = PAGE_SIZE - shmem_page_offset;
  541.                 if ((data_page_offset + page_length) > PAGE_SIZE)
  542.                         page_length = PAGE_SIZE - data_page_offset;
  543.  
  544.                 page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
  545.                 if (IS_ERR(page)) {
  546.                         ret = PTR_ERR(page);
  547.                         goto out;
  548.                 }
  549.  
  550.                 if (do_bit17_swizzling) {
  551.                         slow_shmem_bit17_copy(page,
  552.                                               shmem_page_offset,
  553.                                               user_pages[data_page_index],
  554.                                               data_page_offset,
  555.                                               page_length,
  556.                                               1);
  557.                 } else {
  558.                         slow_shmem_copy(user_pages[data_page_index],
  559.                                         data_page_offset,
  560.                                         page,
  561.                                         shmem_page_offset,
  562.                                         page_length);
  563.                 }
  564.  
  565.                 mark_page_accessed(page);
  566.                 page_cache_release(page);
  567.  
  568.                 remain -= page_length;
  569.                 data_ptr += page_length;
  570.                 offset += page_length;
  571.         }
  572.  
  573. out:
  574.         for (i = 0; i < pinned_pages; i++) {
  575.                 SetPageDirty(user_pages[i]);
  576.                 mark_page_accessed(user_pages[i]);
  577.                 page_cache_release(user_pages[i]);
  578.         }
  579.         drm_free_large(user_pages);
  580.  
  581.         return ret;
  582. }
  583. #endif
  584.  
  585.  
  586.  
  587.  
  588.  
  589.  
  590.  
  591.  
  592.  
  593.  
  594.  
  595.  
  596.  
  597.  
  598.  
  599.  
  600.  
  601.  
  602.  
  603.  
  604.  
  605.  
  606.  
  607.  
  608.  
  609.  
  610.  
  611.  
  612.  
  613.  
  614.  
  615.  
  616.  
  617.  
  618.  
  619.  
  620.  
  621.  
  622.  
  623.  
  624.  
  625.  
  626.  
  627.  
  628.  
  629.  
  630.  
  631.  
  632.  
  633.  
  634.  
  635.  
  636.  
  637.  
  638.  
  639.  
  640.  
  641.  
  642.  
  643.  
  644.  
  645.  
  646.  
  647.  
  648.  
  649.  
  650.  
  651.  
  652.  
  653.  
  654.  
  655.  
  656.  
  657. static uint32_t
  658. i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
  659. {
  660.         uint32_t gtt_size;
  661.  
  662.         if (INTEL_INFO(dev)->gen >= 4 ||
  663.             tiling_mode == I915_TILING_NONE)
  664.                 return size;
  665.  
  666.         /* Previous chips need a power-of-two fence region when tiling */
  667.         if (INTEL_INFO(dev)->gen == 3)
  668.                 gtt_size = 1024*1024;
  669.         else
  670.                 gtt_size = 512*1024;
  671.  
  672.         while (gtt_size < size)
  673.                 gtt_size <<= 1;
  674.  
  675.         return gtt_size;
  676. }
  677.  
  678. /**
  679.  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
  680.  * @obj: object to check
  681.  *
  682.  * Return the required GTT alignment for an object, taking into account
  683.  * potential fence register mapping.
  684.  */
  685. static uint32_t
  686. i915_gem_get_gtt_alignment(struct drm_device *dev,
  687.                            uint32_t size,
  688.                            int tiling_mode)
  689. {
  690.         /*
  691.          * Minimum alignment is 4k (GTT page size), but might be greater
  692.          * if a fence register is needed for the object.
  693.          */
  694.         if (INTEL_INFO(dev)->gen >= 4 ||
  695.             tiling_mode == I915_TILING_NONE)
  696.                 return 4096;
  697.  
  698.         /*
  699.          * Previous chips need to be aligned to the size of the smallest
  700.          * fence register that can contain the object.
  701.          */
  702.         return i915_gem_get_gtt_size(dev, size, tiling_mode);
  703. }
  704.  
  705. /**
  706.  * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
  707.  *                                       unfenced object
  708.  * @dev: the device
  709.  * @size: size of the object
  710.  * @tiling_mode: tiling mode of the object
  711.  *
  712.  * Return the required GTT alignment for an object, only taking into account
  713.  * unfenced tiled surface requirements.
  714.  */
  715. uint32_t
  716. i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
  717.                                     uint32_t size,
  718.                                     int tiling_mode)
  719. {
  720.         /*
  721.          * Minimum alignment is 4k (GTT page size) for sane hw.
  722.          */
  723.         if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
  724.             tiling_mode == I915_TILING_NONE)
  725.                 return 4096;
  726.  
  727.         /* Previous hardware however needs to be aligned to a power-of-two
  728.          * tile height. The simplest method for determining this is to reuse
  729.          * the power-of-tile object size.
  730.          */
  731.         return i915_gem_get_gtt_size(dev, size, tiling_mode);
  732. }
  733.  
  734.  
  735.  
  736.  
  737.  
  738.  
  739.  
  740.  
  741.  
  742.  
  743.  
  744.  
  745.  
  746.  
  747.  
  748. static int
  749. i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
  750.                               gfp_t gfpmask)
  751. {
  752.         int page_count, i;
  753.         struct page *page;
  754.  
  755.         /* Get the list of pages out of our struct file.  They'll be pinned
  756.          * at this point until we release them.
  757.          */
  758.         page_count = obj->base.size / PAGE_SIZE;
  759.         BUG_ON(obj->pages != NULL);
  760.     obj->pages = malloc(page_count * sizeof(struct page *));
  761.         if (obj->pages == NULL)
  762.                 return -ENOMEM;
  763.  
  764.  
  765.         for (i = 0; i < page_count; i++) {
  766.         page = (struct page*)AllocPage(); // oh-oh
  767.                 if (IS_ERR(page))
  768.                         goto err_pages;
  769.  
  770.                 obj->pages[i] = page;
  771.         }
  772.  
  773. //   if (obj->tiling_mode != I915_TILING_NONE)
  774. //       i915_gem_object_do_bit_17_swizzle(obj);
  775.  
  776.  
  777.  
  778.         return 0;
  779.  
  780. err_pages:
  781.     while (i--)
  782.         FreePage((addr_t)obj->pages[i]);
  783.  
  784.     free(obj->pages);
  785.         obj->pages = NULL;
  786.         return PTR_ERR(page);
  787. }
  788.  
  789. static void
  790. i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
  791. {
  792.         int page_count = obj->base.size / PAGE_SIZE;
  793.         int i;
  794.  
  795.         BUG_ON(obj->madv == __I915_MADV_PURGED);
  796.  
  797. //   if (obj->tiling_mode != I915_TILING_NONE)
  798. //       i915_gem_object_save_bit_17_swizzle(obj);
  799.  
  800.         if (obj->madv == I915_MADV_DONTNEED)
  801.                 obj->dirty = 0;
  802.  
  803.         for (i = 0; i < page_count; i++) {
  804.         FreePage((addr_t)obj->pages[i]);
  805.         }
  806.         obj->dirty = 0;
  807.  
  808.     free(obj->pages);
  809.         obj->pages = NULL;
  810. }
  811.  
  812. void
  813. i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
  814.                                struct intel_ring_buffer *ring,
  815.                                u32 seqno)
  816. {
  817.         struct drm_device *dev = obj->base.dev;
  818.         struct drm_i915_private *dev_priv = dev->dev_private;
  819.  
  820.         BUG_ON(ring == NULL);
  821.         obj->ring = ring;
  822.  
  823.         /* Add a reference if we're newly entering the active list. */
  824.         if (!obj->active) {
  825.                 drm_gem_object_reference(&obj->base);
  826.                 obj->active = 1;
  827.         }
  828.  
  829.         /* Move from whatever list we were on to the tail of execution. */
  830.         list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
  831.         list_move_tail(&obj->ring_list, &ring->active_list);
  832.  
  833.         obj->last_rendering_seqno = seqno;
  834.         if (obj->fenced_gpu_access) {
  835.                 struct drm_i915_fence_reg *reg;
  836.  
  837.                 BUG_ON(obj->fence_reg == I915_FENCE_REG_NONE);
  838.  
  839.                 obj->last_fenced_seqno = seqno;
  840.                 obj->last_fenced_ring = ring;
  841.  
  842.                 reg = &dev_priv->fence_regs[obj->fence_reg];
  843.                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
  844.         }
  845. }
  846.  
  847. static void
  848. i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
  849. {
  850.         list_del_init(&obj->ring_list);
  851.         obj->last_rendering_seqno = 0;
  852. }
  853.  
  854. static void
  855. i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
  856. {
  857.         struct drm_device *dev = obj->base.dev;
  858.         drm_i915_private_t *dev_priv = dev->dev_private;
  859.  
  860.         BUG_ON(!obj->active);
  861.         list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
  862.  
  863.         i915_gem_object_move_off_active(obj);
  864. }
  865.  
  866. static void
  867. i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
  868. {
  869.         struct drm_device *dev = obj->base.dev;
  870.         struct drm_i915_private *dev_priv = dev->dev_private;
  871.  
  872.         if (obj->pin_count != 0)
  873.                 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
  874.         else
  875.                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
  876.  
  877.         BUG_ON(!list_empty(&obj->gpu_write_list));
  878.         BUG_ON(!obj->active);
  879.         obj->ring = NULL;
  880.  
  881.         i915_gem_object_move_off_active(obj);
  882.         obj->fenced_gpu_access = false;
  883.  
  884.         obj->active = 0;
  885.         obj->pending_gpu_write = false;
  886.         drm_gem_object_unreference(&obj->base);
  887.  
  888.         WARN_ON(i915_verify_lists(dev));
  889. }
  890.  
  891. /* Immediately discard the backing storage */
  892. static void
  893. i915_gem_object_truncate(struct drm_i915_gem_object *obj)
  894. {
  895.         struct inode *inode;
  896.  
  897.         /* Our goal here is to return as much of the memory as
  898.          * is possible back to the system as we are called from OOM.
  899.          * To do this we must instruct the shmfs to drop all of its
  900.          * backing pages, *now*.
  901.          */
  902.  
  903.         obj->madv = __I915_MADV_PURGED;
  904. }
  905.  
  906. static inline int
  907. i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
  908. {
  909.         return obj->madv == I915_MADV_DONTNEED;
  910. }
  911.  
  912. static void
  913. i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
  914.                                uint32_t flush_domains)
  915. {
  916.         struct drm_i915_gem_object *obj, *next;
  917.  
  918.         list_for_each_entry_safe(obj, next,
  919.                                  &ring->gpu_write_list,
  920.                                  gpu_write_list) {
  921.                 if (obj->base.write_domain & flush_domains) {
  922.                         uint32_t old_write_domain = obj->base.write_domain;
  923.  
  924.                         obj->base.write_domain = 0;
  925.                         list_del_init(&obj->gpu_write_list);
  926.                         i915_gem_object_move_to_active(obj, ring,
  927.                                                        i915_gem_next_request_seqno(ring));
  928.  
  929.                         trace_i915_gem_object_change_domain(obj,
  930.                                                             obj->base.read_domains,
  931.                                                             old_write_domain);
  932.                 }
  933.         }
  934. }
  935.  
  936. int
  937. i915_add_request(struct intel_ring_buffer *ring,
  938.                  struct drm_file *file,
  939.                  struct drm_i915_gem_request *request)
  940. {
  941.         drm_i915_private_t *dev_priv = ring->dev->dev_private;
  942.         uint32_t seqno;
  943.         int was_empty;
  944.         int ret;
  945.  
  946.         BUG_ON(request == NULL);
  947.  
  948.         ret = ring->add_request(ring, &seqno);
  949.         if (ret)
  950.             return ret;
  951.  
  952.         trace_i915_gem_request_add(ring, seqno);
  953.  
  954.         request->seqno = seqno;
  955.         request->ring = ring;
  956.         request->emitted_jiffies = jiffies;
  957.         was_empty = list_empty(&ring->request_list);
  958.         list_add_tail(&request->list, &ring->request_list);
  959.  
  960.  
  961.         ring->outstanding_lazy_request = false;
  962.  
  963.         if (!dev_priv->mm.suspended) {
  964.                 if (i915_enable_hangcheck) {
  965. //                      mod_timer(&dev_priv->hangcheck_timer,
  966. //                                jiffies +
  967. //                                msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
  968.                 }
  969.         if (was_empty)
  970.            queue_delayed_work(dev_priv->wq,
  971.                       &dev_priv->mm.retire_work, HZ);
  972.         }
  973.         return 0;
  974. }
  975.  
  976.  
  977.  
  978.  
  979.  
  980.  
  981.  
  982.  
  983.  
  984.  
  985.  
  986.  
  987.  
  988.  
  989.  
  990.  
  991.  
  992.  
  993.  
  994.  
  995.  
  996.  
  997. /**
  998.  * This function clears the request list as sequence numbers are passed.
  999.  */
  1000. static void
  1001. i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
  1002. {
  1003.         uint32_t seqno;
  1004.         int i;
  1005.  
  1006.         if (list_empty(&ring->request_list))
  1007.                 return;
  1008.  
  1009.         WARN_ON(i915_verify_lists(ring->dev));
  1010.  
  1011.         seqno = ring->get_seqno(ring);
  1012.  
  1013.         for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++)
  1014.                 if (seqno >= ring->sync_seqno[i])
  1015.                         ring->sync_seqno[i] = 0;
  1016.  
  1017.         while (!list_empty(&ring->request_list)) {
  1018.                 struct drm_i915_gem_request *request;
  1019.  
  1020.                 request = list_first_entry(&ring->request_list,
  1021.                                            struct drm_i915_gem_request,
  1022.                                            list);
  1023.  
  1024.                 if (!i915_seqno_passed(seqno, request->seqno))
  1025.                         break;
  1026.  
  1027.                 trace_i915_gem_request_retire(ring, request->seqno);
  1028.  
  1029.                 list_del(&request->list);
  1030.                 kfree(request);
  1031.         }
  1032.  
  1033.         /* Move any buffers on the active list that are no longer referenced
  1034.          * by the ringbuffer to the flushing/inactive lists as appropriate.
  1035.          */
  1036.         while (!list_empty(&ring->active_list)) {
  1037.                 struct drm_i915_gem_object *obj;
  1038.  
  1039.                 obj = list_first_entry(&ring->active_list,
  1040.                                       struct drm_i915_gem_object,
  1041.                                       ring_list);
  1042.  
  1043.                 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
  1044.                         break;
  1045.  
  1046.                 if (obj->base.write_domain != 0)
  1047.                         i915_gem_object_move_to_flushing(obj);
  1048.                 else
  1049.                         i915_gem_object_move_to_inactive(obj);
  1050.         }
  1051.  
  1052.         if (unlikely(ring->trace_irq_seqno &&
  1053.                      i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
  1054.                 ring->irq_put(ring);
  1055.                 ring->trace_irq_seqno = 0;
  1056.         }
  1057.  
  1058.         WARN_ON(i915_verify_lists(ring->dev));
  1059. }
  1060.  
  1061. void
  1062. i915_gem_retire_requests(struct drm_device *dev)
  1063. {
  1064.         drm_i915_private_t *dev_priv = dev->dev_private;
  1065.         int i;
  1066.  
  1067.         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
  1068.             struct drm_i915_gem_object *obj, *next;
  1069.  
  1070.             /* We must be careful that during unbind() we do not
  1071.              * accidentally infinitely recurse into retire requests.
  1072.              * Currently:
  1073.              *   retire -> free -> unbind -> wait -> retire_ring
  1074.              */
  1075.             list_for_each_entry_safe(obj, next,
  1076.                                      &dev_priv->mm.deferred_free_list,
  1077.                                      mm_list)
  1078.                     i915_gem_free_object_tail(obj);
  1079.         }
  1080.  
  1081.         for (i = 0; i < I915_NUM_RINGS; i++)
  1082.                 i915_gem_retire_requests_ring(&dev_priv->ring[i]);
  1083. }
  1084.  
  1085. static void
  1086. i915_gem_retire_work_handler(struct work_struct *work)
  1087. {
  1088.         drm_i915_private_t *dev_priv;
  1089.         struct drm_device *dev;
  1090.         bool idle;
  1091.         int i;
  1092.  
  1093. //    ENTER();
  1094.  
  1095.         dev_priv = container_of(work, drm_i915_private_t,
  1096.                                 mm.retire_work.work);
  1097.         dev = dev_priv->dev;
  1098.  
  1099.         /* Come back later if the device is busy... */
  1100.         if (!mutex_trylock(&dev->struct_mutex)) {
  1101.         queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
  1102. //        LEAVE();
  1103.                 return;
  1104.         }
  1105.  
  1106.         i915_gem_retire_requests(dev);
  1107.  
  1108.         /* Send a periodic flush down the ring so we don't hold onto GEM
  1109.          * objects indefinitely.
  1110.          */
  1111.         idle = true;
  1112.         for (i = 0; i < I915_NUM_RINGS; i++) {
  1113.                 struct intel_ring_buffer *ring = &dev_priv->ring[i];
  1114.  
  1115.                 if (!list_empty(&ring->gpu_write_list)) {
  1116.                         struct drm_i915_gem_request *request;
  1117.                         int ret;
  1118.  
  1119.                         ret = i915_gem_flush_ring(ring,
  1120.                                                   0, I915_GEM_GPU_DOMAINS);
  1121.                         request = kzalloc(sizeof(*request), GFP_KERNEL);
  1122.                         if (ret || request == NULL ||
  1123.                             i915_add_request(ring, NULL, request))
  1124.                             kfree(request);
  1125.                 }
  1126.  
  1127.                 idle &= list_empty(&ring->request_list);
  1128.         }
  1129.  
  1130.    if (!dev_priv->mm.suspended && !idle)
  1131.        queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
  1132.  
  1133.         mutex_unlock(&dev->struct_mutex);
  1134. //    LEAVE();
  1135. }
  1136.  
  1137. /**
  1138.  * Waits for a sequence number to be signaled, and cleans up the
  1139.  * request and object lists appropriately for that event.
  1140.  */
  1141. int
  1142. i915_wait_request(struct intel_ring_buffer *ring,
  1143.                   uint32_t seqno)
  1144. {
  1145.         drm_i915_private_t *dev_priv = ring->dev->dev_private;
  1146.         u32 ier;
  1147.         int ret = 0;
  1148.  
  1149.         BUG_ON(seqno == 0);
  1150.  
  1151. //   if (atomic_read(&dev_priv->mm.wedged)) {
  1152. //       struct completion *x = &dev_priv->error_completion;
  1153. //       bool recovery_complete;
  1154. //       unsigned long flags;
  1155.  
  1156.                 /* Give the error handler a chance to run. */
  1157. //       spin_lock_irqsave(&x->wait.lock, flags);
  1158. //       recovery_complete = x->done > 0;
  1159. //       spin_unlock_irqrestore(&x->wait.lock, flags);
  1160. //
  1161. //       return recovery_complete ? -EIO : -EAGAIN;
  1162. //   }
  1163.  
  1164.         if (seqno == ring->outstanding_lazy_request) {
  1165.                 struct drm_i915_gem_request *request;
  1166.  
  1167.                 request = kzalloc(sizeof(*request), GFP_KERNEL);
  1168.                 if (request == NULL)
  1169.                         return -ENOMEM;
  1170.  
  1171.                 ret = i915_add_request(ring, NULL, request);
  1172.                 if (ret) {
  1173.                         kfree(request);
  1174.                         return ret;
  1175.                 }
  1176.  
  1177.                 seqno = request->seqno;
  1178.         }
  1179.  
  1180.         if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
  1181.                 if (HAS_PCH_SPLIT(ring->dev))
  1182.                         ier = I915_READ(DEIER) | I915_READ(GTIER);
  1183.                 else
  1184.                         ier = I915_READ(IER);
  1185.                 if (!ier) {
  1186.                         DRM_ERROR("something (likely vbetool) disabled "
  1187.                                   "interrupts, re-enabling\n");
  1188. //           ring->dev->driver->irq_preinstall(ring->dev);
  1189. //           ring->dev->driver->irq_postinstall(ring->dev);
  1190.                 }
  1191.  
  1192.                 trace_i915_gem_request_wait_begin(ring, seqno);
  1193.  
  1194.                 ring->waiting_seqno = seqno;
  1195.         if (ring->irq_get(ring)) {
  1196. //            printf("enter wait\n");
  1197.             wait_event(ring->irq_queue,
  1198.                       i915_seqno_passed(ring->get_seqno(ring), seqno)
  1199.                       || atomic_read(&dev_priv->mm.wedged));
  1200.  
  1201.            ring->irq_put(ring);
  1202.         } else if (wait_for_atomic(i915_seqno_passed(ring->get_seqno(ring),
  1203.                                                              seqno) ||
  1204.                                            atomic_read(&dev_priv->mm.wedged), 3000))
  1205.                         ret = -EBUSY;
  1206.                 ring->waiting_seqno = 0;
  1207.  
  1208.                 trace_i915_gem_request_wait_end(ring, seqno);
  1209.         }
  1210.         if (atomic_read(&dev_priv->mm.wedged))
  1211.                 ret = -EAGAIN;
  1212.  
  1213.         if (ret && ret != -ERESTARTSYS)
  1214.                 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
  1215.                           __func__, ret, seqno, ring->get_seqno(ring),
  1216.                           dev_priv->next_seqno);
  1217.  
  1218.         /* Directly dispatch request retiring.  While we have the work queue
  1219.          * to handle this, the waiter on a request often wants an associated
  1220.          * buffer to have made it to the inactive list, and we would need
  1221.          * a separate wait queue to handle that.
  1222.          */
  1223.         if (ret == 0)
  1224.                 i915_gem_retire_requests_ring(ring);
  1225.  
  1226.         return ret;
  1227. }
  1228.  
  1229. /**
  1230.  * Ensures that all rendering to the object has completed and the object is
  1231.  * safe to unbind from the GTT or access from the CPU.
  1232.  */
  1233. int
  1234. i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
  1235. {
  1236.         int ret;
  1237.  
  1238.         /* This function only exists to support waiting for existing rendering,
  1239.          * not for emitting required flushes.
  1240.          */
  1241.         BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
  1242.  
  1243.         /* If there is rendering queued on the buffer being evicted, wait for
  1244.          * it.
  1245.          */
  1246.         if (obj->active) {
  1247.        ret = i915_wait_request(obj->ring, obj->last_rendering_seqno);
  1248.        if (ret)
  1249.            return ret;
  1250.         }
  1251.  
  1252.         return 0;
  1253. }
  1254.  
  1255. static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
  1256. {
  1257.         u32 old_write_domain, old_read_domains;
  1258.  
  1259.         /* Act a barrier for all accesses through the GTT */
  1260.         mb();
  1261.  
  1262.         /* Force a pagefault for domain tracking on next user access */
  1263. //      i915_gem_release_mmap(obj);
  1264.  
  1265.         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
  1266.                 return;
  1267.  
  1268.         old_read_domains = obj->base.read_domains;
  1269.         old_write_domain = obj->base.write_domain;
  1270.  
  1271.         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
  1272.         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
  1273.  
  1274.         trace_i915_gem_object_change_domain(obj,
  1275.                                             old_read_domains,
  1276.                                             old_write_domain);
  1277. }
  1278.  
  1279. /**
  1280.  * Unbinds an object from the GTT aperture.
  1281.  */
  1282. int
  1283. i915_gem_object_unbind(struct drm_i915_gem_object *obj)
  1284. {
  1285.         int ret = 0;
  1286.  
  1287.         if (obj->gtt_space == NULL)
  1288.                 return 0;
  1289.  
  1290.         if (obj->pin_count != 0) {
  1291.                 DRM_ERROR("Attempting to unbind pinned buffer\n");
  1292.                 return -EINVAL;
  1293.         }
  1294.  
  1295.         ret = i915_gem_object_finish_gpu(obj);
  1296.         if (ret == -ERESTARTSYS)
  1297.                 return ret;
  1298.         /* Continue on if we fail due to EIO, the GPU is hung so we
  1299.          * should be safe and we need to cleanup or else we might
  1300.          * cause memory corruption through use-after-free.
  1301.          */
  1302.  
  1303.         i915_gem_object_finish_gtt(obj);
  1304.  
  1305.         /* Move the object to the CPU domain to ensure that
  1306.          * any possible CPU writes while it's not in the GTT
  1307.          * are flushed when we go to remap it.
  1308.          */
  1309.         if (ret == 0)
  1310.                 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
  1311.         if (ret == -ERESTARTSYS)
  1312.                 return ret;
  1313.         if (ret) {
  1314.                 /* In the event of a disaster, abandon all caches and
  1315.                  * hope for the best.
  1316.                  */
  1317.                 i915_gem_clflush_object(obj);
  1318.                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  1319.         }
  1320.  
  1321.         /* release the fence reg _after_ flushing */
  1322.         ret = i915_gem_object_put_fence(obj);
  1323.         if (ret == -ERESTARTSYS)
  1324.                 return ret;
  1325.  
  1326.         trace_i915_gem_object_unbind(obj);
  1327.  
  1328.         i915_gem_gtt_unbind_object(obj);
  1329.         i915_gem_object_put_pages_gtt(obj);
  1330.  
  1331.         list_del_init(&obj->gtt_list);
  1332.         list_del_init(&obj->mm_list);
  1333.         /* Avoid an unnecessary call to unbind on rebind. */
  1334.         obj->map_and_fenceable = true;
  1335.  
  1336.         drm_mm_put_block(obj->gtt_space);
  1337.         obj->gtt_space = NULL;
  1338.         obj->gtt_offset = 0;
  1339.  
  1340.         if (i915_gem_object_is_purgeable(obj))
  1341.                 i915_gem_object_truncate(obj);
  1342.  
  1343.         return ret;
  1344. }
  1345.  
  1346. int
  1347. i915_gem_flush_ring(struct intel_ring_buffer *ring,
  1348.                     uint32_t invalidate_domains,
  1349.                     uint32_t flush_domains)
  1350. {
  1351.         int ret;
  1352.  
  1353.         if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
  1354.                 return 0;
  1355.  
  1356.         trace_i915_gem_ring_flush(ring, invalidate_domains, flush_domains);
  1357.  
  1358.         ret = ring->flush(ring, invalidate_domains, flush_domains);
  1359.         if (ret)
  1360.                 return ret;
  1361.  
  1362.         if (flush_domains & I915_GEM_GPU_DOMAINS)
  1363.                 i915_gem_process_flushing_list(ring, flush_domains);
  1364.  
  1365.         return 0;
  1366. }
  1367.  
  1368. static int i915_ring_idle(struct intel_ring_buffer *ring)
  1369. {
  1370.         int ret;
  1371.  
  1372.         if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
  1373.                 return 0;
  1374.  
  1375.         if (!list_empty(&ring->gpu_write_list)) {
  1376.                 ret = i915_gem_flush_ring(ring,
  1377.                                     I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
  1378.                 if (ret)
  1379.                         return ret;
  1380.         }
  1381.  
  1382.         return i915_wait_request(ring, i915_gem_next_request_seqno(ring));
  1383. }
  1384.  
  1385. int
  1386. i915_gpu_idle(struct drm_device *dev)
  1387. {
  1388.         drm_i915_private_t *dev_priv = dev->dev_private;
  1389.         int ret, i;
  1390.  
  1391.         /* Flush everything onto the inactive list. */
  1392.         for (i = 0; i < I915_NUM_RINGS; i++) {
  1393.                 ret = i915_ring_idle(&dev_priv->ring[i]);
  1394.                 if (ret)
  1395.                         return ret;
  1396.         }
  1397.  
  1398.         return 0;
  1399. }
  1400.  
  1401.  
  1402.  
  1403.  
  1404.  
  1405.  
  1406.  
  1407.  
  1408.  
  1409.  
  1410.  
  1411.  
  1412.  
  1413.  
  1414.  
  1415.  
  1416.  
  1417.  
  1418.  
  1419.  
  1420.  
  1421.  
  1422. static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
  1423. {
  1424.         return i915_seqno_passed(ring->get_seqno(ring), seqno);
  1425. }
  1426.  
  1427. static int
  1428. i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
  1429.                             struct intel_ring_buffer *pipelined)
  1430. {
  1431.         int ret;
  1432.  
  1433.         if (obj->fenced_gpu_access) {
  1434.                 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
  1435.                         ret = i915_gem_flush_ring(obj->last_fenced_ring,
  1436.                                                   0, obj->base.write_domain);
  1437.                         if (ret)
  1438.                                 return ret;
  1439.                 }
  1440.  
  1441.                 obj->fenced_gpu_access = false;
  1442.         }
  1443.  
  1444.         if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
  1445.                 if (!ring_passed_seqno(obj->last_fenced_ring,
  1446.                                        obj->last_fenced_seqno)) {
  1447.                         ret = i915_wait_request(obj->last_fenced_ring,
  1448.                                                 obj->last_fenced_seqno);
  1449.                         if (ret)
  1450.                                 return ret;
  1451.                 }
  1452.  
  1453.                 obj->last_fenced_seqno = 0;
  1454.                 obj->last_fenced_ring = NULL;
  1455.         }
  1456.  
  1457.         /* Ensure that all CPU reads are completed before installing a fence
  1458.          * and all writes before removing the fence.
  1459.          */
  1460.         if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
  1461.                 mb();
  1462.  
  1463.         return 0;
  1464. }
  1465.  
  1466. int
  1467. i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
  1468. {
  1469.         int ret;
  1470.  
  1471. //   if (obj->tiling_mode)
  1472. //       i915_gem_release_mmap(obj);
  1473.  
  1474.         ret = i915_gem_object_flush_fence(obj, NULL);
  1475.         if (ret)
  1476.                 return ret;
  1477.  
  1478.         if (obj->fence_reg != I915_FENCE_REG_NONE) {
  1479.                 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  1480.                 i915_gem_clear_fence_reg(obj->base.dev,
  1481.                                          &dev_priv->fence_regs[obj->fence_reg]);
  1482.  
  1483.                 obj->fence_reg = I915_FENCE_REG_NONE;
  1484.         }
  1485.  
  1486.         return 0;
  1487. }
  1488.  
  1489.  
  1490.  
  1491.  
  1492.  
  1493.  
  1494.  
  1495.  
  1496.  
  1497.  
  1498.  
  1499.  
  1500.  
  1501.  
  1502.  
  1503.  
  1504.  
  1505.  
  1506.  
  1507.  
  1508.  
  1509.  
  1510.  
  1511.  
  1512.  
  1513.  
  1514.  
  1515.  
  1516.  
  1517.  
  1518.  
  1519. /**
  1520.  * i915_gem_clear_fence_reg - clear out fence register info
  1521.  * @obj: object to clear
  1522.  *
  1523.  * Zeroes out the fence register itself and clears out the associated
  1524.  * data structures in dev_priv and obj.
  1525.  */
  1526. static void
  1527. i915_gem_clear_fence_reg(struct drm_device *dev,
  1528.              struct drm_i915_fence_reg *reg)
  1529. {
  1530.     drm_i915_private_t *dev_priv = dev->dev_private;
  1531.     uint32_t fence_reg = reg - dev_priv->fence_regs;
  1532.  
  1533.     switch (INTEL_INFO(dev)->gen) {
  1534.     case 7:
  1535.     case 6:
  1536.         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
  1537.         break;
  1538.     case 5:
  1539.     case 4:
  1540.         I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
  1541.         break;
  1542.     case 3:
  1543.         if (fence_reg >= 8)
  1544.             fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
  1545.         else
  1546.     case 2:
  1547.             fence_reg = FENCE_REG_830_0 + fence_reg * 4;
  1548.  
  1549.         I915_WRITE(fence_reg, 0);
  1550.         break;
  1551.     }
  1552.  
  1553.     list_del_init(&reg->lru_list);
  1554.     reg->obj = NULL;
  1555.     reg->setup_seqno = 0;
  1556. }
  1557.  
  1558. /**
  1559.  * Finds free space in the GTT aperture and binds the object there.
  1560.  */
  1561. static int
  1562. i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  1563.                             unsigned alignment,
  1564.                             bool map_and_fenceable)
  1565. {
  1566.         struct drm_device *dev = obj->base.dev;
  1567.         drm_i915_private_t *dev_priv = dev->dev_private;
  1568.         struct drm_mm_node *free_space;
  1569.     gfp_t gfpmask = 0; //__GFP_NORETRY | __GFP_NOWARN;
  1570.         u32 size, fence_size, fence_alignment, unfenced_alignment;
  1571.         bool mappable, fenceable;
  1572.         int ret;
  1573.  
  1574.         if (obj->madv != I915_MADV_WILLNEED) {
  1575.                 DRM_ERROR("Attempting to bind a purgeable object\n");
  1576.                 return -EINVAL;
  1577.         }
  1578.  
  1579.         fence_size = i915_gem_get_gtt_size(dev,
  1580.                                            obj->base.size,
  1581.                                            obj->tiling_mode);
  1582.         fence_alignment = i915_gem_get_gtt_alignment(dev,
  1583.                                                      obj->base.size,
  1584.                                                      obj->tiling_mode);
  1585.         unfenced_alignment =
  1586.                 i915_gem_get_unfenced_gtt_alignment(dev,
  1587.                                                     obj->base.size,
  1588.                                                     obj->tiling_mode);
  1589.  
  1590.         if (alignment == 0)
  1591.                 alignment = map_and_fenceable ? fence_alignment :
  1592.                                                 unfenced_alignment;
  1593.         if (map_and_fenceable && alignment & (fence_alignment - 1)) {
  1594.                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
  1595.                 return -EINVAL;
  1596.         }
  1597.  
  1598.         size = map_and_fenceable ? fence_size : obj->base.size;
  1599.  
  1600.         /* If the object is bigger than the entire aperture, reject it early
  1601.          * before evicting everything in a vain attempt to find space.
  1602.          */
  1603.         if (obj->base.size >
  1604.             (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
  1605.                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
  1606.                 return -E2BIG;
  1607.         }
  1608.  
  1609.  search_free:
  1610.         if (map_and_fenceable)
  1611.                 free_space =
  1612.                         drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
  1613.                                                     size, alignment, 0,
  1614.                                                     dev_priv->mm.gtt_mappable_end,
  1615.                                                     0);
  1616.         else
  1617.                 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
  1618.                                                 size, alignment, 0);
  1619.  
  1620.         if (free_space != NULL) {
  1621.                 if (map_and_fenceable)
  1622.                         obj->gtt_space =
  1623.                                 drm_mm_get_block_range_generic(free_space,
  1624.                                                                size, alignment, 0,
  1625.                                                                dev_priv->mm.gtt_mappable_end,
  1626.                                                                0);
  1627.                 else
  1628.                         obj->gtt_space =
  1629.                                 drm_mm_get_block(free_space, size, alignment);
  1630.         }
  1631.         if (obj->gtt_space == NULL) {
  1632.                 /* If the gtt is empty and we're still having trouble
  1633.                  * fitting our object in, we're out of memory.
  1634.                  */
  1635.         ret = 1; //i915_gem_evict_something(dev, size, alignment,
  1636.                  //         map_and_fenceable);
  1637.                 if (ret)
  1638.                         return ret;
  1639.  
  1640.                 goto search_free;
  1641.         }
  1642.  
  1643.         ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
  1644.         if (ret) {
  1645.                 drm_mm_put_block(obj->gtt_space);
  1646.                 obj->gtt_space = NULL;
  1647. #if 0
  1648.                 if (ret == -ENOMEM) {
  1649.                         /* first try to reclaim some memory by clearing the GTT */
  1650.                         ret = i915_gem_evict_everything(dev, false);
  1651.                         if (ret) {
  1652.                                 /* now try to shrink everyone else */
  1653.                                 if (gfpmask) {
  1654.                                         gfpmask = 0;
  1655.                                         goto search_free;
  1656.                                 }
  1657.  
  1658.                                 return -ENOMEM;
  1659.                         }
  1660.  
  1661.                         goto search_free;
  1662.                 }
  1663. #endif
  1664.                 return ret;
  1665.         }
  1666.  
  1667.         ret = i915_gem_gtt_bind_object(obj);
  1668.         if (ret) {
  1669.         i915_gem_object_put_pages_gtt(obj);
  1670.                 drm_mm_put_block(obj->gtt_space);
  1671.                 obj->gtt_space = NULL;
  1672.  
  1673. //       if (i915_gem_evict_everything(dev, false))
  1674.                         return ret;
  1675.  
  1676. //       goto search_free;
  1677.         }
  1678.  
  1679.         list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
  1680.         list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
  1681.  
  1682.         /* Assert that the object is not currently in any GPU domain. As it
  1683.          * wasn't in the GTT, there shouldn't be any way it could have been in
  1684.          * a GPU cache
  1685.          */
  1686.         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
  1687.         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
  1688.  
  1689.         obj->gtt_offset = obj->gtt_space->start;
  1690.  
  1691.         fenceable =
  1692.                 obj->gtt_space->size == fence_size &&
  1693.                 (obj->gtt_space->start & (fence_alignment - 1)) == 0;
  1694.  
  1695.         mappable =
  1696.                 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
  1697.  
  1698.         obj->map_and_fenceable = mappable && fenceable;
  1699.  
  1700.         trace_i915_gem_object_bind(obj, map_and_fenceable);
  1701.         return 0;
  1702. }
  1703.  
  1704. void
  1705. i915_gem_clflush_object(struct drm_i915_gem_object *obj)
  1706. {
  1707.         /* If we don't have a page list set up, then we're not pinned
  1708.          * to GPU, and we can ignore the cache flush because it'll happen
  1709.          * again at bind time.
  1710.          */
  1711.         if (obj->pages == NULL)
  1712.                 return;
  1713.  
  1714.         /* If the GPU is snooping the contents of the CPU cache,
  1715.          * we do not need to manually clear the CPU cache lines.  However,
  1716.          * the caches are only snooped when the render cache is
  1717.          * flushed/invalidated.  As we always have to emit invalidations
  1718.          * and flushes when moving into and out of the RENDER domain, correct
  1719.          * snooping behaviour occurs naturally as the result of our domain
  1720.          * tracking.
  1721.          */
  1722.         if (obj->cache_level != I915_CACHE_NONE)
  1723.                 return;
  1724.  
  1725.      if(obj->mapped != NULL)
  1726.      {
  1727.         uint8_t *page_virtual;
  1728.         unsigned int i;
  1729.  
  1730.         page_virtual = obj->mapped;
  1731.         asm volatile("mfence");
  1732.         for (i = 0; i < obj->base.size; i += x86_clflush_size)
  1733.             clflush(page_virtual + i);
  1734.         asm volatile("mfence");
  1735.      }
  1736.      else
  1737.      {
  1738.         uint8_t *page_virtual;
  1739.         unsigned int i;
  1740.         page_virtual = AllocKernelSpace(obj->base.size);
  1741.         if(page_virtual != NULL)
  1742.         {
  1743.             u32_t *src, *dst;
  1744.             u32 count;
  1745.  
  1746. #define page_tabs  0xFDC00000      /* really dirty hack */
  1747.  
  1748.             src =  (u32_t*)obj->pages;
  1749.             dst =  &((u32_t*)page_tabs)[(u32_t)page_virtual >> 12];
  1750.             count = obj->base.size/4096;
  1751.  
  1752.             while(count--)
  1753.             {
  1754.                 *dst++ = (0xFFFFF000 & *src++) | 0x001 ;
  1755.             };
  1756.  
  1757.             asm volatile("mfence");
  1758.             for (i = 0; i < obj->base.size; i += x86_clflush_size)
  1759.                 clflush(page_virtual + i);
  1760.             asm volatile("mfence");
  1761.             FreeKernelSpace(page_virtual);
  1762.         }
  1763.         else
  1764.         {
  1765.             asm volatile (
  1766.             "mfence         \n"
  1767.             "wbinvd         \n"                 /* this is really ugly  */
  1768.             "mfence");
  1769.         }
  1770.      }
  1771. }
  1772.  
  1773. /** Flushes any GPU write domain for the object if it's dirty. */
  1774. static int
  1775. i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
  1776. {
  1777.         if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
  1778.                 return 0;
  1779.  
  1780.         /* Queue the GPU write cache flushing we need. */
  1781.         return i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
  1782. }
  1783.  
  1784. /** Flushes the GTT write domain for the object if it's dirty. */
  1785. static void
  1786. i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
  1787. {
  1788.         uint32_t old_write_domain;
  1789.  
  1790.         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
  1791.                 return;
  1792.  
  1793.         /* No actual flushing is required for the GTT write domain.  Writes
  1794.          * to it immediately go to main memory as far as we know, so there's
  1795.          * no chipset flush.  It also doesn't land in render cache.
  1796.          *
  1797.          * However, we do have to enforce the order so that all writes through
  1798.          * the GTT land before any writes to the device, such as updates to
  1799.          * the GATT itself.
  1800.          */
  1801.         wmb();
  1802.  
  1803.         old_write_domain = obj->base.write_domain;
  1804.         obj->base.write_domain = 0;
  1805.  
  1806.         trace_i915_gem_object_change_domain(obj,
  1807.                                             obj->base.read_domains,
  1808.                                             old_write_domain);
  1809. }
  1810.  
  1811. /** Flushes the CPU write domain for the object if it's dirty. */
  1812. static void
  1813. i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
  1814. {
  1815.         uint32_t old_write_domain;
  1816.  
  1817.         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
  1818.                 return;
  1819.  
  1820.         i915_gem_clflush_object(obj);
  1821.         intel_gtt_chipset_flush();
  1822.         old_write_domain = obj->base.write_domain;
  1823.         obj->base.write_domain = 0;
  1824.  
  1825.         trace_i915_gem_object_change_domain(obj,
  1826.                                             obj->base.read_domains,
  1827.                                             old_write_domain);
  1828. }
  1829.  
  1830. /**
  1831.  * Moves a single object to the GTT read, and possibly write domain.
  1832.  *
  1833.  * This function returns when the move is complete, including waiting on
  1834.  * flushes to occur.
  1835.  */
  1836. int
  1837. i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
  1838. {
  1839.         uint32_t old_write_domain, old_read_domains;
  1840.         int ret;
  1841.  
  1842.         /* Not valid to be called on unbound objects. */
  1843.         if (obj->gtt_space == NULL)
  1844.                 return -EINVAL;
  1845.  
  1846.         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
  1847.                 return 0;
  1848.  
  1849.         ret = i915_gem_object_flush_gpu_write_domain(obj);
  1850.         if (ret)
  1851.                 return ret;
  1852.  
  1853.         if (obj->pending_gpu_write || write) {
  1854.                 ret = i915_gem_object_wait_rendering(obj);
  1855.                 if (ret)
  1856.                         return ret;
  1857.         }
  1858.  
  1859.         i915_gem_object_flush_cpu_write_domain(obj);
  1860.  
  1861.         old_write_domain = obj->base.write_domain;
  1862.         old_read_domains = obj->base.read_domains;
  1863.  
  1864.         /* It should now be out of any other write domains, and we can update
  1865.          * the domain values for our changes.
  1866.          */
  1867.         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
  1868.         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
  1869.         if (write) {
  1870.                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
  1871.                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
  1872.                 obj->dirty = 1;
  1873.         }
  1874.  
  1875.         trace_i915_gem_object_change_domain(obj,
  1876.                                             old_read_domains,
  1877.                                             old_write_domain);
  1878.  
  1879.         return 0;
  1880. }
  1881.  
  1882. int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
  1883.                                     enum i915_cache_level cache_level)
  1884. {
  1885.         int ret;
  1886.  
  1887.         if (obj->cache_level == cache_level)
  1888.                 return 0;
  1889.  
  1890.         if (obj->pin_count) {
  1891.                 DRM_DEBUG("can not change the cache level of pinned objects\n");
  1892.                 return -EBUSY;
  1893.         }
  1894.  
  1895.         if (obj->gtt_space) {
  1896.                 ret = i915_gem_object_finish_gpu(obj);
  1897.                 if (ret)
  1898.                         return ret;
  1899.  
  1900.                 i915_gem_object_finish_gtt(obj);
  1901.  
  1902.                 /* Before SandyBridge, you could not use tiling or fence
  1903.                  * registers with snooped memory, so relinquish any fences
  1904.                  * currently pointing to our region in the aperture.
  1905.                  */
  1906.                 if (INTEL_INFO(obj->base.dev)->gen < 6) {
  1907.                         ret = i915_gem_object_put_fence(obj);
  1908.                         if (ret)
  1909.                                 return ret;
  1910.                 }
  1911.  
  1912.                 i915_gem_gtt_rebind_object(obj, cache_level);
  1913.         }
  1914.  
  1915.         if (cache_level == I915_CACHE_NONE) {
  1916.                 u32 old_read_domains, old_write_domain;
  1917.  
  1918.                 /* If we're coming from LLC cached, then we haven't
  1919.                  * actually been tracking whether the data is in the
  1920.                  * CPU cache or not, since we only allow one bit set
  1921.                  * in obj->write_domain and have been skipping the clflushes.
  1922.                  * Just set it to the CPU cache for now.
  1923.                  */
  1924.                 WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
  1925.                 WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
  1926.  
  1927.                 old_read_domains = obj->base.read_domains;
  1928.                 old_write_domain = obj->base.write_domain;
  1929.  
  1930.                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  1931.                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  1932.  
  1933.                 trace_i915_gem_object_change_domain(obj,
  1934.                                                     old_read_domains,
  1935.                                                     old_write_domain);
  1936.     }
  1937.  
  1938.         obj->cache_level = cache_level;
  1939.         return 0;
  1940. }
  1941.  
  1942. /*
  1943.  * Prepare buffer for display plane (scanout, cursors, etc).
  1944.  * Can be called from an uninterruptible phase (modesetting) and allows
  1945.  * any flushes to be pipelined (for pageflips).
  1946.  *
  1947.  * For the display plane, we want to be in the GTT but out of any write
  1948.  * domains. So in many ways this looks like set_to_gtt_domain() apart from the
  1949.  * ability to pipeline the waits, pinning and any additional subtleties
  1950.  * that may differentiate the display plane from ordinary buffers.
  1951.  */
  1952. int
  1953. i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
  1954.                                      u32 alignment,
  1955.                                      struct intel_ring_buffer *pipelined)
  1956. {
  1957.         u32 old_read_domains, old_write_domain;
  1958.         int ret;
  1959.  
  1960.         ret = i915_gem_object_flush_gpu_write_domain(obj);
  1961.         if (ret)
  1962.                 return ret;
  1963.  
  1964.         if (pipelined != obj->ring) {
  1965.                 ret = i915_gem_object_wait_rendering(obj);
  1966.                 if (ret == -ERESTARTSYS)
  1967.                         return ret;
  1968.         }
  1969.  
  1970.         /* The display engine is not coherent with the LLC cache on gen6.  As
  1971.          * a result, we make sure that the pinning that is about to occur is
  1972.          * done with uncached PTEs. This is lowest common denominator for all
  1973.          * chipsets.
  1974.          *
  1975.          * However for gen6+, we could do better by using the GFDT bit instead
  1976.          * of uncaching, which would allow us to flush all the LLC-cached data
  1977.          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
  1978.          */
  1979.         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
  1980.         if (ret)
  1981.                 return ret;
  1982.  
  1983.         /* As the user may map the buffer once pinned in the display plane
  1984.          * (e.g. libkms for the bootup splash), we have to ensure that we
  1985.          * always use map_and_fenceable for all scanout buffers.
  1986.          */
  1987.         ret = i915_gem_object_pin(obj, alignment, true);
  1988.         if (ret)
  1989.                 return ret;
  1990.  
  1991.         i915_gem_object_flush_cpu_write_domain(obj);
  1992.  
  1993.         old_write_domain = obj->base.write_domain;
  1994.         old_read_domains = obj->base.read_domains;
  1995.  
  1996.         /* It should now be out of any other write domains, and we can update
  1997.          * the domain values for our changes.
  1998.          */
  1999.         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
  2000.         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
  2001.  
  2002.         trace_i915_gem_object_change_domain(obj,
  2003.                                             old_read_domains,
  2004.                                             old_write_domain);
  2005.  
  2006.         return 0;
  2007. }
  2008.  
  2009. int
  2010. i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
  2011. {
  2012.         int ret;
  2013.  
  2014.         if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
  2015.                 return 0;
  2016.  
  2017.         if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
  2018.                 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
  2019.                 if (ret)
  2020.                         return ret;
  2021.         }
  2022.  
  2023.         /* Ensure that we invalidate the GPU's caches and TLBs. */
  2024.         obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
  2025.  
  2026.         return i915_gem_object_wait_rendering(obj);
  2027. }
  2028.  
  2029. /**
  2030.  * Moves a single object to the CPU read, and possibly write domain.
  2031.  *
  2032.  * This function returns when the move is complete, including waiting on
  2033.  * flushes to occur.
  2034.  */
  2035. static int
  2036. i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
  2037. {
  2038.         uint32_t old_write_domain, old_read_domains;
  2039.         int ret;
  2040.  
  2041.         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
  2042.                 return 0;
  2043.  
  2044.         ret = i915_gem_object_flush_gpu_write_domain(obj);
  2045.         if (ret)
  2046.                 return ret;
  2047.  
  2048.         ret = i915_gem_object_wait_rendering(obj);
  2049.         if (ret)
  2050.                 return ret;
  2051.  
  2052.         i915_gem_object_flush_gtt_write_domain(obj);
  2053.  
  2054.  
  2055.         old_write_domain = obj->base.write_domain;
  2056.         old_read_domains = obj->base.read_domains;
  2057.  
  2058.         /* Flush the CPU cache if it's still invalid. */
  2059.         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
  2060.                 i915_gem_clflush_object(obj);
  2061.  
  2062.                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
  2063.         }
  2064.  
  2065.         /* It should now be out of any other write domains, and we can update
  2066.          * the domain values for our changes.
  2067.          */
  2068.         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
  2069.  
  2070.         /* If we're writing through the CPU, then the GPU read domains will
  2071.          * need to be invalidated at next use.
  2072.          */
  2073.         if (write) {
  2074.                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  2075.                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  2076.         }
  2077.  
  2078.         trace_i915_gem_object_change_domain(obj,
  2079.                                             old_read_domains,
  2080.                                             old_write_domain);
  2081.  
  2082.         return 0;
  2083. }
  2084.  
  2085. /**
  2086.  * Moves the object from a partially CPU read to a full one.
  2087.  *
  2088.  * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
  2089.  * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
  2090.  */
  2091. static void
  2092. i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj)
  2093. {
  2094.         if (!obj->page_cpu_valid)
  2095.                 return;
  2096.  
  2097.         /* If we're partially in the CPU read domain, finish moving it in.
  2098.          */
  2099.         if (obj->base.read_domains & I915_GEM_DOMAIN_CPU) {
  2100.         }
  2101.  
  2102.         /* Free the page_cpu_valid mappings which are now stale, whether
  2103.          * or not we've got I915_GEM_DOMAIN_CPU.
  2104.          */
  2105.         kfree(obj->page_cpu_valid);
  2106.         obj->page_cpu_valid = NULL;
  2107. }
  2108.  
  2109.  
  2110.  
  2111.  
  2112. int gem_object_lock(struct drm_i915_gem_object *obj)
  2113. {
  2114.     return i915_gem_object_set_to_cpu_domain(obj, true);
  2115. }
  2116.  
  2117.  
  2118.  
  2119.  
  2120.  
  2121.  
  2122.  
  2123.  
  2124.  
  2125.  
  2126.  
  2127.  
  2128.  
  2129.  
  2130.  
  2131.  
  2132.  
  2133.  
  2134.  
  2135. int
  2136. i915_gem_object_pin(struct drm_i915_gem_object *obj,
  2137.                     uint32_t alignment,
  2138.                     bool map_and_fenceable)
  2139. {
  2140.         struct drm_device *dev = obj->base.dev;
  2141.         struct drm_i915_private *dev_priv = dev->dev_private;
  2142.         int ret;
  2143.  
  2144.         BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
  2145.         WARN_ON(i915_verify_lists(dev));
  2146.  
  2147. #if 0
  2148.         if (obj->gtt_space != NULL) {
  2149.                 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
  2150.                     (map_and_fenceable && !obj->map_and_fenceable)) {
  2151.                         WARN(obj->pin_count,
  2152.                              "bo is already pinned with incorrect alignment:"
  2153.                              " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
  2154.                              " obj->map_and_fenceable=%d\n",
  2155.                              obj->gtt_offset, alignment,
  2156.                              map_and_fenceable,
  2157.                              obj->map_and_fenceable);
  2158.                         ret = i915_gem_object_unbind(obj);
  2159.                         if (ret)
  2160.                                 return ret;
  2161.                 }
  2162.         }
  2163. #endif
  2164.  
  2165.         if (obj->gtt_space == NULL) {
  2166.                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
  2167.                                                   map_and_fenceable);
  2168.                 if (ret)
  2169.                         return ret;
  2170.         }
  2171.  
  2172.         if (obj->pin_count++ == 0) {
  2173.                 if (!obj->active)
  2174.                         list_move_tail(&obj->mm_list,
  2175.                                        &dev_priv->mm.pinned_list);
  2176.         }
  2177.         obj->pin_mappable |= map_and_fenceable;
  2178.  
  2179.         WARN_ON(i915_verify_lists(dev));
  2180.         return 0;
  2181. }
  2182.  
  2183. void
  2184. i915_gem_object_unpin(struct drm_i915_gem_object *obj)
  2185. {
  2186.         struct drm_device *dev = obj->base.dev;
  2187.         drm_i915_private_t *dev_priv = dev->dev_private;
  2188.  
  2189.         WARN_ON(i915_verify_lists(dev));
  2190.         BUG_ON(obj->pin_count == 0);
  2191.         BUG_ON(obj->gtt_space == NULL);
  2192.  
  2193.         if (--obj->pin_count == 0) {
  2194.                 if (!obj->active)
  2195.                         list_move_tail(&obj->mm_list,
  2196.                                        &dev_priv->mm.inactive_list);
  2197.                 obj->pin_mappable = false;
  2198.         }
  2199.         WARN_ON(i915_verify_lists(dev));
  2200. }
  2201.  
  2202.  
  2203.  
  2204.  
  2205.  
  2206.  
  2207.  
  2208.  
  2209.  
  2210.  
  2211.  
  2212.  
  2213.  
  2214.  
  2215.  
  2216.  
  2217.  
  2218.  
  2219.  
  2220.  
  2221.  
  2222.  
  2223.  
  2224.  
  2225.  
  2226.  
  2227.  
  2228.  
  2229.  
  2230.  
  2231. struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
  2232.                                                   size_t size)
  2233. {
  2234.         struct drm_i915_private *dev_priv = dev->dev_private;
  2235.         struct drm_i915_gem_object *obj;
  2236.  
  2237.         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
  2238.         if (obj == NULL)
  2239.                 return NULL;
  2240.  
  2241.         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
  2242.                 kfree(obj);
  2243.                 return NULL;
  2244.         }
  2245.  
  2246.  
  2247.         i915_gem_info_add_obj(dev_priv, size);
  2248.  
  2249.         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  2250.         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  2251.  
  2252.         if (IS_GEN6(dev) || IS_GEN7(dev)) {
  2253.                 /* On Gen6, we can have the GPU use the LLC (the CPU
  2254.                  * cache) for about a 10% performance improvement
  2255.                  * compared to uncached.  Graphics requests other than
  2256.                  * display scanout are coherent with the CPU in
  2257.                  * accessing this cache.  This means in this mode we
  2258.                  * don't need to clflush on the CPU side, and on the
  2259.                  * GPU side we only need to flush internal caches to
  2260.                  * get data visible to the CPU.
  2261.                  *
  2262.                  * However, we maintain the display planes as UC, and so
  2263.                  * need to rebind when first used as such.
  2264.                  */
  2265.                 obj->cache_level = I915_CACHE_LLC;
  2266.         } else
  2267.                 obj->cache_level = I915_CACHE_NONE;
  2268.  
  2269.         obj->base.driver_private = NULL;
  2270.         obj->fence_reg = I915_FENCE_REG_NONE;
  2271.         INIT_LIST_HEAD(&obj->mm_list);
  2272.         INIT_LIST_HEAD(&obj->gtt_list);
  2273.         INIT_LIST_HEAD(&obj->ring_list);
  2274.         INIT_LIST_HEAD(&obj->exec_list);
  2275.         INIT_LIST_HEAD(&obj->gpu_write_list);
  2276.         obj->madv = I915_MADV_WILLNEED;
  2277.         /* Avoid an unnecessary call to unbind on the first bind. */
  2278.         obj->map_and_fenceable = true;
  2279.  
  2280.         return obj;
  2281. }
  2282.  
  2283. int i915_gem_init_object(struct drm_gem_object *obj)
  2284. {
  2285.         BUG();
  2286.  
  2287.         return 0;
  2288. }
  2289.  
  2290. static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
  2291. {
  2292.         struct drm_device *dev = obj->base.dev;
  2293.         drm_i915_private_t *dev_priv = dev->dev_private;
  2294.         int ret;
  2295.  
  2296.         ret = i915_gem_object_unbind(obj);
  2297.         if (ret == -ERESTARTSYS) {
  2298.                 list_move(&obj->mm_list,
  2299.                           &dev_priv->mm.deferred_free_list);
  2300.                 return;
  2301.         }
  2302.  
  2303.         trace_i915_gem_object_destroy(obj);
  2304.  
  2305. //      if (obj->base.map_list.map)
  2306. //              drm_gem_free_mmap_offset(&obj->base);
  2307.  
  2308.         drm_gem_object_release(&obj->base);
  2309.         i915_gem_info_remove_obj(dev_priv, obj->base.size);
  2310.  
  2311.         kfree(obj->page_cpu_valid);
  2312.         kfree(obj->bit_17);
  2313.         kfree(obj);
  2314. }
  2315.  
  2316. void i915_gem_free_object(struct drm_gem_object *gem_obj)
  2317. {
  2318.         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
  2319.         struct drm_device *dev = obj->base.dev;
  2320.  
  2321.     while (obj->pin_count > 0)
  2322.                 i915_gem_object_unpin(obj);
  2323.  
  2324. //      if (obj->phys_obj)
  2325. //              i915_gem_detach_phys_object(dev, obj);
  2326.  
  2327.         i915_gem_free_object_tail(obj);
  2328. }
  2329.  
  2330.  
  2331.  
  2332.  
  2333.  
  2334.  
  2335.  
  2336.  
  2337.  
  2338.  
  2339.  
  2340.  
  2341. int
  2342. i915_gem_init_ringbuffer(struct drm_device *dev)
  2343. {
  2344.         drm_i915_private_t *dev_priv = dev->dev_private;
  2345.         int ret;
  2346.  
  2347.         ret = intel_init_render_ring_buffer(dev);
  2348.         if (ret)
  2349.                 return ret;
  2350.  
  2351.     if (HAS_BSD(dev)) {
  2352.                 ret = intel_init_bsd_ring_buffer(dev);
  2353.                 if (ret)
  2354.                         goto cleanup_render_ring;
  2355.         }
  2356.  
  2357.         if (HAS_BLT(dev)) {
  2358.                 ret = intel_init_blt_ring_buffer(dev);
  2359.                 if (ret)
  2360.                         goto cleanup_bsd_ring;
  2361.         }
  2362.  
  2363.         dev_priv->next_seqno = 1;
  2364.  
  2365.         return 0;
  2366.  
  2367. cleanup_bsd_ring:
  2368.         intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
  2369. cleanup_render_ring:
  2370.         intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
  2371.         return ret;
  2372. }
  2373.  
  2374. #if 0
  2375. void
  2376. i915_gem_cleanup_ringbuffer(struct drm_device *dev)
  2377. {
  2378.         drm_i915_private_t *dev_priv = dev->dev_private;
  2379.         int i;
  2380.  
  2381.         for (i = 0; i < I915_NUM_RINGS; i++)
  2382.                 intel_cleanup_ring_buffer(&dev_priv->ring[i]);
  2383. }
  2384.  
  2385. int
  2386. i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
  2387.                        struct drm_file *file_priv)
  2388. {
  2389.         drm_i915_private_t *dev_priv = dev->dev_private;
  2390.         int ret, i;
  2391.  
  2392.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  2393.                 return 0;
  2394.  
  2395.         if (atomic_read(&dev_priv->mm.wedged)) {
  2396.                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
  2397.                 atomic_set(&dev_priv->mm.wedged, 0);
  2398.         }
  2399.  
  2400.         mutex_lock(&dev->struct_mutex);
  2401.         dev_priv->mm.suspended = 0;
  2402.  
  2403.         ret = i915_gem_init_ringbuffer(dev);
  2404.         if (ret != 0) {
  2405.                 mutex_unlock(&dev->struct_mutex);
  2406.                 return ret;
  2407.         }
  2408.  
  2409.         BUG_ON(!list_empty(&dev_priv->mm.active_list));
  2410.         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
  2411.         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
  2412.         for (i = 0; i < I915_NUM_RINGS; i++) {
  2413.                 BUG_ON(!list_empty(&dev_priv->ring[i].active_list));
  2414.                 BUG_ON(!list_empty(&dev_priv->ring[i].request_list));
  2415.         }
  2416.         mutex_unlock(&dev->struct_mutex);
  2417.  
  2418.         ret = drm_irq_install(dev);
  2419.         if (ret)
  2420.                 goto cleanup_ringbuffer;
  2421.  
  2422.         return 0;
  2423.  
  2424. cleanup_ringbuffer:
  2425.         mutex_lock(&dev->struct_mutex);
  2426.         i915_gem_cleanup_ringbuffer(dev);
  2427.         dev_priv->mm.suspended = 1;
  2428.         mutex_unlock(&dev->struct_mutex);
  2429.  
  2430.         return ret;
  2431. }
  2432.  
  2433. int
  2434. i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
  2435.                        struct drm_file *file_priv)
  2436. {
  2437.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  2438.                 return 0;
  2439.  
  2440.         drm_irq_uninstall(dev);
  2441.         return i915_gem_idle(dev);
  2442. }
  2443.  
  2444. void
  2445. i915_gem_lastclose(struct drm_device *dev)
  2446. {
  2447.         int ret;
  2448.  
  2449.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  2450.                 return;
  2451.  
  2452.         ret = i915_gem_idle(dev);
  2453.         if (ret)
  2454.                 DRM_ERROR("failed to idle hardware: %d\n", ret);
  2455. }
  2456. #endif
  2457.  
  2458. static void
  2459. init_ring_lists(struct intel_ring_buffer *ring)
  2460. {
  2461.     INIT_LIST_HEAD(&ring->active_list);
  2462.     INIT_LIST_HEAD(&ring->request_list);
  2463.     INIT_LIST_HEAD(&ring->gpu_write_list);
  2464. }
  2465.  
  2466. void
  2467. i915_gem_load(struct drm_device *dev)
  2468. {
  2469.     int i;
  2470.     drm_i915_private_t *dev_priv = dev->dev_private;
  2471.  
  2472.     INIT_LIST_HEAD(&dev_priv->mm.active_list);
  2473.     INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
  2474.     INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
  2475.     INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
  2476.     INIT_LIST_HEAD(&dev_priv->mm.fence_list);
  2477.     INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
  2478.     INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
  2479.     for (i = 0; i < I915_NUM_RINGS; i++)
  2480.         init_ring_lists(&dev_priv->ring[i]);
  2481.         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
  2482.         INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
  2483.         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
  2484.                           i915_gem_retire_work_handler);
  2485.  
  2486.     /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
  2487.     if (IS_GEN3(dev)) {
  2488.         u32 tmp = I915_READ(MI_ARB_STATE);
  2489.         if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
  2490.             /* arb state is a masked write, so set bit + bit in mask */
  2491.             tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
  2492.             I915_WRITE(MI_ARB_STATE, tmp);
  2493.         }
  2494.     }
  2495.  
  2496.     dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
  2497.  
  2498.     if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
  2499.         dev_priv->num_fence_regs = 16;
  2500.     else
  2501.         dev_priv->num_fence_regs = 8;
  2502.  
  2503.     /* Initialize fence registers to zero */
  2504.     for (i = 0; i < dev_priv->num_fence_regs; i++) {
  2505.         i915_gem_clear_fence_reg(dev, &dev_priv->fence_regs[i]);
  2506.     }
  2507.  
  2508.     i915_gem_detect_bit_6_swizzle(dev);
  2509.  
  2510.     dev_priv->mm.interruptible = true;
  2511.  
  2512. //    dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
  2513. //    dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
  2514. //    register_shrinker(&dev_priv->mm.inactive_shrinker);
  2515. }
  2516.  
  2517.  
  2518.