Subversion Repositories Kolibri OS

Rev

Rev 2342 | Rev 2351 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright © 2008 Intel Corporation
  3.  *
  4.  * Permission is hereby granted, free of charge, to any person obtaining a
  5.  * copy of this software and associated documentation files (the "Software"),
  6.  * to deal in the Software without restriction, including without limitation
  7.  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8.  * and/or sell copies of the Software, and to permit persons to whom the
  9.  * Software is furnished to do so, subject to the following conditions:
  10.  *
  11.  * The above copyright notice and this permission notice (including the next
  12.  * paragraph) shall be included in all copies or substantial portions of the
  13.  * Software.
  14.  *
  15.  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16.  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17.  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18.  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19.  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20.  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21.  * IN THE SOFTWARE.
  22.  *
  23.  * Authors:
  24.  *    Eric Anholt <eric@anholt.net>
  25.  *
  26.  */
  27.  
  28. #include "drmP.h"
  29. #include "drm.h"
  30. #include "i915_drm.h"
  31. #include "i915_drv.h"
  32. //#include "i915_trace.h"
  33. #include "intel_drv.h"
  34. //#include <linux/shmem_fs.h>
  35. #include <linux/slab.h>
  36. //#include <linux/swap.h>
  37. #include <linux/pci.h>
  38.  
  39. extern int x86_clflush_size;
  40.  
  41. #undef mb
  42. #undef rmb
  43. #undef wmb
  44. #define mb() asm volatile("mfence")
  45. #define rmb() asm volatile ("lfence")
  46. #define wmb() asm volatile ("sfence")
  47.  
  48. static inline void clflush(volatile void *__p)
  49. {
  50.     asm volatile("clflush %0" : "+m" (*(volatile char*)__p));
  51. }
  52.  
  53. #define MAX_ERRNO       4095
  54.  
  55. #define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
  56.  
  57. static inline long IS_ERR(const void *ptr)
  58. {
  59.     return IS_ERR_VALUE((unsigned long)ptr);
  60. }
  61.  
  62. static inline void *ERR_PTR(long error)
  63. {
  64.     return (void *) error;
  65. }
  66.  
  67. static inline long PTR_ERR(const void *ptr)
  68. {
  69.     return (long) ptr;
  70. }
  71.  
  72. void
  73. drm_gem_object_free(struct kref *kref)
  74. {
  75.     struct drm_gem_object *obj = (struct drm_gem_object *) kref;
  76.     struct drm_device *dev = obj->dev;
  77.  
  78.     BUG_ON(!mutex_is_locked(&dev->struct_mutex));
  79.  
  80.     i915_gem_free_object(obj);
  81. }
  82.  
  83. /**
  84.  * Initialize an already allocated GEM object of the specified size with
  85.  * shmfs backing store.
  86.  */
  87. int drm_gem_object_init(struct drm_device *dev,
  88.             struct drm_gem_object *obj, size_t size)
  89. {
  90.     BUG_ON((size & (PAGE_SIZE - 1)) != 0);
  91.  
  92.     obj->dev = dev;
  93.     kref_init(&obj->refcount);
  94.     atomic_set(&obj->handle_count, 0);
  95.     obj->size = size;
  96.  
  97.     return 0;
  98. }
  99.  
  100. void
  101. drm_gem_object_release(struct drm_gem_object *obj)
  102. { }
  103.  
  104.  
  105. #define I915_EXEC_CONSTANTS_MASK        (3<<6)
  106. #define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */
  107. #define I915_EXEC_CONSTANTS_ABSOLUTE    (1<<6)
  108. #define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */
  109.  
  110. static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
  111. static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  112. static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  113. static __must_check int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,
  114.                                                           bool write);
  115. static __must_check int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
  116.                                                                   uint64_t offset,
  117.                                                                   uint64_t size);
  118. static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj);
  119. static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  120.                                                     unsigned alignment,
  121.                                                     bool map_and_fenceable);
  122. static void i915_gem_clear_fence_reg(struct drm_device *dev,
  123.                                      struct drm_i915_fence_reg *reg);
  124. static int i915_gem_phys_pwrite(struct drm_device *dev,
  125.                                 struct drm_i915_gem_object *obj,
  126.                                 struct drm_i915_gem_pwrite *args,
  127.                                 struct drm_file *file);
  128. static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj);
  129.  
  130. static int i915_gem_inactive_shrink(struct shrinker *shrinker,
  131.                                     struct shrink_control *sc);
  132.  
  133. /* some bookkeeping */
  134. static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  135.                                   size_t size)
  136. {
  137.         dev_priv->mm.object_count++;
  138.         dev_priv->mm.object_memory += size;
  139. }
  140.  
  141. static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  142.                                      size_t size)
  143. {
  144.         dev_priv->mm.object_count--;
  145.         dev_priv->mm.object_memory -= size;
  146. }
  147.  
  148. #if 0
  149.  
  150. static int
  151. i915_gem_wait_for_error(struct drm_device *dev)
  152. {
  153.         struct drm_i915_private *dev_priv = dev->dev_private;
  154.         struct completion *x = &dev_priv->error_completion;
  155.         unsigned long flags;
  156.         int ret;
  157.  
  158.         if (!atomic_read(&dev_priv->mm.wedged))
  159.                 return 0;
  160.  
  161.         ret = wait_for_completion_interruptible(x);
  162.         if (ret)
  163.                 return ret;
  164.  
  165.         if (atomic_read(&dev_priv->mm.wedged)) {
  166.                 /* GPU is hung, bump the completion count to account for
  167.                  * the token we just consumed so that we never hit zero and
  168.                  * end up waiting upon a subsequent completion event that
  169.                  * will never happen.
  170.                  */
  171.                 spin_lock_irqsave(&x->wait.lock, flags);
  172.                 x->done++;
  173.                 spin_unlock_irqrestore(&x->wait.lock, flags);
  174.         }
  175.         return 0;
  176. }
  177.  
  178. int i915_mutex_lock_interruptible(struct drm_device *dev)
  179. {
  180.         int ret;
  181.  
  182.         ret = i915_gem_wait_for_error(dev);
  183.         if (ret)
  184.                 return ret;
  185.  
  186.         ret = mutex_lock_interruptible(&dev->struct_mutex);
  187.         if (ret)
  188.                 return ret;
  189.  
  190.         WARN_ON(i915_verify_lists(dev));
  191.         return 0;
  192. }
  193.  
  194. static inline bool
  195. i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
  196. {
  197.         return obj->gtt_space && !obj->active && obj->pin_count == 0;
  198. }
  199.  
  200. #endif
  201.  
  202. void i915_gem_do_init(struct drm_device *dev,
  203.                       unsigned long start,
  204.                       unsigned long mappable_end,
  205.                       unsigned long end)
  206. {
  207.         drm_i915_private_t *dev_priv = dev->dev_private;
  208.  
  209.         drm_mm_init(&dev_priv->mm.gtt_space, start, end - start);
  210.  
  211.         dev_priv->mm.gtt_start = start;
  212.         dev_priv->mm.gtt_mappable_end = mappable_end;
  213.         dev_priv->mm.gtt_end = end;
  214.         dev_priv->mm.gtt_total = end - start;
  215.         dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
  216.  
  217.         /* Take over this portion of the GTT */
  218.         intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
  219. }
  220.  
  221. #if 0
  222.  
  223. int
  224. i915_gem_init_ioctl(struct drm_device *dev, void *data,
  225.                     struct drm_file *file)
  226. {
  227.         struct drm_i915_gem_init *args = data;
  228.  
  229.         if (args->gtt_start >= args->gtt_end ||
  230.             (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
  231.                 return -EINVAL;
  232.  
  233.         mutex_lock(&dev->struct_mutex);
  234.         i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
  235.         mutex_unlock(&dev->struct_mutex);
  236.  
  237.         return 0;
  238. }
  239.  
  240. int
  241. i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  242.                             struct drm_file *file)
  243. {
  244.         struct drm_i915_private *dev_priv = dev->dev_private;
  245.         struct drm_i915_gem_get_aperture *args = data;
  246.         struct drm_i915_gem_object *obj;
  247.         size_t pinned;
  248.  
  249.         if (!(dev->driver->driver_features & DRIVER_GEM))
  250.                 return -ENODEV;
  251.  
  252.         pinned = 0;
  253.         mutex_lock(&dev->struct_mutex);
  254.         list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
  255.                 pinned += obj->gtt_space->size;
  256.         mutex_unlock(&dev->struct_mutex);
  257.  
  258.         args->aper_size = dev_priv->mm.gtt_total;
  259.         args->aper_available_size = args->aper_size - pinned;
  260.  
  261.         return 0;
  262. }
  263.  
  264. static int
  265. i915_gem_create(struct drm_file *file,
  266.                 struct drm_device *dev,
  267.                 uint64_t size,
  268.                 uint32_t *handle_p)
  269. {
  270.         struct drm_i915_gem_object *obj;
  271.         int ret;
  272.         u32 handle;
  273.  
  274.         size = roundup(size, PAGE_SIZE);
  275.         if (size == 0)
  276.                 return -EINVAL;
  277.  
  278.         /* Allocate the new object */
  279.         obj = i915_gem_alloc_object(dev, size);
  280.         if (obj == NULL)
  281.                 return -ENOMEM;
  282.  
  283.         ret = drm_gem_handle_create(file, &obj->base, &handle);
  284.         if (ret) {
  285.                 drm_gem_object_release(&obj->base);
  286.                 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
  287.                 kfree(obj);
  288.                 return ret;
  289.         }
  290.  
  291.         /* drop reference from allocate - handle holds it now */
  292.         drm_gem_object_unreference(&obj->base);
  293.  
  294.         *handle_p = handle;
  295.         return 0;
  296. }
  297.  
  298. int
  299. i915_gem_dumb_create(struct drm_file *file,
  300.                      struct drm_device *dev,
  301.                      struct drm_mode_create_dumb *args)
  302. {
  303.         /* have to work out size/pitch and return them */
  304.         args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
  305.         args->size = args->pitch * args->height;
  306.         return i915_gem_create(file, dev,
  307.                                args->size, &args->handle);
  308. }
  309.  
  310. int i915_gem_dumb_destroy(struct drm_file *file,
  311.                           struct drm_device *dev,
  312.                           uint32_t handle)
  313. {
  314.         return drm_gem_handle_delete(file, handle);
  315. }
  316.  
  317. /**
  318.  * Creates a new mm object and returns a handle to it.
  319.  */
  320. int
  321. i915_gem_create_ioctl(struct drm_device *dev, void *data,
  322.                       struct drm_file *file)
  323. {
  324.         struct drm_i915_gem_create *args = data;
  325.         return i915_gem_create(file, dev,
  326.                                args->size, &args->handle);
  327. }
  328.  
  329. static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
  330. {
  331.         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
  332.  
  333.         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
  334.                 obj->tiling_mode != I915_TILING_NONE;
  335. }
  336.  
  337. static inline void
  338. slow_shmem_copy(struct page *dst_page,
  339.                 int dst_offset,
  340.                 struct page *src_page,
  341.                 int src_offset,
  342.                 int length)
  343. {
  344.         char *dst_vaddr, *src_vaddr;
  345.  
  346.         dst_vaddr = kmap(dst_page);
  347.         src_vaddr = kmap(src_page);
  348.  
  349.         memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
  350.  
  351.         kunmap(src_page);
  352.         kunmap(dst_page);
  353. }
  354.  
  355. static inline void
  356. slow_shmem_bit17_copy(struct page *gpu_page,
  357.                       int gpu_offset,
  358.                       struct page *cpu_page,
  359.                       int cpu_offset,
  360.                       int length,
  361.                       int is_read)
  362. {
  363.         char *gpu_vaddr, *cpu_vaddr;
  364.  
  365.         /* Use the unswizzled path if this page isn't affected. */
  366.         if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
  367.                 if (is_read)
  368.                         return slow_shmem_copy(cpu_page, cpu_offset,
  369.                                                gpu_page, gpu_offset, length);
  370.                 else
  371.                         return slow_shmem_copy(gpu_page, gpu_offset,
  372.                                                cpu_page, cpu_offset, length);
  373.         }
  374.  
  375.         gpu_vaddr = kmap(gpu_page);
  376.         cpu_vaddr = kmap(cpu_page);
  377.  
  378.         /* Copy the data, XORing A6 with A17 (1). The user already knows he's
  379.          * XORing with the other bits (A9 for Y, A9 and A10 for X)
  380.          */
  381.         while (length > 0) {
  382.                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
  383.                 int this_length = min(cacheline_end - gpu_offset, length);
  384.                 int swizzled_gpu_offset = gpu_offset ^ 64;
  385.  
  386.                 if (is_read) {
  387.                         memcpy(cpu_vaddr + cpu_offset,
  388.                                gpu_vaddr + swizzled_gpu_offset,
  389.                                this_length);
  390.                 } else {
  391.                         memcpy(gpu_vaddr + swizzled_gpu_offset,
  392.                                cpu_vaddr + cpu_offset,
  393.                                this_length);
  394.                 }
  395.                 cpu_offset += this_length;
  396.                 gpu_offset += this_length;
  397.                 length -= this_length;
  398.         }
  399.  
  400.         kunmap(cpu_page);
  401.         kunmap(gpu_page);
  402. }
  403.  
  404. /**
  405.  * This is the fast shmem pread path, which attempts to copy_from_user directly
  406.  * from the backing pages of the object to the user's address space.  On a
  407.  * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
  408.  */
  409. static int
  410. i915_gem_shmem_pread_fast(struct drm_device *dev,
  411.                           struct drm_i915_gem_object *obj,
  412.                           struct drm_i915_gem_pread *args,
  413.                           struct drm_file *file)
  414. {
  415.         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
  416.         ssize_t remain;
  417.         loff_t offset;
  418.         char __user *user_data;
  419.         int page_offset, page_length;
  420.  
  421.         user_data = (char __user *) (uintptr_t) args->data_ptr;
  422.         remain = args->size;
  423.  
  424.         offset = args->offset;
  425.  
  426.         while (remain > 0) {
  427.                 struct page *page;
  428.                 char *vaddr;
  429.                 int ret;
  430.  
  431.                 /* Operation in this page
  432.                  *
  433.                  * page_offset = offset within page
  434.                  * page_length = bytes to copy for this page
  435.                  */
  436.                 page_offset = offset_in_page(offset);
  437.                 page_length = remain;
  438.                 if ((page_offset + remain) > PAGE_SIZE)
  439.                         page_length = PAGE_SIZE - page_offset;
  440.  
  441.                 page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
  442.                 if (IS_ERR(page))
  443.                         return PTR_ERR(page);
  444.  
  445.                 vaddr = kmap_atomic(page);
  446.                 ret = __copy_to_user_inatomic(user_data,
  447.                                               vaddr + page_offset,
  448.                                               page_length);
  449.                 kunmap_atomic(vaddr);
  450.  
  451.                 mark_page_accessed(page);
  452.                 page_cache_release(page);
  453.                 if (ret)
  454.                         return -EFAULT;
  455.  
  456.                 remain -= page_length;
  457.                 user_data += page_length;
  458.                 offset += page_length;
  459.         }
  460.  
  461.         return 0;
  462. }
  463.  
  464. /**
  465.  * This is the fallback shmem pread path, which allocates temporary storage
  466.  * in kernel space to copy_to_user into outside of the struct_mutex, so we
  467.  * can copy out of the object's backing pages while holding the struct mutex
  468.  * and not take page faults.
  469.  */
  470. static int
  471. i915_gem_shmem_pread_slow(struct drm_device *dev,
  472.                           struct drm_i915_gem_object *obj,
  473.                           struct drm_i915_gem_pread *args,
  474.                           struct drm_file *file)
  475. {
  476.         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
  477.         struct mm_struct *mm = current->mm;
  478.         struct page **user_pages;
  479.         ssize_t remain;
  480.         loff_t offset, pinned_pages, i;
  481.         loff_t first_data_page, last_data_page, num_pages;
  482.         int shmem_page_offset;
  483.         int data_page_index, data_page_offset;
  484.         int page_length;
  485.         int ret;
  486.         uint64_t data_ptr = args->data_ptr;
  487.         int do_bit17_swizzling;
  488.  
  489.         remain = args->size;
  490.  
  491.         /* Pin the user pages containing the data.  We can't fault while
  492.          * holding the struct mutex, yet we want to hold it while
  493.          * dereferencing the user data.
  494.          */
  495.         first_data_page = data_ptr / PAGE_SIZE;
  496.         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
  497.         num_pages = last_data_page - first_data_page + 1;
  498.  
  499.         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
  500.         if (user_pages == NULL)
  501.                 return -ENOMEM;
  502.  
  503.         mutex_unlock(&dev->struct_mutex);
  504.         down_read(&mm->mmap_sem);
  505.         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
  506.                                       num_pages, 1, 0, user_pages, NULL);
  507.         up_read(&mm->mmap_sem);
  508.         mutex_lock(&dev->struct_mutex);
  509.         if (pinned_pages < num_pages) {
  510.                 ret = -EFAULT;
  511.                 goto out;
  512.         }
  513.  
  514.         ret = i915_gem_object_set_cpu_read_domain_range(obj,
  515.                                                         args->offset,
  516.                                                         args->size);
  517.         if (ret)
  518.                 goto out;
  519.  
  520.         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
  521.  
  522.         offset = args->offset;
  523.  
  524.         while (remain > 0) {
  525.                 struct page *page;
  526.  
  527.                 /* Operation in this page
  528.                  *
  529.                  * shmem_page_offset = offset within page in shmem file
  530.                  * data_page_index = page number in get_user_pages return
  531.                  * data_page_offset = offset with data_page_index page.
  532.                  * page_length = bytes to copy for this page
  533.                  */
  534.                 shmem_page_offset = offset_in_page(offset);
  535.                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
  536.                 data_page_offset = offset_in_page(data_ptr);
  537.  
  538.                 page_length = remain;
  539.                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
  540.                         page_length = PAGE_SIZE - shmem_page_offset;
  541.                 if ((data_page_offset + page_length) > PAGE_SIZE)
  542.                         page_length = PAGE_SIZE - data_page_offset;
  543.  
  544.                 page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
  545.                 if (IS_ERR(page)) {
  546.                         ret = PTR_ERR(page);
  547.                         goto out;
  548.                 }
  549.  
  550.                 if (do_bit17_swizzling) {
  551.                         slow_shmem_bit17_copy(page,
  552.                                               shmem_page_offset,
  553.                                               user_pages[data_page_index],
  554.                                               data_page_offset,
  555.                                               page_length,
  556.                                               1);
  557.                 } else {
  558.                         slow_shmem_copy(user_pages[data_page_index],
  559.                                         data_page_offset,
  560.                                         page,
  561.                                         shmem_page_offset,
  562.                                         page_length);
  563.                 }
  564.  
  565.                 mark_page_accessed(page);
  566.                 page_cache_release(page);
  567.  
  568.                 remain -= page_length;
  569.                 data_ptr += page_length;
  570.                 offset += page_length;
  571.         }
  572.  
  573. out:
  574.         for (i = 0; i < pinned_pages; i++) {
  575.                 SetPageDirty(user_pages[i]);
  576.                 mark_page_accessed(user_pages[i]);
  577.                 page_cache_release(user_pages[i]);
  578.         }
  579.         drm_free_large(user_pages);
  580.  
  581.         return ret;
  582. }
  583. #endif
  584.  
  585.  
  586.  
  587.  
  588.  
  589.  
  590.  
  591.  
  592.  
  593.  
  594.  
  595.  
  596.  
  597.  
  598.  
  599.  
  600.  
  601.  
  602.  
  603.  
  604.  
  605.  
  606.  
  607.  
  608.  
  609.  
  610.  
  611.  
  612.  
  613.  
  614.  
  615.  
  616.  
  617.  
  618.  
  619.  
  620.  
  621.  
  622.  
  623.  
  624.  
  625.  
  626.  
  627.  
  628.  
  629.  
  630.  
  631.  
  632.  
  633.  
  634.  
  635.  
  636.  
  637.  
  638.  
  639.  
  640.  
  641.  
  642.  
  643.  
  644.  
  645.  
  646.  
  647.  
  648.  
  649.  
  650.  
  651.  
  652.  
  653.  
  654.  
  655.  
  656.  
  657. static uint32_t
  658. i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
  659. {
  660.         uint32_t gtt_size;
  661.  
  662.         if (INTEL_INFO(dev)->gen >= 4 ||
  663.             tiling_mode == I915_TILING_NONE)
  664.                 return size;
  665.  
  666.         /* Previous chips need a power-of-two fence region when tiling */
  667.         if (INTEL_INFO(dev)->gen == 3)
  668.                 gtt_size = 1024*1024;
  669.         else
  670.                 gtt_size = 512*1024;
  671.  
  672.         while (gtt_size < size)
  673.                 gtt_size <<= 1;
  674.  
  675.         return gtt_size;
  676. }
  677.  
  678. /**
  679.  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
  680.  * @obj: object to check
  681.  *
  682.  * Return the required GTT alignment for an object, taking into account
  683.  * potential fence register mapping.
  684.  */
  685. static uint32_t
  686. i915_gem_get_gtt_alignment(struct drm_device *dev,
  687.                            uint32_t size,
  688.                            int tiling_mode)
  689. {
  690.         /*
  691.          * Minimum alignment is 4k (GTT page size), but might be greater
  692.          * if a fence register is needed for the object.
  693.          */
  694.         if (INTEL_INFO(dev)->gen >= 4 ||
  695.             tiling_mode == I915_TILING_NONE)
  696.                 return 4096;
  697.  
  698.         /*
  699.          * Previous chips need to be aligned to the size of the smallest
  700.          * fence register that can contain the object.
  701.          */
  702.         return i915_gem_get_gtt_size(dev, size, tiling_mode);
  703. }
  704.  
  705. /**
  706.  * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
  707.  *                                       unfenced object
  708.  * @dev: the device
  709.  * @size: size of the object
  710.  * @tiling_mode: tiling mode of the object
  711.  *
  712.  * Return the required GTT alignment for an object, only taking into account
  713.  * unfenced tiled surface requirements.
  714.  */
  715. uint32_t
  716. i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
  717.                                     uint32_t size,
  718.                                     int tiling_mode)
  719. {
  720.         /*
  721.          * Minimum alignment is 4k (GTT page size) for sane hw.
  722.          */
  723.         if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
  724.             tiling_mode == I915_TILING_NONE)
  725.                 return 4096;
  726.  
  727.         /* Previous hardware however needs to be aligned to a power-of-two
  728.          * tile height. The simplest method for determining this is to reuse
  729.          * the power-of-tile object size.
  730.          */
  731.         return i915_gem_get_gtt_size(dev, size, tiling_mode);
  732. }
  733.  
  734.  
  735.  
  736.  
  737.  
  738.  
  739.  
  740.  
  741.  
  742.  
  743.  
  744.  
  745.  
  746.  
  747.  
  748.  
  749.  
  750. static int
  751. i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
  752.                               gfp_t gfpmask)
  753. {
  754.         int page_count, i;
  755.         struct page *page;
  756.  
  757.         /* Get the list of pages out of our struct file.  They'll be pinned
  758.          * at this point until we release them.
  759.          */
  760.         page_count = obj->base.size / PAGE_SIZE;
  761.         BUG_ON(obj->pages != NULL);
  762.     obj->pages = malloc(page_count * sizeof(struct page *));
  763.         if (obj->pages == NULL)
  764.                 return -ENOMEM;
  765.  
  766.  
  767.         for (i = 0; i < page_count; i++) {
  768.         page = (struct page*)AllocPage(); // oh-oh
  769.                 if (IS_ERR(page))
  770.                         goto err_pages;
  771.  
  772.                 obj->pages[i] = page;
  773.         }
  774.  
  775. //   if (obj->tiling_mode != I915_TILING_NONE)
  776. //       i915_gem_object_do_bit_17_swizzle(obj);
  777.  
  778.  
  779.  
  780.         return 0;
  781.  
  782. err_pages:
  783.     while (i--)
  784.         FreePage(obj->pages[i]);
  785.  
  786.     free(obj->pages);
  787.         obj->pages = NULL;
  788.         return PTR_ERR(page);
  789. }
  790.  
  791. static void
  792. i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
  793. {
  794.         int page_count = obj->base.size / PAGE_SIZE;
  795.         int i;
  796.  
  797.     ENTER();
  798.  
  799.         BUG_ON(obj->madv == __I915_MADV_PURGED);
  800.  
  801. //   if (obj->tiling_mode != I915_TILING_NONE)
  802. //       i915_gem_object_save_bit_17_swizzle(obj);
  803.  
  804.         if (obj->madv == I915_MADV_DONTNEED)
  805.                 obj->dirty = 0;
  806.  
  807.         for (i = 0; i < page_count; i++) {
  808.         FreePage(obj->pages[i]);
  809.         }
  810.         obj->dirty = 0;
  811.  
  812.     free(obj->pages);
  813.         obj->pages = NULL;
  814.  
  815.     LEAVE();
  816. }
  817.  
  818. void
  819. i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
  820.                                struct intel_ring_buffer *ring,
  821.                                u32 seqno)
  822. {
  823.         struct drm_device *dev = obj->base.dev;
  824.         struct drm_i915_private *dev_priv = dev->dev_private;
  825.  
  826.         BUG_ON(ring == NULL);
  827.         obj->ring = ring;
  828.  
  829.         /* Add a reference if we're newly entering the active list. */
  830.         if (!obj->active) {
  831.                 drm_gem_object_reference(&obj->base);
  832.                 obj->active = 1;
  833.         }
  834.  
  835.         /* Move from whatever list we were on to the tail of execution. */
  836.         list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
  837.         list_move_tail(&obj->ring_list, &ring->active_list);
  838.  
  839.         obj->last_rendering_seqno = seqno;
  840.         if (obj->fenced_gpu_access) {
  841.                 struct drm_i915_fence_reg *reg;
  842.  
  843.                 BUG_ON(obj->fence_reg == I915_FENCE_REG_NONE);
  844.  
  845.                 obj->last_fenced_seqno = seqno;
  846.                 obj->last_fenced_ring = ring;
  847.  
  848.                 reg = &dev_priv->fence_regs[obj->fence_reg];
  849.                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
  850.         }
  851. }
  852.  
  853. static void
  854. i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
  855. {
  856.         list_del_init(&obj->ring_list);
  857.         obj->last_rendering_seqno = 0;
  858. }
  859.  
  860. static void
  861. i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
  862. {
  863.         struct drm_device *dev = obj->base.dev;
  864.         drm_i915_private_t *dev_priv = dev->dev_private;
  865.  
  866.         BUG_ON(!obj->active);
  867.         list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
  868.  
  869.         i915_gem_object_move_off_active(obj);
  870. }
  871.  
  872.  
  873.  
  874.  
  875.  
  876. /* Immediately discard the backing storage */
  877. static void
  878. i915_gem_object_truncate(struct drm_i915_gem_object *obj)
  879. {
  880.         struct inode *inode;
  881.  
  882.         /* Our goal here is to return as much of the memory as
  883.          * is possible back to the system as we are called from OOM.
  884.          * To do this we must instruct the shmfs to drop all of its
  885.          * backing pages, *now*.
  886.          */
  887.  
  888.         obj->madv = __I915_MADV_PURGED;
  889. }
  890.  
  891. static inline int
  892. i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
  893. {
  894.         return obj->madv == I915_MADV_DONTNEED;
  895. }
  896.  
  897. static void
  898. i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
  899.                                uint32_t flush_domains)
  900. {
  901.         struct drm_i915_gem_object *obj, *next;
  902.  
  903.         list_for_each_entry_safe(obj, next,
  904.                                  &ring->gpu_write_list,
  905.                                  gpu_write_list) {
  906.                 if (obj->base.write_domain & flush_domains) {
  907.                         uint32_t old_write_domain = obj->base.write_domain;
  908.  
  909.                         obj->base.write_domain = 0;
  910.                         list_del_init(&obj->gpu_write_list);
  911.                         i915_gem_object_move_to_active(obj, ring,
  912.                                                        i915_gem_next_request_seqno(ring));
  913.  
  914.                 }
  915.         }
  916. }
  917.  
  918.  
  919.  
  920.  
  921.  
  922.  
  923.  
  924.  
  925.  
  926.  
  927.  
  928.  
  929.  
  930.  
  931.  
  932.  
  933.  
  934.  
  935.  
  936.  
  937.  
  938.  
  939.  
  940.  
  941.  
  942.  
  943.  
  944.  
  945.  
  946.  
  947.  
  948.  
  949.  
  950.  
  951.  
  952.  
  953.  
  954.  
  955.  
  956. /**
  957.  * Ensures that all rendering to the object has completed and the object is
  958.  * safe to unbind from the GTT or access from the CPU.
  959.  */
  960. int
  961. i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
  962. {
  963.         int ret;
  964.  
  965.         /* This function only exists to support waiting for existing rendering,
  966.          * not for emitting required flushes.
  967.          */
  968.         BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
  969.  
  970.         /* If there is rendering queued on the buffer being evicted, wait for
  971.          * it.
  972.          */
  973.         if (obj->active) {
  974. //              ret = i915_wait_request(obj->ring, obj->last_rendering_seqno);
  975. //              if (ret)
  976. //                      return ret;
  977.         }
  978.  
  979.         return 0;
  980. }
  981.  
  982. static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
  983. {
  984.         u32 old_write_domain, old_read_domains;
  985.  
  986.         /* Act a barrier for all accesses through the GTT */
  987.         mb();
  988.  
  989.         /* Force a pagefault for domain tracking on next user access */
  990. //      i915_gem_release_mmap(obj);
  991.  
  992.         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
  993.                 return;
  994.  
  995.         old_read_domains = obj->base.read_domains;
  996.         old_write_domain = obj->base.write_domain;
  997.         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
  998.         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
  999.  
  1000. }
  1001.  
  1002. /**
  1003.  * Unbinds an object from the GTT aperture.
  1004.  */
  1005. int
  1006. i915_gem_object_unbind(struct drm_i915_gem_object *obj)
  1007. {
  1008.         int ret = 0;
  1009.  
  1010.     ENTER();
  1011.         if (obj->gtt_space == NULL)
  1012.                 return 0;
  1013.  
  1014.         if (obj->pin_count != 0) {
  1015.                 DRM_ERROR("Attempting to unbind pinned buffer\n");
  1016.                 return -EINVAL;
  1017.         }
  1018.  
  1019.         ret = i915_gem_object_finish_gpu(obj);
  1020.         if (ret == -ERESTARTSYS)
  1021.                 return ret;
  1022.         /* Continue on if we fail due to EIO, the GPU is hung so we
  1023.          * should be safe and we need to cleanup or else we might
  1024.          * cause memory corruption through use-after-free.
  1025.          */
  1026.  
  1027.         i915_gem_object_finish_gtt(obj);
  1028.  
  1029.         /* Move the object to the CPU domain to ensure that
  1030.          * any possible CPU writes while it's not in the GTT
  1031.          * are flushed when we go to remap it.
  1032.          */
  1033.         if (ret == 0)
  1034.                 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
  1035.         if (ret == -ERESTARTSYS)
  1036.                 return ret;
  1037.         if (ret) {
  1038.                 /* In the event of a disaster, abandon all caches and
  1039.                  * hope for the best.
  1040.                  */
  1041.                 i915_gem_clflush_object(obj);
  1042.                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  1043.         }
  1044.  
  1045.         /* release the fence reg _after_ flushing */
  1046.         ret = i915_gem_object_put_fence(obj);
  1047.         if (ret == -ERESTARTSYS)
  1048.                 return ret;
  1049.  
  1050.  
  1051.         i915_gem_gtt_unbind_object(obj);
  1052.         i915_gem_object_put_pages_gtt(obj);
  1053.  
  1054.         list_del_init(&obj->gtt_list);
  1055.         list_del_init(&obj->mm_list);
  1056.         /* Avoid an unnecessary call to unbind on rebind. */
  1057.         obj->map_and_fenceable = true;
  1058.  
  1059.         drm_mm_put_block(obj->gtt_space);
  1060.         obj->gtt_space = NULL;
  1061.         obj->gtt_offset = 0;
  1062.  
  1063.         if (i915_gem_object_is_purgeable(obj))
  1064.                 i915_gem_object_truncate(obj);
  1065.  
  1066.     LEAVE();
  1067.         return ret;
  1068. }
  1069.  
  1070. int
  1071. i915_gem_flush_ring(struct intel_ring_buffer *ring,
  1072.                     uint32_t invalidate_domains,
  1073.                     uint32_t flush_domains)
  1074. {
  1075.         int ret;
  1076.  
  1077.         if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
  1078.                 return 0;
  1079.  
  1080.  
  1081.         ret = ring->flush(ring, invalidate_domains, flush_domains);
  1082.         if (ret)
  1083.                 return ret;
  1084.  
  1085.         if (flush_domains & I915_GEM_GPU_DOMAINS)
  1086.                 i915_gem_process_flushing_list(ring, flush_domains);
  1087.  
  1088.         return 0;
  1089. }
  1090.  
  1091. static int i915_ring_idle(struct intel_ring_buffer *ring)
  1092. {
  1093.         int ret;
  1094.  
  1095.         if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
  1096.                 return 0;
  1097.  
  1098.         if (!list_empty(&ring->gpu_write_list)) {
  1099.                 ret = i915_gem_flush_ring(ring,
  1100.                                     I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
  1101.                 if (ret)
  1102.                         return ret;
  1103.         }
  1104.  
  1105.         return 0; //i915_wait_request(ring, i915_gem_next_request_seqno(ring));
  1106. }
  1107.  
  1108. int
  1109. i915_gpu_idle(struct drm_device *dev)
  1110. {
  1111.         drm_i915_private_t *dev_priv = dev->dev_private;
  1112.         int ret, i;
  1113.  
  1114.         /* Flush everything onto the inactive list. */
  1115.         for (i = 0; i < I915_NUM_RINGS; i++) {
  1116.                 ret = i915_ring_idle(&dev_priv->ring[i]);
  1117.                 if (ret)
  1118.                         return ret;
  1119.         }
  1120.  
  1121.         return 0;
  1122. }
  1123.  
  1124.  
  1125.  
  1126.  
  1127.  
  1128.  
  1129.  
  1130.  
  1131.  
  1132.  
  1133.  
  1134.  
  1135.  
  1136.  
  1137.  
  1138.  
  1139.  
  1140.  
  1141.  
  1142.  
  1143.  
  1144.  
  1145. static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
  1146. {
  1147.         return i915_seqno_passed(ring->get_seqno(ring), seqno);
  1148. }
  1149.  
  1150. static int
  1151. i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
  1152.                             struct intel_ring_buffer *pipelined)
  1153. {
  1154.         int ret;
  1155.  
  1156.         if (obj->fenced_gpu_access) {
  1157.                 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
  1158.                         ret = i915_gem_flush_ring(obj->last_fenced_ring,
  1159.                                                   0, obj->base.write_domain);
  1160.                         if (ret)
  1161.                                 return ret;
  1162.                 }
  1163.  
  1164.                 obj->fenced_gpu_access = false;
  1165.         }
  1166.  
  1167.         if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
  1168.                 if (!ring_passed_seqno(obj->last_fenced_ring,
  1169.                                        obj->last_fenced_seqno)) {
  1170. //           ret = i915_wait_request(obj->last_fenced_ring,
  1171. //                       obj->last_fenced_seqno);
  1172. //           if (ret)
  1173. //               return ret;
  1174.                 }
  1175.  
  1176.                 obj->last_fenced_seqno = 0;
  1177.                 obj->last_fenced_ring = NULL;
  1178.         }
  1179.  
  1180.         /* Ensure that all CPU reads are completed before installing a fence
  1181.          * and all writes before removing the fence.
  1182.          */
  1183.         if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
  1184.                 mb();
  1185.  
  1186.         return 0;
  1187. }
  1188.  
  1189. int
  1190. i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
  1191. {
  1192.         int ret;
  1193.  
  1194. //   if (obj->tiling_mode)
  1195. //       i915_gem_release_mmap(obj);
  1196.  
  1197.         ret = i915_gem_object_flush_fence(obj, NULL);
  1198.         if (ret)
  1199.                 return ret;
  1200.  
  1201.         if (obj->fence_reg != I915_FENCE_REG_NONE) {
  1202.                 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
  1203.                 i915_gem_clear_fence_reg(obj->base.dev,
  1204.                                          &dev_priv->fence_regs[obj->fence_reg]);
  1205.  
  1206.                 obj->fence_reg = I915_FENCE_REG_NONE;
  1207.         }
  1208.  
  1209.         return 0;
  1210. }
  1211.  
  1212.  
  1213.  
  1214.  
  1215.  
  1216.  
  1217.  
  1218.  
  1219.  
  1220.  
  1221.  
  1222.  
  1223.  
  1224.  
  1225.  
  1226.  
  1227.  
  1228.  
  1229.  
  1230.  
  1231.  
  1232.  
  1233.  
  1234.  
  1235.  
  1236.  
  1237.  
  1238.  
  1239.  
  1240.  
  1241.  
  1242.  
  1243.  
  1244.  
  1245. /**
  1246.  * i915_gem_clear_fence_reg - clear out fence register info
  1247.  * @obj: object to clear
  1248.  *
  1249.  * Zeroes out the fence register itself and clears out the associated
  1250.  * data structures in dev_priv and obj.
  1251.  */
  1252. static void
  1253. i915_gem_clear_fence_reg(struct drm_device *dev,
  1254.              struct drm_i915_fence_reg *reg)
  1255. {
  1256.     drm_i915_private_t *dev_priv = dev->dev_private;
  1257.     uint32_t fence_reg = reg - dev_priv->fence_regs;
  1258.  
  1259.     switch (INTEL_INFO(dev)->gen) {
  1260.     case 7:
  1261.     case 6:
  1262.         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
  1263.         break;
  1264.     case 5:
  1265.     case 4:
  1266.         I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
  1267.         break;
  1268.     case 3:
  1269.         if (fence_reg >= 8)
  1270.             fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
  1271.         else
  1272.     case 2:
  1273.             fence_reg = FENCE_REG_830_0 + fence_reg * 4;
  1274.  
  1275.         I915_WRITE(fence_reg, 0);
  1276.         break;
  1277.     }
  1278.  
  1279.     list_del_init(&reg->lru_list);
  1280.     reg->obj = NULL;
  1281.     reg->setup_seqno = 0;
  1282. }
  1283.  
  1284. /**
  1285.  * Finds free space in the GTT aperture and binds the object there.
  1286.  */
  1287. static int
  1288. i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  1289.                             unsigned alignment,
  1290.                             bool map_and_fenceable)
  1291. {
  1292.         struct drm_device *dev = obj->base.dev;
  1293.         drm_i915_private_t *dev_priv = dev->dev_private;
  1294.         struct drm_mm_node *free_space;
  1295.     gfp_t gfpmask = 0; //__GFP_NORETRY | __GFP_NOWARN;
  1296.         u32 size, fence_size, fence_alignment, unfenced_alignment;
  1297.         bool mappable, fenceable;
  1298.         int ret;
  1299.  
  1300.         if (obj->madv != I915_MADV_WILLNEED) {
  1301.                 DRM_ERROR("Attempting to bind a purgeable object\n");
  1302.                 return -EINVAL;
  1303.         }
  1304.  
  1305.         fence_size = i915_gem_get_gtt_size(dev,
  1306.                                            obj->base.size,
  1307.                                            obj->tiling_mode);
  1308.         fence_alignment = i915_gem_get_gtt_alignment(dev,
  1309.                                                      obj->base.size,
  1310.                                                      obj->tiling_mode);
  1311.         unfenced_alignment =
  1312.                 i915_gem_get_unfenced_gtt_alignment(dev,
  1313.                                                     obj->base.size,
  1314.                                                     obj->tiling_mode);
  1315.  
  1316.         if (alignment == 0)
  1317.                 alignment = map_and_fenceable ? fence_alignment :
  1318.                                                 unfenced_alignment;
  1319.         if (map_and_fenceable && alignment & (fence_alignment - 1)) {
  1320.                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
  1321.                 return -EINVAL;
  1322.         }
  1323.  
  1324.         size = map_and_fenceable ? fence_size : obj->base.size;
  1325.  
  1326.         /* If the object is bigger than the entire aperture, reject it early
  1327.          * before evicting everything in a vain attempt to find space.
  1328.          */
  1329.         if (obj->base.size >
  1330.             (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
  1331.                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
  1332.                 return -E2BIG;
  1333.         }
  1334.  
  1335.  search_free:
  1336.         if (map_and_fenceable)
  1337.                 free_space =
  1338.                         drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
  1339.                                                     size, alignment, 0,
  1340.                                                     dev_priv->mm.gtt_mappable_end,
  1341.                                                     0);
  1342.         else
  1343.                 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
  1344.                                                 size, alignment, 0);
  1345.  
  1346.         if (free_space != NULL) {
  1347.                 if (map_and_fenceable)
  1348.                         obj->gtt_space =
  1349.                                 drm_mm_get_block_range_generic(free_space,
  1350.                                                                size, alignment, 0,
  1351.                                                                dev_priv->mm.gtt_mappable_end,
  1352.                                                                0);
  1353.                 else
  1354.                         obj->gtt_space =
  1355.                                 drm_mm_get_block(free_space, size, alignment);
  1356.         }
  1357.         if (obj->gtt_space == NULL) {
  1358.                 /* If the gtt is empty and we're still having trouble
  1359.                  * fitting our object in, we're out of memory.
  1360.                  */
  1361.         ret = 1; //i915_gem_evict_something(dev, size, alignment,
  1362.                  //         map_and_fenceable);
  1363.                 if (ret)
  1364.                         return ret;
  1365.  
  1366.                 goto search_free;
  1367.         }
  1368.  
  1369.         ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
  1370.         if (ret) {
  1371.                 drm_mm_put_block(obj->gtt_space);
  1372.                 obj->gtt_space = NULL;
  1373. #if 0
  1374.                 if (ret == -ENOMEM) {
  1375.                         /* first try to reclaim some memory by clearing the GTT */
  1376.                         ret = i915_gem_evict_everything(dev, false);
  1377.                         if (ret) {
  1378.                                 /* now try to shrink everyone else */
  1379.                                 if (gfpmask) {
  1380.                                         gfpmask = 0;
  1381.                                         goto search_free;
  1382.                                 }
  1383.  
  1384.                                 return -ENOMEM;
  1385.                         }
  1386.  
  1387.                         goto search_free;
  1388.                 }
  1389. #endif
  1390.                 return ret;
  1391.         }
  1392.  
  1393.         ret = i915_gem_gtt_bind_object(obj);
  1394.         if (ret) {
  1395.         i915_gem_object_put_pages_gtt(obj);
  1396.                 drm_mm_put_block(obj->gtt_space);
  1397.                 obj->gtt_space = NULL;
  1398.  
  1399. //       if (i915_gem_evict_everything(dev, false))
  1400.                         return ret;
  1401.  
  1402. //       goto search_free;
  1403.         }
  1404.  
  1405.         list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
  1406.         list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
  1407.  
  1408.         /* Assert that the object is not currently in any GPU domain. As it
  1409.          * wasn't in the GTT, there shouldn't be any way it could have been in
  1410.          * a GPU cache
  1411.          */
  1412.         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
  1413.         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
  1414.  
  1415.         obj->gtt_offset = obj->gtt_space->start;
  1416.  
  1417.         fenceable =
  1418.                 obj->gtt_space->size == fence_size &&
  1419.                 (obj->gtt_space->start & (fence_alignment - 1)) == 0;
  1420.  
  1421.         mappable =
  1422.                 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
  1423.  
  1424.         obj->map_and_fenceable = mappable && fenceable;
  1425.  
  1426.         return 0;
  1427. }
  1428.  
  1429. void
  1430. i915_gem_clflush_object(struct drm_i915_gem_object *obj)
  1431. {
  1432.         /* If we don't have a page list set up, then we're not pinned
  1433.          * to GPU, and we can ignore the cache flush because it'll happen
  1434.          * again at bind time.
  1435.          */
  1436.         if (obj->pages == NULL)
  1437.                 return;
  1438.  
  1439.         /* If the GPU is snooping the contents of the CPU cache,
  1440.          * we do not need to manually clear the CPU cache lines.  However,
  1441.          * the caches are only snooped when the render cache is
  1442.          * flushed/invalidated.  As we always have to emit invalidations
  1443.          * and flushes when moving into and out of the RENDER domain, correct
  1444.          * snooping behaviour occurs naturally as the result of our domain
  1445.          * tracking.
  1446.          */
  1447.         if (obj->cache_level != I915_CACHE_NONE)
  1448.                 return;
  1449.  
  1450.      if(obj->mapped != NULL)
  1451.      {
  1452.         uint8_t *page_virtual;
  1453.         unsigned int i;
  1454.  
  1455.         page_virtual = obj->mapped;
  1456.         asm volatile("mfence");
  1457.         for (i = 0; i < obj->base.size; i += x86_clflush_size)
  1458.             clflush(page_virtual + i);
  1459.         asm volatile("mfence");
  1460.      }
  1461.      else
  1462.      {
  1463.         uint8_t *page_virtual;
  1464.         unsigned int i;
  1465.         page_virtual = AllocKernelSpace(obj->base.size);
  1466.         if(page_virtual != NULL)
  1467.         {
  1468.             u32_t *src, *dst;
  1469.             u32 count;
  1470.  
  1471. #define page_tabs  0xFDC00000      /* really dirty hack */
  1472.  
  1473.             src =  (u32_t*)obj->pages;
  1474.             dst =  &((u32_t*)page_tabs)[(u32_t)page_virtual >> 12];
  1475.             count = obj->base.size/4096;
  1476.  
  1477.             while(count--)
  1478.             {
  1479.                 *dst++ = (0xFFFFF000 & *src++) | 0x001 ;
  1480.             };
  1481.  
  1482.             asm volatile("mfence");
  1483.             for (i = 0; i < obj->base.size; i += x86_clflush_size)
  1484.                 clflush(page_virtual + i);
  1485.             asm volatile("mfence");
  1486.             FreeKernelSpace(page_virtual);
  1487.         }
  1488.         else
  1489.         {
  1490.             asm volatile (
  1491.             "mfence         \n"
  1492.             "wbinvd         \n"                 /* this is really ugly  */
  1493.             "mfence");
  1494.         }
  1495.      }
  1496. }
  1497.  
  1498. /** Flushes any GPU write domain for the object if it's dirty. */
  1499. static int
  1500. i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
  1501. {
  1502.         if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
  1503.                 return 0;
  1504.  
  1505.         /* Queue the GPU write cache flushing we need. */
  1506.         return i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
  1507. }
  1508.  
  1509. /** Flushes the GTT write domain for the object if it's dirty. */
  1510. static void
  1511. i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
  1512. {
  1513.         uint32_t old_write_domain;
  1514.  
  1515.         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
  1516.                 return;
  1517.  
  1518.         /* No actual flushing is required for the GTT write domain.  Writes
  1519.          * to it immediately go to main memory as far as we know, so there's
  1520.          * no chipset flush.  It also doesn't land in render cache.
  1521.          *
  1522.          * However, we do have to enforce the order so that all writes through
  1523.          * the GTT land before any writes to the device, such as updates to
  1524.          * the GATT itself.
  1525.          */
  1526.         wmb();
  1527.  
  1528.         old_write_domain = obj->base.write_domain;
  1529.         obj->base.write_domain = 0;
  1530.  
  1531. }
  1532.  
  1533. /** Flushes the CPU write domain for the object if it's dirty. */
  1534. static void
  1535. i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
  1536. {
  1537.         uint32_t old_write_domain;
  1538.  
  1539.         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
  1540.                 return;
  1541.  
  1542.         i915_gem_clflush_object(obj);
  1543.         intel_gtt_chipset_flush();
  1544.         old_write_domain = obj->base.write_domain;
  1545.         obj->base.write_domain = 0;
  1546.  
  1547. }
  1548.  
  1549. /**
  1550.  * Moves a single object to the GTT read, and possibly write domain.
  1551.  *
  1552.  * This function returns when the move is complete, including waiting on
  1553.  * flushes to occur.
  1554.  */
  1555. int
  1556. i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
  1557. {
  1558.         uint32_t old_write_domain, old_read_domains;
  1559.         int ret;
  1560.  
  1561.         /* Not valid to be called on unbound objects. */
  1562.         if (obj->gtt_space == NULL)
  1563.                 return -EINVAL;
  1564.  
  1565.         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
  1566.                 return 0;
  1567.  
  1568.         ret = i915_gem_object_flush_gpu_write_domain(obj);
  1569.         if (ret)
  1570.                 return ret;
  1571.  
  1572.         if (obj->pending_gpu_write || write) {
  1573.                 ret = i915_gem_object_wait_rendering(obj);
  1574.                 if (ret)
  1575.                         return ret;
  1576.         }
  1577.  
  1578.         i915_gem_object_flush_cpu_write_domain(obj);
  1579.  
  1580.         old_write_domain = obj->base.write_domain;
  1581.         old_read_domains = obj->base.read_domains;
  1582.  
  1583.         /* It should now be out of any other write domains, and we can update
  1584.          * the domain values for our changes.
  1585.          */
  1586.         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
  1587.         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
  1588.         if (write) {
  1589.                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
  1590.                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
  1591.                 obj->dirty = 1;
  1592.         }
  1593.  
  1594.         return 0;
  1595. }
  1596.  
  1597. #if 0
  1598. int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
  1599.                                     enum i915_cache_level cache_level)
  1600. {
  1601.         int ret;
  1602.  
  1603.         if (obj->cache_level == cache_level)
  1604.                 return 0;
  1605.  
  1606.         if (obj->pin_count) {
  1607.                 DRM_DEBUG("can not change the cache level of pinned objects\n");
  1608.                 return -EBUSY;
  1609.         }
  1610.  
  1611.         if (obj->gtt_space) {
  1612.                 ret = i915_gem_object_finish_gpu(obj);
  1613.                 if (ret)
  1614.                         return ret;
  1615.  
  1616.                 i915_gem_object_finish_gtt(obj);
  1617.  
  1618.                 /* Before SandyBridge, you could not use tiling or fence
  1619.                  * registers with snooped memory, so relinquish any fences
  1620.                  * currently pointing to our region in the aperture.
  1621.                  */
  1622.                 if (INTEL_INFO(obj->base.dev)->gen < 6) {
  1623.                         ret = i915_gem_object_put_fence(obj);
  1624.                         if (ret)
  1625.                                 return ret;
  1626.                 }
  1627.  
  1628.                 i915_gem_gtt_rebind_object(obj, cache_level);
  1629.         }
  1630.  
  1631.         if (cache_level == I915_CACHE_NONE) {
  1632.                 u32 old_read_domains, old_write_domain;
  1633.  
  1634.                 /* If we're coming from LLC cached, then we haven't
  1635.                  * actually been tracking whether the data is in the
  1636.                  * CPU cache or not, since we only allow one bit set
  1637.                  * in obj->write_domain and have been skipping the clflushes.
  1638.                  * Just set it to the CPU cache for now.
  1639.                  */
  1640.                 WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
  1641.                 WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
  1642.  
  1643.                 old_read_domains = obj->base.read_domains;
  1644.                 old_write_domain = obj->base.write_domain;
  1645.  
  1646.                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  1647.                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  1648.  
  1649.     }
  1650.  
  1651.         obj->cache_level = cache_level;
  1652.         return 0;
  1653. }
  1654. #endif
  1655.  
  1656. /*
  1657.  * Prepare buffer for display plane (scanout, cursors, etc).
  1658.  * Can be called from an uninterruptible phase (modesetting) and allows
  1659.  * any flushes to be pipelined (for pageflips).
  1660.  *
  1661.  * For the display plane, we want to be in the GTT but out of any write
  1662.  * domains. So in many ways this looks like set_to_gtt_domain() apart from the
  1663.  * ability to pipeline the waits, pinning and any additional subtleties
  1664.  * that may differentiate the display plane from ordinary buffers.
  1665.  */
  1666. int
  1667. i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
  1668.                                      u32 alignment,
  1669.                                      struct intel_ring_buffer *pipelined)
  1670. {
  1671.         u32 old_read_domains, old_write_domain;
  1672.         int ret;
  1673.  
  1674.         ret = i915_gem_object_flush_gpu_write_domain(obj);
  1675.         if (ret)
  1676.                 return ret;
  1677.  
  1678.         if (pipelined != obj->ring) {
  1679.                 ret = i915_gem_object_wait_rendering(obj);
  1680.                 if (ret == -ERESTARTSYS)
  1681.                         return ret;
  1682.         }
  1683.  
  1684.         /* The display engine is not coherent with the LLC cache on gen6.  As
  1685.          * a result, we make sure that the pinning that is about to occur is
  1686.          * done with uncached PTEs. This is lowest common denominator for all
  1687.          * chipsets.
  1688.          *
  1689.          * However for gen6+, we could do better by using the GFDT bit instead
  1690.          * of uncaching, which would allow us to flush all the LLC-cached data
  1691.          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
  1692.          */
  1693. //   ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
  1694. //   if (ret)
  1695. //       return ret;
  1696.  
  1697.         /* As the user may map the buffer once pinned in the display plane
  1698.          * (e.g. libkms for the bootup splash), we have to ensure that we
  1699.          * always use map_and_fenceable for all scanout buffers.
  1700.          */
  1701.         ret = i915_gem_object_pin(obj, alignment, true);
  1702.         if (ret)
  1703.                 return ret;
  1704.  
  1705.         i915_gem_object_flush_cpu_write_domain(obj);
  1706.  
  1707.         old_write_domain = obj->base.write_domain;
  1708.         old_read_domains = obj->base.read_domains;
  1709.  
  1710.         /* It should now be out of any other write domains, and we can update
  1711.          * the domain values for our changes.
  1712.          */
  1713.         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
  1714.         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
  1715.  
  1716.  
  1717.         return 0;
  1718. }
  1719.  
  1720. int
  1721. i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
  1722. {
  1723.         int ret;
  1724.  
  1725.         if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
  1726.                 return 0;
  1727.  
  1728.         if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
  1729.                 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
  1730.                 if (ret)
  1731.                         return ret;
  1732.         }
  1733.  
  1734.         /* Ensure that we invalidate the GPU's caches and TLBs. */
  1735.         obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
  1736.  
  1737.         return i915_gem_object_wait_rendering(obj);
  1738. }
  1739.  
  1740. /**
  1741.  * Moves a single object to the CPU read, and possibly write domain.
  1742.  *
  1743.  * This function returns when the move is complete, including waiting on
  1744.  * flushes to occur.
  1745.  */
  1746. static int
  1747. i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
  1748. {
  1749.         uint32_t old_write_domain, old_read_domains;
  1750.         int ret;
  1751.  
  1752.         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
  1753.                 return 0;
  1754.  
  1755.         ret = i915_gem_object_flush_gpu_write_domain(obj);
  1756.         if (ret)
  1757.                 return ret;
  1758.  
  1759.         ret = i915_gem_object_wait_rendering(obj);
  1760.         if (ret)
  1761.                 return ret;
  1762.  
  1763.         i915_gem_object_flush_gtt_write_domain(obj);
  1764.  
  1765.         /* If we have a partially-valid cache of the object in the CPU,
  1766.          * finish invalidating it and free the per-page flags.
  1767.          */
  1768.         i915_gem_object_set_to_full_cpu_read_domain(obj);
  1769.  
  1770.         old_write_domain = obj->base.write_domain;
  1771.         old_read_domains = obj->base.read_domains;
  1772.  
  1773.         /* Flush the CPU cache if it's still invalid. */
  1774.         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
  1775.                 i915_gem_clflush_object(obj);
  1776.  
  1777.                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
  1778.         }
  1779.  
  1780.         /* It should now be out of any other write domains, and we can update
  1781.          * the domain values for our changes.
  1782.          */
  1783.         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
  1784.  
  1785.         /* If we're writing through the CPU, then the GPU read domains will
  1786.          * need to be invalidated at next use.
  1787.          */
  1788.         if (write) {
  1789.                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  1790.                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  1791.         }
  1792.  
  1793.  
  1794.         return 0;
  1795. }
  1796.  
  1797. /**
  1798.  * Moves the object from a partially CPU read to a full one.
  1799.  *
  1800.  * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
  1801.  * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
  1802.  */
  1803. static void
  1804. i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj)
  1805. {
  1806.         if (!obj->page_cpu_valid)
  1807.                 return;
  1808.  
  1809.         /* If we're partially in the CPU read domain, finish moving it in.
  1810.          */
  1811.         if (obj->base.read_domains & I915_GEM_DOMAIN_CPU) {
  1812.         }
  1813.  
  1814.         /* Free the page_cpu_valid mappings which are now stale, whether
  1815.          * or not we've got I915_GEM_DOMAIN_CPU.
  1816.          */
  1817.         kfree(obj->page_cpu_valid);
  1818.         obj->page_cpu_valid = NULL;
  1819. }
  1820.  
  1821.  
  1822.  
  1823.  
  1824.  
  1825.  
  1826.  
  1827.  
  1828.  
  1829.  
  1830.  
  1831.  
  1832.  
  1833.  
  1834.  
  1835.  
  1836.  
  1837.  
  1838.  
  1839.  
  1840.  
  1841.  
  1842.  
  1843.  
  1844. int
  1845. i915_gem_object_pin(struct drm_i915_gem_object *obj,
  1846.                     uint32_t alignment,
  1847.                     bool map_and_fenceable)
  1848. {
  1849.         struct drm_device *dev = obj->base.dev;
  1850.         struct drm_i915_private *dev_priv = dev->dev_private;
  1851.         int ret;
  1852.  
  1853.         BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
  1854.  
  1855. #if 0
  1856.         if (obj->gtt_space != NULL) {
  1857.                 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
  1858.                     (map_and_fenceable && !obj->map_and_fenceable)) {
  1859.                         WARN(obj->pin_count,
  1860.                              "bo is already pinned with incorrect alignment:"
  1861.                              " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
  1862.                              " obj->map_and_fenceable=%d\n",
  1863.                              obj->gtt_offset, alignment,
  1864.                              map_and_fenceable,
  1865.                              obj->map_and_fenceable);
  1866.                         ret = i915_gem_object_unbind(obj);
  1867.                         if (ret)
  1868.                                 return ret;
  1869.                 }
  1870.         }
  1871. #endif
  1872.  
  1873.         if (obj->gtt_space == NULL) {
  1874.                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
  1875.                                                   map_and_fenceable);
  1876.                 if (ret)
  1877.                         return ret;
  1878.         }
  1879.  
  1880.         if (obj->pin_count++ == 0) {
  1881.                 if (!obj->active)
  1882.                         list_move_tail(&obj->mm_list,
  1883.                                        &dev_priv->mm.pinned_list);
  1884.         }
  1885.         obj->pin_mappable |= map_and_fenceable;
  1886.  
  1887.         return 0;
  1888. }
  1889.  
  1890. void
  1891. i915_gem_object_unpin(struct drm_i915_gem_object *obj)
  1892. {
  1893.         struct drm_device *dev = obj->base.dev;
  1894.         drm_i915_private_t *dev_priv = dev->dev_private;
  1895.  
  1896.         BUG_ON(obj->pin_count == 0);
  1897.         BUG_ON(obj->gtt_space == NULL);
  1898.  
  1899.         if (--obj->pin_count == 0) {
  1900.                 if (!obj->active)
  1901.                         list_move_tail(&obj->mm_list,
  1902.                                        &dev_priv->mm.inactive_list);
  1903.                 obj->pin_mappable = false;
  1904.         }
  1905. }
  1906.  
  1907.  
  1908.  
  1909.  
  1910.  
  1911.  
  1912.  
  1913.  
  1914.  
  1915.  
  1916.  
  1917.  
  1918.  
  1919.  
  1920.  
  1921.  
  1922.  
  1923.  
  1924.  
  1925.  
  1926.  
  1927.  
  1928.  
  1929.  
  1930.  
  1931.  
  1932.  
  1933.  
  1934.  
  1935.  
  1936.  
  1937.  
  1938. struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
  1939.                                                   size_t size)
  1940. {
  1941.         struct drm_i915_private *dev_priv = dev->dev_private;
  1942.         struct drm_i915_gem_object *obj;
  1943.  
  1944.         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
  1945.         if (obj == NULL)
  1946.                 return NULL;
  1947.  
  1948.         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
  1949.                 kfree(obj);
  1950.                 return NULL;
  1951.         }
  1952.  
  1953.  
  1954.         i915_gem_info_add_obj(dev_priv, size);
  1955.  
  1956.         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  1957.         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
  1958.  
  1959.         if (IS_GEN6(dev) || IS_GEN7(dev)) {
  1960.                 /* On Gen6, we can have the GPU use the LLC (the CPU
  1961.                  * cache) for about a 10% performance improvement
  1962.                  * compared to uncached.  Graphics requests other than
  1963.                  * display scanout are coherent with the CPU in
  1964.                  * accessing this cache.  This means in this mode we
  1965.                  * don't need to clflush on the CPU side, and on the
  1966.                  * GPU side we only need to flush internal caches to
  1967.                  * get data visible to the CPU.
  1968.                  *
  1969.                  * However, we maintain the display planes as UC, and so
  1970.                  * need to rebind when first used as such.
  1971.                  */
  1972.                 obj->cache_level = I915_CACHE_LLC;
  1973.         } else
  1974.                 obj->cache_level = I915_CACHE_NONE;
  1975.  
  1976.         obj->base.driver_private = NULL;
  1977.         obj->fence_reg = I915_FENCE_REG_NONE;
  1978.         INIT_LIST_HEAD(&obj->mm_list);
  1979.         INIT_LIST_HEAD(&obj->gtt_list);
  1980.         INIT_LIST_HEAD(&obj->ring_list);
  1981.         INIT_LIST_HEAD(&obj->exec_list);
  1982.         INIT_LIST_HEAD(&obj->gpu_write_list);
  1983.         obj->madv = I915_MADV_WILLNEED;
  1984.         /* Avoid an unnecessary call to unbind on the first bind. */
  1985.         obj->map_and_fenceable = true;
  1986.  
  1987.         return obj;
  1988. }
  1989.  
  1990. int i915_gem_init_object(struct drm_gem_object *obj)
  1991. {
  1992.         BUG();
  1993.  
  1994.         return 0;
  1995. }
  1996.  
  1997. static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
  1998. {
  1999.         struct drm_device *dev = obj->base.dev;
  2000.         drm_i915_private_t *dev_priv = dev->dev_private;
  2001.         int ret;
  2002.  
  2003.     ENTER();
  2004.  
  2005.         ret = i915_gem_object_unbind(obj);
  2006.         if (ret == -ERESTARTSYS) {
  2007.                 list_move(&obj->mm_list,
  2008.                           &dev_priv->mm.deferred_free_list);
  2009.                 return;
  2010.         }
  2011.  
  2012.  
  2013. //      if (obj->base.map_list.map)
  2014. //              drm_gem_free_mmap_offset(&obj->base);
  2015.  
  2016.         drm_gem_object_release(&obj->base);
  2017.         i915_gem_info_remove_obj(dev_priv, obj->base.size);
  2018.  
  2019.         kfree(obj->page_cpu_valid);
  2020.         kfree(obj->bit_17);
  2021.         kfree(obj);
  2022.     LEAVE();
  2023. }
  2024.  
  2025. void i915_gem_free_object(struct drm_gem_object *gem_obj)
  2026. {
  2027.         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
  2028.         struct drm_device *dev = obj->base.dev;
  2029.  
  2030.     ENTER();
  2031.         while (obj->pin_count > 0)
  2032.                 i915_gem_object_unpin(obj);
  2033.  
  2034. //      if (obj->phys_obj)
  2035. //              i915_gem_detach_phys_object(dev, obj);
  2036.  
  2037.         i915_gem_free_object_tail(obj);
  2038.     LEAVE();
  2039. }
  2040.  
  2041.  
  2042.  
  2043.  
  2044.  
  2045.  
  2046.  
  2047.  
  2048.  
  2049.  
  2050.  
  2051.  
  2052. int
  2053. i915_gem_init_ringbuffer(struct drm_device *dev)
  2054. {
  2055.         drm_i915_private_t *dev_priv = dev->dev_private;
  2056.         int ret;
  2057.     ENTER();
  2058.         ret = intel_init_render_ring_buffer(dev);
  2059.         if (ret)
  2060.                 return ret;
  2061.  
  2062.     if (HAS_BSD(dev)) {
  2063.                 ret = intel_init_bsd_ring_buffer(dev);
  2064.                 if (ret)
  2065.                         goto cleanup_render_ring;
  2066.         }
  2067.  
  2068.         if (HAS_BLT(dev)) {
  2069.                 ret = intel_init_blt_ring_buffer(dev);
  2070.                 if (ret)
  2071.                         goto cleanup_bsd_ring;
  2072.         }
  2073.  
  2074.         dev_priv->next_seqno = 1;
  2075.     LEAVE();
  2076.         return 0;
  2077.  
  2078. cleanup_bsd_ring:
  2079.         intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
  2080. cleanup_render_ring:
  2081.         intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
  2082.         return ret;
  2083. }
  2084.  
  2085. #if 0
  2086. void
  2087. i915_gem_cleanup_ringbuffer(struct drm_device *dev)
  2088. {
  2089.         drm_i915_private_t *dev_priv = dev->dev_private;
  2090.         int i;
  2091.  
  2092.         for (i = 0; i < I915_NUM_RINGS; i++)
  2093.                 intel_cleanup_ring_buffer(&dev_priv->ring[i]);
  2094. }
  2095.  
  2096. int
  2097. i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
  2098.                        struct drm_file *file_priv)
  2099. {
  2100.         drm_i915_private_t *dev_priv = dev->dev_private;
  2101.         int ret, i;
  2102.  
  2103.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  2104.                 return 0;
  2105.  
  2106.         if (atomic_read(&dev_priv->mm.wedged)) {
  2107.                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
  2108.                 atomic_set(&dev_priv->mm.wedged, 0);
  2109.         }
  2110.  
  2111.         mutex_lock(&dev->struct_mutex);
  2112.         dev_priv->mm.suspended = 0;
  2113.  
  2114.         ret = i915_gem_init_ringbuffer(dev);
  2115.         if (ret != 0) {
  2116.                 mutex_unlock(&dev->struct_mutex);
  2117.                 return ret;
  2118.         }
  2119.  
  2120.         BUG_ON(!list_empty(&dev_priv->mm.active_list));
  2121.         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
  2122.         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
  2123.         for (i = 0; i < I915_NUM_RINGS; i++) {
  2124.                 BUG_ON(!list_empty(&dev_priv->ring[i].active_list));
  2125.                 BUG_ON(!list_empty(&dev_priv->ring[i].request_list));
  2126.         }
  2127.         mutex_unlock(&dev->struct_mutex);
  2128.  
  2129.         ret = drm_irq_install(dev);
  2130.         if (ret)
  2131.                 goto cleanup_ringbuffer;
  2132.  
  2133.         return 0;
  2134.  
  2135. cleanup_ringbuffer:
  2136.         mutex_lock(&dev->struct_mutex);
  2137.         i915_gem_cleanup_ringbuffer(dev);
  2138.         dev_priv->mm.suspended = 1;
  2139.         mutex_unlock(&dev->struct_mutex);
  2140.  
  2141.         return ret;
  2142. }
  2143.  
  2144. int
  2145. i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
  2146.                        struct drm_file *file_priv)
  2147. {
  2148.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  2149.                 return 0;
  2150.  
  2151.         drm_irq_uninstall(dev);
  2152.         return i915_gem_idle(dev);
  2153. }
  2154.  
  2155. void
  2156. i915_gem_lastclose(struct drm_device *dev)
  2157. {
  2158.         int ret;
  2159.  
  2160.         if (drm_core_check_feature(dev, DRIVER_MODESET))
  2161.                 return;
  2162.  
  2163.         ret = i915_gem_idle(dev);
  2164.         if (ret)
  2165.                 DRM_ERROR("failed to idle hardware: %d\n", ret);
  2166. }
  2167. #endif
  2168.  
  2169. static void
  2170. init_ring_lists(struct intel_ring_buffer *ring)
  2171. {
  2172.     INIT_LIST_HEAD(&ring->active_list);
  2173.     INIT_LIST_HEAD(&ring->request_list);
  2174.     INIT_LIST_HEAD(&ring->gpu_write_list);
  2175. }
  2176.  
  2177. void
  2178. i915_gem_load(struct drm_device *dev)
  2179. {
  2180.     int i;
  2181.     drm_i915_private_t *dev_priv = dev->dev_private;
  2182.  
  2183.     INIT_LIST_HEAD(&dev_priv->mm.active_list);
  2184.     INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
  2185.     INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
  2186.     INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
  2187.     INIT_LIST_HEAD(&dev_priv->mm.fence_list);
  2188.     INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
  2189.     INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
  2190.     for (i = 0; i < I915_NUM_RINGS; i++)
  2191.         init_ring_lists(&dev_priv->ring[i]);
  2192.         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
  2193.         INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
  2194.  
  2195.     /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
  2196.     if (IS_GEN3(dev)) {
  2197.         u32 tmp = I915_READ(MI_ARB_STATE);
  2198.         if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
  2199.             /* arb state is a masked write, so set bit + bit in mask */
  2200.             tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
  2201.             I915_WRITE(MI_ARB_STATE, tmp);
  2202.         }
  2203.     }
  2204.  
  2205.     dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
  2206.  
  2207.     if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
  2208.         dev_priv->num_fence_regs = 16;
  2209.     else
  2210.         dev_priv->num_fence_regs = 8;
  2211.  
  2212.     /* Initialize fence registers to zero */
  2213.     for (i = 0; i < dev_priv->num_fence_regs; i++) {
  2214.         i915_gem_clear_fence_reg(dev, &dev_priv->fence_regs[i]);
  2215.     }
  2216.  
  2217.     i915_gem_detect_bit_6_swizzle(dev);
  2218.  
  2219.     dev_priv->mm.interruptible = true;
  2220.  
  2221. //    dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
  2222. //    dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
  2223. //    register_shrinker(&dev_priv->mm.inactive_shrinker);
  2224. }
  2225.  
  2226.  
  2227.  
  2228.