Subversion Repositories Kolibri OS

Rev

Rev 3746 | Rev 4104 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright © 2010 Daniel Vetter
  3.  *
  4.  * Permission is hereby granted, free of charge, to any person obtaining a
  5.  * copy of this software and associated documentation files (the "Software"),
  6.  * to deal in the Software without restriction, including without limitation
  7.  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8.  * and/or sell copies of the Software, and to permit persons to whom the
  9.  * Software is furnished to do so, subject to the following conditions:
  10.  *
  11.  * The above copyright notice and this permission notice (including the next
  12.  * paragraph) shall be included in all copies or substantial portions of the
  13.  * Software.
  14.  *
  15.  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16.  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17.  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18.  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19.  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20.  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21.  * IN THE SOFTWARE.
  22.  *
  23.  */
  24.  
  25.  
  26. #define AGP_NORMAL_MEMORY 0
  27.  
  28. #define AGP_USER_TYPES (1 << 16)
  29. #define AGP_USER_MEMORY (AGP_USER_TYPES)
  30. #define AGP_USER_CACHED_MEMORY (AGP_USER_TYPES + 1)
  31.  
  32. #include <drm/drmP.h>
  33. #include <drm/i915_drm.h>
  34. #include "i915_drv.h"
  35. #include "i915_trace.h"
  36. #include "intel_drv.h"
  37.  
  38. typedef uint32_t gen6_gtt_pte_t;
  39.  
  40. /* PPGTT stuff */
  41. #define GEN6_GTT_ADDR_ENCODE(addr)      ((addr) | (((addr) >> 28) & 0xff0))
  42.  
  43. #define GEN6_PDE_VALID                  (1 << 0)
  44. /* gen6+ has bit 11-4 for physical addr bit 39-32 */
  45. #define GEN6_PDE_ADDR_ENCODE(addr)      GEN6_GTT_ADDR_ENCODE(addr)
  46.  
  47. #define GEN6_PTE_VALID                  (1 << 0)
  48. #define GEN6_PTE_UNCACHED               (1 << 1)
  49. #define HSW_PTE_UNCACHED                (0)
  50. #define GEN6_PTE_CACHE_LLC              (2 << 1)
  51. #define GEN6_PTE_CACHE_LLC_MLC          (3 << 1)
  52. #define GEN6_PTE_ADDR_ENCODE(addr)      GEN6_GTT_ADDR_ENCODE(addr)
  53.  
  54. static inline gen6_gtt_pte_t gen6_pte_encode(struct drm_device *dev,
  55.                                    dma_addr_t addr,
  56.                                    enum i915_cache_level level)
  57. {
  58.         gen6_gtt_pte_t pte = GEN6_PTE_VALID;
  59.         pte |= GEN6_PTE_ADDR_ENCODE(addr);
  60.  
  61.         switch (level) {
  62.         case I915_CACHE_LLC_MLC:
  63.                 /* Haswell doesn't set L3 this way */
  64.                 if (IS_HASWELL(dev))
  65.                         pte |= GEN6_PTE_CACHE_LLC;
  66.                 else
  67.                         pte |= GEN6_PTE_CACHE_LLC_MLC;
  68.                 break;
  69.         case I915_CACHE_LLC:
  70.                 pte |= GEN6_PTE_CACHE_LLC;
  71.                 break;
  72.         case I915_CACHE_NONE:
  73.                 if (IS_HASWELL(dev))
  74.                         pte |= HSW_PTE_UNCACHED;
  75.                 else
  76.                         pte |= GEN6_PTE_UNCACHED;
  77.                 break;
  78.         default:
  79.                 BUG();
  80.         }
  81.  
  82.         return pte;
  83. }
  84.  
  85. static int gen6_ppgtt_enable(struct drm_device *dev)
  86. {
  87.         drm_i915_private_t *dev_priv = dev->dev_private;
  88.         uint32_t pd_offset;
  89.         struct intel_ring_buffer *ring;
  90.         struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
  91.         gen6_gtt_pte_t __iomem *pd_addr;
  92.         uint32_t pd_entry;
  93.         int i;
  94.  
  95.         pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm +
  96.                 ppgtt->pd_offset / sizeof(gen6_gtt_pte_t);
  97.         for (i = 0; i < ppgtt->num_pd_entries; i++) {
  98.                 dma_addr_t pt_addr;
  99.  
  100.                 pt_addr = ppgtt->pt_dma_addr[i];
  101.                 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
  102.                 pd_entry |= GEN6_PDE_VALID;
  103.  
  104.                 writel(pd_entry, pd_addr + i);
  105.         }
  106.         readl(pd_addr);
  107.  
  108.         pd_offset = ppgtt->pd_offset;
  109.         pd_offset /= 64; /* in cachelines, */
  110.         pd_offset <<= 16;
  111.  
  112.         if (INTEL_INFO(dev)->gen == 6) {
  113.                 uint32_t ecochk, gab_ctl, ecobits;
  114.  
  115.                 ecobits = I915_READ(GAC_ECO_BITS);
  116.                 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
  117.                                          ECOBITS_PPGTT_CACHE64B);
  118.  
  119.                 gab_ctl = I915_READ(GAB_CTL);
  120.                 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
  121.  
  122.                 ecochk = I915_READ(GAM_ECOCHK);
  123.                 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT |
  124.                                        ECOCHK_PPGTT_CACHE64B);
  125.                 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
  126.         } else if (INTEL_INFO(dev)->gen >= 7) {
  127.                 uint32_t ecochk, ecobits;
  128.  
  129.                 ecobits = I915_READ(GAC_ECO_BITS);
  130.                 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
  131.  
  132.                 ecochk = I915_READ(GAM_ECOCHK);
  133.                 if (IS_HASWELL(dev)) {
  134.                         ecochk |= ECOCHK_PPGTT_WB_HSW;
  135.                 } else {
  136.                         ecochk |= ECOCHK_PPGTT_LLC_IVB;
  137.                         ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
  138.                 }
  139.                 I915_WRITE(GAM_ECOCHK, ecochk);
  140.                 /* GFX_MODE is per-ring on gen7+ */
  141.         }
  142.  
  143.         for_each_ring(ring, dev_priv, i) {
  144.                 if (INTEL_INFO(dev)->gen >= 7)
  145.                         I915_WRITE(RING_MODE_GEN7(ring),
  146.                                    _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
  147.  
  148.                 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
  149.                 I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
  150.         }
  151.         return 0;
  152. }
  153.  
  154. /* PPGTT support for Sandybdrige/Gen6 and later */
  155. static void gen6_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt,
  156.                                    unsigned first_entry,
  157.                                    unsigned num_entries)
  158. {
  159.         gen6_gtt_pte_t *pt_vaddr, scratch_pte;
  160.         unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
  161.         unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
  162.         unsigned last_pte, i;
  163.  
  164.         scratch_pte = gen6_pte_encode(ppgtt->dev,
  165.                                       ppgtt->scratch_page_dma_addr,
  166.                                  I915_CACHE_LLC);
  167.  
  168.     pt_vaddr = AllocKernelSpace(4096);
  169.  
  170.     if(pt_vaddr == NULL)
  171.         return;
  172.  
  173.         while (num_entries) {
  174.             last_pte = first_pte + num_entries;
  175.             if (last_pte > I915_PPGTT_PT_ENTRIES)
  176.                 last_pte = I915_PPGTT_PT_ENTRIES;
  177.  
  178.             MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3);
  179.  
  180.             for (i = first_pte; i < last_pte; i++)
  181.                 pt_vaddr[i] = scratch_pte;
  182.  
  183.             num_entries -= last_pte - first_pte;
  184.             first_pte = 0;
  185.             act_pt++;
  186.     };
  187.  
  188.     FreeKernelSpace(pt_vaddr);
  189. }
  190.  
  191. static void gen6_ppgtt_insert_entries(struct i915_hw_ppgtt *ppgtt,
  192.                                       struct sg_table *pages,
  193.                                       unsigned first_entry,
  194.                                       enum i915_cache_level cache_level)
  195. {
  196.         gen6_gtt_pte_t *pt_vaddr;
  197.         unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
  198.         unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES;
  199.         struct sg_page_iter sg_iter;
  200.         dma_addr_t page_addr;
  201.  
  202.  
  203.     pt_vaddr = AllocKernelSpace(4096);
  204.  
  205.     if(pt_vaddr == NULL)
  206.         return;
  207.  
  208.     MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3);
  209.         for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
  210.                 dma_addr_t page_addr;
  211.  
  212.                 page_addr = sg_page_iter_dma_address(&sg_iter);
  213.                 pt_vaddr[act_pte] = gen6_pte_encode(ppgtt->dev, page_addr,
  214.                                                       cache_level);
  215.                 if (++act_pte == I915_PPGTT_PT_ENTRIES) {
  216.                         act_pt++;
  217.                 MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3);
  218.                         act_pte = 0;
  219.  
  220.                 }
  221.         }
  222.         FreeKernelSpace(pt_vaddr);
  223. }
  224.  
  225. static void gen6_ppgtt_cleanup(struct i915_hw_ppgtt *ppgtt)
  226. {
  227.         int i;
  228.  
  229.         if (ppgtt->pt_dma_addr) {
  230.                 for (i = 0; i < ppgtt->num_pd_entries; i++)
  231.                         pci_unmap_page(ppgtt->dev->pdev,
  232.                                        ppgtt->pt_dma_addr[i],
  233.                                        4096, PCI_DMA_BIDIRECTIONAL);
  234.         }
  235.  
  236.         kfree(ppgtt->pt_dma_addr);
  237.         for (i = 0; i < ppgtt->num_pd_entries; i++)
  238.                 __free_page(ppgtt->pt_pages[i]);
  239.         kfree(ppgtt->pt_pages);
  240.         kfree(ppgtt);
  241. }
  242.  
  243. static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
  244. {
  245.         struct drm_device *dev = ppgtt->dev;
  246.         struct drm_i915_private *dev_priv = dev->dev_private;
  247.         unsigned first_pd_entry_in_global_pt;
  248.         int i;
  249.         int ret = -ENOMEM;
  250.  
  251.         /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024
  252.          * entries. For aliasing ppgtt support we just steal them at the end for
  253.          * now. */
  254.        first_pd_entry_in_global_pt = gtt_total_entries(dev_priv->gtt);
  255.  
  256.         ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES;
  257.         ppgtt->enable = gen6_ppgtt_enable;
  258.         ppgtt->clear_range = gen6_ppgtt_clear_range;
  259.         ppgtt->insert_entries = gen6_ppgtt_insert_entries;
  260.         ppgtt->cleanup = gen6_ppgtt_cleanup;
  261.         ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries,
  262.                                   GFP_KERNEL);
  263.         if (!ppgtt->pt_pages)
  264.                 return -ENOMEM;
  265.  
  266.         for (i = 0; i < ppgtt->num_pd_entries; i++) {
  267.                 ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
  268.                 if (!ppgtt->pt_pages[i])
  269.                         goto err_pt_alloc;
  270.         }
  271.  
  272.         ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t) *ppgtt->num_pd_entries,
  273.                                              GFP_KERNEL);
  274.                 if (!ppgtt->pt_dma_addr)
  275.                         goto err_pt_alloc;
  276.  
  277.                 for (i = 0; i < ppgtt->num_pd_entries; i++) {
  278.                         dma_addr_t pt_addr;
  279.  
  280.                 pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 0, 4096,
  281.                                                PCI_DMA_BIDIRECTIONAL);
  282.  
  283.                         ppgtt->pt_dma_addr[i] = pt_addr;
  284.                 }
  285.  
  286.         ppgtt->clear_range(ppgtt, 0,
  287.                                ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES);
  288.  
  289.         ppgtt->pd_offset = first_pd_entry_in_global_pt * sizeof(gen6_gtt_pte_t);
  290.  
  291.         return 0;
  292.  
  293. err_pd_pin:
  294.         if (ppgtt->pt_dma_addr) {
  295.                 for (i--; i >= 0; i--)
  296.                         pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
  297.                                        4096, PCI_DMA_BIDIRECTIONAL);
  298.         }
  299. err_pt_alloc:
  300.         kfree(ppgtt->pt_dma_addr);
  301.         for (i = 0; i < ppgtt->num_pd_entries; i++) {
  302.                 if (ppgtt->pt_pages[i])
  303.                         __free_page(ppgtt->pt_pages[i]);
  304.         }
  305.         kfree(ppgtt->pt_pages);
  306.  
  307.         return ret;
  308. }
  309.  
  310. static int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
  311. {
  312.         struct drm_i915_private *dev_priv = dev->dev_private;
  313.         struct i915_hw_ppgtt *ppgtt;
  314.         int ret;
  315.  
  316.         ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
  317.         if (!ppgtt)
  318.                 return -ENOMEM;
  319.  
  320.         ppgtt->dev = dev;
  321.         ppgtt->scratch_page_dma_addr = dev_priv->gtt.scratch_page_dma;
  322.  
  323.         if (INTEL_INFO(dev)->gen < 8)
  324.         ret = gen6_ppgtt_init(ppgtt);
  325.         else
  326.                 BUG();
  327.  
  328.         if (ret)
  329.         kfree(ppgtt);
  330.         else
  331.                 dev_priv->mm.aliasing_ppgtt = ppgtt;
  332.  
  333.         return ret;
  334. }
  335.  
  336. void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
  337. {
  338.         struct drm_i915_private *dev_priv = dev->dev_private;
  339.         struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
  340.  
  341.         if (!ppgtt)
  342.                 return;
  343.  
  344.         ppgtt->cleanup(ppgtt);
  345.         dev_priv->mm.aliasing_ppgtt = NULL;
  346. }
  347.  
  348. void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
  349.                             struct drm_i915_gem_object *obj,
  350.                             enum i915_cache_level cache_level)
  351. {
  352.         ppgtt->insert_entries(ppgtt, obj->pages,
  353.                                      obj->gtt_space->start >> PAGE_SHIFT,
  354.                                      cache_level);
  355. }
  356.  
  357. void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
  358.                               struct drm_i915_gem_object *obj)
  359. {
  360.         ppgtt->clear_range(ppgtt,
  361.                                obj->gtt_space->start >> PAGE_SHIFT,
  362.                                obj->base.size >> PAGE_SHIFT);
  363. }
  364.  
  365. extern int intel_iommu_gfx_mapped;
  366. /* Certain Gen5 chipsets require require idling the GPU before
  367.  * unmapping anything from the GTT when VT-d is enabled.
  368.  */
  369. static inline bool needs_idle_maps(struct drm_device *dev)
  370. {
  371. #ifdef CONFIG_INTEL_IOMMU
  372.         /* Query intel_iommu to see if we need the workaround. Presumably that
  373.          * was loaded first.
  374.          */
  375.         if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped)
  376.                 return true;
  377. #endif
  378.         return false;
  379. }
  380.  
  381. static bool do_idling(struct drm_i915_private *dev_priv)
  382. {
  383.         bool ret = dev_priv->mm.interruptible;
  384.  
  385.         if (unlikely(dev_priv->gtt.do_idle_maps)) {
  386.                 dev_priv->mm.interruptible = false;
  387.                 if (i915_gpu_idle(dev_priv->dev)) {
  388.                         DRM_ERROR("Couldn't idle GPU\n");
  389.                         /* Wait a bit, in hopes it avoids the hang */
  390.                         udelay(10);
  391.                 }
  392.         }
  393.  
  394.         return ret;
  395. }
  396.  
  397. static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
  398. {
  399.         if (unlikely(dev_priv->gtt.do_idle_maps))
  400.                 dev_priv->mm.interruptible = interruptible;
  401. }
  402.  
  403. void i915_gem_restore_gtt_mappings(struct drm_device *dev)
  404. {
  405.         struct drm_i915_private *dev_priv = dev->dev_private;
  406.         struct drm_i915_gem_object *obj;
  407.  
  408.         /* First fill our portion of the GTT with scratch pages */
  409.         dev_priv->gtt.gtt_clear_range(dev, dev_priv->gtt.start / PAGE_SIZE,
  410.                                       dev_priv->gtt.total / PAGE_SIZE);
  411.  
  412.         list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) {
  413.                 i915_gem_clflush_object(obj);
  414.                 i915_gem_gtt_bind_object(obj, obj->cache_level);
  415.         }
  416.  
  417.         i915_gem_chipset_flush(dev);
  418. }
  419.  
  420. int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
  421. {
  422.         if (obj->has_dma_mapping)
  423.                 return 0;
  424.  
  425.         if (!dma_map_sg(&obj->base.dev->pdev->dev,
  426.                         obj->pages->sgl, obj->pages->nents,
  427.                         PCI_DMA_BIDIRECTIONAL))
  428.                 return -ENOSPC;
  429.  
  430.         return 0;
  431. }
  432.  
  433. /*
  434.  * Binds an object into the global gtt with the specified cache level. The object
  435.  * will be accessible to the GPU via commands whose operands reference offsets
  436.  * within the global GTT as well as accessible by the GPU through the GMADR
  437.  * mapped BAR (dev_priv->mm.gtt->gtt).
  438.  */
  439. static void gen6_ggtt_insert_entries(struct drm_device *dev,
  440.                                      struct sg_table *st,
  441.                                      unsigned int first_entry,
  442.                                   enum i915_cache_level level)
  443. {
  444.         struct drm_i915_private *dev_priv = dev->dev_private;
  445.         gen6_gtt_pte_t __iomem *gtt_entries =
  446.                 (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
  447.         int i = 0;
  448.         struct sg_page_iter sg_iter;
  449.         dma_addr_t addr;
  450.  
  451.         for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
  452.                 addr = sg_page_iter_dma_address(&sg_iter);
  453.                 iowrite32(gen6_pte_encode(dev, addr, level), &gtt_entries[i]);
  454.                         i++;
  455.                 }
  456.  
  457.         /* XXX: This serves as a posting read to make sure that the PTE has
  458.          * actually been updated. There is some concern that even though
  459.          * registers and PTEs are within the same BAR that they are potentially
  460.          * of NUMA access patterns. Therefore, even with the way we assume
  461.          * hardware should work, we must keep this posting read for paranoia.
  462.          */
  463.         if (i != 0)
  464.                 WARN_ON(readl(&gtt_entries[i-1])
  465.                         != gen6_pte_encode(dev, addr, level));
  466.  
  467.         /* This next bit makes the above posting read even more important. We
  468.          * want to flush the TLBs only after we're certain all the PTE updates
  469.          * have finished.
  470.          */
  471.         I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
  472.         POSTING_READ(GFX_FLSH_CNTL_GEN6);
  473. }
  474.  
  475. static void gen6_ggtt_clear_range(struct drm_device *dev,
  476.                                   unsigned int first_entry,
  477.                                   unsigned int num_entries)
  478. {
  479.         struct drm_i915_private *dev_priv = dev->dev_private;
  480.         gen6_gtt_pte_t scratch_pte, __iomem *gtt_base =
  481.                 (gen6_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
  482.         const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
  483.         int i;
  484.  
  485. //   if (WARN(num_entries > max_entries,
  486. //        "First entry = %d; Num entries = %d (max=%d)\n",
  487. //        first_entry, num_entries, max_entries))
  488.    if (num_entries > max_entries)
  489.                 num_entries = max_entries;
  490.  
  491.         scratch_pte = gen6_pte_encode(dev, dev_priv->gtt.scratch_page_dma,
  492.                                       I915_CACHE_LLC);
  493.         for (i = 0; i < num_entries; i++)
  494.                 iowrite32(scratch_pte, &gtt_base[i]);
  495.         readl(gtt_base);
  496. }
  497.  
  498.  
  499. static void i915_ggtt_insert_entries(struct drm_device *dev,
  500.                                      struct sg_table *st,
  501.                                      unsigned int pg_start,
  502.                                      enum i915_cache_level cache_level)
  503. {
  504.         unsigned int flags = (cache_level == I915_CACHE_NONE) ?
  505.                 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
  506.  
  507.         intel_gtt_insert_sg_entries(st, pg_start, flags);
  508.  
  509. }
  510.  
  511. static void i915_ggtt_clear_range(struct drm_device *dev,
  512.                                   unsigned int first_entry,
  513.                                   unsigned int num_entries)
  514. {
  515.         intel_gtt_clear_range(first_entry, num_entries);
  516. }
  517.  
  518.  
  519. void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
  520.                                 enum i915_cache_level cache_level)
  521. {
  522.         struct drm_device *dev = obj->base.dev;
  523.         struct drm_i915_private *dev_priv = dev->dev_private;
  524.  
  525.         dev_priv->gtt.gtt_insert_entries(dev, obj->pages,
  526.                                             obj->gtt_space->start >> PAGE_SHIFT,
  527.                                          cache_level);
  528.  
  529.         obj->has_global_gtt_mapping = 1;
  530. }
  531.  
  532. void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
  533. {
  534.         struct drm_device *dev = obj->base.dev;
  535.         struct drm_i915_private *dev_priv = dev->dev_private;
  536.  
  537.         dev_priv->gtt.gtt_clear_range(obj->base.dev,
  538.                               obj->gtt_space->start >> PAGE_SHIFT,
  539.                               obj->base.size >> PAGE_SHIFT);
  540.  
  541.         obj->has_global_gtt_mapping = 0;
  542. }
  543.  
  544. void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
  545. {
  546.         struct drm_device *dev = obj->base.dev;
  547.         struct drm_i915_private *dev_priv = dev->dev_private;
  548.         bool interruptible;
  549.  
  550.         interruptible = do_idling(dev_priv);
  551.  
  552.         if (!obj->has_dma_mapping)
  553.                 dma_unmap_sg(&dev->pdev->dev,
  554.                              obj->pages->sgl, obj->pages->nents,
  555.                              PCI_DMA_BIDIRECTIONAL);
  556.  
  557.         undo_idling(dev_priv, interruptible);
  558. }
  559.  
  560. static void i915_gtt_color_adjust(struct drm_mm_node *node,
  561.                                   unsigned long color,
  562.                                   unsigned long *start,
  563.                                   unsigned long *end)
  564. {
  565.         if (node->color != color)
  566.                 *start += 4096;
  567.  
  568.         if (!list_empty(&node->node_list)) {
  569.                 node = list_entry(node->node_list.next,
  570.                                   struct drm_mm_node,
  571.                                   node_list);
  572.                 if (node->allocated && node->color != color)
  573.                         *end -= 4096;
  574.         }
  575. }
  576. void i915_gem_setup_global_gtt(struct drm_device *dev,
  577.                               unsigned long start,
  578.                               unsigned long mappable_end,
  579.                               unsigned long end)
  580. {
  581.         /* Let GEM Manage all of the aperture.
  582.          *
  583.          * However, leave one page at the end still bound to the scratch page.
  584.          * There are a number of places where the hardware apparently prefetches
  585.          * past the end of the object, and we've seen multiple hangs with the
  586.          * GPU head pointer stuck in a batchbuffer bound at the last page of the
  587.          * aperture.  One page should be enough to keep any prefetching inside
  588.          * of the aperture.
  589.          */
  590.         drm_i915_private_t *dev_priv = dev->dev_private;
  591.         struct drm_mm_node *entry;
  592.         struct drm_i915_gem_object *obj;
  593.         unsigned long hole_start, hole_end;
  594.  
  595.         BUG_ON(mappable_end > end);
  596.  
  597.         /* Subtract the guard page ... */
  598.         drm_mm_init(&dev_priv->mm.gtt_space, start, end - start - PAGE_SIZE);
  599.         if (!HAS_LLC(dev))
  600.                 dev_priv->mm.gtt_space.color_adjust = i915_gtt_color_adjust;
  601.  
  602.         /* Mark any preallocated objects as occupied */
  603.         list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) {
  604.                 DRM_DEBUG_KMS("reserving preallocated space: %x + %zx\n",
  605.                               obj->gtt_offset, obj->base.size);
  606.  
  607.                 BUG_ON(obj->gtt_space != I915_GTT_RESERVED);
  608.                 obj->gtt_space = drm_mm_create_block(&dev_priv->mm.gtt_space,
  609.                                                      obj->gtt_offset,
  610.                                                      obj->base.size,
  611.                                                      false);
  612.                 obj->has_global_gtt_mapping = 1;
  613.         }
  614.  
  615.         dev_priv->gtt.start = start;
  616.         dev_priv->gtt.total = end - start;
  617.  
  618.         /* Clear any non-preallocated blocks */
  619.         drm_mm_for_each_hole(entry, &dev_priv->mm.gtt_space,
  620.                              hole_start, hole_end) {
  621.                 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
  622.                               hole_start, hole_end);
  623.                 dev_priv->gtt.gtt_clear_range(dev, hole_start / PAGE_SIZE,
  624.                                               (hole_end-hole_start) / PAGE_SIZE);
  625.         }
  626.  
  627.         /* And finally clear the reserved guard page */
  628.         dev_priv->gtt.gtt_clear_range(dev, end / PAGE_SIZE - 1, 1);
  629. }
  630.  
  631. static bool
  632. intel_enable_ppgtt(struct drm_device *dev)
  633. {
  634.         if (i915_enable_ppgtt >= 0)
  635.                 return i915_enable_ppgtt;
  636.  
  637. #ifdef CONFIG_INTEL_IOMMU
  638.         /* Disable ppgtt on SNB if VT-d is on. */
  639.         if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
  640.                 return false;
  641. #endif
  642.  
  643.         return true;
  644. }
  645.  
  646. void i915_gem_init_global_gtt(struct drm_device *dev)
  647. {
  648.         struct drm_i915_private *dev_priv = dev->dev_private;
  649.         unsigned long gtt_size, mappable_size;
  650.  
  651.         gtt_size = dev_priv->gtt.total;
  652.         mappable_size = dev_priv->gtt.mappable_end;
  653.  
  654. #if 0
  655.         if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
  656.                 int ret;
  657.  
  658.                 if (INTEL_INFO(dev)->gen <= 7) {
  659.                 /* PPGTT pdes are stolen from global gtt ptes, so shrink the
  660.                  * aperture accordingly when using aliasing ppgtt. */
  661.         gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
  662.                 }
  663.  
  664.         i915_gem_setup_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size);
  665.  
  666.                 ret = i915_gem_init_aliasing_ppgtt(dev);
  667.                 if (!ret)
  668.                         return;
  669.  
  670.                 DRM_ERROR("Aliased PPGTT setup failed %d\n", ret);
  671.                 drm_mm_takedown(&dev_priv->mm.gtt_space);
  672.                 gtt_size += I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
  673. }
  674. #endif
  675.  
  676.     i915_gem_setup_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size);
  677. }
  678.  
  679. static int setup_scratch_page(struct drm_device *dev)
  680. {
  681.         struct drm_i915_private *dev_priv = dev->dev_private;
  682.         struct page *page;
  683.         dma_addr_t dma_addr;
  684.  
  685.         page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
  686.         if (page == NULL)
  687.                 return -ENOMEM;
  688.         get_page(page);
  689.         set_pages_uc(page, 1);
  690.  
  691. #ifdef CONFIG_INTEL_IOMMU
  692.         dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE,
  693.                                 PCI_DMA_BIDIRECTIONAL);
  694.         if (pci_dma_mapping_error(dev->pdev, dma_addr))
  695.                 return -EINVAL;
  696. #else
  697.         dma_addr = page_to_phys(page);
  698. #endif
  699.         dev_priv->gtt.scratch_page = page;
  700.         dev_priv->gtt.scratch_page_dma = dma_addr;
  701.  
  702.         return 0;
  703. }
  704.  
  705. static void teardown_scratch_page(struct drm_device *dev)
  706. {
  707.         struct drm_i915_private *dev_priv = dev->dev_private;
  708.         set_pages_wb(dev_priv->gtt.scratch_page, 1);
  709.         pci_unmap_page(dev->pdev, dev_priv->gtt.scratch_page_dma,
  710.                        PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
  711.         put_page(dev_priv->gtt.scratch_page);
  712.         __free_page(dev_priv->gtt.scratch_page);
  713. }
  714.  
  715. static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
  716. {
  717.         snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
  718.         snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
  719.         return snb_gmch_ctl << 20;
  720. }
  721.  
  722. static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
  723. {
  724.         snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
  725.         snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
  726.         return snb_gmch_ctl << 25; /* 32 MB units */
  727. }
  728.  
  729. static int gen6_gmch_probe(struct drm_device *dev,
  730.                            size_t *gtt_total,
  731.                            size_t *stolen,
  732.                            phys_addr_t *mappable_base,
  733.                            unsigned long *mappable_end)
  734. {
  735.         struct drm_i915_private *dev_priv = dev->dev_private;
  736.         phys_addr_t gtt_bus_addr;
  737.         unsigned int gtt_size;
  738.         u16 snb_gmch_ctl;
  739.         int ret;
  740.  
  741.         *mappable_base = pci_resource_start(dev->pdev, 2);
  742.         *mappable_end = pci_resource_len(dev->pdev, 2);
  743.  
  744.         /* 64/512MB is the current min/max we actually know of, but this is just
  745.          * a coarse sanity check.
  746.          */
  747.         if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) {
  748.                 DRM_ERROR("Unknown GMADR size (%lx)\n",
  749.                           dev_priv->gtt.mappable_end);
  750.                 return -ENXIO;
  751.                 }
  752.  
  753.         if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40)))
  754.                 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40));
  755.         pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
  756.         gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl);
  757.  
  758.                 *stolen = gen6_get_stolen_size(snb_gmch_ctl);
  759.         *gtt_total = (gtt_size / sizeof(gen6_gtt_pte_t)) << PAGE_SHIFT;
  760.  
  761.         /* For Modern GENs the PTEs and register space are split in the BAR */
  762.         gtt_bus_addr = pci_resource_start(dev->pdev, 0) +
  763.                 (pci_resource_len(dev->pdev, 0) / 2);
  764.  
  765.         dev_priv->gtt.gsm = ioremap_wc(gtt_bus_addr, gtt_size);
  766.         if (!dev_priv->gtt.gsm) {
  767.                 DRM_ERROR("Failed to map the gtt page table\n");
  768.                 return -ENOMEM;
  769.         }
  770.  
  771.         ret = setup_scratch_page(dev);
  772.         if (ret)
  773.                 DRM_ERROR("Scratch setup failed\n");
  774.  
  775.         dev_priv->gtt.gtt_clear_range = gen6_ggtt_clear_range;
  776.         dev_priv->gtt.gtt_insert_entries = gen6_ggtt_insert_entries;
  777.  
  778.         return ret;
  779. }
  780.  
  781. static void gen6_gmch_remove(struct drm_device *dev)
  782. {
  783.         struct drm_i915_private *dev_priv = dev->dev_private;
  784.         iounmap(dev_priv->gtt.gsm);
  785.         teardown_scratch_page(dev_priv->dev);
  786. }
  787.  
  788. static int i915_gmch_probe(struct drm_device *dev,
  789.                            size_t *gtt_total,
  790.                            size_t *stolen,
  791.                            phys_addr_t *mappable_base,
  792.                            unsigned long *mappable_end)
  793. {
  794.         struct drm_i915_private *dev_priv = dev->dev_private;
  795.         int ret;
  796.  
  797.         ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL);
  798.         if (!ret) {
  799.                 DRM_ERROR("failed to set up gmch\n");
  800.                 return -EIO;
  801.         }
  802.  
  803.         intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end);
  804.  
  805.         dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev);
  806.         dev_priv->gtt.gtt_clear_range = i915_ggtt_clear_range;
  807.         dev_priv->gtt.gtt_insert_entries = i915_ggtt_insert_entries;
  808.  
  809.         return 0;
  810. }
  811.  
  812. static void i915_gmch_remove(struct drm_device *dev)
  813. {
  814. //   intel_gmch_remove();
  815. }
  816.  
  817. int i915_gem_gtt_init(struct drm_device *dev)
  818. {
  819.         struct drm_i915_private *dev_priv = dev->dev_private;
  820.         struct i915_gtt *gtt = &dev_priv->gtt;
  821.         int ret;
  822.  
  823.         if (INTEL_INFO(dev)->gen <= 5) {
  824.                 dev_priv->gtt.gtt_probe = i915_gmch_probe;
  825.                 dev_priv->gtt.gtt_remove = i915_gmch_remove;
  826.         } else {
  827.                 dev_priv->gtt.gtt_probe = gen6_gmch_probe;
  828.                 dev_priv->gtt.gtt_remove = gen6_gmch_remove;
  829.         }
  830.  
  831.         ret = dev_priv->gtt.gtt_probe(dev, &dev_priv->gtt.total,
  832.                                      &dev_priv->gtt.stolen_size,
  833.                                      &gtt->mappable_base,
  834.                                      &gtt->mappable_end);
  835.         if (ret)
  836.         return ret;
  837.  
  838.         /* GMADR is the PCI mmio aperture into the global GTT. */
  839.         DRM_INFO("Memory usable by graphics device = %zdM\n",
  840.                  dev_priv->gtt.total >> 20);
  841.         DRM_DEBUG_DRIVER("GMADR size = %ldM\n",
  842.                          dev_priv->gtt.mappable_end >> 20);
  843.         DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n",
  844.                          dev_priv->gtt.stolen_size >> 20);
  845.  
  846.         return 0;
  847. }
  848.  
  849. struct scatterlist *sg_next(struct scatterlist *sg)
  850. {
  851.     if (sg_is_last(sg))
  852.         return NULL;
  853.  
  854.     sg++;
  855.     if (unlikely(sg_is_chain(sg)))
  856.             sg = sg_chain_ptr(sg);
  857.  
  858.     return sg;
  859. }
  860.  
  861.  
  862. void __sg_free_table(struct sg_table *table, unsigned int max_ents,
  863.                      sg_free_fn *free_fn)
  864. {
  865.     struct scatterlist *sgl, *next;
  866.  
  867.     if (unlikely(!table->sgl))
  868.             return;
  869.  
  870.     sgl = table->sgl;
  871.     while (table->orig_nents) {
  872.         unsigned int alloc_size = table->orig_nents;
  873.         unsigned int sg_size;
  874.  
  875.         /*
  876.          * If we have more than max_ents segments left,
  877.          * then assign 'next' to the sg table after the current one.
  878.          * sg_size is then one less than alloc size, since the last
  879.          * element is the chain pointer.
  880.          */
  881.         if (alloc_size > max_ents) {
  882.                 next = sg_chain_ptr(&sgl[max_ents - 1]);
  883.                 alloc_size = max_ents;
  884.                 sg_size = alloc_size - 1;
  885.         } else {
  886.                 sg_size = alloc_size;
  887.                 next = NULL;
  888.         }
  889.  
  890.         table->orig_nents -= sg_size;
  891.         kfree(sgl);
  892.         sgl = next;
  893.     }
  894.  
  895.     table->sgl = NULL;
  896. }
  897.  
  898. void sg_free_table(struct sg_table *table)
  899. {
  900.     __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL);
  901. }
  902.  
  903. int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask)
  904. {
  905.     struct scatterlist *sg, *prv;
  906.     unsigned int left;
  907.     unsigned int max_ents = SG_MAX_SINGLE_ALLOC;
  908.  
  909. #ifndef ARCH_HAS_SG_CHAIN
  910.     BUG_ON(nents > max_ents);
  911. #endif
  912.  
  913.     memset(table, 0, sizeof(*table));
  914.  
  915.     left = nents;
  916.     prv = NULL;
  917.     do {
  918.         unsigned int sg_size, alloc_size = left;
  919.  
  920.         if (alloc_size > max_ents) {
  921.                 alloc_size = max_ents;
  922.                 sg_size = alloc_size - 1;
  923.         } else
  924.                 sg_size = alloc_size;
  925.  
  926.         left -= sg_size;
  927.  
  928.         sg = kmalloc(alloc_size * sizeof(struct scatterlist), gfp_mask);
  929.         if (unlikely(!sg)) {
  930.                 /*
  931.                  * Adjust entry count to reflect that the last
  932.                  * entry of the previous table won't be used for
  933.                  * linkage.  Without this, sg_kfree() may get
  934.                  * confused.
  935.                  */
  936.                 if (prv)
  937.                         table->nents = ++table->orig_nents;
  938.  
  939.                 goto err;
  940.         }
  941.  
  942.         sg_init_table(sg, alloc_size);
  943.         table->nents = table->orig_nents += sg_size;
  944.  
  945.         /*
  946.          * If this is the first mapping, assign the sg table header.
  947.          * If this is not the first mapping, chain previous part.
  948.          */
  949.         if (prv)
  950.                 sg_chain(prv, max_ents, sg);
  951.         else
  952.                 table->sgl = sg;
  953.  
  954.         /*
  955.          * If no more entries after this one, mark the end
  956.          */
  957.         if (!left)
  958.                 sg_mark_end(&sg[sg_size - 1]);
  959.  
  960.         prv = sg;
  961.     } while (left);
  962.  
  963.     return 0;
  964.  
  965. err:
  966.     __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL);
  967.  
  968.     return -ENOMEM;
  969. }
  970.  
  971.  
  972. void sg_init_table(struct scatterlist *sgl, unsigned int nents)
  973. {
  974.     memset(sgl, 0, sizeof(*sgl) * nents);
  975. #ifdef CONFIG_DEBUG_SG
  976.     {
  977.             unsigned int i;
  978.             for (i = 0; i < nents; i++)
  979.                     sgl[i].sg_magic = SG_MAGIC;
  980.     }
  981. #endif
  982.     sg_mark_end(&sgl[nents - 1]);
  983. }
  984.  
  985.  
  986. void __sg_page_iter_start(struct sg_page_iter *piter,
  987.               struct scatterlist *sglist, unsigned int nents,
  988.               unsigned long pgoffset)
  989. {
  990.     piter->__pg_advance = 0;
  991.     piter->__nents = nents;
  992.  
  993.     piter->sg = sglist;
  994.     piter->sg_pgoffset = pgoffset;
  995. }
  996.  
  997. static int sg_page_count(struct scatterlist *sg)
  998. {
  999.     return PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT;
  1000. }
  1001.  
  1002. bool __sg_page_iter_next(struct sg_page_iter *piter)
  1003. {
  1004.     if (!piter->__nents || !piter->sg)
  1005.         return false;
  1006.  
  1007.     piter->sg_pgoffset += piter->__pg_advance;
  1008.     piter->__pg_advance = 1;
  1009.  
  1010.     while (piter->sg_pgoffset >= sg_page_count(piter->sg)) {
  1011.         piter->sg_pgoffset -= sg_page_count(piter->sg);
  1012.         piter->sg = sg_next(piter->sg);
  1013.         if (!--piter->__nents || !piter->sg)
  1014.             return false;
  1015.     }
  1016.  
  1017.     return true;
  1018. }
  1019. EXPORT_SYMBOL(__sg_page_iter_next);
  1020.  
  1021.  
  1022.