Subversion Repositories Kolibri OS

Rev

Rev 3243 | Rev 3746 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright © 2010 Daniel Vetter
  3.  *
  4.  * Permission is hereby granted, free of charge, to any person obtaining a
  5.  * copy of this software and associated documentation files (the "Software"),
  6.  * to deal in the Software without restriction, including without limitation
  7.  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8.  * and/or sell copies of the Software, and to permit persons to whom the
  9.  * Software is furnished to do so, subject to the following conditions:
  10.  *
  11.  * The above copyright notice and this permission notice (including the next
  12.  * paragraph) shall be included in all copies or substantial portions of the
  13.  * Software.
  14.  *
  15.  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16.  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17.  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18.  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19.  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20.  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21.  * IN THE SOFTWARE.
  22.  *
  23.  */
  24.  
  25. #define iowrite32(v, addr)      writel((v), (addr))
  26.  
  27. #define AGP_NORMAL_MEMORY 0
  28.  
  29. #define AGP_USER_TYPES (1 << 16)
  30. #define AGP_USER_MEMORY (AGP_USER_TYPES)
  31. #define AGP_USER_CACHED_MEMORY (AGP_USER_TYPES + 1)
  32.  
  33. #include <drm/drmP.h>
  34. #include <drm/i915_drm.h>
  35. #include "i915_drv.h"
  36. #include "i915_trace.h"
  37. #include "intel_drv.h"
  38.  
  39. typedef uint32_t gtt_pte_t;
  40.  
  41. /* PPGTT stuff */
  42. #define GEN6_GTT_ADDR_ENCODE(addr)      ((addr) | (((addr) >> 28) & 0xff0))
  43.  
  44. #define GEN6_PDE_VALID                  (1 << 0)
  45. /* gen6+ has bit 11-4 for physical addr bit 39-32 */
  46. #define GEN6_PDE_ADDR_ENCODE(addr)      GEN6_GTT_ADDR_ENCODE(addr)
  47.  
  48. #define GEN6_PTE_VALID                  (1 << 0)
  49. #define GEN6_PTE_UNCACHED               (1 << 1)
  50. #define HSW_PTE_UNCACHED                (0)
  51. #define GEN6_PTE_CACHE_LLC              (2 << 1)
  52. #define GEN6_PTE_CACHE_LLC_MLC          (3 << 1)
  53. #define GEN6_PTE_ADDR_ENCODE(addr)      GEN6_GTT_ADDR_ENCODE(addr)
  54.  
  55. static inline gtt_pte_t gen6_pte_encode(struct drm_device *dev,
  56.                                    dma_addr_t addr,
  57.                                    enum i915_cache_level level)
  58. {
  59.         gtt_pte_t pte = GEN6_PTE_VALID;
  60.         pte |= GEN6_PTE_ADDR_ENCODE(addr);
  61.  
  62.         switch (level) {
  63.         case I915_CACHE_LLC_MLC:
  64.                 /* Haswell doesn't set L3 this way */
  65.                 if (IS_HASWELL(dev))
  66.                         pte |= GEN6_PTE_CACHE_LLC;
  67.                 else
  68.                         pte |= GEN6_PTE_CACHE_LLC_MLC;
  69.                 break;
  70.         case I915_CACHE_LLC:
  71.                 pte |= GEN6_PTE_CACHE_LLC;
  72.                 break;
  73.         case I915_CACHE_NONE:
  74.                 if (IS_HASWELL(dev))
  75.                         pte |= HSW_PTE_UNCACHED;
  76.                 else
  77.                         pte |= GEN6_PTE_UNCACHED;
  78.                 break;
  79.         default:
  80.                 BUG();
  81.         }
  82.  
  83.  
  84.         return pte;
  85. }
  86.  
  87. /* PPGTT support for Sandybdrige/Gen6 and later */
  88. static void gen6_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt,
  89.                                    unsigned first_entry,
  90.                                    unsigned num_entries)
  91. {
  92.         gtt_pte_t *pt_vaddr;
  93.         gtt_pte_t scratch_pte;
  94.         unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
  95.         unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
  96.         unsigned last_pte, i;
  97.  
  98.         scratch_pte = gen6_pte_encode(ppgtt->dev,
  99.                                       ppgtt->scratch_page_dma_addr,
  100.                                  I915_CACHE_LLC);
  101.  
  102.     pt_vaddr = AllocKernelSpace(4096);
  103.  
  104.     if(pt_vaddr == NULL)
  105.         return;
  106.  
  107.         while (num_entries) {
  108.             last_pte = first_pte + num_entries;
  109.             if (last_pte > I915_PPGTT_PT_ENTRIES)
  110.                 last_pte = I915_PPGTT_PT_ENTRIES;
  111.  
  112.             MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pd]), 3);
  113.  
  114.             for (i = first_pte; i < last_pte; i++)
  115.                 pt_vaddr[i] = scratch_pte;
  116.  
  117.             num_entries -= last_pte - first_pte;
  118.             first_pte = 0;
  119.             act_pd++;
  120.     };
  121.  
  122.     FreeKernelSpace(pt_vaddr);
  123. }
  124.  
  125. static void gen6_ppgtt_insert_entries(struct i915_hw_ppgtt *ppgtt,
  126.                                       struct sg_table *pages,
  127.                                       unsigned first_entry,
  128.                                       enum i915_cache_level cache_level)
  129. {
  130.         gtt_pte_t *pt_vaddr;
  131.         unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
  132.         unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
  133.         unsigned i, j, m, segment_len;
  134.         dma_addr_t page_addr;
  135.         struct scatterlist *sg;
  136.  
  137.         /* init sg walking */
  138.         sg = pages->sgl;
  139.         i = 0;
  140.         segment_len = sg_dma_len(sg) >> PAGE_SHIFT;
  141.         m = 0;
  142.  
  143.     pt_vaddr = AllocKernelSpace(4096);
  144.  
  145.     if(pt_vaddr == NULL)
  146.         return;
  147.  
  148.         while (i < pages->nents) {
  149.         MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pd]), 3);
  150.  
  151.                 for (j = first_pte; j < I915_PPGTT_PT_ENTRIES; j++) {
  152.                         page_addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
  153.                         pt_vaddr[j] = gen6_pte_encode(ppgtt->dev, page_addr,
  154.                                                       cache_level);
  155.  
  156.                         /* grab the next page */
  157.                         if (++m == segment_len) {
  158.                                 if (++i == pages->nents)
  159.                                         break;
  160.  
  161.                                 sg = sg_next(sg);
  162.                                 segment_len = sg_dma_len(sg) >> PAGE_SHIFT;
  163.                                 m = 0;
  164.                         }
  165.                 }
  166.  
  167.                 first_pte = 0;
  168.                 act_pd++;
  169.         }
  170.         FreeKernelSpace(pt_vaddr);
  171. }
  172.  
  173. static void gen6_ppgtt_cleanup(struct i915_hw_ppgtt *ppgtt)
  174. {
  175.         int i;
  176.  
  177.         if (ppgtt->pt_dma_addr) {
  178.                 for (i = 0; i < ppgtt->num_pd_entries; i++)
  179.                         pci_unmap_page(ppgtt->dev->pdev,
  180.                                        ppgtt->pt_dma_addr[i],
  181.                                        4096, PCI_DMA_BIDIRECTIONAL);
  182.         }
  183.  
  184.         kfree(ppgtt->pt_dma_addr);
  185.         for (i = 0; i < ppgtt->num_pd_entries; i++)
  186.                 __free_page(ppgtt->pt_pages[i]);
  187.         kfree(ppgtt->pt_pages);
  188.         kfree(ppgtt);
  189. }
  190.  
  191. static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
  192. {
  193.         struct drm_device *dev = ppgtt->dev;
  194.         struct drm_i915_private *dev_priv = dev->dev_private;
  195.         unsigned first_pd_entry_in_global_pt;
  196.         int i;
  197.         int ret = -ENOMEM;
  198.  
  199.         /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024
  200.          * entries. For aliasing ppgtt support we just steal them at the end for
  201.          * now. */
  202.         first_pd_entry_in_global_pt =
  203.                 gtt_total_entries(dev_priv->gtt) - I915_PPGTT_PD_ENTRIES;
  204.  
  205.         ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES;
  206.         ppgtt->clear_range = gen6_ppgtt_clear_range;
  207.         ppgtt->insert_entries = gen6_ppgtt_insert_entries;
  208.         ppgtt->cleanup = gen6_ppgtt_cleanup;
  209.         ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries,
  210.                                   GFP_KERNEL);
  211.         if (!ppgtt->pt_pages)
  212.                 return -ENOMEM;
  213.  
  214.         for (i = 0; i < ppgtt->num_pd_entries; i++) {
  215.                 ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
  216.                 if (!ppgtt->pt_pages[i])
  217.                         goto err_pt_alloc;
  218.         }
  219.  
  220.         ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t) *ppgtt->num_pd_entries,
  221.                                              GFP_KERNEL);
  222.                 if (!ppgtt->pt_dma_addr)
  223.                         goto err_pt_alloc;
  224.  
  225.                 for (i = 0; i < ppgtt->num_pd_entries; i++) {
  226.                         dma_addr_t pt_addr;
  227.  
  228.                 pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 0, 4096,
  229.                                                PCI_DMA_BIDIRECTIONAL);
  230.  
  231.                         ppgtt->pt_dma_addr[i] = pt_addr;
  232.                 }
  233.  
  234.         ppgtt->scratch_page_dma_addr = dev_priv->gtt.scratch_page_dma;
  235.  
  236.         ppgtt->clear_range(ppgtt, 0,
  237.                                ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES);
  238.  
  239.         ppgtt->pd_offset = (first_pd_entry_in_global_pt)*sizeof(gtt_pte_t);
  240.  
  241.         return 0;
  242.  
  243. err_pd_pin:
  244.         if (ppgtt->pt_dma_addr) {
  245.                 for (i--; i >= 0; i--)
  246.                         pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
  247.                                        4096, PCI_DMA_BIDIRECTIONAL);
  248.         }
  249. err_pt_alloc:
  250.         kfree(ppgtt->pt_dma_addr);
  251.         for (i = 0; i < ppgtt->num_pd_entries; i++) {
  252.                 if (ppgtt->pt_pages[i])
  253.                         __free_page(ppgtt->pt_pages[i]);
  254.         }
  255.         kfree(ppgtt->pt_pages);
  256.  
  257.         return ret;
  258. }
  259.  
  260. static int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
  261. {
  262.         struct drm_i915_private *dev_priv = dev->dev_private;
  263.         struct i915_hw_ppgtt *ppgtt;
  264.         int ret;
  265.  
  266.         ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
  267.         if (!ppgtt)
  268.                 return -ENOMEM;
  269.  
  270.         ppgtt->dev = dev;
  271.  
  272.         ret = gen6_ppgtt_init(ppgtt);
  273.         if (ret)
  274.         kfree(ppgtt);
  275.         else
  276.                 dev_priv->mm.aliasing_ppgtt = ppgtt;
  277.  
  278.         return ret;
  279. }
  280.  
  281. void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
  282. {
  283.         struct drm_i915_private *dev_priv = dev->dev_private;
  284.         struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
  285.  
  286.         if (!ppgtt)
  287.                 return;
  288.  
  289.         ppgtt->cleanup(ppgtt);
  290. }
  291.  
  292. void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
  293.                             struct drm_i915_gem_object *obj,
  294.                             enum i915_cache_level cache_level)
  295. {
  296.         ppgtt->insert_entries(ppgtt, obj->pages,
  297.                                      obj->gtt_space->start >> PAGE_SHIFT,
  298.                                      cache_level);
  299. }
  300.  
  301. void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
  302.                               struct drm_i915_gem_object *obj)
  303. {
  304.         ppgtt->clear_range(ppgtt,
  305.                                obj->gtt_space->start >> PAGE_SHIFT,
  306.                                obj->base.size >> PAGE_SHIFT);
  307. }
  308.  
  309. void i915_gem_init_ppgtt(struct drm_device *dev)
  310. {
  311.         drm_i915_private_t *dev_priv = dev->dev_private;
  312.         uint32_t pd_offset;
  313.         struct intel_ring_buffer *ring;
  314.         struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
  315.         gtt_pte_t __iomem *pd_addr;
  316.         uint32_t pd_entry;
  317.         int i;
  318.  
  319.         if (!dev_priv->mm.aliasing_ppgtt)
  320.                 return;
  321.  
  322.  
  323.         pd_addr = (gtt_pte_t __iomem*)dev_priv->gtt.gsm + ppgtt->pd_offset/sizeof(gtt_pte_t);
  324.         for (i = 0; i < ppgtt->num_pd_entries; i++) {
  325.                 dma_addr_t pt_addr;
  326.  
  327.                         pt_addr = ppgtt->pt_dma_addr[i];
  328.                 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
  329.                 pd_entry |= GEN6_PDE_VALID;
  330.  
  331.                 writel(pd_entry, pd_addr + i);
  332.         }
  333.         readl(pd_addr);
  334.  
  335.         pd_offset = ppgtt->pd_offset;
  336.         pd_offset /= 64; /* in cachelines, */
  337.         pd_offset <<= 16;
  338.  
  339.         if (INTEL_INFO(dev)->gen == 6) {
  340.                 uint32_t ecochk, gab_ctl, ecobits;
  341.  
  342.                 ecobits = I915_READ(GAC_ECO_BITS);
  343.                 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
  344.  
  345.                 gab_ctl = I915_READ(GAB_CTL);
  346.                 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
  347.  
  348.                 ecochk = I915_READ(GAM_ECOCHK);
  349.                 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT |
  350.                                        ECOCHK_PPGTT_CACHE64B);
  351.                 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
  352.         } else if (INTEL_INFO(dev)->gen >= 7) {
  353.                 I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B);
  354.                 /* GFX_MODE is per-ring on gen7+ */
  355.         }
  356.  
  357.         for_each_ring(ring, dev_priv, i) {
  358.                 if (INTEL_INFO(dev)->gen >= 7)
  359.                         I915_WRITE(RING_MODE_GEN7(ring),
  360.                                    _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
  361.  
  362.                 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
  363.                 I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
  364.         }
  365. }
  366.  
  367. extern int intel_iommu_gfx_mapped;
  368. /* Certain Gen5 chipsets require require idling the GPU before
  369.  * unmapping anything from the GTT when VT-d is enabled.
  370.  */
  371. static inline bool needs_idle_maps(struct drm_device *dev)
  372. {
  373. #ifdef CONFIG_INTEL_IOMMU
  374.         /* Query intel_iommu to see if we need the workaround. Presumably that
  375.          * was loaded first.
  376.          */
  377.         if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped)
  378.                 return true;
  379. #endif
  380.         return false;
  381. }
  382.  
  383. static bool do_idling(struct drm_i915_private *dev_priv)
  384. {
  385.         bool ret = dev_priv->mm.interruptible;
  386.  
  387.         if (unlikely(dev_priv->gtt.do_idle_maps)) {
  388.                 dev_priv->mm.interruptible = false;
  389.                 if (i915_gpu_idle(dev_priv->dev)) {
  390.                         DRM_ERROR("Couldn't idle GPU\n");
  391.                         /* Wait a bit, in hopes it avoids the hang */
  392.                         udelay(10);
  393.                 }
  394.         }
  395.  
  396.         return ret;
  397. }
  398.  
  399. static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
  400. {
  401.         if (unlikely(dev_priv->gtt.do_idle_maps))
  402.                 dev_priv->mm.interruptible = interruptible;
  403. }
  404.  
  405. void i915_gem_restore_gtt_mappings(struct drm_device *dev)
  406. {
  407.         struct drm_i915_private *dev_priv = dev->dev_private;
  408.         struct drm_i915_gem_object *obj;
  409.  
  410.         /* First fill our portion of the GTT with scratch pages */
  411.         dev_priv->gtt.gtt_clear_range(dev, dev_priv->gtt.start / PAGE_SIZE,
  412.                                       dev_priv->gtt.total / PAGE_SIZE);
  413.  
  414.         list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) {
  415.                 i915_gem_clflush_object(obj);
  416.                 i915_gem_gtt_bind_object(obj, obj->cache_level);
  417.         }
  418.  
  419.         i915_gem_chipset_flush(dev);
  420. }
  421.  
  422. int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
  423. {
  424.         if (obj->has_dma_mapping)
  425.                 return 0;
  426.  
  427.         if (!dma_map_sg(&obj->base.dev->pdev->dev,
  428.                         obj->pages->sgl, obj->pages->nents,
  429.                         PCI_DMA_BIDIRECTIONAL))
  430.                 return -ENOSPC;
  431.  
  432.         return 0;
  433. }
  434.  
  435. /*
  436.  * Binds an object into the global gtt with the specified cache level. The object
  437.  * will be accessible to the GPU via commands whose operands reference offsets
  438.  * within the global GTT as well as accessible by the GPU through the GMADR
  439.  * mapped BAR (dev_priv->mm.gtt->gtt).
  440.  */
  441. static void gen6_ggtt_insert_entries(struct drm_device *dev,
  442.                                      struct sg_table *st,
  443.                                      unsigned int first_entry,
  444.                                   enum i915_cache_level level)
  445. {
  446.         struct drm_i915_private *dev_priv = dev->dev_private;
  447.         struct scatterlist *sg = st->sgl;
  448.         gtt_pte_t __iomem *gtt_entries =
  449.                 (gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
  450.         int unused, i = 0;
  451.         unsigned int len, m = 0;
  452.         dma_addr_t addr;
  453.  
  454.         for_each_sg(st->sgl, sg, st->nents, unused) {
  455.                 len = sg_dma_len(sg) >> PAGE_SHIFT;
  456.                 for (m = 0; m < len; m++) {
  457.                         addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
  458.                         iowrite32(gen6_pte_encode(dev, addr, level),
  459.                                   &gtt_entries[i]);
  460.                         i++;
  461.                 }
  462.         }
  463.  
  464.         /* XXX: This serves as a posting read to make sure that the PTE has
  465.          * actually been updated. There is some concern that even though
  466.          * registers and PTEs are within the same BAR that they are potentially
  467.          * of NUMA access patterns. Therefore, even with the way we assume
  468.          * hardware should work, we must keep this posting read for paranoia.
  469.          */
  470.         if (i != 0)
  471.                 WARN_ON(readl(&gtt_entries[i-1])
  472.                         != gen6_pte_encode(dev, addr, level));
  473.  
  474.         /* This next bit makes the above posting read even more important. We
  475.          * want to flush the TLBs only after we're certain all the PTE updates
  476.          * have finished.
  477.          */
  478.         I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
  479.         POSTING_READ(GFX_FLSH_CNTL_GEN6);
  480. }
  481.  
  482. static void gen6_ggtt_clear_range(struct drm_device *dev,
  483.                                   unsigned int first_entry,
  484.                                   unsigned int num_entries)
  485. {
  486.         struct drm_i915_private *dev_priv = dev->dev_private;
  487.         gtt_pte_t scratch_pte;
  488.         gtt_pte_t __iomem *gtt_base = (gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
  489.         const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
  490.         int i;
  491.  
  492.         if (WARN(num_entries > max_entries,
  493.                  "First entry = %d; Num entries = %d (max=%d)\n",
  494.                  first_entry, num_entries, max_entries))
  495.                 num_entries = max_entries;
  496.  
  497.         scratch_pte = gen6_pte_encode(dev, dev_priv->gtt.scratch_page_dma,
  498.                                       I915_CACHE_LLC);
  499.         for (i = 0; i < num_entries; i++)
  500.                 iowrite32(scratch_pte, &gtt_base[i]);
  501.         readl(gtt_base);
  502. }
  503.  
  504.  
  505. static void i915_ggtt_insert_entries(struct drm_device *dev,
  506.                                      struct sg_table *st,
  507.                                      unsigned int pg_start,
  508.                                      enum i915_cache_level cache_level)
  509. {
  510.         unsigned int flags = (cache_level == I915_CACHE_NONE) ?
  511.                 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
  512.  
  513.         intel_gtt_insert_sg_entries(st, pg_start, flags);
  514.  
  515. }
  516.  
  517. static void i915_ggtt_clear_range(struct drm_device *dev,
  518.                                   unsigned int first_entry,
  519.                                   unsigned int num_entries)
  520. {
  521.         intel_gtt_clear_range(first_entry, num_entries);
  522. }
  523.  
  524.  
  525. void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
  526.                                 enum i915_cache_level cache_level)
  527. {
  528.         struct drm_device *dev = obj->base.dev;
  529.         struct drm_i915_private *dev_priv = dev->dev_private;
  530.  
  531.         dev_priv->gtt.gtt_insert_entries(dev, obj->pages,
  532.                                             obj->gtt_space->start >> PAGE_SHIFT,
  533.                                          cache_level);
  534.  
  535.         obj->has_global_gtt_mapping = 1;
  536. }
  537.  
  538. void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
  539. {
  540.         struct drm_device *dev = obj->base.dev;
  541.         struct drm_i915_private *dev_priv = dev->dev_private;
  542.  
  543.         dev_priv->gtt.gtt_clear_range(obj->base.dev,
  544.                               obj->gtt_space->start >> PAGE_SHIFT,
  545.                               obj->base.size >> PAGE_SHIFT);
  546.  
  547.         obj->has_global_gtt_mapping = 0;
  548. }
  549.  
  550. void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
  551. {
  552.         struct drm_device *dev = obj->base.dev;
  553.         struct drm_i915_private *dev_priv = dev->dev_private;
  554.         bool interruptible;
  555.  
  556.         interruptible = do_idling(dev_priv);
  557.  
  558.         if (!obj->has_dma_mapping)
  559.                 dma_unmap_sg(&dev->pdev->dev,
  560.                              obj->pages->sgl, obj->pages->nents,
  561.                              PCI_DMA_BIDIRECTIONAL);
  562.  
  563.         undo_idling(dev_priv, interruptible);
  564. }
  565.  
  566. static void i915_gtt_color_adjust(struct drm_mm_node *node,
  567.                                   unsigned long color,
  568.                                   unsigned long *start,
  569.                                   unsigned long *end)
  570. {
  571.         if (node->color != color)
  572.                 *start += 4096;
  573.  
  574.         if (!list_empty(&node->node_list)) {
  575.                 node = list_entry(node->node_list.next,
  576.                                   struct drm_mm_node,
  577.                                   node_list);
  578.                 if (node->allocated && node->color != color)
  579.                         *end -= 4096;
  580.         }
  581. }
  582. void i915_gem_setup_global_gtt(struct drm_device *dev,
  583.                               unsigned long start,
  584.                               unsigned long mappable_end,
  585.                               unsigned long end)
  586. {
  587.         /* Let GEM Manage all of the aperture.
  588.          *
  589.          * However, leave one page at the end still bound to the scratch page.
  590.          * There are a number of places where the hardware apparently prefetches
  591.          * past the end of the object, and we've seen multiple hangs with the
  592.          * GPU head pointer stuck in a batchbuffer bound at the last page of the
  593.          * aperture.  One page should be enough to keep any prefetching inside
  594.          * of the aperture.
  595.          */
  596.         drm_i915_private_t *dev_priv = dev->dev_private;
  597.         struct drm_mm_node *entry;
  598.         struct drm_i915_gem_object *obj;
  599.         unsigned long hole_start, hole_end;
  600.  
  601.         BUG_ON(mappable_end > end);
  602.  
  603.         /* Subtract the guard page ... */
  604.         drm_mm_init(&dev_priv->mm.gtt_space, start, end - start - PAGE_SIZE);
  605.         if (!HAS_LLC(dev))
  606.                 dev_priv->mm.gtt_space.color_adjust = i915_gtt_color_adjust;
  607.  
  608.         /* Mark any preallocated objects as occupied */
  609.         list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) {
  610.                 DRM_DEBUG_KMS("reserving preallocated space: %x + %zx\n",
  611.                               obj->gtt_offset, obj->base.size);
  612.  
  613.                 BUG_ON(obj->gtt_space != I915_GTT_RESERVED);
  614.                 obj->gtt_space = drm_mm_create_block(&dev_priv->mm.gtt_space,
  615.                                                      obj->gtt_offset,
  616.                                                      obj->base.size,
  617.                                                      false);
  618.                 obj->has_global_gtt_mapping = 1;
  619.         }
  620.  
  621.         dev_priv->gtt.start = start;
  622.         dev_priv->gtt.total = end - start;
  623.  
  624.         /* Clear any non-preallocated blocks */
  625.         drm_mm_for_each_hole(entry, &dev_priv->mm.gtt_space,
  626.                              hole_start, hole_end) {
  627.                 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
  628.                               hole_start, hole_end);
  629.                 dev_priv->gtt.gtt_clear_range(dev, hole_start / PAGE_SIZE,
  630.                                               (hole_end-hole_start) / PAGE_SIZE);
  631.         }
  632.  
  633.         /* And finally clear the reserved guard page */
  634.         dev_priv->gtt.gtt_clear_range(dev, end / PAGE_SIZE - 1, 1);
  635. }
  636.  
  637. static bool
  638. intel_enable_ppgtt(struct drm_device *dev)
  639. {
  640.         if (i915_enable_ppgtt >= 0)
  641.                 return i915_enable_ppgtt;
  642.  
  643. #ifdef CONFIG_INTEL_IOMMU
  644.         /* Disable ppgtt on SNB if VT-d is on. */
  645.         if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
  646.                 return false;
  647. #endif
  648.  
  649.         return true;
  650. }
  651.  
  652. void i915_gem_init_global_gtt(struct drm_device *dev)
  653. {
  654.         struct drm_i915_private *dev_priv = dev->dev_private;
  655.         unsigned long gtt_size, mappable_size;
  656.  
  657.         gtt_size = dev_priv->gtt.total;
  658.         mappable_size = dev_priv->gtt.mappable_end;
  659.  
  660.         if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
  661.                 int ret;
  662.                 /* PPGTT pdes are stolen from global gtt ptes, so shrink the
  663.                  * aperture accordingly when using aliasing ppgtt. */
  664.         gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
  665.         gtt_size -= LFB_SIZE;
  666.  
  667.         i915_gem_setup_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size);
  668.  
  669.                 ret = i915_gem_init_aliasing_ppgtt(dev);
  670.                 if (!ret)
  671.                         return;
  672.  
  673.                 DRM_ERROR("Aliased PPGTT setup failed %d\n", ret);
  674.                 drm_mm_takedown(&dev_priv->mm.gtt_space);
  675.                 gtt_size += I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
  676.         }
  677.     i915_gem_setup_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size);
  678. }
  679.  
  680. static int setup_scratch_page(struct drm_device *dev)
  681. {
  682.         struct drm_i915_private *dev_priv = dev->dev_private;
  683.         struct page *page;
  684.         dma_addr_t dma_addr;
  685.  
  686.         page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
  687.         if (page == NULL)
  688.                 return -ENOMEM;
  689.         get_page(page);
  690.         set_pages_uc(page, 1);
  691.  
  692. #ifdef CONFIG_INTEL_IOMMU
  693.         dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE,
  694.                                 PCI_DMA_BIDIRECTIONAL);
  695.         if (pci_dma_mapping_error(dev->pdev, dma_addr))
  696.                 return -EINVAL;
  697. #else
  698.         dma_addr = page_to_phys(page);
  699. #endif
  700.         dev_priv->gtt.scratch_page = page;
  701.         dev_priv->gtt.scratch_page_dma = dma_addr;
  702.  
  703.         return 0;
  704. }
  705.  
  706. static void teardown_scratch_page(struct drm_device *dev)
  707. {
  708.         struct drm_i915_private *dev_priv = dev->dev_private;
  709.         set_pages_wb(dev_priv->gtt.scratch_page, 1);
  710.         pci_unmap_page(dev->pdev, dev_priv->gtt.scratch_page_dma,
  711.                        PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
  712.         put_page(dev_priv->gtt.scratch_page);
  713.         __free_page(dev_priv->gtt.scratch_page);
  714. }
  715.  
  716. static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
  717. {
  718.         snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
  719.         snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
  720.         return snb_gmch_ctl << 20;
  721. }
  722.  
  723. static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
  724. {
  725.         snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
  726.         snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
  727.         return snb_gmch_ctl << 25; /* 32 MB units */
  728. }
  729.  
  730. static inline size_t gen7_get_stolen_size(u16 snb_gmch_ctl)
  731. {
  732.         static const int stolen_decoder[] = {
  733.                 0, 0, 0, 0, 0, 32, 48, 64, 128, 256, 96, 160, 224, 352};
  734.         snb_gmch_ctl >>= IVB_GMCH_GMS_SHIFT;
  735.         snb_gmch_ctl &= IVB_GMCH_GMS_MASK;
  736.         return stolen_decoder[snb_gmch_ctl] << 20;
  737. }
  738.  
  739. static int gen6_gmch_probe(struct drm_device *dev,
  740.                            size_t *gtt_total,
  741.                            size_t *stolen,
  742.                            phys_addr_t *mappable_base,
  743.                            unsigned long *mappable_end)
  744. {
  745.         struct drm_i915_private *dev_priv = dev->dev_private;
  746.         phys_addr_t gtt_bus_addr;
  747.         unsigned int gtt_size;
  748.         u16 snb_gmch_ctl;
  749.         int ret;
  750.  
  751.         *mappable_base = pci_resource_start(dev->pdev, 2);
  752.         *mappable_end = pci_resource_len(dev->pdev, 2);
  753.  
  754.         /* 64/512MB is the current min/max we actually know of, but this is just
  755.          * a coarse sanity check.
  756.          */
  757.         if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) {
  758.                 DRM_ERROR("Unknown GMADR size (%lx)\n",
  759.                           dev_priv->gtt.mappable_end);
  760.                 return -ENXIO;
  761.                 }
  762.  
  763.         if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40)))
  764.                 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40));
  765.         pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
  766.         gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl);
  767.  
  768.         if (IS_GEN7(dev))
  769.                 *stolen = gen7_get_stolen_size(snb_gmch_ctl);
  770.         else
  771.                 *stolen = gen6_get_stolen_size(snb_gmch_ctl);
  772.  
  773.         *gtt_total = (gtt_size / sizeof(gtt_pte_t)) << PAGE_SHIFT;
  774.  
  775.         /* For GEN6+ the PTEs for the ggtt live at 2MB + BAR0 */
  776.         gtt_bus_addr = pci_resource_start(dev->pdev, 0) + (2<<20);
  777.         dev_priv->gtt.gsm = ioremap_wc(gtt_bus_addr, gtt_size);
  778.         if (!dev_priv->gtt.gsm) {
  779.                 DRM_ERROR("Failed to map the gtt page table\n");
  780.                 return -ENOMEM;
  781.         }
  782.  
  783.         ret = setup_scratch_page(dev);
  784.         if (ret)
  785.                 DRM_ERROR("Scratch setup failed\n");
  786.  
  787.         dev_priv->gtt.gtt_clear_range = gen6_ggtt_clear_range;
  788.         dev_priv->gtt.gtt_insert_entries = gen6_ggtt_insert_entries;
  789.  
  790.         return ret;
  791. }
  792.  
  793. static void gen6_gmch_remove(struct drm_device *dev)
  794. {
  795.         struct drm_i915_private *dev_priv = dev->dev_private;
  796.         iounmap(dev_priv->gtt.gsm);
  797.         teardown_scratch_page(dev_priv->dev);
  798. }
  799.  
  800. static int i915_gmch_probe(struct drm_device *dev,
  801.                            size_t *gtt_total,
  802.                            size_t *stolen,
  803.                            phys_addr_t *mappable_base,
  804.                            unsigned long *mappable_end)
  805. {
  806.         struct drm_i915_private *dev_priv = dev->dev_private;
  807.         int ret;
  808.  
  809.         ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL);
  810.         if (!ret) {
  811.                 DRM_ERROR("failed to set up gmch\n");
  812.                 return -EIO;
  813.         }
  814.  
  815.         intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end);
  816.  
  817.         dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev);
  818.         dev_priv->gtt.gtt_clear_range = i915_ggtt_clear_range;
  819.         dev_priv->gtt.gtt_insert_entries = i915_ggtt_insert_entries;
  820.  
  821.         return 0;
  822. }
  823.  
  824. static void i915_gmch_remove(struct drm_device *dev)
  825. {
  826. //   intel_gmch_remove();
  827. }
  828.  
  829. int i915_gem_gtt_init(struct drm_device *dev)
  830. {
  831.         struct drm_i915_private *dev_priv = dev->dev_private;
  832.         struct i915_gtt *gtt = &dev_priv->gtt;
  833.         unsigned long gtt_size;
  834.         int ret;
  835.  
  836.         if (INTEL_INFO(dev)->gen <= 5) {
  837.                 dev_priv->gtt.gtt_probe = i915_gmch_probe;
  838.                 dev_priv->gtt.gtt_remove = i915_gmch_remove;
  839.         } else {
  840.                 dev_priv->gtt.gtt_probe = gen6_gmch_probe;
  841.                 dev_priv->gtt.gtt_remove = gen6_gmch_remove;
  842.         }
  843.  
  844.         ret = dev_priv->gtt.gtt_probe(dev, &dev_priv->gtt.total,
  845.                                      &dev_priv->gtt.stolen_size,
  846.                                      &gtt->mappable_base,
  847.                                      &gtt->mappable_end);
  848.         if (ret)
  849.         return ret;
  850.  
  851.         gtt_size = (dev_priv->gtt.total >> PAGE_SHIFT) * sizeof(gtt_pte_t);
  852.  
  853.         /* GMADR is the PCI mmio aperture into the global GTT. */
  854.         DRM_INFO("Memory usable by graphics device = %zdM\n",
  855.                  dev_priv->gtt.total >> 20);
  856.         DRM_DEBUG_DRIVER("GMADR size = %ldM\n",
  857.                          dev_priv->gtt.mappable_end >> 20);
  858.         DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n",
  859.                          dev_priv->gtt.stolen_size >> 20);
  860.  
  861.         return 0;
  862. }
  863.  
  864. struct scatterlist *sg_next(struct scatterlist *sg)
  865. {
  866.     if (sg_is_last(sg))
  867.         return NULL;
  868.  
  869.     sg++;
  870.     if (unlikely(sg_is_chain(sg)))
  871.             sg = sg_chain_ptr(sg);
  872.  
  873.     return sg;
  874. }
  875.  
  876.  
  877. void __sg_free_table(struct sg_table *table, unsigned int max_ents,
  878.                      sg_free_fn *free_fn)
  879. {
  880.     struct scatterlist *sgl, *next;
  881.  
  882.     if (unlikely(!table->sgl))
  883.             return;
  884.  
  885.     sgl = table->sgl;
  886.     while (table->orig_nents) {
  887.         unsigned int alloc_size = table->orig_nents;
  888.         unsigned int sg_size;
  889.  
  890.         /*
  891.          * If we have more than max_ents segments left,
  892.          * then assign 'next' to the sg table after the current one.
  893.          * sg_size is then one less than alloc size, since the last
  894.          * element is the chain pointer.
  895.          */
  896.         if (alloc_size > max_ents) {
  897.                 next = sg_chain_ptr(&sgl[max_ents - 1]);
  898.                 alloc_size = max_ents;
  899.                 sg_size = alloc_size - 1;
  900.         } else {
  901.                 sg_size = alloc_size;
  902.                 next = NULL;
  903.         }
  904.  
  905.         table->orig_nents -= sg_size;
  906.         kfree(sgl);
  907.         sgl = next;
  908.     }
  909.  
  910.     table->sgl = NULL;
  911. }
  912.  
  913. void sg_free_table(struct sg_table *table)
  914. {
  915.     __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL);
  916. }
  917.  
  918. int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask)
  919. {
  920.     struct scatterlist *sg, *prv;
  921.     unsigned int left;
  922.     unsigned int max_ents = SG_MAX_SINGLE_ALLOC;
  923.  
  924. #ifndef ARCH_HAS_SG_CHAIN
  925.     BUG_ON(nents > max_ents);
  926. #endif
  927.  
  928.     memset(table, 0, sizeof(*table));
  929.  
  930.     left = nents;
  931.     prv = NULL;
  932.     do {
  933.         unsigned int sg_size, alloc_size = left;
  934.  
  935.         if (alloc_size > max_ents) {
  936.                 alloc_size = max_ents;
  937.                 sg_size = alloc_size - 1;
  938.         } else
  939.                 sg_size = alloc_size;
  940.  
  941.         left -= sg_size;
  942.  
  943.         sg = kmalloc(alloc_size * sizeof(struct scatterlist), gfp_mask);
  944.         if (unlikely(!sg)) {
  945.                 /*
  946.                  * Adjust entry count to reflect that the last
  947.                  * entry of the previous table won't be used for
  948.                  * linkage.  Without this, sg_kfree() may get
  949.                  * confused.
  950.                  */
  951.                 if (prv)
  952.                         table->nents = ++table->orig_nents;
  953.  
  954.                 goto err;
  955.         }
  956.  
  957.         sg_init_table(sg, alloc_size);
  958.         table->nents = table->orig_nents += sg_size;
  959.  
  960.         /*
  961.          * If this is the first mapping, assign the sg table header.
  962.          * If this is not the first mapping, chain previous part.
  963.          */
  964.         if (prv)
  965.                 sg_chain(prv, max_ents, sg);
  966.         else
  967.                 table->sgl = sg;
  968.  
  969.         /*
  970.          * If no more entries after this one, mark the end
  971.          */
  972.         if (!left)
  973.                 sg_mark_end(&sg[sg_size - 1]);
  974.  
  975.         prv = sg;
  976.     } while (left);
  977.  
  978.     return 0;
  979.  
  980. err:
  981.     __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL);
  982.  
  983.     return -ENOMEM;
  984. }
  985.  
  986.  
  987. void sg_init_table(struct scatterlist *sgl, unsigned int nents)
  988. {
  989.     memset(sgl, 0, sizeof(*sgl) * nents);
  990. #ifdef CONFIG_DEBUG_SG
  991.     {
  992.             unsigned int i;
  993.             for (i = 0; i < nents; i++)
  994.                     sgl[i].sg_magic = SG_MAGIC;
  995.     }
  996. #endif
  997.     sg_mark_end(&sgl[nents - 1]);
  998. }
  999.  
  1000.