Subversion Repositories Kolibri OS

Rev

Rev 4126 | Rev 4539 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright © 2010 Daniel Vetter
  3.  *
  4.  * Permission is hereby granted, free of charge, to any person obtaining a
  5.  * copy of this software and associated documentation files (the "Software"),
  6.  * to deal in the Software without restriction, including without limitation
  7.  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8.  * and/or sell copies of the Software, and to permit persons to whom the
  9.  * Software is furnished to do so, subject to the following conditions:
  10.  *
  11.  * The above copyright notice and this permission notice (including the next
  12.  * paragraph) shall be included in all copies or substantial portions of the
  13.  * Software.
  14.  *
  15.  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16.  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17.  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18.  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19.  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20.  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21.  * IN THE SOFTWARE.
  22.  *
  23.  */
  24.  
  25.  
  26. #define AGP_NORMAL_MEMORY 0
  27.  
  28. #define AGP_USER_TYPES (1 << 16)
  29. #define AGP_USER_MEMORY (AGP_USER_TYPES)
  30. #define AGP_USER_CACHED_MEMORY (AGP_USER_TYPES + 1)
  31.  
  32. #include <drm/drmP.h>
  33. #include <drm/i915_drm.h>
  34. #include "i915_drv.h"
  35. #include "i915_trace.h"
  36. #include "intel_drv.h"
  37.  
  38. #define GEN6_PPGTT_PD_ENTRIES 512
  39. #define I915_PPGTT_PT_ENTRIES (PAGE_SIZE / sizeof(gen6_gtt_pte_t))
  40.  
  41. /* PPGTT stuff */
  42. #define GEN6_GTT_ADDR_ENCODE(addr)      ((addr) | (((addr) >> 28) & 0xff0))
  43. #define HSW_GTT_ADDR_ENCODE(addr)       ((addr) | (((addr) >> 28) & 0x7f0))
  44.  
  45. #define GEN6_PDE_VALID                  (1 << 0)
  46. /* gen6+ has bit 11-4 for physical addr bit 39-32 */
  47. #define GEN6_PDE_ADDR_ENCODE(addr)      GEN6_GTT_ADDR_ENCODE(addr)
  48.  
  49. #define GEN6_PTE_VALID                  (1 << 0)
  50. #define GEN6_PTE_UNCACHED               (1 << 1)
  51. #define HSW_PTE_UNCACHED                (0)
  52. #define GEN6_PTE_CACHE_LLC              (2 << 1)
  53. #define GEN7_PTE_CACHE_L3_LLC           (3 << 1)
  54. #define GEN6_PTE_ADDR_ENCODE(addr)      GEN6_GTT_ADDR_ENCODE(addr)
  55. #define HSW_PTE_ADDR_ENCODE(addr)       HSW_GTT_ADDR_ENCODE(addr)
  56.  
  57. /* Cacheability Control is a 4-bit value. The low three bits are stored in *
  58.  * bits 3:1 of the PTE, while the fourth bit is stored in bit 11 of the PTE.
  59.  */
  60. #define HSW_CACHEABILITY_CONTROL(bits)  ((((bits) & 0x7) << 1) | \
  61.                                          (((bits) & 0x8) << (11 - 3)))
  62. #define HSW_WB_LLC_AGE3                 HSW_CACHEABILITY_CONTROL(0x2)
  63. #define HSW_WB_LLC_AGE0                 HSW_CACHEABILITY_CONTROL(0x3)
  64. #define HSW_WB_ELLC_LLC_AGE0            HSW_CACHEABILITY_CONTROL(0xb)
  65. #define HSW_WT_ELLC_LLC_AGE0            HSW_CACHEABILITY_CONTROL(0x6)
  66.  
  67. static gen6_gtt_pte_t snb_pte_encode(dma_addr_t addr,
  68.                                      enum i915_cache_level level,
  69.                                      bool valid)
  70. {
  71.         gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
  72.         pte |= GEN6_PTE_ADDR_ENCODE(addr);
  73.  
  74.         switch (level) {
  75.         case I915_CACHE_L3_LLC:
  76.         case I915_CACHE_LLC:
  77.                 pte |= GEN6_PTE_CACHE_LLC;
  78.                 break;
  79.         case I915_CACHE_NONE:
  80.                 pte |= GEN6_PTE_UNCACHED;
  81.                 break;
  82.         default:
  83.                 WARN_ON(1);
  84.         }
  85.  
  86.         return pte;
  87. }
  88.  
  89. static gen6_gtt_pte_t ivb_pte_encode(dma_addr_t addr,
  90.                                      enum i915_cache_level level,
  91.                                      bool valid)
  92. {
  93.         gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
  94.         pte |= GEN6_PTE_ADDR_ENCODE(addr);
  95.  
  96.         switch (level) {
  97.         case I915_CACHE_L3_LLC:
  98.                 pte |= GEN7_PTE_CACHE_L3_LLC;
  99.                 break;
  100.         case I915_CACHE_LLC:
  101.                 pte |= GEN6_PTE_CACHE_LLC;
  102.                 break;
  103.         case I915_CACHE_NONE:
  104.                         pte |= GEN6_PTE_UNCACHED;
  105.                 break;
  106.         default:
  107.                 WARN_ON(1);
  108.         }
  109.  
  110.         return pte;
  111. }
  112.  
  113. #define BYT_PTE_WRITEABLE               (1 << 1)
  114. #define BYT_PTE_SNOOPED_BY_CPU_CACHES   (1 << 2)
  115.  
  116. static gen6_gtt_pte_t byt_pte_encode(dma_addr_t addr,
  117.                                      enum i915_cache_level level,
  118.                                      bool valid)
  119. {
  120.         gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
  121.         pte |= GEN6_PTE_ADDR_ENCODE(addr);
  122.  
  123.         /* Mark the page as writeable.  Other platforms don't have a
  124.          * setting for read-only/writable, so this matches that behavior.
  125.          */
  126.         pte |= BYT_PTE_WRITEABLE;
  127.  
  128.         if (level != I915_CACHE_NONE)
  129.                 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
  130.  
  131.         return pte;
  132. }
  133.  
  134. static gen6_gtt_pte_t hsw_pte_encode(dma_addr_t addr,
  135.                                      enum i915_cache_level level,
  136.                                      bool valid)
  137. {
  138.         gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
  139.         pte |= HSW_PTE_ADDR_ENCODE(addr);
  140.  
  141.         if (level != I915_CACHE_NONE)
  142.                 pte |= HSW_WB_LLC_AGE3;
  143.  
  144.         return pte;
  145. }
  146.  
  147. static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,
  148.                                       enum i915_cache_level level,
  149.                                       bool valid)
  150. {
  151.         gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
  152.         pte |= HSW_PTE_ADDR_ENCODE(addr);
  153.  
  154.         switch (level) {
  155.         case I915_CACHE_NONE:
  156.                 break;
  157.         case I915_CACHE_WT:
  158.                 pte |= HSW_WT_ELLC_LLC_AGE0;
  159.                 break;
  160.         default:
  161.                 pte |= HSW_WB_ELLC_LLC_AGE0;
  162.                 break;
  163.         }
  164.  
  165.         return pte;
  166. }
  167.  
  168. static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)
  169. {
  170.         struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;
  171.         gen6_gtt_pte_t __iomem *pd_addr;
  172.         uint32_t pd_entry;
  173.         int i;
  174.  
  175.         WARN_ON(ppgtt->pd_offset & 0x3f);
  176.         pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm +
  177.                 ppgtt->pd_offset / sizeof(gen6_gtt_pte_t);
  178.         for (i = 0; i < ppgtt->num_pd_entries; i++) {
  179.                 dma_addr_t pt_addr;
  180.  
  181.                 pt_addr = ppgtt->pt_dma_addr[i];
  182.                 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
  183.                 pd_entry |= GEN6_PDE_VALID;
  184.  
  185.                 writel(pd_entry, pd_addr + i);
  186.         }
  187.         readl(pd_addr);
  188. }
  189.  
  190. static int gen6_ppgtt_enable(struct drm_device *dev)
  191. {
  192.         drm_i915_private_t *dev_priv = dev->dev_private;
  193.         uint32_t pd_offset;
  194.         struct intel_ring_buffer *ring;
  195.         struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
  196.         int i;
  197.  
  198.         BUG_ON(ppgtt->pd_offset & 0x3f);
  199.  
  200.         gen6_write_pdes(ppgtt);
  201.  
  202.         pd_offset = ppgtt->pd_offset;
  203.         pd_offset /= 64; /* in cachelines, */
  204.         pd_offset <<= 16;
  205.  
  206.         if (INTEL_INFO(dev)->gen == 6) {
  207.                 uint32_t ecochk, gab_ctl, ecobits;
  208.  
  209.                 ecobits = I915_READ(GAC_ECO_BITS);
  210.                 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
  211.                                          ECOBITS_PPGTT_CACHE64B);
  212.  
  213.                 gab_ctl = I915_READ(GAB_CTL);
  214.                 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
  215.  
  216.                 ecochk = I915_READ(GAM_ECOCHK);
  217.                 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT |
  218.                                        ECOCHK_PPGTT_CACHE64B);
  219.                 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
  220.         } else if (INTEL_INFO(dev)->gen >= 7) {
  221.                 uint32_t ecochk, ecobits;
  222.  
  223.                 ecobits = I915_READ(GAC_ECO_BITS);
  224.                 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
  225.  
  226.                 ecochk = I915_READ(GAM_ECOCHK);
  227.                 if (IS_HASWELL(dev)) {
  228.                         ecochk |= ECOCHK_PPGTT_WB_HSW;
  229.                 } else {
  230.                         ecochk |= ECOCHK_PPGTT_LLC_IVB;
  231.                         ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
  232.                 }
  233.                 I915_WRITE(GAM_ECOCHK, ecochk);
  234.                 /* GFX_MODE is per-ring on gen7+ */
  235.         }
  236.  
  237.         for_each_ring(ring, dev_priv, i) {
  238.                 if (INTEL_INFO(dev)->gen >= 7)
  239.                         I915_WRITE(RING_MODE_GEN7(ring),
  240.                                    _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
  241.  
  242.                 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
  243.                 I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
  244.         }
  245.         return 0;
  246. }
  247.  
  248. /* PPGTT support for Sandybdrige/Gen6 and later */
  249. static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
  250.                                    unsigned first_entry,
  251.                                    unsigned num_entries,
  252.                                    bool use_scratch)
  253. {
  254.         struct i915_hw_ppgtt *ppgtt =
  255.                 container_of(vm, struct i915_hw_ppgtt, base);
  256.         gen6_gtt_pte_t *pt_vaddr, scratch_pte;
  257.         unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
  258.         unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
  259.         unsigned last_pte, i;
  260.  
  261.         scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, true);
  262.  
  263.     pt_vaddr = AllocKernelSpace(4096);
  264.  
  265.     if(pt_vaddr == NULL)
  266.         return;
  267.  
  268.         while (num_entries) {
  269.             last_pte = first_pte + num_entries;
  270.             if (last_pte > I915_PPGTT_PT_ENTRIES)
  271.                 last_pte = I915_PPGTT_PT_ENTRIES;
  272.  
  273.             MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3);
  274.  
  275.             for (i = first_pte; i < last_pte; i++)
  276.                 pt_vaddr[i] = scratch_pte;
  277.  
  278.             num_entries -= last_pte - first_pte;
  279.             first_pte = 0;
  280.             act_pt++;
  281.     };
  282.  
  283.     FreeKernelSpace(pt_vaddr);
  284. }
  285.  
  286. static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
  287.                                       struct sg_table *pages,
  288.                                       unsigned first_entry,
  289.                                       enum i915_cache_level cache_level)
  290. {
  291.         struct i915_hw_ppgtt *ppgtt =
  292.                 container_of(vm, struct i915_hw_ppgtt, base);
  293.         gen6_gtt_pte_t *pt_vaddr;
  294.         unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
  295.         unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES;
  296.         struct sg_page_iter sg_iter;
  297.         dma_addr_t page_addr;
  298.  
  299.  
  300.     pt_vaddr = AllocKernelSpace(4096);
  301.  
  302.     if(pt_vaddr == NULL)
  303.         return;
  304.  
  305.     MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3);
  306.         for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
  307.                 dma_addr_t page_addr;
  308.  
  309.                 page_addr = sg_page_iter_dma_address(&sg_iter);
  310.                 pt_vaddr[act_pte] = vm->pte_encode(page_addr, cache_level, true);
  311.                 if (++act_pte == I915_PPGTT_PT_ENTRIES) {
  312.                         act_pt++;
  313.                 MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3);
  314.                         act_pte = 0;
  315.  
  316.                         }
  317.                 }
  318.     FreeKernelSpace(pt_vaddr);
  319. }
  320.  
  321. static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
  322. {
  323.         struct i915_hw_ppgtt *ppgtt =
  324.                 container_of(vm, struct i915_hw_ppgtt, base);
  325.         int i;
  326.  
  327.         drm_mm_takedown(&ppgtt->base.mm);
  328.  
  329.         if (ppgtt->pt_dma_addr) {
  330.                 for (i = 0; i < ppgtt->num_pd_entries; i++)
  331.                         pci_unmap_page(ppgtt->base.dev->pdev,
  332.                                        ppgtt->pt_dma_addr[i],
  333.                                        4096, PCI_DMA_BIDIRECTIONAL);
  334.         }
  335.  
  336.         kfree(ppgtt->pt_dma_addr);
  337.         for (i = 0; i < ppgtt->num_pd_entries; i++)
  338.                 __free_page(ppgtt->pt_pages[i]);
  339.         kfree(ppgtt->pt_pages);
  340.         kfree(ppgtt);
  341. }
  342.  
  343. static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
  344. {
  345.         struct drm_device *dev = ppgtt->base.dev;
  346.         struct drm_i915_private *dev_priv = dev->dev_private;
  347.         unsigned first_pd_entry_in_global_pt;
  348.         int i;
  349.         int ret = -ENOMEM;
  350.  
  351.         /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024
  352.          * entries. For aliasing ppgtt support we just steal them at the end for
  353.          * now. */
  354.        first_pd_entry_in_global_pt = gtt_total_entries(dev_priv->gtt);
  355.  
  356.         ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode;
  357.         ppgtt->num_pd_entries = GEN6_PPGTT_PD_ENTRIES;
  358.         ppgtt->enable = gen6_ppgtt_enable;
  359.         ppgtt->base.clear_range = gen6_ppgtt_clear_range;
  360.         ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
  361.         ppgtt->base.cleanup = gen6_ppgtt_cleanup;
  362.         ppgtt->base.scratch = dev_priv->gtt.base.scratch;
  363.         ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries,
  364.                                   GFP_KERNEL);
  365.         if (!ppgtt->pt_pages)
  366.                 return -ENOMEM;
  367.  
  368.         for (i = 0; i < ppgtt->num_pd_entries; i++) {
  369.                 ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
  370.                 if (!ppgtt->pt_pages[i])
  371.                         goto err_pt_alloc;
  372.         }
  373.  
  374.         ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t) *ppgtt->num_pd_entries,
  375.                                              GFP_KERNEL);
  376.     if (!ppgtt->pt_dma_addr)
  377.         goto err_pt_alloc;
  378.  
  379.     for (i = 0; i < ppgtt->num_pd_entries; i++) {
  380.         dma_addr_t pt_addr;
  381.  
  382.                 pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 0, 4096,
  383.                                                PCI_DMA_BIDIRECTIONAL);
  384.  
  385.         ppgtt->pt_dma_addr[i] = pt_addr;
  386.     }
  387.  
  388.         ppgtt->base.clear_range(&ppgtt->base, 0,
  389.                                 ppgtt->num_pd_entries * I915_PPGTT_PT_ENTRIES, true);
  390.  
  391.         ppgtt->pd_offset = first_pd_entry_in_global_pt * sizeof(gen6_gtt_pte_t);
  392.  
  393.         return 0;
  394.  
  395. err_pd_pin:
  396.         if (ppgtt->pt_dma_addr) {
  397.                 for (i--; i >= 0; i--)
  398.                         pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
  399.                                        4096, PCI_DMA_BIDIRECTIONAL);
  400.         }
  401. err_pt_alloc:
  402.         kfree(ppgtt->pt_dma_addr);
  403.         for (i = 0; i < ppgtt->num_pd_entries; i++) {
  404.                 if (ppgtt->pt_pages[i])
  405.                         __free_page(ppgtt->pt_pages[i]);
  406.         }
  407.         kfree(ppgtt->pt_pages);
  408.  
  409.         return ret;
  410. }
  411.  
  412. static int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
  413. {
  414.         struct drm_i915_private *dev_priv = dev->dev_private;
  415.         struct i915_hw_ppgtt *ppgtt;
  416.         int ret;
  417.  
  418.         ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
  419.         if (!ppgtt)
  420.                 return -ENOMEM;
  421.  
  422.         ppgtt->base.dev = dev;
  423.  
  424.         if (INTEL_INFO(dev)->gen < 8)
  425.         ret = gen6_ppgtt_init(ppgtt);
  426.         else
  427.                 BUG();
  428.  
  429.         if (ret)
  430.         kfree(ppgtt);
  431.         else {
  432.                 dev_priv->mm.aliasing_ppgtt = ppgtt;
  433.                 drm_mm_init(&ppgtt->base.mm, ppgtt->base.start,
  434.                             ppgtt->base.total);
  435.         }
  436.  
  437.         return ret;
  438. }
  439.  
  440. void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
  441. {
  442.         struct drm_i915_private *dev_priv = dev->dev_private;
  443.         struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
  444.  
  445.         if (!ppgtt)
  446.                 return;
  447.  
  448.         ppgtt->base.cleanup(&ppgtt->base);
  449.         dev_priv->mm.aliasing_ppgtt = NULL;
  450. }
  451.  
  452. void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
  453.                             struct drm_i915_gem_object *obj,
  454.                             enum i915_cache_level cache_level)
  455. {
  456.         ppgtt->base.insert_entries(&ppgtt->base, obj->pages,
  457.                                    i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT,
  458.                                      cache_level);
  459. }
  460.  
  461. void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
  462.                               struct drm_i915_gem_object *obj)
  463. {
  464.         ppgtt->base.clear_range(&ppgtt->base,
  465.                                 i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT,
  466.                                 obj->base.size >> PAGE_SHIFT,
  467.                                 true);
  468. }
  469.  
  470. extern int intel_iommu_gfx_mapped;
  471. /* Certain Gen5 chipsets require require idling the GPU before
  472.  * unmapping anything from the GTT when VT-d is enabled.
  473.  */
  474. static inline bool needs_idle_maps(struct drm_device *dev)
  475. {
  476. #ifdef CONFIG_INTEL_IOMMU
  477.         /* Query intel_iommu to see if we need the workaround. Presumably that
  478.          * was loaded first.
  479.          */
  480.         if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped)
  481.                 return true;
  482. #endif
  483.         return false;
  484. }
  485.  
  486. static bool do_idling(struct drm_i915_private *dev_priv)
  487. {
  488.         bool ret = dev_priv->mm.interruptible;
  489.  
  490.         if (unlikely(dev_priv->gtt.do_idle_maps)) {
  491.                 dev_priv->mm.interruptible = false;
  492.                 if (i915_gpu_idle(dev_priv->dev)) {
  493.                         DRM_ERROR("Couldn't idle GPU\n");
  494.                         /* Wait a bit, in hopes it avoids the hang */
  495.                         udelay(10);
  496.                 }
  497.         }
  498.  
  499.         return ret;
  500. }
  501.  
  502. static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
  503. {
  504.         if (unlikely(dev_priv->gtt.do_idle_maps))
  505.                 dev_priv->mm.interruptible = interruptible;
  506. }
  507.  
  508. void i915_check_and_clear_faults(struct drm_device *dev)
  509. {
  510.         struct drm_i915_private *dev_priv = dev->dev_private;
  511.         struct intel_ring_buffer *ring;
  512.         int i;
  513.  
  514.         if (INTEL_INFO(dev)->gen < 6)
  515.                 return;
  516.  
  517.         for_each_ring(ring, dev_priv, i) {
  518.                 u32 fault_reg;
  519.                 fault_reg = I915_READ(RING_FAULT_REG(ring));
  520.                 if (fault_reg & RING_FAULT_VALID) {
  521.                         DRM_DEBUG_DRIVER("Unexpected fault\n"
  522.                                          "\tAddr: 0x%08lx\\n"
  523.                                          "\tAddress space: %s\n"
  524.                                          "\tSource ID: %d\n"
  525.                                          "\tType: %d\n",
  526.                                          fault_reg & PAGE_MASK,
  527.                                          fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT",
  528.                                          RING_FAULT_SRCID(fault_reg),
  529.                                          RING_FAULT_FAULT_TYPE(fault_reg));
  530.                         I915_WRITE(RING_FAULT_REG(ring),
  531.                                    fault_reg & ~RING_FAULT_VALID);
  532.                 }
  533.         }
  534.         POSTING_READ(RING_FAULT_REG(&dev_priv->ring[RCS]));
  535. }
  536.  
  537. void i915_gem_suspend_gtt_mappings(struct drm_device *dev)
  538. {
  539.         struct drm_i915_private *dev_priv = dev->dev_private;
  540.  
  541.         /* Don't bother messing with faults pre GEN6 as we have little
  542.          * documentation supporting that it's a good idea.
  543.          */
  544.         if (INTEL_INFO(dev)->gen < 6)
  545.                 return;
  546.  
  547.         i915_check_and_clear_faults(dev);
  548.  
  549.         dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
  550.                                        dev_priv->gtt.base.start / PAGE_SIZE,
  551.                                        dev_priv->gtt.base.total / PAGE_SIZE,
  552.                                        false);
  553. }
  554.  
  555. void i915_gem_restore_gtt_mappings(struct drm_device *dev)
  556. {
  557.         struct drm_i915_private *dev_priv = dev->dev_private;
  558.         struct drm_i915_gem_object *obj;
  559.  
  560.         i915_check_and_clear_faults(dev);
  561.  
  562.         /* First fill our portion of the GTT with scratch pages */
  563.         dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
  564.                                        dev_priv->gtt.base.start / PAGE_SIZE,
  565.                                        dev_priv->gtt.base.total / PAGE_SIZE,
  566.                                        true);
  567.  
  568.         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
  569.                 i915_gem_clflush_object(obj, obj->pin_display);
  570.                 i915_gem_gtt_bind_object(obj, obj->cache_level);
  571.         }
  572.  
  573.         i915_gem_chipset_flush(dev);
  574. }
  575.  
  576. int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
  577. {
  578.         if (obj->has_dma_mapping)
  579.                 return 0;
  580.  
  581.         if (!dma_map_sg(&obj->base.dev->pdev->dev,
  582.                         obj->pages->sgl, obj->pages->nents,
  583.                         PCI_DMA_BIDIRECTIONAL))
  584.                 return -ENOSPC;
  585.  
  586.         return 0;
  587. }
  588.  
  589. /*
  590.  * Binds an object into the global gtt with the specified cache level. The object
  591.  * will be accessible to the GPU via commands whose operands reference offsets
  592.  * within the global GTT as well as accessible by the GPU through the GMADR
  593.  * mapped BAR (dev_priv->mm.gtt->gtt).
  594.  */
  595. static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
  596.                                      struct sg_table *st,
  597.                                      unsigned int first_entry,
  598.                                   enum i915_cache_level level)
  599. {
  600.         struct drm_i915_private *dev_priv = vm->dev->dev_private;
  601.         gen6_gtt_pte_t __iomem *gtt_entries =
  602.                 (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
  603.         int i = 0;
  604.         struct sg_page_iter sg_iter;
  605.         dma_addr_t addr;
  606.  
  607.         for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
  608.                 addr = sg_page_iter_dma_address(&sg_iter);
  609.                 iowrite32(vm->pte_encode(addr, level, true), &gtt_entries[i]);
  610.                         i++;
  611.                 }
  612.  
  613.         /* XXX: This serves as a posting read to make sure that the PTE has
  614.          * actually been updated. There is some concern that even though
  615.          * registers and PTEs are within the same BAR that they are potentially
  616.          * of NUMA access patterns. Therefore, even with the way we assume
  617.          * hardware should work, we must keep this posting read for paranoia.
  618.          */
  619.         if (i != 0)
  620.                 WARN_ON(readl(&gtt_entries[i-1]) !=
  621.                         vm->pte_encode(addr, level, true));
  622.  
  623.         /* This next bit makes the above posting read even more important. We
  624.          * want to flush the TLBs only after we're certain all the PTE updates
  625.          * have finished.
  626.          */
  627.         I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
  628.         POSTING_READ(GFX_FLSH_CNTL_GEN6);
  629. }
  630.  
  631. static void gen6_ggtt_clear_range(struct i915_address_space *vm,
  632.                                   unsigned int first_entry,
  633.                                   unsigned int num_entries,
  634.                                   bool use_scratch)
  635. {
  636.         struct drm_i915_private *dev_priv = vm->dev->dev_private;
  637.         gen6_gtt_pte_t scratch_pte, __iomem *gtt_base =
  638.                 (gen6_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
  639.         const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
  640.         int i;
  641.  
  642.         if (WARN(num_entries > max_entries,
  643.                  "First entry = %d; Num entries = %d (max=%d)\n",
  644.                  first_entry, num_entries, max_entries))
  645.         num_entries = max_entries;
  646.  
  647.         scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, use_scratch);
  648.  
  649.         for (i = 0; i < num_entries; i++)
  650.                 iowrite32(scratch_pte, &gtt_base[i]);
  651.         readl(gtt_base);
  652. }
  653.  
  654.  
  655. static void i915_ggtt_insert_entries(struct i915_address_space *vm,
  656.                                      struct sg_table *st,
  657.                                      unsigned int pg_start,
  658.                                      enum i915_cache_level cache_level)
  659. {
  660.         unsigned int flags = (cache_level == I915_CACHE_NONE) ?
  661.                 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
  662.  
  663.         intel_gtt_insert_sg_entries(st, pg_start, flags);
  664.  
  665. }
  666.  
  667. static void i915_ggtt_clear_range(struct i915_address_space *vm,
  668.                                   unsigned int first_entry,
  669.                                   unsigned int num_entries,
  670.                                   bool unused)
  671. {
  672.         intel_gtt_clear_range(first_entry, num_entries);
  673. }
  674.  
  675.  
  676. void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
  677.                                 enum i915_cache_level cache_level)
  678. {
  679.         struct drm_device *dev = obj->base.dev;
  680.         struct drm_i915_private *dev_priv = dev->dev_private;
  681.         const unsigned long entry = i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT;
  682.  
  683.         dev_priv->gtt.base.insert_entries(&dev_priv->gtt.base, obj->pages,
  684.                                           entry,
  685.                                          cache_level);
  686.  
  687.         obj->has_global_gtt_mapping = 1;
  688. }
  689.  
  690. void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
  691. {
  692.         struct drm_device *dev = obj->base.dev;
  693.         struct drm_i915_private *dev_priv = dev->dev_private;
  694.         const unsigned long entry = i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT;
  695.  
  696.         dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
  697.                                        entry,
  698.                                        obj->base.size >> PAGE_SHIFT,
  699.                                        true);
  700.  
  701.         obj->has_global_gtt_mapping = 0;
  702. }
  703.  
  704. void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
  705. {
  706.         struct drm_device *dev = obj->base.dev;
  707.         struct drm_i915_private *dev_priv = dev->dev_private;
  708.         bool interruptible;
  709.  
  710.         interruptible = do_idling(dev_priv);
  711.  
  712.         if (!obj->has_dma_mapping)
  713.                 dma_unmap_sg(&dev->pdev->dev,
  714.                              obj->pages->sgl, obj->pages->nents,
  715.                              PCI_DMA_BIDIRECTIONAL);
  716.  
  717.         undo_idling(dev_priv, interruptible);
  718. }
  719.  
  720. static void i915_gtt_color_adjust(struct drm_mm_node *node,
  721.                                   unsigned long color,
  722.                                   unsigned long *start,
  723.                                   unsigned long *end)
  724. {
  725.         if (node->color != color)
  726.                 *start += 4096;
  727.  
  728.         if (!list_empty(&node->node_list)) {
  729.                 node = list_entry(node->node_list.next,
  730.                                   struct drm_mm_node,
  731.                                   node_list);
  732.                 if (node->allocated && node->color != color)
  733.                         *end -= 4096;
  734.         }
  735. }
  736. void i915_gem_setup_global_gtt(struct drm_device *dev,
  737.                               unsigned long start,
  738.                               unsigned long mappable_end,
  739.                               unsigned long end)
  740. {
  741.         /* Let GEM Manage all of the aperture.
  742.          *
  743.          * However, leave one page at the end still bound to the scratch page.
  744.          * There are a number of places where the hardware apparently prefetches
  745.          * past the end of the object, and we've seen multiple hangs with the
  746.          * GPU head pointer stuck in a batchbuffer bound at the last page of the
  747.          * aperture.  One page should be enough to keep any prefetching inside
  748.          * of the aperture.
  749.          */
  750.         struct drm_i915_private *dev_priv = dev->dev_private;
  751.         struct i915_address_space *ggtt_vm = &dev_priv->gtt.base;
  752.         struct drm_mm_node *entry;
  753.         struct drm_i915_gem_object *obj;
  754.         unsigned long hole_start, hole_end;
  755.  
  756.         BUG_ON(mappable_end > end);
  757.  
  758.         /* Subtract the guard page ... */
  759.         drm_mm_init(&ggtt_vm->mm, start, end - start - PAGE_SIZE);
  760.         if (!HAS_LLC(dev))
  761.                 dev_priv->gtt.base.mm.color_adjust = i915_gtt_color_adjust;
  762.  
  763.         /* Mark any preallocated objects as occupied */
  764.         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
  765.                 struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
  766.                 int ret;
  767.                 DRM_DEBUG_KMS("reserving preallocated space: %lx + %zx\n",
  768.                               i915_gem_obj_ggtt_offset(obj), obj->base.size);
  769.  
  770.                 WARN_ON(i915_gem_obj_ggtt_bound(obj));
  771.                 ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node);
  772.                 if (ret)
  773.                         DRM_DEBUG_KMS("Reservation failed\n");
  774.                 obj->has_global_gtt_mapping = 1;
  775.                 list_add(&vma->vma_link, &obj->vma_list);
  776.         }
  777.  
  778.         dev_priv->gtt.base.start = start;
  779.         dev_priv->gtt.base.total = end - start;
  780.  
  781.         /* Clear any non-preallocated blocks */
  782.         drm_mm_for_each_hole(entry, &ggtt_vm->mm, hole_start, hole_end) {
  783.                 const unsigned long count = (hole_end - hole_start) / PAGE_SIZE;
  784.                 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
  785.                               hole_start, hole_end);
  786.                 ggtt_vm->clear_range(ggtt_vm, hole_start / PAGE_SIZE, count, true);
  787.         }
  788.  
  789.         /* And finally clear the reserved guard page */
  790.         ggtt_vm->clear_range(ggtt_vm, end / PAGE_SIZE - 1, 1, true);
  791. }
  792.  
  793. static bool
  794. intel_enable_ppgtt(struct drm_device *dev)
  795. {
  796.         if (i915_enable_ppgtt >= 0)
  797.                 return i915_enable_ppgtt;
  798.  
  799. #ifdef CONFIG_INTEL_IOMMU
  800.         /* Disable ppgtt on SNB if VT-d is on. */
  801.         if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
  802.                 return false;
  803. #endif
  804.  
  805.         return true;
  806. }
  807.  
  808. void i915_gem_init_global_gtt(struct drm_device *dev)
  809. {
  810.         struct drm_i915_private *dev_priv = dev->dev_private;
  811.         unsigned long gtt_size, mappable_size;
  812.  
  813.         gtt_size = dev_priv->gtt.base.total;
  814.         mappable_size = dev_priv->gtt.mappable_end;
  815.  
  816.         if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
  817.                 int ret;
  818.  
  819.                 if (INTEL_INFO(dev)->gen <= 7) {
  820.                 /* PPGTT pdes are stolen from global gtt ptes, so shrink the
  821.                  * aperture accordingly when using aliasing ppgtt. */
  822.                         gtt_size -= GEN6_PPGTT_PD_ENTRIES * PAGE_SIZE;
  823.                 }
  824.  
  825.                 i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
  826.  
  827.                 ret = i915_gem_init_aliasing_ppgtt(dev);
  828.                 if (!ret)
  829.                         return;
  830.  
  831.                 DRM_ERROR("Aliased PPGTT setup failed %d\n", ret);
  832.                 drm_mm_takedown(&dev_priv->gtt.base.mm);
  833.                 gtt_size += GEN6_PPGTT_PD_ENTRIES * PAGE_SIZE;
  834. }
  835.         i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
  836. }
  837.  
  838. static int setup_scratch_page(struct drm_device *dev)
  839. {
  840.         struct drm_i915_private *dev_priv = dev->dev_private;
  841.         struct page *page;
  842.         dma_addr_t dma_addr;
  843.  
  844.         page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
  845.         if (page == NULL)
  846.                 return -ENOMEM;
  847.     get_page(page);
  848.         set_pages_uc(page, 1);
  849.  
  850. #ifdef CONFIG_INTEL_IOMMU
  851.         dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE,
  852.                                 PCI_DMA_BIDIRECTIONAL);
  853.         if (pci_dma_mapping_error(dev->pdev, dma_addr))
  854.                 return -EINVAL;
  855. #else
  856.         dma_addr = page_to_phys(page);
  857. #endif
  858.         dev_priv->gtt.base.scratch.page = page;
  859.         dev_priv->gtt.base.scratch.addr = dma_addr;
  860.  
  861.         return 0;
  862. }
  863.  
  864. static void teardown_scratch_page(struct drm_device *dev)
  865. {
  866.         struct drm_i915_private *dev_priv = dev->dev_private;
  867.         struct page *page = dev_priv->gtt.base.scratch.page;
  868.  
  869.         set_pages_wb(page, 1);
  870.         pci_unmap_page(dev->pdev, dev_priv->gtt.base.scratch.addr,
  871.                        PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
  872.         put_page(page);
  873.         __free_page(page);
  874. }
  875.  
  876. static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
  877. {
  878.         snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
  879.         snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
  880.         return snb_gmch_ctl << 20;
  881. }
  882.  
  883. static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
  884. {
  885.         snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
  886.         snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
  887.         return snb_gmch_ctl << 25; /* 32 MB units */
  888. }
  889.  
  890. static int gen6_gmch_probe(struct drm_device *dev,
  891.                            size_t *gtt_total,
  892.                            size_t *stolen,
  893.                            phys_addr_t *mappable_base,
  894.                            unsigned long *mappable_end)
  895. {
  896.         struct drm_i915_private *dev_priv = dev->dev_private;
  897.         phys_addr_t gtt_bus_addr;
  898.         unsigned int gtt_size;
  899.         u16 snb_gmch_ctl;
  900.         int ret;
  901.  
  902.         *mappable_base = pci_resource_start(dev->pdev, 2);
  903.         *mappable_end = pci_resource_len(dev->pdev, 2);
  904.  
  905.         /* 64/512MB is the current min/max we actually know of, but this is just
  906.          * a coarse sanity check.
  907.          */
  908.         if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) {
  909.                 DRM_ERROR("Unknown GMADR size (%lx)\n",
  910.                           dev_priv->gtt.mappable_end);
  911.                 return -ENXIO;
  912.                 }
  913.  
  914.         if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40)))
  915.                 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40));
  916.         pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
  917.         gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl);
  918.  
  919.         *stolen = gen6_get_stolen_size(snb_gmch_ctl);
  920.         *gtt_total = (gtt_size / sizeof(gen6_gtt_pte_t)) << PAGE_SHIFT;
  921.  
  922.         /* For Modern GENs the PTEs and register space are split in the BAR */
  923.         gtt_bus_addr = pci_resource_start(dev->pdev, 0) +
  924.                 (pci_resource_len(dev->pdev, 0) / 2);
  925.  
  926.         dev_priv->gtt.gsm = ioremap_wc(gtt_bus_addr, gtt_size);
  927.         if (!dev_priv->gtt.gsm) {
  928.                 DRM_ERROR("Failed to map the gtt page table\n");
  929.                 return -ENOMEM;
  930.         }
  931.  
  932.         ret = setup_scratch_page(dev);
  933.         if (ret)
  934.                 DRM_ERROR("Scratch setup failed\n");
  935.  
  936.         dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range;
  937.         dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries;
  938.  
  939.         return ret;
  940. }
  941.  
  942. static void gen6_gmch_remove(struct i915_address_space *vm)
  943. {
  944.  
  945.         struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base);
  946.         iounmap(gtt->gsm);
  947.         teardown_scratch_page(vm->dev);
  948. }
  949.  
  950. static int i915_gmch_probe(struct drm_device *dev,
  951.                            size_t *gtt_total,
  952.                            size_t *stolen,
  953.                            phys_addr_t *mappable_base,
  954.                            unsigned long *mappable_end)
  955. {
  956.         struct drm_i915_private *dev_priv = dev->dev_private;
  957.         int ret;
  958.  
  959.         ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL);
  960.         if (!ret) {
  961.                 DRM_ERROR("failed to set up gmch\n");
  962.                 return -EIO;
  963.         }
  964.  
  965.         intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end);
  966.  
  967.         dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev);
  968.         dev_priv->gtt.base.clear_range = i915_ggtt_clear_range;
  969.         dev_priv->gtt.base.insert_entries = i915_ggtt_insert_entries;
  970.  
  971.         return 0;
  972. }
  973.  
  974. static void i915_gmch_remove(struct i915_address_space *vm)
  975. {
  976. //   intel_gmch_remove();
  977. }
  978.  
  979. int i915_gem_gtt_init(struct drm_device *dev)
  980. {
  981.         struct drm_i915_private *dev_priv = dev->dev_private;
  982.         struct i915_gtt *gtt = &dev_priv->gtt;
  983.         int ret;
  984.  
  985.         if (INTEL_INFO(dev)->gen <= 5) {
  986.                 gtt->gtt_probe = i915_gmch_probe;
  987.                 gtt->base.cleanup = i915_gmch_remove;
  988.         } else {
  989.                 gtt->gtt_probe = gen6_gmch_probe;
  990.                 gtt->base.cleanup = gen6_gmch_remove;
  991.                 if (IS_HASWELL(dev) && dev_priv->ellc_size)
  992.                         gtt->base.pte_encode = iris_pte_encode;
  993.                 else if (IS_HASWELL(dev))
  994.                         gtt->base.pte_encode = hsw_pte_encode;
  995.                 else if (IS_VALLEYVIEW(dev))
  996.                         gtt->base.pte_encode = byt_pte_encode;
  997.                 else if (INTEL_INFO(dev)->gen >= 7)
  998.                         gtt->base.pte_encode = ivb_pte_encode;
  999.                 else
  1000.                         gtt->base.pte_encode = snb_pte_encode;
  1001.         }
  1002.  
  1003.         ret = gtt->gtt_probe(dev, &gtt->base.total, &gtt->stolen_size,
  1004.                              &gtt->mappable_base, &gtt->mappable_end);
  1005.         if (ret)
  1006.                 return ret;
  1007.  
  1008.         gtt->base.dev = dev;
  1009.  
  1010.         /* GMADR is the PCI mmio aperture into the global GTT. */
  1011.         DRM_INFO("Memory usable by graphics device = %zdM\n",
  1012.                  gtt->base.total >> 20);
  1013.         DRM_DEBUG_DRIVER("GMADR size = %ldM\n", gtt->mappable_end >> 20);
  1014.         DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20);
  1015.  
  1016.         return 0;
  1017. }
  1018.  
  1019. struct scatterlist *sg_next(struct scatterlist *sg)
  1020. {
  1021.     if (sg_is_last(sg))
  1022.         return NULL;
  1023.  
  1024.     sg++;
  1025.     if (unlikely(sg_is_chain(sg)))
  1026.             sg = sg_chain_ptr(sg);
  1027.  
  1028.     return sg;
  1029. }
  1030.  
  1031.  
  1032. void __sg_free_table(struct sg_table *table, unsigned int max_ents,
  1033.                      sg_free_fn *free_fn)
  1034. {
  1035.     struct scatterlist *sgl, *next;
  1036.  
  1037.     if (unlikely(!table->sgl))
  1038.             return;
  1039.  
  1040.     sgl = table->sgl;
  1041.     while (table->orig_nents) {
  1042.         unsigned int alloc_size = table->orig_nents;
  1043.         unsigned int sg_size;
  1044.  
  1045.         /*
  1046.          * If we have more than max_ents segments left,
  1047.          * then assign 'next' to the sg table after the current one.
  1048.          * sg_size is then one less than alloc size, since the last
  1049.          * element is the chain pointer.
  1050.          */
  1051.         if (alloc_size > max_ents) {
  1052.                 next = sg_chain_ptr(&sgl[max_ents - 1]);
  1053.                 alloc_size = max_ents;
  1054.                 sg_size = alloc_size - 1;
  1055.         } else {
  1056.                 sg_size = alloc_size;
  1057.                 next = NULL;
  1058.         }
  1059.  
  1060.         table->orig_nents -= sg_size;
  1061.         kfree(sgl);
  1062.         sgl = next;
  1063.     }
  1064.  
  1065.     table->sgl = NULL;
  1066. }
  1067.  
  1068. void sg_free_table(struct sg_table *table)
  1069. {
  1070.     __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL);
  1071. }
  1072.  
  1073. int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask)
  1074. {
  1075.     struct scatterlist *sg, *prv;
  1076.     unsigned int left;
  1077.     unsigned int max_ents = SG_MAX_SINGLE_ALLOC;
  1078.  
  1079. #ifndef ARCH_HAS_SG_CHAIN
  1080.     BUG_ON(nents > max_ents);
  1081. #endif
  1082.  
  1083.     memset(table, 0, sizeof(*table));
  1084.  
  1085.     left = nents;
  1086.     prv = NULL;
  1087.     do {
  1088.         unsigned int sg_size, alloc_size = left;
  1089.  
  1090.         if (alloc_size > max_ents) {
  1091.                 alloc_size = max_ents;
  1092.                 sg_size = alloc_size - 1;
  1093.         } else
  1094.                 sg_size = alloc_size;
  1095.  
  1096.         left -= sg_size;
  1097.  
  1098.         sg = kmalloc(alloc_size * sizeof(struct scatterlist), gfp_mask);
  1099.         if (unlikely(!sg)) {
  1100.                 /*
  1101.                  * Adjust entry count to reflect that the last
  1102.                  * entry of the previous table won't be used for
  1103.                  * linkage.  Without this, sg_kfree() may get
  1104.                  * confused.
  1105.                  */
  1106.                 if (prv)
  1107.                         table->nents = ++table->orig_nents;
  1108.  
  1109.                 goto err;
  1110.         }
  1111.  
  1112.         sg_init_table(sg, alloc_size);
  1113.         table->nents = table->orig_nents += sg_size;
  1114.  
  1115.         /*
  1116.          * If this is the first mapping, assign the sg table header.
  1117.          * If this is not the first mapping, chain previous part.
  1118.          */
  1119.         if (prv)
  1120.                 sg_chain(prv, max_ents, sg);
  1121.         else
  1122.                 table->sgl = sg;
  1123.  
  1124.         /*
  1125.          * If no more entries after this one, mark the end
  1126.          */
  1127.         if (!left)
  1128.                 sg_mark_end(&sg[sg_size - 1]);
  1129.  
  1130.         prv = sg;
  1131.     } while (left);
  1132.  
  1133.     return 0;
  1134.  
  1135. err:
  1136.     __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL);
  1137.  
  1138.     return -ENOMEM;
  1139. }
  1140.  
  1141.  
  1142. void sg_init_table(struct scatterlist *sgl, unsigned int nents)
  1143. {
  1144.     memset(sgl, 0, sizeof(*sgl) * nents);
  1145. #ifdef CONFIG_DEBUG_SG
  1146.     {
  1147.             unsigned int i;
  1148.             for (i = 0; i < nents; i++)
  1149.                     sgl[i].sg_magic = SG_MAGIC;
  1150.     }
  1151. #endif
  1152.     sg_mark_end(&sgl[nents - 1]);
  1153. }
  1154.  
  1155.  
  1156. void __sg_page_iter_start(struct sg_page_iter *piter,
  1157.               struct scatterlist *sglist, unsigned int nents,
  1158.               unsigned long pgoffset)
  1159. {
  1160.     piter->__pg_advance = 0;
  1161.     piter->__nents = nents;
  1162.  
  1163.     piter->sg = sglist;
  1164.     piter->sg_pgoffset = pgoffset;
  1165. }
  1166.  
  1167. static int sg_page_count(struct scatterlist *sg)
  1168. {
  1169.     return PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT;
  1170. }
  1171.  
  1172. bool __sg_page_iter_next(struct sg_page_iter *piter)
  1173. {
  1174.     if (!piter->__nents || !piter->sg)
  1175.         return false;
  1176.  
  1177.     piter->sg_pgoffset += piter->__pg_advance;
  1178.     piter->__pg_advance = 1;
  1179.  
  1180.     while (piter->sg_pgoffset >= sg_page_count(piter->sg)) {
  1181.         piter->sg_pgoffset -= sg_page_count(piter->sg);
  1182.         piter->sg = sg_next(piter->sg);
  1183.         if (!--piter->__nents || !piter->sg)
  1184.             return false;
  1185.     }
  1186.  
  1187.     return true;
  1188. }
  1189. EXPORT_SYMBOL(__sg_page_iter_next);
  1190.  
  1191.  
  1192.