WebSVN – Kolibri OS – Diff – /drivers/video/drm/i915/i915_gem_gtt.c

+ *
  */
+#define iowrite32(v, addr)      writel((v), (addr))
+#define AGP_NORMAL_MEMORY 0
+#define AGP_USER_TYPES (1 << 16)
+#define AGP_USER_MEMORY (AGP_USER_TYPES)
-#define iowrite32(v, addr)      writel((v), (addr))
+#define AGP_USER_CACHED_MEMORY (AGP_USER_TYPES + 1)
 #include
 #include
 #include "i915_drv.h"
-#include "i915_trace.h"
-#include "intel_drv.h"
-#define AGP_USER_TYPES          (1 << 16)
-#define AGP_USER_MEMORY         (AGP_USER_TYPES)
+#include "i915_trace.h"
-#define AGP_USER_CACHED_MEMORY  (AGP_USER_TYPES + 1)
+#include "intel_drv.h"
 #define HSW_PTE_UNCACHED		(0)
 #define GEN6_PTE_CACHE_LLC		(2 << 1)
 #define GEN6_PTE_CACHE_LLC_MLC		(3 << 1)
 #define GEN6_PTE_ADDR_ENCODE(addr)	GEN6_GTT_ADDR_ENCODE(addr)
-static inline gtt_pte_t pte_encode(struct drm_device *dev,
+static inline gtt_pte_t gen6_pte_encode(struct drm_device *dev,
 				   dma_addr_t addr,
 				   enum i915_cache_level level)
+{
 	gtt_pte_t pte = GEN6_PTE_VALID;
 	return pte;
+}
 /* PPGTT support for Sandybdrige/Gen6 and later */
-static void i915_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt,
+static void gen6_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt,
 				   unsigned first_entry,
 				   unsigned num_entries)
+{
 	gtt_pte_t *pt_vaddr;
 	gtt_pte_t scratch_pte;
 	unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
+	unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
-	unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
+	unsigned last_pte, i;
 	unsigned last_pte, i;
 	scratch_pte = gen6_pte_encode(ppgtt->dev,
-	scratch_pte = pte_encode(ppgtt->dev, ppgtt->scratch_page_dma_addr,
+				      ppgtt->scratch_page_dma_addr,
 				 I915_CACHE_LLC);
+    pt_vaddr = AllocKernelSpace(4096);
-    pt_vaddr = AllocKernelSpace(4096);
-    if(pt_vaddr != NULL)
+    if(pt_vaddr == NULL)
         return;
         while (num_entries)
                 pt_vaddr[i] = scratch_pte;
             num_entries -= last_pte - first_pte;
             first_pte = 0;
+            act_pd++;
+    };
+    FreeKernelSpace(pt_vaddr);
+}
+static void gen6_ppgtt_insert_entries(struct i915_hw_ppgtt *ppgtt,
+				      struct sg_table *pages,
+				      unsigned first_entry,
+				      enum i915_cache_level cache_level)
+{
+	gtt_pte_t *pt_vaddr;
+	unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
+	unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
+	unsigned i, j, m, segment_len;
+	dma_addr_t page_addr;
+	struct scatterlist *sg;
+	/* init sg walking */
+	sg = pages->sgl;
+	i = 0;
+	segment_len = sg_dma_len(sg) >> PAGE_SHIFT;
+	m = 0;
+    pt_vaddr = AllocKernelSpace(4096);
+    if(pt_vaddr == NULL)
+        return;
+	while (i < pages->nents) {
+        MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pd]), 3);
+		for (j = first_pte; j < I915_PPGTT_PT_ENTRIES; j++) {
+			page_addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
+			pt_vaddr[j] = gen6_pte_encode(ppgtt->dev, page_addr,
+						      cache_level);
+			/* grab the next page */
+			if (++m == segment_len) {
+				if (++i == pages->nents)
+					break;
+				sg = sg_next(sg);
+				segment_len = sg_dma_len(sg) >> PAGE_SHIFT;
+				m = 0;
+			}
+		}
+		first_pte = 0;
-            act_pd++;
+		act_pd++;
+        }
-        FreeKernelSpace(pt_vaddr);
-    };
+        FreeKernelSpace(pt_vaddr);
 }
+static void gen6_ppgtt_cleanup(struct i915_hw_ppgtt *ppgtt)
+{
+	int i;
+	if (ppgtt->pt_dma_addr) {
+		for (i = 0; i < ppgtt->num_pd_entries; i++)
+			pci_unmap_page(ppgtt->dev->pdev,
+				       ppgtt->pt_dma_addr[i],
+, PCI_DMA_BIDIRECTIONAL);
+	}
+	kfree(ppgtt->pt_dma_addr);
+	for (i = 0; i < ppgtt->num_pd_entries; i++)
+		__free_page(ppgtt->pt_pages[i]);
+	kfree(ppgtt->pt_pages);
+	kfree(ppgtt);
+}
+static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
 int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_device *dev = ppgtt->dev;
-	struct i915_hw_ppgtt *ppgtt;
+	struct drm_i915_private *dev_priv = dev->dev_private;
 	unsigned first_pd_entry_in_global_pt;
 	int i;
 	int ret = -ENOMEM;
 	/* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024
-	 * entries. For aliasing ppgtt support we just steal them at the end for
-	 * now. */
+	 * entries. For aliasing ppgtt support we just steal them at the end for
-	first_pd_entry_in_global_pt = dev_priv->mm.gtt->gtt_total_entries - I915_PPGTT_PD_ENTRIES;
-	ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
+	 * now. */
+	first_pd_entry_in_global_pt =
+		gtt_total_entries(dev_priv->gtt) - I915_PPGTT_PD_ENTRIES;
-	if (!ppgtt)
+	ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES;
-		return ret;
+	ppgtt->clear_range = gen6_ppgtt_clear_range;
 	ppgtt->insert_entries = gen6_ppgtt_insert_entries;
-	ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES;
+	ppgtt->cleanup = gen6_ppgtt_cleanup;
 	ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries,
 				  GFP_KERNEL);
 	if (!ppgtt->pt_pages)
-		goto err_ppgtt;
+		return -ENOMEM;
-	for (i = 0; i < ppgtt->num_pd_entries; i++) {
-		ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
-		if (!ppgtt->pt_pages[i])
+	for (i = 0; i < ppgtt->num_pd_entries; i++) {
-			goto err_pt_alloc;
 		ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
 		if (!ppgtt->pt_pages[i])
-/*
+			goto err_pt_alloc;
 	if (dev_priv->mm.gtt->needs_dmar) {
 		ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t)
-						*ppgtt->num_pd_entries,
+	ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t) *ppgtt->num_pd_entries,
-					     GFP_KERNEL);
-		if (!ppgtt->pt_dma_addr)
+					     GFP_KERNEL);
-			goto err_pt_alloc;
-		for (i = 0; i < ppgtt->num_pd_entries; i++) {
-			dma_addr_t pt_addr;
-			pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i],
-, 4096,
+		if (!ppgtt->pt_dma_addr)
-					       PCI_DMA_BIDIRECTIONAL);
+			goto err_pt_alloc;
-			if (pci_dma_mapping_error(dev->pdev,
-						  pt_addr)) {
+		for (i = 0; i < ppgtt->num_pd_entries; i++) {
-				ret = -EIO;
+			dma_addr_t pt_addr;
 				goto err_pd_pin;
 		pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 0, 4096,
-			}
-			ppgtt->pt_dma_addr[i] = pt_addr;
 					       PCI_DMA_BIDIRECTIONAL);
-*/
+			ppgtt->pt_dma_addr[i] = pt_addr;
-	ppgtt->scratch_page_dma_addr = dev_priv->mm.gtt->scratch_page_dma;
+		}
-	i915_ppgtt_clear_range(ppgtt, 0,
+	ppgtt->scratch_page_dma_addr = dev_priv->gtt.scratch_page_dma;
-			       ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES);
 	ppgtt->clear_range(ppgtt, 0,
-	ppgtt->pd_offset = (first_pd_entry_in_global_pt)*sizeof(gtt_pte_t);
+			       ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES);
-	dev_priv->mm.aliasing_ppgtt = ppgtt;
+	ppgtt->pd_offset = (first_pd_entry_in_global_pt)*sizeof(gtt_pte_t);
 	return 0;
-err_pd_pin:
-//   if (ppgtt->pt_dma_addr) {
-//       for (i--; i >= 0; i--)
+err_pd_pin:
-//           pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
+	if (ppgtt->pt_dma_addr) {
-//                      4096, PCI_DMA_BIDIRECTIONAL);
+		for (i--; i >= 0; i--)
-//   }
+			pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
-err_pt_alloc:
+, PCI_DMA_BIDIRECTIONAL);
-//   kfree(ppgtt->pt_dma_addr);
+	}
-	for (i = 0; i < ppgtt->num_pd_entries; i++) {
+err_pt_alloc:
+	kfree(ppgtt->pt_dma_addr);
-		if (ppgtt->pt_pages[i])
+	for (i = 0; i < ppgtt->num_pd_entries; i++) {
-            FreePage((addr_t)(ppgtt->pt_pages[i]));
+		if (ppgtt->pt_pages[i])
-	}
+			__free_page(ppgtt->pt_pages[i]);
-	kfree(ppgtt->pt_pages);
-err_ppgtt:
-	kfree(ppgtt);
-	return ret;
+	}
-}
-void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
+	kfree(ppgtt->pt_pages);
+	return ret;
+}
+static int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
+	struct drm_i915_private *dev_priv = dev->dev_private;
-	int i;
-	if (!ppgtt)
-		return;
+	struct i915_hw_ppgtt *ppgtt;
-//   if (ppgtt->pt_dma_addr) {
+	int ret;
-//       for (i = 0; i < ppgtt->num_pd_entries; i++)
-//           pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
-//                      4096, PCI_DMA_BIDIRECTIONAL);
-//   }
-//   kfree(ppgtt->pt_dma_addr);
-	for (i = 0; i < ppgtt->num_pd_entries; i++)
-        FreePage((addr_t)(ppgtt->pt_pages[i]));
-	kfree(ppgtt->pt_pages);
-	kfree(ppgtt);
-}
+	ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
-static void i915_ppgtt_insert_sg_entries(struct i915_hw_ppgtt *ppgtt,
+	if (!ppgtt)
-					 const struct sg_table *pages,
-					 unsigned first_entry,
-					 enum i915_cache_level cache_level)
-{
-	gtt_pte_t *pt_vaddr;
-	unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
-	unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
-	unsigned i, j, m, segment_len;
-	dma_addr_t page_addr;
-	struct scatterlist *sg;
-	/* init sg walking */
-	sg = pages->sgl;
-	i = 0;
+		return -ENOMEM;
-	segment_len = sg_dma_len(sg) >> PAGE_SHIFT;
-	m = 0;
-    pt_vaddr = AllocKernelSpace(4096);
-	if( pt_vaddr == NULL)
-		return;
-	while (i < pages->nents) {
-		MapPage(pt_vaddr,(addr_t)ppgtt->pt_pages[act_pd], 3);
         for (j = first_pte; j < I915_PPGTT_PT_ENTRIES; j++) {
-			page_addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
+	ppgtt->dev = dev;
 			pt_vaddr[j] = pte_encode(ppgtt->dev, page_addr,
-						 cache_level);
+	ret = gen6_ppgtt_init(ppgtt);
 	if (ret)
-			/* grab the next page */
+	kfree(ppgtt);
-			if (++m == segment_len) {
-				if (++i == pages->nents)
+	else
-					break;
+		dev_priv->mm.aliasing_ppgtt = ppgtt;
-				sg = sg_next(sg);
+	return ret;
 				segment_len = sg_dma_len(sg) >> PAGE_SHIFT;
 				m = 0;
-			}
+void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
+{
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
         first_pte = 0;
-        act_pd++;
+	if (!ppgtt)
 		return;
         FreeKernelSpace(pt_vaddr);
 	ppgtt->cleanup(ppgtt);
+}
-void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
-			    struct drm_i915_gem_object *obj,
+void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
-			    enum i915_cache_level cache_level)
+			    struct drm_i915_gem_object *obj,
 			    enum i915_cache_level cache_level)
 	i915_ppgtt_insert_sg_entries(ppgtt,
-				     obj->pages,
+	ppgtt->insert_entries(ppgtt, obj->pages,
 				     obj->gtt_space->start >> PAGE_SHIFT,
 				     cache_level);
-}
+}
-void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
-			      struct drm_i915_gem_object *obj)
-{
 	i915_ppgtt_clear_range(ppgtt,
-			       obj->gtt_space->start >> PAGE_SHIFT,
+void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
-			       obj->base.size >> PAGE_SHIFT);
+			      struct drm_i915_gem_object *obj)
+{
 		I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
 		I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
+	}
+}
+extern int intel_iommu_gfx_mapped;
+/* Certain Gen5 chipsets require require idling the GPU before
+ * unmapping anything from the GTT when VT-d is enabled.
+ */
+static inline bool needs_idle_maps(struct drm_device *dev)
+{
+#ifdef CONFIG_INTEL_IOMMU
+	/* Query intel_iommu to see if we need the workaround. Presumably that
+	 * was loaded first.
+	 */
+	if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped)
+		return true;
+#endif
+	return false;
+}
 static bool do_idling(struct drm_i915_private *dev_priv)
+{
 	bool ret = dev_priv->mm.interruptible;
-	if (unlikely(dev_priv->mm.gtt->do_idle_maps)) {
+	if (unlikely(dev_priv->gtt.do_idle_maps)) {
 		dev_priv->mm.interruptible = false;
 		if (i915_gpu_idle(dev_priv->dev)) {
 			DRM_ERROR("Couldn't idle GPU\n");
 	return ret;
+}
 static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
 {
-	if (unlikely(dev_priv->mm.gtt->do_idle_maps))
+	if (unlikely(dev_priv->gtt.do_idle_maps))
 		dev_priv->mm.interruptible = interruptible;
-}
-static void i915_ggtt_clear_range(struct drm_device *dev,
-				 unsigned first_entry,
-				 unsigned num_entries)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	gtt_pte_t scratch_pte;
-	gtt_pte_t __iomem *gtt_base = dev_priv->mm.gtt->gtt + first_entry;
-	const int max_entries = dev_priv->mm.gtt->gtt_total_entries - first_entry;
-	int i;
-	if (INTEL_INFO(dev)->gen < 6) {
-		intel_gtt_clear_range(first_entry, num_entries);
-		return;
-	}
-	if (WARN(num_entries > max_entries,
-		 "First entry = %d; Num entries = %d (max=%d)\n",
-		 first_entry, num_entries, max_entries))
-		num_entries = max_entries;
-	scratch_pte = pte_encode(dev, dev_priv->mm.gtt->scratch_page_dma, I915_CACHE_LLC);
-	for (i = 0; i < num_entries; i++)
-		iowrite32(scratch_pte, >t_base[i]);
-	readl(gtt_base);
-}
+}
 #if 0
 void i915_gem_restore_gtt_mappings(struct drm_device *dev)
+{
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct drm_i915_gem_object *obj;
 	/* First fill our portion of the GTT with scratch pages */
-	i915_ggtt_clear_range(dev, dev_priv->mm.gtt_start / PAGE_SIZE,
+	dev_priv->gtt.gtt_clear_range(dev, dev_priv->gtt.start / PAGE_SIZE,
-			      (dev_priv->mm.gtt_end - dev_priv->mm.gtt_start) / PAGE_SIZE);
+				      dev_priv->gtt.total / PAGE_SIZE);
 	list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) {
 		i915_gem_clflush_object(obj);
-		i915_gem_gtt_bind_object(obj, obj->cache_level);
 		i915_gem_gtt_bind_object(obj, obj->cache_level);
+	}
-	i915_gem_chipset_flush(dev);
-}
-#endif
 int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
 	i915_gem_chipset_flush(dev);
-    struct scatterlist *sg, *s;
+}
-    unsigned int nents ;
-    int i;
-	if (obj->has_dma_mapping)
-		return 0;
+int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
-    sg    = obj->pages->sgl;
+{
-    nents = obj->pages->nents;
-    WARN_ON(nents == 0 || sg[0].length == 0);
 	if (obj->has_dma_mapping)
-    for_each_sg(sg, s, nents, i) {
+		return 0;
              BUG_ON(!sg_page(s));
-             s->dma_address = sg_phys(s);
+	if (!dma_map_sg(&obj->base.dev->pdev->dev,
 			obj->pages->sgl, obj->pages->nents,
 			PCI_DMA_BIDIRECTIONAL))
-    asm volatile("lock; addl $0,0(%%esp)": : :"memory");
+		return -ENOSPC;
 	return 0;
+}
 /*
  * Binds an object into the global gtt with the specified cache level. The object
-/*
- * Binds an object into the global gtt with the specified cache level. The object
+ * will be accessible to the GPU via commands whose operands reference offsets
- * will be accessible to the GPU via commands whose operands reference offsets
  * within the global GTT as well as accessible by the GPU through the GMADR
- * mapped BAR (dev_priv->mm.gtt->gtt).
- */
+ * mapped BAR (dev_priv->mm.gtt->gtt).
-static void gen6_ggtt_bind_object(struct drm_i915_gem_object *obj,
+ */
-				  enum i915_cache_level level)
+static void gen6_ggtt_insert_entries(struct drm_device *dev,
 				     struct sg_table *st,
-	struct drm_device *dev = obj->base.dev;
+				     unsigned int first_entry,
-	struct drm_i915_private *dev_priv = dev->dev_private;
+				  enum i915_cache_level level)
 	struct sg_table *st = obj->pages;
-	struct scatterlist *sg = st->sgl;
+	struct drm_i915_private *dev_priv = dev->dev_private;
-	const int first_entry = obj->gtt_space->start >> PAGE_SHIFT;
+	struct scatterlist *sg = st->sgl;
-	const int max_entries = dev_priv->mm.gtt->gtt_total_entries - first_entry;
+	gtt_pte_t __iomem *gtt_entries =
+		(gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
-	gtt_pte_t __iomem *gtt_entries = dev_priv->mm.gtt->gtt + first_entry;
+	int unused, i = 0;
-	int unused, i = 0;
+	unsigned int len, m = 0;
-	unsigned int len, m = 0;
+	dma_addr_t addr;
-	dma_addr_t addr;
-	for_each_sg(st->sgl, sg, st->nents, unused) {
 		len = sg_dma_len(sg) >> PAGE_SHIFT;
-		for (m = 0; m < len; m++) {
+	for_each_sg(st->sgl, sg, st->nents, unused) {
-			addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
+		len = sg_dma_len(sg) >> PAGE_SHIFT;
-			iowrite32(pte_encode(dev, addr, level), >t_entries[i]);
+		for (m = 0; m < len; m++) {
-			i++;
+			addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
 			iowrite32(gen6_pte_encode(dev, addr, level),
 				  >t_entries[i]);
+			i++;
+		}
 	BUG_ON(i > max_entries);
 	BUG_ON(i != obj->base.size / PAGE_SIZE);
 	/* XXX: This serves as a posting read to make sure that the PTE has
-	/* XXX: This serves as a posting read to make sure that the PTE has
+	 * actually been updated. There is some concern that even though
-	 * actually been updated. There is some concern that even though
+	 * registers and PTEs are within the same BAR that they are potentially
-	 * registers and PTEs are within the same BAR that they are potentially
+	 * of NUMA access patterns. Therefore, even with the way we assume
-	 * of NUMA access patterns. Therefore, even with the way we assume
+	 * hardware should work, we must keep this posting read for paranoia.
+	 */
+	if (i != 0)
+		WARN_ON(readl(>t_entries[i-1])
+			!= gen6_pte_encode(dev, addr, level));
+	/* This next bit makes the above posting read even more important. We
+	 * want to flush the TLBs only after we're certain all the PTE updates
+	 * have finished.
+	 */
+	I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
+	POSTING_READ(GFX_FLSH_CNTL_GEN6);
+}
+static void gen6_ggtt_clear_range(struct drm_device *dev,
+				  unsigned int first_entry,
+				  unsigned int num_entries)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	gtt_pte_t scratch_pte;
+	gtt_pte_t __iomem *gtt_base = (gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
+	const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
+	int i;
-	 * hardware should work, we must keep this posting read for paranoia.
+	if (WARN(num_entries > max_entries,
+		 "First entry = %d; Num entries = %d (max=%d)\n",
+		 first_entry, num_entries, max_entries))
-	 */
+		num_entries = max_entries;
 	if (i != 0)
-		WARN_ON(readl(>t_entries[i-1]) != pte_encode(dev, addr, level));
-	/* This next bit makes the above posting read even more important. We
+	scratch_pte = gen6_pte_encode(dev, dev_priv->gtt.scratch_page_dma,
-	 * want to flush the TLBs only after we're certain all the PTE updates
+				      I915_CACHE_LLC);
+	for (i = 0; i < num_entries; i++)
-	 * have finished.
+		iowrite32(scratch_pte, >t_base[i]);
+	readl(gtt_base);
+}
-	 */
+static void i915_ggtt_insert_entries(struct drm_device *dev,
-	I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
+				     struct sg_table *st,
-	POSTING_READ(GFX_FLSH_CNTL_GEN6);
+				     unsigned int pg_start,
-}
+				     enum i915_cache_level cache_level)
+{
+	unsigned int flags = (cache_level == I915_CACHE_NONE) ?
+		AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
+	intel_gtt_insert_sg_entries(st, pg_start, flags);
+}
+static void i915_ggtt_clear_range(struct drm_device *dev,
+				  unsigned int first_entry,
+				  unsigned int num_entries)
+{
-void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
+	intel_gtt_clear_range(first_entry, num_entries);
 				enum i915_cache_level cache_level)
 	struct drm_device *dev = obj->base.dev;
+void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
+				enum i915_cache_level cache_level)
+{
-	if (INTEL_INFO(dev)->gen < 6) {
+	struct drm_device *dev = obj->base.dev;
-		unsigned int flags = (cache_level == I915_CACHE_NONE) ?
+	struct drm_i915_private *dev_priv = dev->dev_private;
 			AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
-		intel_gtt_insert_sg_entries(obj->pages,
+	dev_priv->gtt.gtt_insert_entries(dev, obj->pages,
 					    obj->gtt_space->start >> PAGE_SHIFT,
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	bool interruptible;
 	interruptible = do_idling(dev_priv);
-//   if (!obj->has_dma_mapping)
+	if (!obj->has_dma_mapping)
-//       dma_unmap_sg(&dev->pdev->dev,
+		dma_unmap_sg(&dev->pdev->dev,
-//                obj->pages->sgl, obj->pages->nents,
+			     obj->pages->sgl, obj->pages->nents,
-//                PCI_DMA_BIDIRECTIONAL);
+			     PCI_DMA_BIDIRECTIONAL);
 				  node_list);
 		if (node->allocated && node->color != color)
 			*end -= 4096;
+	}
+}
-void i915_gem_init_global_gtt(struct drm_device *dev,
+void i915_gem_setup_global_gtt(struct drm_device *dev,
 			      unsigned long start,
 			      unsigned long mappable_end,
 			      unsigned long end)
+{
+	/* Let GEM Manage all of the aperture.
+	 *
+	 * However, leave one page at the end still bound to the scratch page.
+	 * There are a number of places where the hardware apparently prefetches
+	 * past the end of the object, and we've seen multiple hangs with the
+	 * GPU head pointer stuck in a batchbuffer bound at the last page of the
+	 * aperture.  One page should be enough to keep any prefetching inside
+	 * of the aperture.
+	 */
 	drm_i915_private_t *dev_priv = dev->dev_private;
+	struct drm_mm_node *entry;
+	struct drm_i915_gem_object *obj;
+	unsigned long hole_start, hole_end;
+	BUG_ON(mappable_end > end);
-	/* Substract the guard page ... */
+	/* Subtract the guard page ... */
 	drm_mm_init(&dev_priv->mm.gtt_space, start, end - start - PAGE_SIZE);
 	if (!HAS_LLC(dev))
+		dev_priv->mm.gtt_space.color_adjust = i915_gtt_color_adjust;
+	/* Mark any preallocated objects as occupied */
+	list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) {
+		DRM_DEBUG_KMS("reserving preallocated space: %x + %zx\n",
+			      obj->gtt_offset, obj->base.size);
+		BUG_ON(obj->gtt_space != I915_GTT_RESERVED);
+		obj->gtt_space = drm_mm_create_block(&dev_priv->mm.gtt_space,
+						     obj->gtt_offset,
+						     obj->base.size,
+						     false);
+		obj->has_global_gtt_mapping = 1;
-		dev_priv->mm.gtt_space.color_adjust = i915_gtt_color_adjust;
+	}
+	dev_priv->gtt.start = start;
+	dev_priv->gtt.total = end - start;
+	/* Clear any non-preallocated blocks */
+	drm_mm_for_each_hole(entry, &dev_priv->mm.gtt_space,
-	dev_priv->mm.gtt_start = start;
+			     hole_start, hole_end) {
-	dev_priv->mm.gtt_mappable_end = mappable_end;
+		DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
+			      hole_start, hole_end);
+		dev_priv->gtt.gtt_clear_range(dev, hole_start / PAGE_SIZE,
+					      (hole_end-hole_start) / PAGE_SIZE);
+	}
-	dev_priv->mm.gtt_end = end;
+	/* And finally clear the reserved guard page */
+	dev_priv->gtt.gtt_clear_range(dev, end / PAGE_SIZE - 1, 1);
+}
+static bool
+intel_enable_ppgtt(struct drm_device *dev)
+{
+	if (i915_enable_ppgtt >= 0)
+		return i915_enable_ppgtt;
+#ifdef CONFIG_INTEL_IOMMU
+	/* Disable ppgtt on SNB if VT-d is on. */
+	if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
+		return false;
+#endif
+	return true;
+}
+void i915_gem_init_global_gtt(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	unsigned long gtt_size, mappable_size;
+	gtt_size = dev_priv->gtt.total;
+	mappable_size = dev_priv->gtt.mappable_end;
+	if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
+		int ret;
+		/* PPGTT pdes are stolen from global gtt ptes, so shrink the
+		 * aperture accordingly when using aliasing ppgtt. */
+        gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
+        gtt_size -= LFB_SIZE;
+        i915_gem_setup_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size);
+		ret = i915_gem_init_aliasing_ppgtt(dev);
-	dev_priv->mm.gtt_total = end - start;
+		if (!ret)
+			return;
+		DRM_ERROR("Aliased PPGTT setup failed %d\n", ret);
-	dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
+		drm_mm_takedown(&dev_priv->mm.gtt_space);
 		gtt_size += I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
 	/* ... but ensure that we clear the entire range. */
-	i915_ggtt_clear_range(dev, start / PAGE_SIZE, (end-start) / PAGE_SIZE);
+    i915_gem_setup_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size);
+}
 	dma_addr_t dma_addr;
 	page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
 	if (page == NULL)
+		return -ENOMEM;
+	get_page(page);
-		return -ENOMEM;
+	set_pages_uc(page, 1);
 #ifdef CONFIG_INTEL_IOMMU
 	dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE,
 				PCI_DMA_BIDIRECTIONAL);
 	if (pci_dma_mapping_error(dev->pdev, dma_addr))
 		return -EINVAL;
 #else
 	dma_addr = page_to_phys(page);
 #endif
-	dev_priv->mm.gtt->scratch_page = page;
+	dev_priv->gtt.scratch_page = page;
-	dev_priv->mm.gtt->scratch_page_dma = dma_addr;
+	dev_priv->gtt.scratch_page_dma = dma_addr;
+	return 0;
+}
+static void teardown_scratch_page(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	set_pages_wb(dev_priv->gtt.scratch_page, 1);
+	pci_unmap_page(dev->pdev, dev_priv->gtt.scratch_page_dma,
 		       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
-	return 0;
+	put_page(dev_priv->gtt.scratch_page);
 	__free_page(dev_priv->gtt.scratch_page);
+}
 static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
 {
 	snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
 	snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
 	return snb_gmch_ctl << 20;
+}
-static inline unsigned int gen6_get_stolen_size(u16 snb_gmch_ctl)
+static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
+{
 	snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
 	snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
 	return snb_gmch_ctl << 25; /* 32 MB units */
+}
-static inline unsigned int gen7_get_stolen_size(u16 snb_gmch_ctl)
+static inline size_t gen7_get_stolen_size(u16 snb_gmch_ctl)
 {
+	static const int stolen_decoder[] = {
+, 0, 0, 0, 0, 32, 48, 64, 128, 256, 96, 160, 224, 352};
+	snb_gmch_ctl >>= IVB_GMCH_GMS_SHIFT;
+	snb_gmch_ctl &= IVB_GMCH_GMS_MASK;
-	static const int stolen_decoder[] = {
+	return stolen_decoder[snb_gmch_ctl] << 20;
 , 0, 0, 0, 0, 32, 48, 64, 128, 256, 96, 160, 224, 352};
 	snb_gmch_ctl >>= IVB_GMCH_GMS_SHIFT;
+static int gen6_gmch_probe(struct drm_device *dev,
-	snb_gmch_ctl &= IVB_GMCH_GMS_MASK;
+			   size_t *gtt_total,
-	return stolen_decoder[snb_gmch_ctl] << 20;
+			   size_t *stolen,
-}
+			   phys_addr_t *mappable_base,
-int i915_gem_gtt_init(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
+			   unsigned long *mappable_end)
-	phys_addr_t gtt_bus_addr;
-	u16 snb_gmch_ctl;
-	int ret;
-	/* On modern platforms we need not worry ourself with the legacy
+{
-	 * hostbridge query stuff. Skip it entirely
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	phys_addr_t gtt_bus_addr;
+	unsigned int gtt_size;
-	 */
+	u16 snb_gmch_ctl;
-	if (INTEL_INFO(dev)->gen < 6) {
+	int ret;
-		ret = intel_gmch_probe(dev_priv->bridge_dev, dev->pdev, NULL);
-		if (!ret) {
-			DRM_ERROR("failed to set up gmch\n");
+	*mappable_base = pci_resource_start(dev->pdev, 2);
+	*mappable_end = pci_resource_len(dev->pdev, 2);
-			return -EIO;
-		}
+	/* 64/512MB is the current min/max we actually know of, but this is just
+	 * a coarse sanity check.
+	 */
+	if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) {
+		DRM_ERROR("Unknown GMADR size (%lx)\n",
+			  dev_priv->gtt.mappable_end);
-		dev_priv->mm.gtt = intel_gtt_get();
-		if (!dev_priv->mm.gtt) {
+		return -ENXIO;
-			DRM_ERROR("Failed to initialize GTT\n");
 			return -ENODEV;
-		return 0;
+	if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40)))
-	}
-	dev_priv->mm.gtt = kzalloc(sizeof(*dev_priv->mm.gtt), GFP_KERNEL);
-	if (!dev_priv->mm.gtt)
-		return -ENOMEM;
+		pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40));
-#ifdef CONFIG_INTEL_IOMMU
-	dev_priv->mm.gtt->needs_dmar = 1;
-#endif
-	/* For GEN6+ the PTEs for the ggtt live at 2MB + BAR0 */
-	gtt_bus_addr = pci_resource_start(dev->pdev, 0) + (2<<20);
-	dev_priv->mm.gtt->gma_bus_addr = pci_resource_start(dev->pdev, 2);
-	/* i9xx_setup */
 	pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
-	dev_priv->mm.gtt->gtt_total_entries =
-		gen6_get_total_gtt_size(snb_gmch_ctl) / sizeof(gtt_pte_t);
+	gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl);
-	if (INTEL_INFO(dev)->gen < 7)
 		dev_priv->mm.gtt->stolen_size = gen6_get_stolen_size(snb_gmch_ctl);
-	else
+	if (IS_GEN7(dev))
-		dev_priv->mm.gtt->stolen_size = gen7_get_stolen_size(snb_gmch_ctl);
+		*stolen = gen7_get_stolen_size(snb_gmch_ctl);
 	else
+		*stolen = gen6_get_stolen_size(snb_gmch_ctl);
+	*gtt_total = (gtt_size / sizeof(gtt_pte_t)) << PAGE_SHIFT;
-	dev_priv->mm.gtt->gtt_mappable_entries = pci_resource_len(dev->pdev, 2) >> PAGE_SHIFT;
+	/* For GEN6+ the PTEs for the ggtt live at 2MB + BAR0 */
-	/* 64/512MB is the current min/max we actually know of, but this is just a
+	gtt_bus_addr = pci_resource_start(dev->pdev, 0) + (2<<20);
-	 * coarse sanity check.
+	dev_priv->gtt.gsm = ioremap_wc(gtt_bus_addr, gtt_size);
+	if (!dev_priv->gtt.gsm) {
-	 */
+		DRM_ERROR("Failed to map the gtt page table\n");
-	if ((dev_priv->mm.gtt->gtt_mappable_entries >> 8) < 64 ||
+		return -ENOMEM;
-	    dev_priv->mm.gtt->gtt_mappable_entries > dev_priv->mm.gtt->gtt_total_entries) {
+	}
-		DRM_ERROR("Unknown GMADR entries (%d)\n",
-			  dev_priv->mm.gtt->gtt_mappable_entries);
 		ret = -ENXIO;
+	ret = setup_scratch_page(dev);
+	if (ret)
+		DRM_ERROR("Scratch setup failed\n");
+	dev_priv->gtt.gtt_clear_range = gen6_ggtt_clear_range;
+	dev_priv->gtt.gtt_insert_entries = gen6_ggtt_insert_entries;
-		goto err_out;
+	return ret;
+}
-	}
+static void gen6_gmch_remove(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	iounmap(dev_priv->gtt.gsm);
+	teardown_scratch_page(dev_priv->dev);
+}
-	ret = setup_scratch_page(dev);
+static int i915_gmch_probe(struct drm_device *dev,
+			   size_t *gtt_total,
+			   size_t *stolen,
-	if (ret) {
+			   phys_addr_t *mappable_base,
+			   unsigned long *mappable_end)
+{
-		DRM_ERROR("Scratch setup failed\n");
+	struct drm_i915_private *dev_priv = dev->dev_private;
-		goto err_out;
+	int ret;
-	}
+	ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL);
+	if (!ret) {
+		DRM_ERROR("failed to set up gmch\n");
+		return -EIO;
+	}
+	intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end);
+	dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev);
+	dev_priv->gtt.gtt_clear_range = i915_ggtt_clear_range;
+	dev_priv->gtt.gtt_insert_entries = i915_ggtt_insert_entries;
+	return 0;
+}
+static void i915_gmch_remove(struct drm_device *dev)
+{
+//   intel_gmch_remove();
+}
+int i915_gem_gtt_init(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct i915_gtt *gtt = &dev_priv->gtt;
+	unsigned long gtt_size;
+	int ret;
+	if (INTEL_INFO(dev)->gen <= 5) {
+		dev_priv->gtt.gtt_probe = i915_gmch_probe;
+		dev_priv->gtt.gtt_remove = i915_gmch_remove;
+	} else {
+		dev_priv->gtt.gtt_probe = gen6_gmch_probe;
+		dev_priv->gtt.gtt_remove = gen6_gmch_remove;
+	}
 	dev_priv->mm.gtt->gtt = ioremap(gtt_bus_addr,
-					   dev_priv->mm.gtt->gtt_total_entries * sizeof(gtt_pte_t));
+	ret = dev_priv->gtt.gtt_probe(dev, &dev_priv->gtt.total,
-	if (!dev_priv->mm.gtt->gtt) {
+				     &dev_priv->gtt.stolen_size,
-		DRM_ERROR("Failed to map the gtt page table\n");
+				     >t->mappable_base,

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem_gtt.c – Rev 3243 → 3480