WebSVN – Kolibri OS – Diff – /drivers/video/drm/i915/i915_gem_gtt.c


/*
 * Copyright © 2010 Daniel Vetter
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 */
 
 
#define AGP_NORMAL_MEMORY 0
 
#define AGP_USER_TYPES (1 << 16)
#define AGP_USER_MEMORY (AGP_USER_TYPES)
#define AGP_USER_CACHED_MEMORY (AGP_USER_TYPES + 1)
 
#include 
#include 
#include "i915_drv.h"
#include "i915_trace.h"
#include "intel_drv.h"
 
#define GEN6_PPGTT_PD_ENTRIES 512
#define I915_PPGTT_PT_ENTRIES (PAGE_SIZE / sizeof(gen6_gtt_pte_t))
 
/* PPGTT stuff */
#define GEN6_GTT_ADDR_ENCODE(addr)	((addr) | (((addr) >> 28) & 0xff0))
#define HSW_GTT_ADDR_ENCODE(addr)	((addr) | (((addr) >> 28) & 0x7f0))
 
#define GEN6_PDE_VALID			(1 << 0)
/* gen6+ has bit 11-4 for physical addr bit 39-32 */
#define GEN6_PDE_ADDR_ENCODE(addr)	GEN6_GTT_ADDR_ENCODE(addr)
 
#define GEN6_PTE_VALID			(1 << 0)
#define GEN6_PTE_UNCACHED		(1 << 1)
#define HSW_PTE_UNCACHED		(0)
#define GEN6_PTE_CACHE_LLC		(2 << 1)
#define GEN7_PTE_CACHE_L3_LLC		(3 << 1)
#define GEN6_PTE_ADDR_ENCODE(addr)	GEN6_GTT_ADDR_ENCODE(addr)
#define HSW_PTE_ADDR_ENCODE(addr)	HSW_GTT_ADDR_ENCODE(addr)
 
/* Cacheability Control is a 4-bit value. The low three bits are stored in *
 * bits 3:1 of the PTE, while the fourth bit is stored in bit 11 of the PTE.
 */
#define HSW_CACHEABILITY_CONTROL(bits)	((((bits) & 0x7) << 1) | \
					 (((bits) & 0x8) << (11 - 3)))
#define HSW_WB_LLC_AGE3			HSW_CACHEABILITY_CONTROL(0x2)
#define HSW_WB_LLC_AGE0			HSW_CACHEABILITY_CONTROL(0x3)
#define HSW_WB_ELLC_LLC_AGE0		HSW_CACHEABILITY_CONTROL(0xb)
#define HSW_WT_ELLC_LLC_AGE0		HSW_CACHEABILITY_CONTROL(0x6)
 
static gen6_gtt_pte_t snb_pte_encode(dma_addr_t addr,
				     enum i915_cache_level level)
{
	gen6_gtt_pte_t pte = GEN6_PTE_VALID;
	pte |= GEN6_PTE_ADDR_ENCODE(addr);
 
	switch (level) {
	case I915_CACHE_L3_LLC:
	case I915_CACHE_LLC:
		pte |= GEN6_PTE_CACHE_LLC;
		break;
	case I915_CACHE_NONE:
		pte |= GEN6_PTE_UNCACHED;
		break;
	default:
		WARN_ON(1);
	}
 
	return pte;
}
 
static gen6_gtt_pte_t ivb_pte_encode(dma_addr_t addr,
				   enum i915_cache_level level)
{
	gen6_gtt_pte_t pte = GEN6_PTE_VALID;
	pte |= GEN6_PTE_ADDR_ENCODE(addr);
 
	switch (level) {
	case I915_CACHE_L3_LLC:
		pte |= GEN7_PTE_CACHE_L3_LLC;
		break;
	case I915_CACHE_LLC:
		pte |= GEN6_PTE_CACHE_LLC;
		break;
	case I915_CACHE_NONE:
			pte |= GEN6_PTE_UNCACHED;
		break;
	default:
		WARN_ON(1);
	}
 
	return pte;
}
 
#define BYT_PTE_WRITEABLE		(1 << 1)
#define BYT_PTE_SNOOPED_BY_CPU_CACHES	(1 << 2)
 
static gen6_gtt_pte_t byt_pte_encode(dma_addr_t addr,
				     enum i915_cache_level level)
{
	gen6_gtt_pte_t pte = GEN6_PTE_VALID;
	pte |= GEN6_PTE_ADDR_ENCODE(addr);
 
	/* Mark the page as writeable.  Other platforms don't have a
	 * setting for read-only/writable, so this matches that behavior.
	 */
	pte |= BYT_PTE_WRITEABLE;
 
	if (level != I915_CACHE_NONE)
		pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
 
	return pte;
}
 
static gen6_gtt_pte_t hsw_pte_encode(dma_addr_t addr,
				     enum i915_cache_level level)
{
	gen6_gtt_pte_t pte = GEN6_PTE_VALID;
	pte |= HSW_PTE_ADDR_ENCODE(addr);
 
	if (level != I915_CACHE_NONE)
		pte |= HSW_WB_LLC_AGE3;
 
	return pte;
}
 
static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,
				      enum i915_cache_level level)
{
	gen6_gtt_pte_t pte = GEN6_PTE_VALID;
	pte |= HSW_PTE_ADDR_ENCODE(addr);
 
	switch (level) {
	case I915_CACHE_NONE:
		break;
	case I915_CACHE_WT:
		pte |= HSW_WT_ELLC_LLC_AGE0;
		break;
	default:
		pte |= HSW_WB_ELLC_LLC_AGE0;
		break;
	}
 
	return pte;
}
 
static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)
{
	struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;
	gen6_gtt_pte_t __iomem *pd_addr;
	uint32_t pd_entry;
	int i;
 
	WARN_ON(ppgtt->pd_offset & 0x3f);
	pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm +
		ppgtt->pd_offset / sizeof(gen6_gtt_pte_t);
	for (i = 0; i < ppgtt->num_pd_entries; i++) {
		dma_addr_t pt_addr;
 
		pt_addr = ppgtt->pt_dma_addr[i];
		pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
		pd_entry |= GEN6_PDE_VALID;
 
		writel(pd_entry, pd_addr + i);
	}
	readl(pd_addr);
}
 
static int gen6_ppgtt_enable(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	uint32_t pd_offset;
	struct intel_ring_buffer *ring;
	struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
	int i;
 
	BUG_ON(ppgtt->pd_offset & 0x3f);
 
	gen6_write_pdes(ppgtt);
 
	pd_offset = ppgtt->pd_offset;
	pd_offset /= 64; /* in cachelines, */
	pd_offset <<= 16;
 
	if (INTEL_INFO(dev)->gen == 6) {
		uint32_t ecochk, gab_ctl, ecobits;
 
		ecobits = I915_READ(GAC_ECO_BITS);
		I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
					 ECOBITS_PPGTT_CACHE64B);
 
		gab_ctl = I915_READ(GAB_CTL);
		I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
 
		ecochk = I915_READ(GAM_ECOCHK);
		I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT |
				       ECOCHK_PPGTT_CACHE64B);
		I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
	} else if (INTEL_INFO(dev)->gen >= 7) {
		uint32_t ecochk, ecobits;
 
		ecobits = I915_READ(GAC_ECO_BITS);
		I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
 
		ecochk = I915_READ(GAM_ECOCHK);
		if (IS_HASWELL(dev)) {
			ecochk |= ECOCHK_PPGTT_WB_HSW;
		} else {
			ecochk |= ECOCHK_PPGTT_LLC_IVB;
			ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
		}
		I915_WRITE(GAM_ECOCHK, ecochk);
		/* GFX_MODE is per-ring on gen7+ */
	}
 
	for_each_ring(ring, dev_priv, i) {
		if (INTEL_INFO(dev)->gen >= 7)
			I915_WRITE(RING_MODE_GEN7(ring),
				   _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
 
		I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
		I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
	}
	return 0;
}
 
/* PPGTT support for Sandybdrige/Gen6 and later */
static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
				   unsigned first_entry,
				   unsigned num_entries)
{
	struct i915_hw_ppgtt *ppgtt =
		container_of(vm, struct i915_hw_ppgtt, base);
	gen6_gtt_pte_t *pt_vaddr, scratch_pte;
	unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
	unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
	unsigned last_pte, i;
 
	scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC);
 
    pt_vaddr = AllocKernelSpace(4096);
 
    if(pt_vaddr == NULL)
        return;
 
	while (num_entries) {
            last_pte = first_pte + num_entries;
            if (last_pte > I915_PPGTT_PT_ENTRIES)
                last_pte = I915_PPGTT_PT_ENTRIES;
 
            MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3);
 
            for (i = first_pte; i < last_pte; i++)
                pt_vaddr[i] = scratch_pte;
 
            num_entries -= last_pte - first_pte;
            first_pte = 0;
            act_pt++;
    };
 
    FreeKernelSpace(pt_vaddr);
}
 
static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
				      struct sg_table *pages,
				      unsigned first_entry,
				      enum i915_cache_level cache_level)
{
	struct i915_hw_ppgtt *ppgtt =
		container_of(vm, struct i915_hw_ppgtt, base);
	gen6_gtt_pte_t *pt_vaddr;
	unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
	unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES;
	struct sg_page_iter sg_iter;
	dma_addr_t page_addr;
 
 
    pt_vaddr = AllocKernelSpace(4096);
 
    if(pt_vaddr == NULL)
        return;
 
    MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3);
	for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
		dma_addr_t page_addr;
 
		page_addr = sg_page_iter_dma_address(&sg_iter);
		pt_vaddr[act_pte] = vm->pte_encode(page_addr, cache_level);
		if (++act_pte == I915_PPGTT_PT_ENTRIES) {
			act_pt++;
    		MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3);
			act_pte = 0;
 
			}
		}
    FreeKernelSpace(pt_vaddr);
}
 
static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
{
	struct i915_hw_ppgtt *ppgtt =
		container_of(vm, struct i915_hw_ppgtt, base);
	int i;
 
	drm_mm_takedown(&ppgtt->base.mm);
 
	if (ppgtt->pt_dma_addr) {
		for (i = 0; i < ppgtt->num_pd_entries; i++)
			pci_unmap_page(ppgtt->base.dev->pdev,
				       ppgtt->pt_dma_addr[i],
				       4096, PCI_DMA_BIDIRECTIONAL);
	}
 
	kfree(ppgtt->pt_dma_addr);
	for (i = 0; i < ppgtt->num_pd_entries; i++)
		__free_page(ppgtt->pt_pages[i]);
	kfree(ppgtt->pt_pages);
	kfree(ppgtt);
}
 
static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
{
	struct drm_device *dev = ppgtt->base.dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
	unsigned first_pd_entry_in_global_pt;
	int i;
	int ret = -ENOMEM;
 
	/* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024
	 * entries. For aliasing ppgtt support we just steal them at the end for
	 * now. */
       first_pd_entry_in_global_pt = gtt_total_entries(dev_priv->gtt);
 
	ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode;
	ppgtt->num_pd_entries = GEN6_PPGTT_PD_ENTRIES;
	ppgtt->enable = gen6_ppgtt_enable;
	ppgtt->base.clear_range = gen6_ppgtt_clear_range;
	ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
	ppgtt->base.cleanup = gen6_ppgtt_cleanup;
	ppgtt->base.scratch = dev_priv->gtt.base.scratch;
	ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries,
				  GFP_KERNEL);
	if (!ppgtt->pt_pages)
		return -ENOMEM;
 
	for (i = 0; i < ppgtt->num_pd_entries; i++) {
		ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
		if (!ppgtt->pt_pages[i])
			goto err_pt_alloc;
	}
 
	ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t) *ppgtt->num_pd_entries,
					     GFP_KERNEL);
    if (!ppgtt->pt_dma_addr)
        goto err_pt_alloc;
 
    for (i = 0; i < ppgtt->num_pd_entries; i++) {
        dma_addr_t pt_addr;
 
		pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 0, 4096,
					       PCI_DMA_BIDIRECTIONAL);
 
        ppgtt->pt_dma_addr[i] = pt_addr;
    }
 
	ppgtt->base.clear_range(&ppgtt->base, 0,
			       ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES);
 
	ppgtt->pd_offset = first_pd_entry_in_global_pt * sizeof(gen6_gtt_pte_t);
 
	return 0;
 
err_pd_pin:
	if (ppgtt->pt_dma_addr) {
		for (i--; i >= 0; i--)
			pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
				       4096, PCI_DMA_BIDIRECTIONAL);
	}
err_pt_alloc:
	kfree(ppgtt->pt_dma_addr);
	for (i = 0; i < ppgtt->num_pd_entries; i++) {
		if (ppgtt->pt_pages[i])
			__free_page(ppgtt->pt_pages[i]);
	}
	kfree(ppgtt->pt_pages);
 
	return ret;
}
 
static int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct i915_hw_ppgtt *ppgtt;
	int ret;
 
	ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
	if (!ppgtt)
		return -ENOMEM;
 
	ppgtt->base.dev = dev;
 
	if (INTEL_INFO(dev)->gen < 8)
	ret = gen6_ppgtt_init(ppgtt);
	else
		BUG();
 
	if (ret)
        kfree(ppgtt);
	else {
		dev_priv->mm.aliasing_ppgtt = ppgtt;
		drm_mm_init(&ppgtt->base.mm, ppgtt->base.start,
			    ppgtt->base.total);
	}
 
	return ret;
}
 
void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
 
	if (!ppgtt)
		return;
 
	ppgtt->base.cleanup(&ppgtt->base);
	dev_priv->mm.aliasing_ppgtt = NULL;
}
 
void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
			    struct drm_i915_gem_object *obj,
			    enum i915_cache_level cache_level)
{
	ppgtt->base.insert_entries(&ppgtt->base, obj->pages,
				   i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT,
				     cache_level);
}
 
void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
			      struct drm_i915_gem_object *obj)
{
	ppgtt->base.clear_range(&ppgtt->base,
				i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT,
			       obj->base.size >> PAGE_SHIFT);
}
 
extern int intel_iommu_gfx_mapped;
/* Certain Gen5 chipsets require require idling the GPU before
 * unmapping anything from the GTT when VT-d is enabled.
 */
static inline bool needs_idle_maps(struct drm_device *dev)
{
#ifdef CONFIG_INTEL_IOMMU
	/* Query intel_iommu to see if we need the workaround. Presumably that
	 * was loaded first.
	 */
	if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped)
		return true;
#endif
	return false;
}
 
static bool do_idling(struct drm_i915_private *dev_priv)
{
	bool ret = dev_priv->mm.interruptible;
 
	if (unlikely(dev_priv->gtt.do_idle_maps)) {
		dev_priv->mm.interruptible = false;
		if (i915_gpu_idle(dev_priv->dev)) {
			DRM_ERROR("Couldn't idle GPU\n");
			/* Wait a bit, in hopes it avoids the hang */
			udelay(10);
		}
	}
 
	return ret;
}
 
static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
{
	if (unlikely(dev_priv->gtt.do_idle_maps))
		dev_priv->mm.interruptible = interruptible;
}
 
void i915_gem_restore_gtt_mappings(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_i915_gem_object *obj;
 
	/* First fill our portion of the GTT with scratch pages */
	dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
				       dev_priv->gtt.base.start / PAGE_SIZE,
				       dev_priv->gtt.base.total / PAGE_SIZE);
 
	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
		i915_gem_clflush_object(obj, obj->pin_display);
		i915_gem_gtt_bind_object(obj, obj->cache_level);
	}
 
	i915_gem_chipset_flush(dev);
}
 
int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
{
	if (obj->has_dma_mapping)
		return 0;
 
	if (!dma_map_sg(&obj->base.dev->pdev->dev,
			obj->pages->sgl, obj->pages->nents,
			PCI_DMA_BIDIRECTIONAL))
		return -ENOSPC;
 
	return 0;
}
 
/*
 * Binds an object into the global gtt with the specified cache level. The object
 * will be accessible to the GPU via commands whose operands reference offsets
 * within the global GTT as well as accessible by the GPU through the GMADR
 * mapped BAR (dev_priv->mm.gtt->gtt).
 */
static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
				     struct sg_table *st,
				     unsigned int first_entry,
				  enum i915_cache_level level)
{
	struct drm_i915_private *dev_priv = vm->dev->dev_private;
	gen6_gtt_pte_t __iomem *gtt_entries =
		(gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
	int i = 0;
	struct sg_page_iter sg_iter;
	dma_addr_t addr;
 
	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
		addr = sg_page_iter_dma_address(&sg_iter);
		iowrite32(vm->pte_encode(addr, level), >t_entries[i]);
			i++;
		}
 
	/* XXX: This serves as a posting read to make sure that the PTE has
	 * actually been updated. There is some concern that even though
	 * registers and PTEs are within the same BAR that they are potentially
	 * of NUMA access patterns. Therefore, even with the way we assume
	 * hardware should work, we must keep this posting read for paranoia.
	 */
	if (i != 0)
		WARN_ON(readl(>t_entries[i-1]) !=
			vm->pte_encode(addr, level));
 
	/* This next bit makes the above posting read even more important. We
	 * want to flush the TLBs only after we're certain all the PTE updates
	 * have finished.
	 */
	I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
	POSTING_READ(GFX_FLSH_CNTL_GEN6);
}
 
static void gen6_ggtt_clear_range(struct i915_address_space *vm,
				  unsigned int first_entry,
				  unsigned int num_entries)
{
	struct drm_i915_private *dev_priv = vm->dev->dev_private;
	gen6_gtt_pte_t scratch_pte, __iomem *gtt_base =
		(gen6_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
	const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
	int i;
 
	if (WARN(num_entries > max_entries,
		 "First entry = %d; Num entries = %d (max=%d)\n",
 
		 first_entry, num_entries, max_entries))
        num_entries = max_entries;
 
	scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC);
	for (i = 0; i < num_entries; i++)
		iowrite32(scratch_pte, >t_base[i]);
	readl(gtt_base);
}
 
 
static void i915_ggtt_insert_entries(struct i915_address_space *vm,
				     struct sg_table *st,
				     unsigned int pg_start,
				     enum i915_cache_level cache_level)
{
	unsigned int flags = (cache_level == I915_CACHE_NONE) ?
		AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
 
	intel_gtt_insert_sg_entries(st, pg_start, flags);
 
}
 
static void i915_ggtt_clear_range(struct i915_address_space *vm,
				  unsigned int first_entry,
				  unsigned int num_entries)
{
	intel_gtt_clear_range(first_entry, num_entries);
}
 
 
void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
				enum i915_cache_level cache_level)
{
	struct drm_device *dev = obj->base.dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
	const unsigned long entry = i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT;
 
	dev_priv->gtt.base.insert_entries(&dev_priv->gtt.base, obj->pages,
					  entry,
					 cache_level);
 
	obj->has_global_gtt_mapping = 1;
}
 
void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
{
	struct drm_device *dev = obj->base.dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
	const unsigned long entry = i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT;
 
	dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
				       entry,
			      obj->base.size >> PAGE_SHIFT);
 
	obj->has_global_gtt_mapping = 0;
}
 
void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
{
	struct drm_device *dev = obj->base.dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
	bool interruptible;
 
	interruptible = do_idling(dev_priv);
 
	if (!obj->has_dma_mapping)
		dma_unmap_sg(&dev->pdev->dev,
			     obj->pages->sgl, obj->pages->nents,
			     PCI_DMA_BIDIRECTIONAL);
 
	undo_idling(dev_priv, interruptible);
}
 
static void i915_gtt_color_adjust(struct drm_mm_node *node,
				  unsigned long color,
				  unsigned long *start,
				  unsigned long *end)
{
	if (node->color != color)
		*start += 4096;
 
	if (!list_empty(&node->node_list)) {
		node = list_entry(node->node_list.next,
				  struct drm_mm_node,
				  node_list);
		if (node->allocated && node->color != color)
			*end -= 4096;
	}
}
void i915_gem_setup_global_gtt(struct drm_device *dev,
			      unsigned long start,
			      unsigned long mappable_end,
			      unsigned long end)
{
	/* Let GEM Manage all of the aperture.
	 *
	 * However, leave one page at the end still bound to the scratch page.
	 * There are a number of places where the hardware apparently prefetches
	 * past the end of the object, and we've seen multiple hangs with the
	 * GPU head pointer stuck in a batchbuffer bound at the last page of the
	 * aperture.  One page should be enough to keep any prefetching inside
	 * of the aperture.
	 */
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct i915_address_space *ggtt_vm = &dev_priv->gtt.base;
	struct drm_mm_node *entry;
	struct drm_i915_gem_object *obj;
	unsigned long hole_start, hole_end;
 
	BUG_ON(mappable_end > end);
 
	/* Subtract the guard page ... */
	drm_mm_init(&ggtt_vm->mm, start, end - start - PAGE_SIZE);
	if (!HAS_LLC(dev))
		dev_priv->gtt.base.mm.color_adjust = i915_gtt_color_adjust;
 
	/* Mark any preallocated objects as occupied */
	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
		struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
		int ret;
		DRM_DEBUG_KMS("reserving preallocated space: %lx + %zx\n",
			      i915_gem_obj_ggtt_offset(obj), obj->base.size);
 
		WARN_ON(i915_gem_obj_ggtt_bound(obj));
		ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node);
		if (ret)
			DRM_DEBUG_KMS("Reservation failed\n");
		obj->has_global_gtt_mapping = 1;
		list_add(&vma->vma_link, &obj->vma_list);
	}
 
	dev_priv->gtt.base.start = start;
	dev_priv->gtt.base.total = end - start;
 
	/* Clear any non-preallocated blocks */
	drm_mm_for_each_hole(entry, &ggtt_vm->mm, hole_start, hole_end) {
		const unsigned long count = (hole_end - hole_start) / PAGE_SIZE;
		DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
			      hole_start, hole_end);
		ggtt_vm->clear_range(ggtt_vm, hole_start / PAGE_SIZE, count);
	}
 
	/* And finally clear the reserved guard page */
	ggtt_vm->clear_range(ggtt_vm, end / PAGE_SIZE - 1, 1);
}
 
static bool
intel_enable_ppgtt(struct drm_device *dev)
{
	if (i915_enable_ppgtt >= 0)
		return i915_enable_ppgtt;
 
#ifdef CONFIG_INTEL_IOMMU
	/* Disable ppgtt on SNB if VT-d is on. */
	if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
		return false;
#endif
 
	return true;
}
 
void i915_gem_init_global_gtt(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	unsigned long gtt_size, mappable_size;
 
	gtt_size = dev_priv->gtt.base.total;
	mappable_size = dev_priv->gtt.mappable_end;
 
#if 0
	if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
		int ret;
 
		if (INTEL_INFO(dev)->gen <= 7) {
		/* PPGTT pdes are stolen from global gtt ptes, so shrink the
		 * aperture accordingly when using aliasing ppgtt. */
			gtt_size -= GEN6_PPGTT_PD_ENTRIES * PAGE_SIZE;
		}
 
        i915_gem_setup_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size-LFB_SIZE);
 
		ret = i915_gem_init_aliasing_ppgtt(dev);
		if (!ret)
			return;
 
		DRM_ERROR("Aliased PPGTT setup failed %d\n", ret);
		drm_mm_takedown(&dev_priv->gtt.base.mm);
		gtt_size += GEN6_PPGTT_PD_ENTRIES * PAGE_SIZE;
}
#endif
 
    i915_gem_setup_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size-LFB_SIZE);
}
 
static int setup_scratch_page(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct page *page;
	dma_addr_t dma_addr;
 
	page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
	if (page == NULL)
		return -ENOMEM;
    get_page(page);
	set_pages_uc(page, 1);
 
#ifdef CONFIG_INTEL_IOMMU
	dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE,
				PCI_DMA_BIDIRECTIONAL);
	if (pci_dma_mapping_error(dev->pdev, dma_addr))
		return -EINVAL;
#else
	dma_addr = page_to_phys(page);
#endif
	dev_priv->gtt.base.scratch.page = page;
	dev_priv->gtt.base.scratch.addr = dma_addr;
 
	return 0;
}
 
static void teardown_scratch_page(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct page *page = dev_priv->gtt.base.scratch.page;
 
	set_pages_wb(page, 1);
	pci_unmap_page(dev->pdev, dev_priv->gtt.base.scratch.addr,
		       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
	put_page(page);
	__free_page(page);
}
 
static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
{
	snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
	snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
	return snb_gmch_ctl << 20;
}
 
static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
{
	snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
	snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
	return snb_gmch_ctl << 25; /* 32 MB units */
}
 
static int gen6_gmch_probe(struct drm_device *dev,
			   size_t *gtt_total,
			   size_t *stolen,
			   phys_addr_t *mappable_base,
			   unsigned long *mappable_end)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	phys_addr_t gtt_bus_addr;
	unsigned int gtt_size;
	u16 snb_gmch_ctl;
	int ret;
 
	*mappable_base = pci_resource_start(dev->pdev, 2);
	*mappable_end = pci_resource_len(dev->pdev, 2);
 
	/* 64/512MB is the current min/max we actually know of, but this is just
	 * a coarse sanity check.
	 */
	if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) {
		DRM_ERROR("Unknown GMADR size (%lx)\n",
			  dev_priv->gtt.mappable_end);
		return -ENXIO;
		}
 
	if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40)))
		pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40));
	pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
	gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl);
 
	*stolen = gen6_get_stolen_size(snb_gmch_ctl);
	*gtt_total = (gtt_size / sizeof(gen6_gtt_pte_t)) << PAGE_SHIFT;
 
	/* For Modern GENs the PTEs and register space are split in the BAR */
	gtt_bus_addr = pci_resource_start(dev->pdev, 0) +
		(pci_resource_len(dev->pdev, 0) / 2);
 
	dev_priv->gtt.gsm = ioremap_wc(gtt_bus_addr, gtt_size);
	if (!dev_priv->gtt.gsm) {
		DRM_ERROR("Failed to map the gtt page table\n");
		return -ENOMEM;
	}
 
	ret = setup_scratch_page(dev);
	if (ret)
		DRM_ERROR("Scratch setup failed\n");
 
	dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range;
	dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries;
 
	return ret;
}
 
static void gen6_gmch_remove(struct i915_address_space *vm)
{
 
	struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base);
	iounmap(gtt->gsm);
	teardown_scratch_page(vm->dev);
}
 
static int i915_gmch_probe(struct drm_device *dev,
			   size_t *gtt_total,
			   size_t *stolen,
			   phys_addr_t *mappable_base,
			   unsigned long *mappable_end)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int ret;
 
	ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL);
	if (!ret) {
		DRM_ERROR("failed to set up gmch\n");
		return -EIO;
	}
 
	intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end);
 
	dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev);
	dev_priv->gtt.base.clear_range = i915_ggtt_clear_range;
	dev_priv->gtt.base.insert_entries = i915_ggtt_insert_entries;
 
	return 0;
}
 
static void i915_gmch_remove(struct i915_address_space *vm)
{
//   intel_gmch_remove();
}
 
int i915_gem_gtt_init(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct i915_gtt *gtt = &dev_priv->gtt;
	int ret;
 
	if (INTEL_INFO(dev)->gen <= 5) {
		gtt->gtt_probe = i915_gmch_probe;
		gtt->base.cleanup = i915_gmch_remove;
	} else {
		gtt->gtt_probe = gen6_gmch_probe;
		gtt->base.cleanup = gen6_gmch_remove;
		if (IS_HASWELL(dev) && dev_priv->ellc_size)
			gtt->base.pte_encode = iris_pte_encode;
		else if (IS_HASWELL(dev))
			gtt->base.pte_encode = hsw_pte_encode;
		else if (IS_VALLEYVIEW(dev))
			gtt->base.pte_encode = byt_pte_encode;
		else if (INTEL_INFO(dev)->gen >= 7)
			gtt->base.pte_encode = ivb_pte_encode;
		else
			gtt->base.pte_encode = snb_pte_encode;
	}
 
	ret = gtt->gtt_probe(dev, >t->base.total, >t->stolen_size,
			     >t->mappable_base, >t->mappable_end);
	if (ret)
		return ret;
 
	gtt->base.dev = dev;
 
	/* GMADR is the PCI mmio aperture into the global GTT. */
	DRM_INFO("Memory usable by graphics device = %zdM\n",
		 gtt->base.total >> 20);
	DRM_DEBUG_DRIVER("GMADR size = %ldM\n", gtt->mappable_end >> 20);
	DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20);
 
	return 0;
}
 
struct scatterlist *sg_next(struct scatterlist *sg)
{
    if (sg_is_last(sg))
        return NULL;
 
    sg++;
    if (unlikely(sg_is_chain(sg)))
            sg = sg_chain_ptr(sg);
 
    return sg;
}
 
 
void __sg_free_table(struct sg_table *table, unsigned int max_ents,
                     sg_free_fn *free_fn)
{
    struct scatterlist *sgl, *next;
 
    if (unlikely(!table->sgl))
            return;
 
    sgl = table->sgl;
    while (table->orig_nents) {
        unsigned int alloc_size = table->orig_nents;
        unsigned int sg_size;
 
        /*
         * If we have more than max_ents segments left,
         * then assign 'next' to the sg table after the current one.
         * sg_size is then one less than alloc size, since the last
         * element is the chain pointer.
         */
        if (alloc_size > max_ents) {
                next = sg_chain_ptr(&sgl[max_ents - 1]);
                alloc_size = max_ents;
                sg_size = alloc_size - 1;
        } else {
                sg_size = alloc_size;
                next = NULL;
        }
 
        table->orig_nents -= sg_size;
        kfree(sgl);
        sgl = next;
    }
 
    table->sgl = NULL;
}
 
void sg_free_table(struct sg_table *table)
{
    __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL);
}
 
int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask)
{
    struct scatterlist *sg, *prv;
    unsigned int left;
    unsigned int max_ents = SG_MAX_SINGLE_ALLOC;
 
#ifndef ARCH_HAS_SG_CHAIN
    BUG_ON(nents > max_ents);
#endif
 
    memset(table, 0, sizeof(*table));
 
    left = nents;
    prv = NULL;
    do {
        unsigned int sg_size, alloc_size = left;
 
        if (alloc_size > max_ents) {
                alloc_size = max_ents;
                sg_size = alloc_size - 1;
        } else
                sg_size = alloc_size;
 
        left -= sg_size;
 
        sg = kmalloc(alloc_size * sizeof(struct scatterlist), gfp_mask);
        if (unlikely(!sg)) {
                /*
                 * Adjust entry count to reflect that the last
                 * entry of the previous table won't be used for
                 * linkage.  Without this, sg_kfree() may get
                 * confused.
                 */
                if (prv)
                        table->nents = ++table->orig_nents;
 
                goto err;
        }
 
        sg_init_table(sg, alloc_size);
        table->nents = table->orig_nents += sg_size;
 
        /*
         * If this is the first mapping, assign the sg table header.
         * If this is not the first mapping, chain previous part.
         */
        if (prv)
                sg_chain(prv, max_ents, sg);
        else
                table->sgl = sg;
 
        /*
         * If no more entries after this one, mark the end
         */
        if (!left)
                sg_mark_end(&sg[sg_size - 1]);
 
        prv = sg;
    } while (left);
 
    return 0;
 
err:
    __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL);
 
    return -ENOMEM;
}
 
 
void sg_init_table(struct scatterlist *sgl, unsigned int nents)
{
    memset(sgl, 0, sizeof(*sgl) * nents);
#ifdef CONFIG_DEBUG_SG
    {
            unsigned int i;
            for (i = 0; i < nents; i++)
                    sgl[i].sg_magic = SG_MAGIC;
    }
#endif
    sg_mark_end(&sgl[nents - 1]);
}
 
 
void __sg_page_iter_start(struct sg_page_iter *piter,
              struct scatterlist *sglist, unsigned int nents,
              unsigned long pgoffset)
{
    piter->__pg_advance = 0;
    piter->__nents = nents;
 
    piter->sg = sglist;
    piter->sg_pgoffset = pgoffset;
}
 
static int sg_page_count(struct scatterlist *sg)
{
    return PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT;
}
 
bool __sg_page_iter_next(struct sg_page_iter *piter)
{
    if (!piter->__nents || !piter->sg)
        return false;
 
    piter->sg_pgoffset += piter->__pg_advance;
    piter->__pg_advance = 1;
 
    while (piter->sg_pgoffset >= sg_page_count(piter->sg)) {
        piter->sg_pgoffset -= sg_page_count(piter->sg);
        piter->sg = sg_next(piter->sg);
        if (!--piter->__nents || !piter->sg)
            return false;
    }
 
    return true;
}
EXPORT_SYMBOL(__sg_page_iter_next);

Rev 4104	Rev 4126
1	/*	1	/*
2	* Copyright © 2010 Daniel Vetter	2	* Copyright © 2010 Daniel Vetter
3	*	3	*
4	* Permission is hereby granted, free of charge, to any person obtaining a	4	* Permission is hereby granted, free of charge, to any person obtaining a
5	* copy of this software and associated documentation files (the "Software"),	5	* copy of this software and associated documentation files (the "Software"),
6	* to deal in the Software without restriction, including without limitation	6	* to deal in the Software without restriction, including without limitation
7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8	* and/or sell copies of the Software, and to permit persons to whom the	8	* and/or sell copies of the Software, and to permit persons to whom the
9	* Software is furnished to do so, subject to the following conditions:	9	* Software is furnished to do so, subject to the following conditions:
10	*	10	*
11	* The above copyright notice and this permission notice (including the next	11	* The above copyright notice and this permission notice (including the next
12	* paragraph) shall be included in all copies or substantial portions of the	12	* paragraph) shall be included in all copies or substantial portions of the
13	* Software.	13	* Software.
14	*	14	*
15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR	15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,	16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL	17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER	18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING	19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS	20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21	* IN THE SOFTWARE.	21	* IN THE SOFTWARE.
22	*	22	*
23	*/	23	*/
24		24
25		25
26	#define AGP_NORMAL_MEMORY 0	26	#define AGP_NORMAL_MEMORY 0
27		27
28	#define AGP_USER_TYPES (1 << 16)	28	#define AGP_USER_TYPES (1 << 16)
29	#define AGP_USER_MEMORY (AGP_USER_TYPES)	29	#define AGP_USER_MEMORY (AGP_USER_TYPES)
30	#define AGP_USER_CACHED_MEMORY (AGP_USER_TYPES + 1)	30	#define AGP_USER_CACHED_MEMORY (AGP_USER_TYPES + 1)
31		31
32	#include	32	#include
33	#include	33	#include
34	#include "i915_drv.h"	34	#include "i915_drv.h"
35	#include "i915_trace.h"	35	#include "i915_trace.h"
36	#include "intel_drv.h"	36	#include "intel_drv.h"
37		37
38	#define GEN6_PPGTT_PD_ENTRIES 512	38	#define GEN6_PPGTT_PD_ENTRIES 512
39	#define I915_PPGTT_PT_ENTRIES (PAGE_SIZE / sizeof(gen6_gtt_pte_t))	39	#define I915_PPGTT_PT_ENTRIES (PAGE_SIZE / sizeof(gen6_gtt_pte_t))
40		40
41	/* PPGTT stuff */	41	/* PPGTT stuff */
42	#define GEN6_GTT_ADDR_ENCODE(addr) ((addr) \| (((addr) >> 28) & 0xff0))	42	#define GEN6_GTT_ADDR_ENCODE(addr) ((addr) \| (((addr) >> 28) & 0xff0))
43	#define HSW_GTT_ADDR_ENCODE(addr) ((addr) \| (((addr) >> 28) & 0x7f0))	43	#define HSW_GTT_ADDR_ENCODE(addr) ((addr) \| (((addr) >> 28) & 0x7f0))
44		44
45	#define GEN6_PDE_VALID (1 << 0)	45	#define GEN6_PDE_VALID (1 << 0)
46	/* gen6+ has bit 11-4 for physical addr bit 39-32 */	46	/* gen6+ has bit 11-4 for physical addr bit 39-32 */
47	#define GEN6_PDE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr)	47	#define GEN6_PDE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr)
48		48
49	#define GEN6_PTE_VALID (1 << 0)	49	#define GEN6_PTE_VALID (1 << 0)
50	#define GEN6_PTE_UNCACHED (1 << 1)	50	#define GEN6_PTE_UNCACHED (1 << 1)
51	#define HSW_PTE_UNCACHED (0)	51	#define HSW_PTE_UNCACHED (0)
52	#define GEN6_PTE_CACHE_LLC (2 << 1)	52	#define GEN6_PTE_CACHE_LLC (2 << 1)
53	#define GEN7_PTE_CACHE_L3_LLC (3 << 1)	53	#define GEN7_PTE_CACHE_L3_LLC (3 << 1)
54	#define GEN6_PTE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr)	54	#define GEN6_PTE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr)
55	#define HSW_PTE_ADDR_ENCODE(addr) HSW_GTT_ADDR_ENCODE(addr)	55	#define HSW_PTE_ADDR_ENCODE(addr) HSW_GTT_ADDR_ENCODE(addr)
56		56
57	/* Cacheability Control is a 4-bit value. The low three bits are stored in *	57	/* Cacheability Control is a 4-bit value. The low three bits are stored in *
58	* bits 3:1 of the PTE, while the fourth bit is stored in bit 11 of the PTE.	58	* bits 3:1 of the PTE, while the fourth bit is stored in bit 11 of the PTE.
59	*/	59	*/
60	#define HSW_CACHEABILITY_CONTROL(bits) ((((bits) & 0x7) << 1) \| \	60	#define HSW_CACHEABILITY_CONTROL(bits) ((((bits) & 0x7) << 1) \| \
61	(((bits) & 0x8) << (11 - 3)))	61	(((bits) & 0x8) << (11 - 3)))
62	#define HSW_WB_LLC_AGE3 HSW_CACHEABILITY_CONTROL(0x2)	62	#define HSW_WB_LLC_AGE3 HSW_CACHEABILITY_CONTROL(0x2)
63	#define HSW_WB_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0x3)	63	#define HSW_WB_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0x3)
64	#define HSW_WB_ELLC_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0xb)	64	#define HSW_WB_ELLC_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0xb)
65	#define HSW_WT_ELLC_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0x6)	65	#define HSW_WT_ELLC_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0x6)
66		66
67	static gen6_gtt_pte_t snb_pte_encode(dma_addr_t addr,	67	static gen6_gtt_pte_t snb_pte_encode(dma_addr_t addr,
68	enum i915_cache_level level)	68	enum i915_cache_level level)
69	{	69	{
70	gen6_gtt_pte_t pte = GEN6_PTE_VALID;	70	gen6_gtt_pte_t pte = GEN6_PTE_VALID;
71	pte \|= GEN6_PTE_ADDR_ENCODE(addr);	71	pte \|= GEN6_PTE_ADDR_ENCODE(addr);
72		72
73	switch (level) {	73	switch (level) {
74	case I915_CACHE_L3_LLC:	74	case I915_CACHE_L3_LLC:
75	case I915_CACHE_LLC:	75	case I915_CACHE_LLC:
76	pte \|= GEN6_PTE_CACHE_LLC;	76	pte \|= GEN6_PTE_CACHE_LLC;
77	break;	77	break;
78	case I915_CACHE_NONE:	78	case I915_CACHE_NONE:
79	pte \|= GEN6_PTE_UNCACHED;	79	pte \|= GEN6_PTE_UNCACHED;
80	break;	80	break;
81	default:	81	default:
82	WARN_ON(1);	82	WARN_ON(1);
83	}	83	}
84		84
85	return pte;	85	return pte;
86	}	86	}
87		87
88	static gen6_gtt_pte_t ivb_pte_encode(dma_addr_t addr,	88	static gen6_gtt_pte_t ivb_pte_encode(dma_addr_t addr,
89	enum i915_cache_level level)	89	enum i915_cache_level level)
90	{	90	{
91	gen6_gtt_pte_t pte = GEN6_PTE_VALID;	91	gen6_gtt_pte_t pte = GEN6_PTE_VALID;
92	pte \|= GEN6_PTE_ADDR_ENCODE(addr);	92	pte \|= GEN6_PTE_ADDR_ENCODE(addr);
93		93
94	switch (level) {	94	switch (level) {
95	case I915_CACHE_L3_LLC:	95	case I915_CACHE_L3_LLC:
96	pte \|= GEN7_PTE_CACHE_L3_LLC;	96	pte \|= GEN7_PTE_CACHE_L3_LLC;
97	break;	97	break;
98	case I915_CACHE_LLC:	98	case I915_CACHE_LLC:
99	pte \|= GEN6_PTE_CACHE_LLC;	99	pte \|= GEN6_PTE_CACHE_LLC;
100	break;	100	break;
101	case I915_CACHE_NONE:	101	case I915_CACHE_NONE:
102	pte \|= GEN6_PTE_UNCACHED;	102	pte \|= GEN6_PTE_UNCACHED;
103	break;	103	break;
104	default:	104	default:
105	WARN_ON(1);	105	WARN_ON(1);
106	}	106	}
107		107
108	return pte;	108	return pte;
109	}	109	}
110		110
111	#define BYT_PTE_WRITEABLE (1 << 1)	111	#define BYT_PTE_WRITEABLE (1 << 1)
112	#define BYT_PTE_SNOOPED_BY_CPU_CACHES (1 << 2)	112	#define BYT_PTE_SNOOPED_BY_CPU_CACHES (1 << 2)
113		113
114	static gen6_gtt_pte_t byt_pte_encode(dma_addr_t addr,	114	static gen6_gtt_pte_t byt_pte_encode(dma_addr_t addr,
115	enum i915_cache_level level)	115	enum i915_cache_level level)
116	{	116	{
117	gen6_gtt_pte_t pte = GEN6_PTE_VALID;	117	gen6_gtt_pte_t pte = GEN6_PTE_VALID;
118	pte \|= GEN6_PTE_ADDR_ENCODE(addr);	118	pte \|= GEN6_PTE_ADDR_ENCODE(addr);
119		119
120	/* Mark the page as writeable. Other platforms don't have a	120	/* Mark the page as writeable. Other platforms don't have a
121	* setting for read-only/writable, so this matches that behavior.	121	* setting for read-only/writable, so this matches that behavior.
122	*/	122	*/
123	pte \|= BYT_PTE_WRITEABLE;	123	pte \|= BYT_PTE_WRITEABLE;
124		124
125	if (level != I915_CACHE_NONE)	125	if (level != I915_CACHE_NONE)
126	pte \|= BYT_PTE_SNOOPED_BY_CPU_CACHES;	126	pte \|= BYT_PTE_SNOOPED_BY_CPU_CACHES;
127		127
128	return pte;	128	return pte;
129	}	129	}
130		130
131	static gen6_gtt_pte_t hsw_pte_encode(dma_addr_t addr,	131	static gen6_gtt_pte_t hsw_pte_encode(dma_addr_t addr,
132	enum i915_cache_level level)	132	enum i915_cache_level level)
133	{	133	{
134	gen6_gtt_pte_t pte = GEN6_PTE_VALID;	134	gen6_gtt_pte_t pte = GEN6_PTE_VALID;
135	pte \|= HSW_PTE_ADDR_ENCODE(addr);	135	pte \|= HSW_PTE_ADDR_ENCODE(addr);
136		136
137	if (level != I915_CACHE_NONE)	137	if (level != I915_CACHE_NONE)
138	pte \|= HSW_WB_LLC_AGE3;	138	pte \|= HSW_WB_LLC_AGE3;
139		139
140	return pte;	140	return pte;
141	}	141	}
142		142
143	static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,	143	static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,
144	enum i915_cache_level level)	144	enum i915_cache_level level)
145	{	145	{
146	gen6_gtt_pte_t pte = GEN6_PTE_VALID;	146	gen6_gtt_pte_t pte = GEN6_PTE_VALID;
147	pte \|= HSW_PTE_ADDR_ENCODE(addr);	147	pte \|= HSW_PTE_ADDR_ENCODE(addr);
148		148
149	switch (level) {	149	switch (level) {
150	case I915_CACHE_NONE:	150	case I915_CACHE_NONE:
151	break;	151	break;
152	case I915_CACHE_WT:	152	case I915_CACHE_WT:
153	pte \|= HSW_WT_ELLC_LLC_AGE0;	153	pte \|= HSW_WT_ELLC_LLC_AGE0;
154	break;	154	break;
155	default:	155	default:
156	pte \|= HSW_WB_ELLC_LLC_AGE0;	156	pte \|= HSW_WB_ELLC_LLC_AGE0;
157	break;	157	break;
158	}	158	}
159		159
160	return pte;	160	return pte;
161	}	161	}
162		162
163	static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)	163	static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)
164	{	164	{
165	struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;	165	struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;
166	gen6_gtt_pte_t __iomem *pd_addr;	166	gen6_gtt_pte_t __iomem *pd_addr;
167	uint32_t pd_entry;	167	uint32_t pd_entry;
168	int i;	168	int i;
169		169
170	WARN_ON(ppgtt->pd_offset & 0x3f);	170	WARN_ON(ppgtt->pd_offset & 0x3f);
171	pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm +	171	pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm +
172	ppgtt->pd_offset / sizeof(gen6_gtt_pte_t);	172	ppgtt->pd_offset / sizeof(gen6_gtt_pte_t);
173	for (i = 0; i < ppgtt->num_pd_entries; i++) {	173	for (i = 0; i < ppgtt->num_pd_entries; i++) {
174	dma_addr_t pt_addr;	174	dma_addr_t pt_addr;
175		175
176	pt_addr = ppgtt->pt_dma_addr[i];	176	pt_addr = ppgtt->pt_dma_addr[i];
177	pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);	177	pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
178	pd_entry \|= GEN6_PDE_VALID;	178	pd_entry \|= GEN6_PDE_VALID;
179		179
180	writel(pd_entry, pd_addr + i);	180	writel(pd_entry, pd_addr + i);
181	}	181	}
182	readl(pd_addr);	182	readl(pd_addr);
183	}	183	}
184		184
185	static int gen6_ppgtt_enable(struct drm_device *dev)	185	static int gen6_ppgtt_enable(struct drm_device *dev)
186	{	186	{
187	drm_i915_private_t *dev_priv = dev->dev_private;	187	drm_i915_private_t *dev_priv = dev->dev_private;
188	uint32_t pd_offset;	188	uint32_t pd_offset;
189	struct intel_ring_buffer *ring;	189	struct intel_ring_buffer *ring;
190	struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;	190	struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
191	int i;	191	int i;
192		192
193	BUG_ON(ppgtt->pd_offset & 0x3f);	193	BUG_ON(ppgtt->pd_offset & 0x3f);
194		194
195	gen6_write_pdes(ppgtt);	195	gen6_write_pdes(ppgtt);
196		196
197	pd_offset = ppgtt->pd_offset;	197	pd_offset = ppgtt->pd_offset;
198	pd_offset /= 64; /* in cachelines, */	198	pd_offset /= 64; /* in cachelines, */
199	pd_offset <<= 16;	199	pd_offset <<= 16;
200		200
201	if (INTEL_INFO(dev)->gen == 6) {	201	if (INTEL_INFO(dev)->gen == 6) {
202	uint32_t ecochk, gab_ctl, ecobits;	202	uint32_t ecochk, gab_ctl, ecobits;
203		203
204	ecobits = I915_READ(GAC_ECO_BITS);	204	ecobits = I915_READ(GAC_ECO_BITS);
205	I915_WRITE(GAC_ECO_BITS, ecobits \| ECOBITS_SNB_BIT \|	205	I915_WRITE(GAC_ECO_BITS, ecobits \| ECOBITS_SNB_BIT \|
206	ECOBITS_PPGTT_CACHE64B);	206	ECOBITS_PPGTT_CACHE64B);
207		207
208	gab_ctl = I915_READ(GAB_CTL);	208	gab_ctl = I915_READ(GAB_CTL);
209	I915_WRITE(GAB_CTL, gab_ctl \| GAB_CTL_CONT_AFTER_PAGEFAULT);	209	I915_WRITE(GAB_CTL, gab_ctl \| GAB_CTL_CONT_AFTER_PAGEFAULT);
210		210
211	ecochk = I915_READ(GAM_ECOCHK);	211	ecochk = I915_READ(GAM_ECOCHK);
212	I915_WRITE(GAM_ECOCHK, ecochk \| ECOCHK_SNB_BIT \|	212	I915_WRITE(GAM_ECOCHK, ecochk \| ECOCHK_SNB_BIT \|
213	ECOCHK_PPGTT_CACHE64B);	213	ECOCHK_PPGTT_CACHE64B);
214	I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));	214	I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
215	} else if (INTEL_INFO(dev)->gen >= 7) {	215	} else if (INTEL_INFO(dev)->gen >= 7) {
216	uint32_t ecochk, ecobits;	216	uint32_t ecochk, ecobits;
217		217
218	ecobits = I915_READ(GAC_ECO_BITS);	218	ecobits = I915_READ(GAC_ECO_BITS);
219	I915_WRITE(GAC_ECO_BITS, ecobits \| ECOBITS_PPGTT_CACHE64B);	219	I915_WRITE(GAC_ECO_BITS, ecobits \| ECOBITS_PPGTT_CACHE64B);
220		220
221	ecochk = I915_READ(GAM_ECOCHK);	221	ecochk = I915_READ(GAM_ECOCHK);
222	if (IS_HASWELL(dev)) {	222	if (IS_HASWELL(dev)) {
223	ecochk \|= ECOCHK_PPGTT_WB_HSW;	223	ecochk \|= ECOCHK_PPGTT_WB_HSW;
224	} else {	224	} else {
225	ecochk \|= ECOCHK_PPGTT_LLC_IVB;	225	ecochk \|= ECOCHK_PPGTT_LLC_IVB;
226	ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;	226	ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
227	}	227	}
228	I915_WRITE(GAM_ECOCHK, ecochk);	228	I915_WRITE(GAM_ECOCHK, ecochk);
229	/* GFX_MODE is per-ring on gen7+ */	229	/* GFX_MODE is per-ring on gen7+ */
230	}	230	}
231		231
232	for_each_ring(ring, dev_priv, i) {	232	for_each_ring(ring, dev_priv, i) {
233	if (INTEL_INFO(dev)->gen >= 7)	233	if (INTEL_INFO(dev)->gen >= 7)
234	I915_WRITE(RING_MODE_GEN7(ring),	234	I915_WRITE(RING_MODE_GEN7(ring),
235	_MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));	235	_MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
236		236
237	I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);	237	I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
238	I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);	238	I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
239	}	239	}
240	return 0;	240	return 0;
241	}	241	}
242		242
243	/* PPGTT support for Sandybdrige/Gen6 and later */	243	/* PPGTT support for Sandybdrige/Gen6 and later */
244	static void gen6_ppgtt_clear_range(struct i915_address_space *vm,	244	static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
245	unsigned first_entry,	245	unsigned first_entry,
246	unsigned num_entries)	246	unsigned num_entries)
247	{	247	{
248	struct i915_hw_ppgtt *ppgtt =	248	struct i915_hw_ppgtt *ppgtt =
249	container_of(vm, struct i915_hw_ppgtt, base);	249	container_of(vm, struct i915_hw_ppgtt, base);
250	gen6_gtt_pte_t *pt_vaddr, scratch_pte;	250	gen6_gtt_pte_t *pt_vaddr, scratch_pte;
251	unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;	251	unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
252	unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;	252	unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
253	unsigned last_pte, i;	253	unsigned last_pte, i;
254		254
255	scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC);	255	scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC);
256		256
257	pt_vaddr = AllocKernelSpace(4096);	257	pt_vaddr = AllocKernelSpace(4096);
258		258
259	if(pt_vaddr == NULL)	259	if(pt_vaddr == NULL)
260	return;	260	return;
261		261
262	while (num_entries) {	262	while (num_entries) {
263	last_pte = first_pte + num_entries;	263	last_pte = first_pte + num_entries;
264	if (last_pte > I915_PPGTT_PT_ENTRIES)	264	if (last_pte > I915_PPGTT_PT_ENTRIES)
265	last_pte = I915_PPGTT_PT_ENTRIES;	265	last_pte = I915_PPGTT_PT_ENTRIES;
266		266
267	MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3);	267	MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3);
268		268
269	for (i = first_pte; i < last_pte; i++)	269	for (i = first_pte; i < last_pte; i++)
270	pt_vaddr[i] = scratch_pte;	270	pt_vaddr[i] = scratch_pte;
271		271
272	num_entries -= last_pte - first_pte;	272	num_entries -= last_pte - first_pte;
273	first_pte = 0;	273	first_pte = 0;
274	act_pt++;	274	act_pt++;
275	};	275	};
276		276
277	FreeKernelSpace(pt_vaddr);	277	FreeKernelSpace(pt_vaddr);
278	}	278	}
279		279
280	static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,	280	static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
281	struct sg_table *pages,	281	struct sg_table *pages,
282	unsigned first_entry,	282	unsigned first_entry,
283	enum i915_cache_level cache_level)	283	enum i915_cache_level cache_level)
284	{	284	{
285	struct i915_hw_ppgtt *ppgtt =	285	struct i915_hw_ppgtt *ppgtt =
286	container_of(vm, struct i915_hw_ppgtt, base);	286	container_of(vm, struct i915_hw_ppgtt, base);
287	gen6_gtt_pte_t *pt_vaddr;	287	gen6_gtt_pte_t *pt_vaddr;
288	unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;	288	unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
289	unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES;	289	unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES;
290	struct sg_page_iter sg_iter;	290	struct sg_page_iter sg_iter;
291	dma_addr_t page_addr;	291	dma_addr_t page_addr;
292		292
293		293
294	pt_vaddr = AllocKernelSpace(4096);	294	pt_vaddr = AllocKernelSpace(4096);
295		295
296	if(pt_vaddr == NULL)	296	if(pt_vaddr == NULL)
297	return;	297	return;
298		298
299	MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3);	299	MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3);
300	for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {	300	for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
301	dma_addr_t page_addr;	301	dma_addr_t page_addr;
302		302
303	page_addr = sg_page_iter_dma_address(&sg_iter);	303	page_addr = sg_page_iter_dma_address(&sg_iter);
304	pt_vaddr[act_pte] = vm->pte_encode(page_addr, cache_level);	304	pt_vaddr[act_pte] = vm->pte_encode(page_addr, cache_level);
305	if (++act_pte == I915_PPGTT_PT_ENTRIES) {	305	if (++act_pte == I915_PPGTT_PT_ENTRIES) {
306	act_pt++;	306	act_pt++;
307	MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3);	307	MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3);
308	act_pte = 0;	308	act_pte = 0;
309		309
310	}	310	}
311	}	311	}
312	FreeKernelSpace(pt_vaddr);	312	FreeKernelSpace(pt_vaddr);
313	}	313	}
314		314
315	static void gen6_ppgtt_cleanup(struct i915_address_space *vm)	315	static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
316	{	316	{
317	struct i915_hw_ppgtt *ppgtt =	317	struct i915_hw_ppgtt *ppgtt =
318	container_of(vm, struct i915_hw_ppgtt, base);	318	container_of(vm, struct i915_hw_ppgtt, base);
319	int i;	319	int i;
320		320
321	drm_mm_takedown(&ppgtt->base.mm);	321	drm_mm_takedown(&ppgtt->base.mm);
322		322
323	if (ppgtt->pt_dma_addr) {	323	if (ppgtt->pt_dma_addr) {
324	for (i = 0; i < ppgtt->num_pd_entries; i++)	324	for (i = 0; i < ppgtt->num_pd_entries; i++)
325	pci_unmap_page(ppgtt->base.dev->pdev,	325	pci_unmap_page(ppgtt->base.dev->pdev,
326	ppgtt->pt_dma_addr[i],	326	ppgtt->pt_dma_addr[i],
327	4096, PCI_DMA_BIDIRECTIONAL);	327	4096, PCI_DMA_BIDIRECTIONAL);
328	}	328	}
329		329
330	kfree(ppgtt->pt_dma_addr);	330	kfree(ppgtt->pt_dma_addr);
331	for (i = 0; i < ppgtt->num_pd_entries; i++)	331	for (i = 0; i < ppgtt->num_pd_entries; i++)
332	__free_page(ppgtt->pt_pages[i]);	332	__free_page(ppgtt->pt_pages[i]);
333	kfree(ppgtt->pt_pages);	333	kfree(ppgtt->pt_pages);
334	kfree(ppgtt);	334	kfree(ppgtt);
335	}	335	}
336		336
337	static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)	337	static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
338	{	338	{
339	struct drm_device *dev = ppgtt->base.dev;	339	struct drm_device *dev = ppgtt->base.dev;
340	struct drm_i915_private *dev_priv = dev->dev_private;	340	struct drm_i915_private *dev_priv = dev->dev_private;
341	unsigned first_pd_entry_in_global_pt;	341	unsigned first_pd_entry_in_global_pt;
342	int i;	342	int i;
343	int ret = -ENOMEM;	343	int ret = -ENOMEM;
344		344
345	/* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024	345	/* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024
346	* entries. For aliasing ppgtt support we just steal them at the end for	346	* entries. For aliasing ppgtt support we just steal them at the end for
347	* now. */	347	* now. */
348	first_pd_entry_in_global_pt = gtt_total_entries(dev_priv->gtt);	348	first_pd_entry_in_global_pt = gtt_total_entries(dev_priv->gtt);
349		349
350	ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode;	350	ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode;
351	ppgtt->num_pd_entries = GEN6_PPGTT_PD_ENTRIES;	351	ppgtt->num_pd_entries = GEN6_PPGTT_PD_ENTRIES;
352	ppgtt->enable = gen6_ppgtt_enable;	352	ppgtt->enable = gen6_ppgtt_enable;
353	ppgtt->base.clear_range = gen6_ppgtt_clear_range;	353	ppgtt->base.clear_range = gen6_ppgtt_clear_range;
354	ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;	354	ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
355	ppgtt->base.cleanup = gen6_ppgtt_cleanup;	355	ppgtt->base.cleanup = gen6_ppgtt_cleanup;
356	ppgtt->base.scratch = dev_priv->gtt.base.scratch;	356	ppgtt->base.scratch = dev_priv->gtt.base.scratch;
357	ppgtt->pt_pages = kzalloc(sizeof(struct page )ppgtt->num_pd_entries,	357	ppgtt->pt_pages = kzalloc(sizeof(struct page )ppgtt->num_pd_entries,
358	GFP_KERNEL);	358	GFP_KERNEL);
359	if (!ppgtt->pt_pages)	359	if (!ppgtt->pt_pages)
360	return -ENOMEM;	360	return -ENOMEM;
361		361
362	for (i = 0; i < ppgtt->num_pd_entries; i++) {	362	for (i = 0; i < ppgtt->num_pd_entries; i++) {
363	ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);	363	ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
364	if (!ppgtt->pt_pages[i])	364	if (!ppgtt->pt_pages[i])
365	goto err_pt_alloc;	365	goto err_pt_alloc;
366	}	366	}
367		367
368	ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t) *ppgtt->num_pd_entries,	368	ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t) *ppgtt->num_pd_entries,
369	GFP_KERNEL);	369	GFP_KERNEL);
370	if (!ppgtt->pt_dma_addr)	370	if (!ppgtt->pt_dma_addr)
371	goto err_pt_alloc;	371	goto err_pt_alloc;
372		372
373	for (i = 0; i < ppgtt->num_pd_entries; i++) {	373	for (i = 0; i < ppgtt->num_pd_entries; i++) {
374	dma_addr_t pt_addr;	374	dma_addr_t pt_addr;
375		375
376	pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 0, 4096,	376	pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 0, 4096,
377	PCI_DMA_BIDIRECTIONAL);	377	PCI_DMA_BIDIRECTIONAL);
378		378
379	ppgtt->pt_dma_addr[i] = pt_addr;	379	ppgtt->pt_dma_addr[i] = pt_addr;
380	}	380	}
381		381
382	ppgtt->base.clear_range(&ppgtt->base, 0,	382	ppgtt->base.clear_range(&ppgtt->base, 0,
383	ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES);	383	ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES);
384		384
385	ppgtt->pd_offset = first_pd_entry_in_global_pt * sizeof(gen6_gtt_pte_t);	385	ppgtt->pd_offset = first_pd_entry_in_global_pt * sizeof(gen6_gtt_pte_t);
386		386
387	return 0;	387	return 0;
388		388
389	err_pd_pin:	389	err_pd_pin:
390	if (ppgtt->pt_dma_addr) {	390	if (ppgtt->pt_dma_addr) {
391	for (i--; i >= 0; i--)	391	for (i--; i >= 0; i--)
392	pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],	392	pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
393	4096, PCI_DMA_BIDIRECTIONAL);	393	4096, PCI_DMA_BIDIRECTIONAL);
394	}	394	}
395	err_pt_alloc:	395	err_pt_alloc:
396	kfree(ppgtt->pt_dma_addr);	396	kfree(ppgtt->pt_dma_addr);
397	for (i = 0; i < ppgtt->num_pd_entries; i++) {	397	for (i = 0; i < ppgtt->num_pd_entries; i++) {
398	if (ppgtt->pt_pages[i])	398	if (ppgtt->pt_pages[i])
399	__free_page(ppgtt->pt_pages[i]);	399	__free_page(ppgtt->pt_pages[i]);
400	}	400	}
401	kfree(ppgtt->pt_pages);	401	kfree(ppgtt->pt_pages);
402		402
403	return ret;	403	return ret;
404	}	404	}
405		405
406	static int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)	406	static int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
407	{	407	{
408	struct drm_i915_private *dev_priv = dev->dev_private;	408	struct drm_i915_private *dev_priv = dev->dev_private;
409	struct i915_hw_ppgtt *ppgtt;	409	struct i915_hw_ppgtt *ppgtt;
410	int ret;	410	int ret;
411		411
412	ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);	412	ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
413	if (!ppgtt)	413	if (!ppgtt)
414	return -ENOMEM;	414	return -ENOMEM;
415		415
416	ppgtt->base.dev = dev;	416	ppgtt->base.dev = dev;
417		417
418	if (INTEL_INFO(dev)->gen < 8)	418	if (INTEL_INFO(dev)->gen < 8)
419	ret = gen6_ppgtt_init(ppgtt);	419	ret = gen6_ppgtt_init(ppgtt);
420	else	420	else
421	BUG();	421	BUG();
422		422
423	if (ret)	423	if (ret)
424	kfree(ppgtt);	424	kfree(ppgtt);
425	else {	425	else {
426	dev_priv->mm.aliasing_ppgtt = ppgtt;	426	dev_priv->mm.aliasing_ppgtt = ppgtt;
427	drm_mm_init(&ppgtt->base.mm, ppgtt->base.start,	427	drm_mm_init(&ppgtt->base.mm, ppgtt->base.start,
428	ppgtt->base.total);	428	ppgtt->base.total);
429	}	429	}
430		430
431	return ret;	431	return ret;
432	}	432	}
433		433
434	void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)	434	void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
435	{	435	{
436	struct drm_i915_private *dev_priv = dev->dev_private;	436	struct drm_i915_private *dev_priv = dev->dev_private;
437	struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;	437	struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
438		438
439	if (!ppgtt)	439	if (!ppgtt)
440	return;	440	return;
441		441
442	ppgtt->base.cleanup(&ppgtt->base);	442	ppgtt->base.cleanup(&ppgtt->base);
443	dev_priv->mm.aliasing_ppgtt = NULL;	443	dev_priv->mm.aliasing_ppgtt = NULL;
444	}	444	}
445		445
446	void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,	446	void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
447	struct drm_i915_gem_object *obj,	447	struct drm_i915_gem_object *obj,
448	enum i915_cache_level cache_level)	448	enum i915_cache_level cache_level)
449	{	449	{
450	ppgtt->base.insert_entries(&ppgtt->base, obj->pages,	450	ppgtt->base.insert_entries(&ppgtt->base, obj->pages,
451	i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT,	451	i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT,
452	cache_level);	452	cache_level);
453	}	453	}
454		454
455	void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,	455	void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
456	struct drm_i915_gem_object *obj)	456	struct drm_i915_gem_object *obj)
457	{	457	{
458	ppgtt->base.clear_range(&ppgtt->base,	458	ppgtt->base.clear_range(&ppgtt->base,
459	i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT,	459	i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT,
460	obj->base.size >> PAGE_SHIFT);	460	obj->base.size >> PAGE_SHIFT);
461	}	461	}
462		462
463	extern int intel_iommu_gfx_mapped;	463	extern int intel_iommu_gfx_mapped;
464	/* Certain Gen5 chipsets require require idling the GPU before	464	/* Certain Gen5 chipsets require require idling the GPU before
465	* unmapping anything from the GTT when VT-d is enabled.	465	* unmapping anything from the GTT when VT-d is enabled.
466	*/	466	*/
467	static inline bool needs_idle_maps(struct drm_device *dev)	467	static inline bool needs_idle_maps(struct drm_device *dev)
468	{	468	{
469	#ifdef CONFIG_INTEL_IOMMU	469	#ifdef CONFIG_INTEL_IOMMU
470	/* Query intel_iommu to see if we need the workaround. Presumably that	470	/* Query intel_iommu to see if we need the workaround. Presumably that
471	* was loaded first.	471	* was loaded first.
472	*/	472	*/
473	if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped)	473	if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped)
474	return true;	474	return true;
475	#endif	475	#endif
476	return false;	476	return false;
477	}	477	}
478		478
479	static bool do_idling(struct drm_i915_private *dev_priv)	479	static bool do_idling(struct drm_i915_private *dev_priv)
480	{	480	{
481	bool ret = dev_priv->mm.interruptible;	481	bool ret = dev_priv->mm.interruptible;
482		482
483	if (unlikely(dev_priv->gtt.do_idle_maps)) {	483	if (unlikely(dev_priv->gtt.do_idle_maps)) {
484	dev_priv->mm.interruptible = false;	484	dev_priv->mm.interruptible = false;
485	if (i915_gpu_idle(dev_priv->dev)) {	485	if (i915_gpu_idle(dev_priv->dev)) {
486	DRM_ERROR("Couldn't idle GPU\n");	486	DRM_ERROR("Couldn't idle GPU\n");
487	/* Wait a bit, in hopes it avoids the hang */	487	/* Wait a bit, in hopes it avoids the hang */
488	udelay(10);	488	udelay(10);
489	}	489	}
490	}	490	}
491		491
492	return ret;	492	return ret;
493	}	493	}
494		494
495	static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)	495	static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
496	{	496	{
497	if (unlikely(dev_priv->gtt.do_idle_maps))	497	if (unlikely(dev_priv->gtt.do_idle_maps))
498	dev_priv->mm.interruptible = interruptible;	498	dev_priv->mm.interruptible = interruptible;
499	}	499	}
500		500

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem_gtt.c – Rev 4104 → 4126