WebSVN – Kolibri OS – Diff – /drivers/video/drm/i915/i915_gem_gtt.c


/*
 * Copyright © 2010 Daniel Vetter
 * Copyright © 2011-2014 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 */
 
 
 
 
 
 
 
#include 
#include 
#include 
#include "i915_drv.h"
#include "i915_trace.h"
#include "intel_drv.h"
 
#include 
 
static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv);
static void chv_setup_private_ppat(struct drm_i915_private *dev_priv);
 
static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)
{
	bool has_aliasing_ppgtt;
	bool has_full_ppgtt;
 
	has_aliasing_ppgtt = INTEL_INFO(dev)->gen >= 6;
	has_full_ppgtt = INTEL_INFO(dev)->gen >= 7;
 
	if (IS_GEN8(dev))
 
		has_full_ppgtt = false; /* XXX why? */
 
	/*
	 * We don't allow disabling PPGTT for gen9+ as it's a requirement for
	 * execlists, the sole mechanism available to submit work.
	 */
	if (INTEL_INFO(dev)->gen < 9 &&
	    (enable_ppgtt == 0 || !has_aliasing_ppgtt))
		return 0;
 
	if (enable_ppgtt == 1)
		return 1;
 
	if (enable_ppgtt == 2 && has_full_ppgtt)
		return 2;
 
#ifdef CONFIG_INTEL_IOMMU
	/* Disable ppgtt on SNB if VT-d is on. */
	if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) {
		DRM_INFO("Disabling PPGTT because VT-d is on\n");
		return 0;
	}
#endif
 
	/* Early VLV doesn't have this */
	if (IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) &&
	    dev->pdev->revision < 0xb) {
		DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n");
		return 0;
	}
 
	return has_aliasing_ppgtt ? 1 : 0;
}
 
 
static void ppgtt_bind_vma(struct i915_vma *vma,
			   enum i915_cache_level cache_level,
			   u32 flags);
static void ppgtt_unbind_vma(struct i915_vma *vma);
 
 
static inline gen8_gtt_pte_t gen8_pte_encode(dma_addr_t addr,
					     enum i915_cache_level level,
					     bool valid)
{
	gen8_gtt_pte_t pte = valid ? _PAGE_PRESENT | _PAGE_RW : 0;
	pte |= addr;
 
	switch (level) {
	case I915_CACHE_NONE:
		pte |= PPAT_UNCACHED_INDEX;
		break;
	case I915_CACHE_WT:
		pte |= PPAT_DISPLAY_ELLC_INDEX;
		break;
	default:
		pte |= PPAT_CACHED_INDEX;
		break;
	}
 
	return pte;
}
 
static inline gen8_ppgtt_pde_t gen8_pde_encode(struct drm_device *dev,
					     dma_addr_t addr,
					     enum i915_cache_level level)
{
	gen8_ppgtt_pde_t pde = _PAGE_PRESENT | _PAGE_RW;
	pde |= addr;
	if (level != I915_CACHE_NONE)
		pde |= PPAT_CACHED_PDE_INDEX;
	else
		pde |= PPAT_UNCACHED_INDEX;
	return pde;
}
 
static gen6_gtt_pte_t snb_pte_encode(dma_addr_t addr,
				     enum i915_cache_level level,
				     bool valid, u32 unused)
{
	gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
	pte |= GEN6_PTE_ADDR_ENCODE(addr);
 
	switch (level) {
	case I915_CACHE_L3_LLC:
	case I915_CACHE_LLC:
		pte |= GEN6_PTE_CACHE_LLC;
		break;
	case I915_CACHE_NONE:
		pte |= GEN6_PTE_UNCACHED;
		break;
	default:
		WARN_ON(1);
	}
 
	return pte;
}
 
static gen6_gtt_pte_t ivb_pte_encode(dma_addr_t addr,
				     enum i915_cache_level level,
				     bool valid, u32 unused)
{
	gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
	pte |= GEN6_PTE_ADDR_ENCODE(addr);
 
	switch (level) {
	case I915_CACHE_L3_LLC:
		pte |= GEN7_PTE_CACHE_L3_LLC;
		break;
	case I915_CACHE_LLC:
		pte |= GEN6_PTE_CACHE_LLC;
		break;
	case I915_CACHE_NONE:
			pte |= GEN6_PTE_UNCACHED;
		break;
	default:
		WARN_ON(1);
	}
 
	return pte;
}
 
static gen6_gtt_pte_t byt_pte_encode(dma_addr_t addr,
				     enum i915_cache_level level,
				     bool valid, u32 flags)
{
	gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
	pte |= GEN6_PTE_ADDR_ENCODE(addr);
 
 
 
 
	if (!(flags & PTE_READ_ONLY))
	pte |= BYT_PTE_WRITEABLE;
 
	if (level != I915_CACHE_NONE)
		pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
 
	return pte;
}
 
static gen6_gtt_pte_t hsw_pte_encode(dma_addr_t addr,
				     enum i915_cache_level level,
				     bool valid, u32 unused)
{
	gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
	pte |= HSW_PTE_ADDR_ENCODE(addr);
 
	if (level != I915_CACHE_NONE)
		pte |= HSW_WB_LLC_AGE3;
 
	return pte;
}
 
static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,
				      enum i915_cache_level level,
				      bool valid, u32 unused)
{
	gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
	pte |= HSW_PTE_ADDR_ENCODE(addr);
 
	switch (level) {
	case I915_CACHE_NONE:
		break;
	case I915_CACHE_WT:
		pte |= HSW_WT_ELLC_LLC_AGE3;
		break;
	default:
		pte |= HSW_WB_ELLC_LLC_AGE3;
		break;
	}
 
	return pte;
}
 
/* Broadwell Page Directory Pointer Descriptors */
static int gen8_write_pdp(struct intel_engine_cs *ring, unsigned entry,
			   uint64_t val)
{
 
	int ret;
 
	BUG_ON(entry >= 4);
 
 
 
 
 
 
 
	ret = intel_ring_begin(ring, 6);
	if (ret)
		return ret;
 
	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
	intel_ring_emit(ring, GEN8_RING_PDP_UDW(ring, entry));
	intel_ring_emit(ring, (u32)(val >> 32));
	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
	intel_ring_emit(ring, GEN8_RING_PDP_LDW(ring, entry));
	intel_ring_emit(ring, (u32)(val));
	intel_ring_advance(ring);
 
	return 0;
}
 
static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,
			  struct intel_engine_cs *ring)
 
{
	int i, ret;
 
	/* bit of a hack to find the actual last used pd */
	int used_pd = ppgtt->num_pd_entries / GEN8_PDES_PER_PAGE;
 
	for (i = used_pd - 1; i >= 0; i--) {
		dma_addr_t addr = ppgtt->pd_dma_addr[i];
		ret = gen8_write_pdp(ring, i, addr);
			if (ret)
			return ret;
	}
 
	return 0;
}
 
static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
				   uint64_t start,
				   uint64_t length,
				   bool use_scratch)
{
	struct i915_hw_ppgtt *ppgtt =
		container_of(vm, struct i915_hw_ppgtt, base);
	gen8_gtt_pte_t *pt_vaddr, scratch_pte;
	unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;
	unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
	unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
	unsigned num_entries = length >> PAGE_SHIFT;
	unsigned last_pte, i;
 
 
 
 
 
    scratch_pte = gen8_pte_encode(ppgtt->base.scratch.addr,
                      I915_CACHE_LLC, use_scratch);
 
	while (num_entries) {
		struct page *page_table = ppgtt->gen8_pt_pages[pdpe][pde];
 
		last_pte = pte + num_entries;
		if (last_pte > GEN8_PTES_PER_PAGE)
			last_pte = GEN8_PTES_PER_PAGE;
 
		pt_vaddr = kmap_atomic(page_table);
 
		for (i = pte; i < last_pte; i++) {
			pt_vaddr[i] = scratch_pte;
			num_entries--;
		}
 
		if (!HAS_LLC(ppgtt->base.dev))
			drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
		kunmap_atomic(pt_vaddr);
 
		pte = 0;
		if (++pde == GEN8_PDES_PER_PAGE) {
			pdpe++;
			pde = 0;
		}
	}
 
}
 
static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
				      struct sg_table *pages,
				      uint64_t start,
				      enum i915_cache_level cache_level, u32 unused)
{
	struct i915_hw_ppgtt *ppgtt =
		container_of(vm, struct i915_hw_ppgtt, base);
	gen8_gtt_pte_t *pt_vaddr;
	unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;
	unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
	unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
	struct sg_page_iter sg_iter;
 
 
 
 
 
	pt_vaddr = NULL;
 
	for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
		if (WARN_ON(pdpe >= GEN8_LEGACY_PDPS))
			break;
 
		if (pt_vaddr == NULL)
			pt_vaddr = kmap_atomic(ppgtt->gen8_pt_pages[pdpe][pde]);
 
		pt_vaddr[pte] =
			gen8_pte_encode(sg_page_iter_dma_address(&sg_iter),
					cache_level, true);
		if (++pte == GEN8_PTES_PER_PAGE) {
			if (!HAS_LLC(ppgtt->base.dev))
				drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
			kunmap_atomic(pt_vaddr);
			pt_vaddr = NULL;
			if (++pde == GEN8_PDES_PER_PAGE) {
				pdpe++;
				pde = 0;
			}
			pte = 0;
 
		}
	}
	if (pt_vaddr) {
		if (!HAS_LLC(ppgtt->base.dev))
			drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
		kunmap_atomic(pt_vaddr);
	}
}
 
static void gen8_free_page_tables(struct page **pt_pages)
{
	int i;
 
	if (pt_pages == NULL)
		return;
 
//   for (i = 0; i < GEN8_PDES_PER_PAGE; i++)
//       if (pt_pages[i])
//           __free_pages(pt_pages[i], 0);
}
 
static void gen8_ppgtt_free(const struct i915_hw_ppgtt *ppgtt)
{
	int i;
 
	for (i = 0; i < ppgtt->num_pd_pages; i++) {
		gen8_free_page_tables(ppgtt->gen8_pt_pages[i]);
		kfree(ppgtt->gen8_pt_pages[i]);
		kfree(ppgtt->gen8_pt_dma_addr[i]);
	}
 
//	__free_pages(ppgtt->pd_pages, get_order(ppgtt->num_pd_pages << PAGE_SHIFT));
}
 
static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)
{
	struct pci_dev *hwdev = ppgtt->base.dev->pdev;
	int i, j;
 
	for (i = 0; i < ppgtt->num_pd_pages; i++) {
		/* TODO: In the future we'll support sparse mappings, so this
		 * will have to change. */
		if (!ppgtt->pd_dma_addr[i])
			continue;
 
		pci_unmap_page(hwdev, ppgtt->pd_dma_addr[i], PAGE_SIZE,
			       PCI_DMA_BIDIRECTIONAL);
 
			for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
				dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j];
				if (addr)
				pci_unmap_page(hwdev, addr, PAGE_SIZE,
						       PCI_DMA_BIDIRECTIONAL);
		}
	}
}
 
static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
{
	struct i915_hw_ppgtt *ppgtt =
		container_of(vm, struct i915_hw_ppgtt, base);
 
 
 
 
	gen8_ppgtt_unmap_pages(ppgtt);
	gen8_ppgtt_free(ppgtt);
}
 
static struct page **__gen8_alloc_page_tables(void)
{
	struct page **pt_pages;
	int i;
 
	pt_pages = kcalloc(GEN8_PDES_PER_PAGE, sizeof(struct page *), GFP_KERNEL);
	if (!pt_pages)
		return ERR_PTR(-ENOMEM);
 
	for (i = 0; i < GEN8_PDES_PER_PAGE; i++) {
		pt_pages[i] = alloc_page(GFP_KERNEL);
		if (!pt_pages[i])
			goto bail;
			}
 
	return pt_pages;
 
bail:
	gen8_free_page_tables(pt_pages);
	kfree(pt_pages);
	return ERR_PTR(-ENOMEM);
}
 
static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt,
					   const int max_pdp)
{
	struct page **pt_pages[GEN8_LEGACY_PDPS];
	int i, ret;
 
	for (i = 0; i < max_pdp; i++) {
		pt_pages[i] = __gen8_alloc_page_tables();
		if (IS_ERR(pt_pages[i])) {
			ret = PTR_ERR(pt_pages[i]);
			goto unwind_out;
		}
	}
 
	/* NB: Avoid touching gen8_pt_pages until last to keep the allocation,
	 * "atomic" - for cleanup purposes.
	 */
	for (i = 0; i < max_pdp; i++)
		ppgtt->gen8_pt_pages[i] = pt_pages[i];
 
	return 0;
 
unwind_out:
	while (i--) {
		gen8_free_page_tables(pt_pages[i]);
		kfree(pt_pages[i]);
	}
 
	return ret;
}
 
static int gen8_ppgtt_allocate_dma(struct i915_hw_ppgtt *ppgtt)
{
	int i;
 
	for (i = 0; i < ppgtt->num_pd_pages; i++) {
		ppgtt->gen8_pt_dma_addr[i] = kcalloc(GEN8_PDES_PER_PAGE,
						     sizeof(dma_addr_t),
						     GFP_KERNEL);
		if (!ppgtt->gen8_pt_dma_addr[i])
			return -ENOMEM;
		}
 
	return 0;
}
 
static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,
						const int max_pdp)
{
//	ppgtt->pd_pages = alloc_pages(GFP_KERNEL, get_order(max_pdp << PAGE_SHIFT));
	if (!ppgtt->pd_pages)
		return -ENOMEM;
 
//   ppgtt->num_pd_pages = 1 << get_order(max_pdp << PAGE_SHIFT);
	BUG_ON(ppgtt->num_pd_pages > GEN8_LEGACY_PDPS);
 
	return 0;
}
 
static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,
			    const int max_pdp)
{
	int ret;
 
	ret = gen8_ppgtt_allocate_page_directories(ppgtt, max_pdp);
	if (ret)
		return ret;
 
	ret = gen8_ppgtt_allocate_page_tables(ppgtt, max_pdp);
	if (ret) {
//		__free_pages(ppgtt->pd_pages, get_order(max_pdp << PAGE_SHIFT));
		return ret;
	}
 
	ppgtt->num_pd_entries = max_pdp * GEN8_PDES_PER_PAGE;
 
	ret = gen8_ppgtt_allocate_dma(ppgtt);
	if (ret)
		gen8_ppgtt_free(ppgtt);
 
	return ret;
}
 
static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,
					     const int pd)
{
	dma_addr_t pd_addr;
	int ret;
 
	pd_addr = pci_map_page(ppgtt->base.dev->pdev,
			       &ppgtt->pd_pages[pd], 0,
			       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 
//   ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pd_addr);
//   if (ret)
//       return ret;
 
	ppgtt->pd_dma_addr[pd] = pd_addr;
 
	return 0;
}
 
static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt,
					const int pd,
					const int pt)
{
	dma_addr_t pt_addr;
	struct page *p;
	int ret;
 
	p = ppgtt->gen8_pt_pages[pd][pt];
	pt_addr = pci_map_page(ppgtt->base.dev->pdev,
			       p, 0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
//   ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pt_addr);
//   if (ret)
//       return ret;
 
	ppgtt->gen8_pt_dma_addr[pd][pt] = pt_addr;
 
	return 0;
}
 
/**
 * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
 * with a net effect resembling a 2-level page table in normal x86 terms. Each
 * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address
 * space.
 *
 * FIXME: split allocation into smaller pieces. For now we only ever do this
 * once, but with full PPGTT, the multiple contiguous allocations will be bad.
 * TODO: Do something with the size parameter
 */
static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
{
	const int max_pdp = DIV_ROUND_UP(size, 1 << 30);
	const int min_pt_pages = GEN8_PDES_PER_PAGE * max_pdp;
	int i, j, ret;
 
 
	if (size % (1<<30))
		DRM_INFO("Pages will be wasted unless GTT size (%llu) is divisible by 1GB\n", size);
 
	/* 1. Do all our allocations for page directories and page tables. */
	ret = gen8_ppgtt_alloc(ppgtt, max_pdp);
	if (ret)
		return ret;
 
	/*
	 * 2. Create DMA mappings for the page directories and page tables.
	 */
	for (i = 0; i < max_pdp; i++) {
		ret = gen8_ppgtt_setup_page_directories(ppgtt, i);
		if (ret)
			goto bail;
 
		for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
			ret = gen8_ppgtt_setup_page_tables(ppgtt, i, j);
			if (ret)
				goto bail;
		}
	}
 
	/*
	 * 3. Map all the page directory entires to point to the page tables
	 * we've allocated.
	 *
	 * For now, the PPGTT helper functions all require that the PDEs are
	 * plugged in correctly. So we do that now/here. For aliasing PPGTT, we
	 * will never need to touch the PDEs again.
	 */
 
 
 
    for (i = 0; i < max_pdp; i++) {
		gen8_ppgtt_pde_t *pd_vaddr;
		pd_vaddr = kmap_atomic(&ppgtt->pd_pages[i]);
		for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
			dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j];
			pd_vaddr[j] = gen8_pde_encode(ppgtt->base.dev, addr,
						      I915_CACHE_LLC);
		}
		if (!HAS_LLC(ppgtt->base.dev))
			drm_clflush_virt_range(pd_vaddr, PAGE_SIZE);
		kunmap_atomic(pd_vaddr);
	}
 
 
 
	ppgtt->switch_mm = gen8_mm_switch;
	ppgtt->base.clear_range = gen8_ppgtt_clear_range;
	ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
	ppgtt->base.cleanup = gen8_ppgtt_cleanup;
	ppgtt->base.start = 0;
	ppgtt->base.total = ppgtt->num_pd_entries * GEN8_PTES_PER_PAGE * PAGE_SIZE;
 
	ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true);
 
	DRM_DEBUG_DRIVER("Allocated %d pages for page directories (%d wasted)\n",
			 ppgtt->num_pd_pages, ppgtt->num_pd_pages - max_pdp);
	DRM_DEBUG_DRIVER("Allocated %d pages for page tables (%lld wasted)\n",
			 ppgtt->num_pd_entries,
			 (ppgtt->num_pd_entries - min_pt_pages) + size % (1<<30));
	return 0;
 
bail:
	gen8_ppgtt_unmap_pages(ppgtt);
	gen8_ppgtt_free(ppgtt);
	return ret;
}
 
static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)
{
	struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;
	gen6_gtt_pte_t __iomem *pd_addr;
	uint32_t pd_entry;
	int i;
 
	WARN_ON(ppgtt->pd_offset & 0x3f);
	pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm +
		ppgtt->pd_offset / sizeof(gen6_gtt_pte_t);
	for (i = 0; i < ppgtt->num_pd_entries; i++) {
		dma_addr_t pt_addr;
 
		pt_addr = ppgtt->pt_dma_addr[i];
		pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
		pd_entry |= GEN6_PDE_VALID;
 
		writel(pd_entry, pd_addr + i);
	}
	readl(pd_addr);
}
 
static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt)
{
	BUG_ON(ppgtt->pd_offset & 0x3f);
 
	return (ppgtt->pd_offset / 64) << 16;
}
 
static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
			 struct intel_engine_cs *ring)
 
{
 
 
	int ret;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
	/* NB: TLBs must be flushed and invalidated before a switch */
	ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
	if (ret)
		return ret;
 
	ret = intel_ring_begin(ring, 6);
	if (ret)
		return ret;
 
	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
	intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
	intel_ring_emit(ring, PP_DIR_DCLV_2G);
	intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
	intel_ring_emit(ring, get_pd_offset(ppgtt));
	intel_ring_emit(ring, MI_NOOP);
	intel_ring_advance(ring);
 
	return 0;
}
 
static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
			  struct intel_engine_cs *ring)
 
{
 
 
	int ret;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
	/* NB: TLBs must be flushed and invalidated before a switch */
	ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
	if (ret)
		return ret;
 
	ret = intel_ring_begin(ring, 6);
	if (ret)
		return ret;
 
	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
	intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
	intel_ring_emit(ring, PP_DIR_DCLV_2G);
	intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
	intel_ring_emit(ring, get_pd_offset(ppgtt));
	intel_ring_emit(ring, MI_NOOP);
	intel_ring_advance(ring);
 
	/* XXX: RCS is the only one to auto invalidate the TLBs? */
	if (ring->id != RCS) {
		ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
		if (ret)
			return ret;
	}
 
	return 0;
}
 
static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,
			  struct intel_engine_cs *ring)
 
{
	struct drm_device *dev = ppgtt->base.dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
 
 
 
 
	I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
	I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
 
	POSTING_READ(RING_PP_DIR_DCLV(ring));
 
	return 0;
}
 
static void gen8_ppgtt_enable(struct drm_device *dev)
 
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct intel_engine_cs *ring;
	int j;
 
	for_each_ring(ring, dev_priv, j) {
		I915_WRITE(RING_MODE_GEN7(ring),
			   _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
 
 
 
 
 
 
 
 
 
	}
 
 
 
 
 
 
 
 
}
 
static void gen7_ppgtt_enable(struct drm_device *dev)
 
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct intel_engine_cs *ring;
		uint32_t ecochk, ecobits;
	int i;
 
		ecobits = I915_READ(GAC_ECO_BITS);
		I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
 
		ecochk = I915_READ(GAM_ECOCHK);
		if (IS_HASWELL(dev)) {
			ecochk |= ECOCHK_PPGTT_WB_HSW;
		} else {
			ecochk |= ECOCHK_PPGTT_LLC_IVB;
			ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
		}
		I915_WRITE(GAM_ECOCHK, ecochk);
 
	for_each_ring(ring, dev_priv, i) {
 
		/* GFX_MODE is per-ring on gen7+ */
			I915_WRITE(RING_MODE_GEN7(ring),
				   _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
 
 
 
 
 
 
 
 
 
	}
 
 
}
 
static void gen6_ppgtt_enable(struct drm_device *dev)
 
{
 
	struct drm_i915_private *dev_priv = dev->dev_private;
 
	uint32_t ecochk, gab_ctl, ecobits;
 
	ecobits = I915_READ(GAC_ECO_BITS);
	I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
		   ECOBITS_PPGTT_CACHE64B);
 
	gab_ctl = I915_READ(GAB_CTL);
	I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
 
	ecochk = I915_READ(GAM_ECOCHK);
	I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);
 
	I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
 
 
 
 
 
 
 
 
}
 
/* PPGTT support for Sandybdrige/Gen6 and later */
static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
				   uint64_t start,
				   uint64_t length,
				   bool use_scratch)
{
	struct i915_hw_ppgtt *ppgtt =
		container_of(vm, struct i915_hw_ppgtt, base);
	gen6_gtt_pte_t *pt_vaddr, scratch_pte;
	unsigned first_entry = start >> PAGE_SHIFT;
	unsigned num_entries = length >> PAGE_SHIFT;
	unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
	unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
	unsigned last_pte, i;
 
	scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, true, 0);
 
 
 
 
 
 
	while (num_entries) {
            last_pte = first_pte + num_entries;
            if (last_pte > I915_PPGTT_PT_ENTRIES)
                last_pte = I915_PPGTT_PT_ENTRIES;
 
		pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pt]);
 
            for (i = first_pte; i < last_pte; i++)
                pt_vaddr[i] = scratch_pte;
 
		kunmap_atomic(pt_vaddr);
 
            num_entries -= last_pte - first_pte;
            first_pte = 0;
            act_pt++;
 
	}
 
}
 
static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
				      struct sg_table *pages,
				      uint64_t start,
				      enum i915_cache_level cache_level, u32 flags)
{
	struct i915_hw_ppgtt *ppgtt =
		container_of(vm, struct i915_hw_ppgtt, base);
	gen6_gtt_pte_t *pt_vaddr;
	unsigned first_entry = start >> PAGE_SHIFT;
	unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
	unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES;
	struct sg_page_iter sg_iter;
 
 
 
 
 
 
	pt_vaddr = NULL;
	for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
		if (pt_vaddr == NULL)
			pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pt]);
 
		pt_vaddr[act_pte] =
			vm->pte_encode(sg_page_iter_dma_address(&sg_iter),
				       cache_level, true, flags);
 
		if (++act_pte == I915_PPGTT_PT_ENTRIES) {
			kunmap_atomic(pt_vaddr);
			pt_vaddr = NULL;
			act_pt++;
 
			act_pte = 0;
			}
		}
	if (pt_vaddr)
		kunmap_atomic(pt_vaddr);
}
 
static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)
{
	int i;
 
	if (ppgtt->pt_dma_addr) {
		for (i = 0; i < ppgtt->num_pd_entries; i++)
			pci_unmap_page(ppgtt->base.dev->pdev,
				       ppgtt->pt_dma_addr[i],
				       4096, PCI_DMA_BIDIRECTIONAL);
	}
}
 
static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
{
	int i;
 
	kfree(ppgtt->pt_dma_addr);
	for (i = 0; i < ppgtt->num_pd_entries; i++)
		__free_page(ppgtt->pt_pages[i]);
	kfree(ppgtt->pt_pages);
}
 
static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
{
	struct i915_hw_ppgtt *ppgtt =
		container_of(vm, struct i915_hw_ppgtt, base);
 
 
 
	drm_mm_remove_node(&ppgtt->node);
 
	gen6_ppgtt_unmap_pages(ppgtt);
	gen6_ppgtt_free(ppgtt);
}
 
static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt)
{
	struct drm_device *dev = ppgtt->base.dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
	bool retried = false;
	int ret;
 
	/* PPGTT PDEs reside in the GGTT and consists of 512 entries. The
	 * allocator works in address space sizes, so it's multiplied by page
	 * size. We allocate at the top of the GTT to avoid fragmentation.
	 */
	BUG_ON(!drm_mm_initialized(&dev_priv->gtt.base.mm));
alloc:
	ret = drm_mm_insert_node_in_range_generic(&dev_priv->gtt.base.mm,
						  &ppgtt->node, GEN6_PD_SIZE,
						  GEN6_PD_ALIGN, 0,
						  0, dev_priv->gtt.base.total,
						  DRM_MM_TOPDOWN);
	if (ret == -ENOSPC && !retried) {
		ret = i915_gem_evict_something(dev, &dev_priv->gtt.base,
					       GEN6_PD_SIZE, GEN6_PD_ALIGN,
					       I915_CACHE_NONE,
					       0, dev_priv->gtt.base.total,
					       0);
		if (ret)
			return ret;
 
		retried = true;
		goto alloc;
	}
 
	if (ppgtt->node.start < dev_priv->gtt.mappable_end)
		DRM_DEBUG("Forced to use aperture for PDEs\n");
 
	ppgtt->num_pd_entries = GEN6_PPGTT_PD_ENTRIES;
	return ret;
}
 
static int gen6_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt)
{
	int i;
 
	ppgtt->pt_pages = kcalloc(ppgtt->num_pd_entries, sizeof(struct page *),
				  GFP_KERNEL);
 
	if (!ppgtt->pt_pages)
		return -ENOMEM;
 
	for (i = 0; i < ppgtt->num_pd_entries; i++) {
		ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
		if (!ppgtt->pt_pages[i]) {
			gen6_ppgtt_free(ppgtt);
			return -ENOMEM;
		}
	}
 
	return 0;
}
 
static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)
{
	int ret;
 
	ret = gen6_ppgtt_allocate_page_directories(ppgtt);
	if (ret)
		return ret;
 
	ret = gen6_ppgtt_allocate_page_tables(ppgtt);
	if (ret) {
		drm_mm_remove_node(&ppgtt->node);
		return ret;
	}
 
	ppgtt->pt_dma_addr = kcalloc(ppgtt->num_pd_entries, sizeof(dma_addr_t),
				     GFP_KERNEL);
	if (!ppgtt->pt_dma_addr) {
		drm_mm_remove_node(&ppgtt->node);
		gen6_ppgtt_free(ppgtt);
		return -ENOMEM;
	}
 
	return 0;
}
 
static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)
{
	struct drm_device *dev = ppgtt->base.dev;
	int i;
 
	for (i = 0; i < ppgtt->num_pd_entries; i++) {
		dma_addr_t pt_addr;
 
		pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 0, 4096,
				       PCI_DMA_BIDIRECTIONAL);
 
//       if (pci_dma_mapping_error(dev->pdev, pt_addr)) {
//           gen6_ppgtt_unmap_pages(ppgtt);
//           return -EIO;
//       }
 
		ppgtt->pt_dma_addr[i] = pt_addr;
	}
 
	return 0;
}
 
static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
{
	struct drm_device *dev = ppgtt->base.dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
	int ret;
 
	ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode;
	if (IS_GEN6(dev)) {
 
		ppgtt->switch_mm = gen6_mm_switch;
	} else if (IS_HASWELL(dev)) {
 
		ppgtt->switch_mm = hsw_mm_switch;
	} else if (IS_GEN7(dev)) {
 
		ppgtt->switch_mm = gen7_mm_switch;
	} else
		BUG();
 
	ret = gen6_ppgtt_alloc(ppgtt);
	if (ret)
		return ret;
 
	ret = gen6_ppgtt_setup_page_tables(ppgtt);
	if (ret) {
		gen6_ppgtt_free(ppgtt);
		return ret;
	}
 
	ppgtt->base.clear_range = gen6_ppgtt_clear_range;
	ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
	ppgtt->base.cleanup = gen6_ppgtt_cleanup;
	ppgtt->base.start = 0;
	ppgtt->base.total =  ppgtt->num_pd_entries * I915_PPGTT_PT_ENTRIES * PAGE_SIZE;
//	ppgtt->debug_dump = gen6_dump_ppgtt;
 
	ppgtt->pd_offset =
		ppgtt->node.start / PAGE_SIZE * sizeof(gen6_gtt_pte_t);
 
	ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true);
 
	DRM_DEBUG_DRIVER("Allocated pde space (%ldM) at GTT entry: %lx\n",
			 ppgtt->node.size >> 20,
			 ppgtt->node.start / PAGE_SIZE);
 
	gen6_write_pdes(ppgtt);
	DRM_DEBUG("Adding PPGTT at offset %x\n",
		  ppgtt->pd_offset << 10);
 
	return 0;
}
 
static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
{
 
	struct drm_i915_private *dev_priv = dev->dev_private;
 
	ppgtt->base.dev = dev;
	ppgtt->base.scratch = dev_priv->gtt.base.scratch;
 
	if (INTEL_INFO(dev)->gen < 8)
		return gen6_ppgtt_init(ppgtt);
	else if (IS_GEN8(dev) || IS_GEN9(dev))
		return gen8_ppgtt_init(ppgtt, dev_priv->gtt.base.total);
	else
		BUG();
}
int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int ret = 0;
 
	ret = __hw_ppgtt_init(dev, ppgtt);
	if (ret == 0) {
		kref_init(&ppgtt->ref);
		drm_mm_init(&ppgtt->base.mm, ppgtt->base.start,
			    ppgtt->base.total);
		i915_init_vm(dev_priv, &ppgtt->base);
 
 
 
 
	}
 
	return ret;
}
 
int i915_ppgtt_init_hw(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct intel_engine_cs *ring;
	struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
	int i, ret = 0;
 
	/* In the case of execlists, PPGTT is enabled by the context descriptor
	 * and the PDPs are contained within the context itself.  We don't
	 * need to do anything here. */
	if (i915.enable_execlists)
		return 0;
 
	if (!USES_PPGTT(dev))
		return 0;
 
	if (IS_GEN6(dev))
		gen6_ppgtt_enable(dev);
	else if (IS_GEN7(dev))
		gen7_ppgtt_enable(dev);
	else if (INTEL_INFO(dev)->gen >= 8)
		gen8_ppgtt_enable(dev);
	else
		WARN_ON(1);
 
	if (ppgtt) {
		for_each_ring(ring, dev_priv, i) {
			ret = ppgtt->switch_mm(ppgtt, ring);
			if (ret != 0)
				return ret;
		}
	}
 
	return ret;
}
struct i915_hw_ppgtt *
i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv)
{
	struct i915_hw_ppgtt *ppgtt;
	int ret;
 
	ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
	if (!ppgtt)
		return ERR_PTR(-ENOMEM);
 
	ret = i915_ppgtt_init(dev, ppgtt);
	if (ret) {
		kfree(ppgtt);
		return ERR_PTR(ret);
	}
 
	ppgtt->file_priv = fpriv;
 
	trace_i915_ppgtt_create(&ppgtt->base);
 
	return ppgtt;
}
 
void  i915_ppgtt_release(struct kref *kref)
{
	struct i915_hw_ppgtt *ppgtt =
		container_of(kref, struct i915_hw_ppgtt, ref);
 
	trace_i915_ppgtt_release(&ppgtt->base);
 
	/* vmas should already be unbound */
	WARN_ON(!list_empty(&ppgtt->base.active_list));
	WARN_ON(!list_empty(&ppgtt->base.inactive_list));
 
	list_del(&ppgtt->base.global_link);
	drm_mm_takedown(&ppgtt->base.mm);
 
	ppgtt->base.cleanup(&ppgtt->base);
	kfree(ppgtt);
}
 
static void
ppgtt_bind_vma(struct i915_vma *vma,
	       enum i915_cache_level cache_level,
	       u32 flags)
{
	/* Currently applicable only to VLV */
	if (vma->obj->gt_ro)
		flags |= PTE_READ_ONLY;
 
	vma->vm->insert_entries(vma->vm, vma->obj->pages, vma->node.start,
				cache_level, flags);
}
 
static void ppgtt_unbind_vma(struct i915_vma *vma)
{
	vma->vm->clear_range(vma->vm,
			     vma->node.start,
			     vma->obj->base.size,
				true);
}
 
extern int intel_iommu_gfx_mapped;
/* Certain Gen5 chipsets require require idling the GPU before
 * unmapping anything from the GTT when VT-d is enabled.
 */
static inline bool needs_idle_maps(struct drm_device *dev)
{
#ifdef CONFIG_INTEL_IOMMU
	/* Query intel_iommu to see if we need the workaround. Presumably that
	 * was loaded first.
	 */
	if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped)
		return true;
#endif
	return false;
}
 
static bool do_idling(struct drm_i915_private *dev_priv)
{
	bool ret = dev_priv->mm.interruptible;
 
	if (unlikely(dev_priv->gtt.do_idle_maps)) {
		dev_priv->mm.interruptible = false;
		if (i915_gpu_idle(dev_priv->dev)) {
			DRM_ERROR("Couldn't idle GPU\n");
			/* Wait a bit, in hopes it avoids the hang */
			udelay(10);
		}
	}
 
	return ret;
}
 
static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
{
	if (unlikely(dev_priv->gtt.do_idle_maps))
		dev_priv->mm.interruptible = interruptible;
}
 
void i915_check_and_clear_faults(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct intel_engine_cs *ring;
	int i;
 
	if (INTEL_INFO(dev)->gen < 6)
		return;
 
	for_each_ring(ring, dev_priv, i) {
		u32 fault_reg;
		fault_reg = I915_READ(RING_FAULT_REG(ring));
		if (fault_reg & RING_FAULT_VALID) {
			DRM_DEBUG_DRIVER("Unexpected fault\n"
					 "\tAddr: 0x%08lx\n"
					 "\tAddress space: %s\n"
					 "\tSource ID: %d\n"
					 "\tType: %d\n",
					 fault_reg & PAGE_MASK,
					 fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT",
					 RING_FAULT_SRCID(fault_reg),
					 RING_FAULT_FAULT_TYPE(fault_reg));
			I915_WRITE(RING_FAULT_REG(ring),
				   fault_reg & ~RING_FAULT_VALID);
		}
	}
	POSTING_READ(RING_FAULT_REG(&dev_priv->ring[RCS]));
}
 
static void i915_ggtt_flush(struct drm_i915_private *dev_priv)
{
	if (INTEL_INFO(dev_priv->dev)->gen < 6) {
		intel_gtt_chipset_flush();
	} else {
		I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
		POSTING_READ(GFX_FLSH_CNTL_GEN6);
	}
}
 
void i915_gem_suspend_gtt_mappings(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
 
	/* Don't bother messing with faults pre GEN6 as we have little
	 * documentation supporting that it's a good idea.
	 */
	if (INTEL_INFO(dev)->gen < 6)
		return;
 
	i915_check_and_clear_faults(dev);
 
	dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
				       dev_priv->gtt.base.start,
				       dev_priv->gtt.base.total,
				       true);
 
	i915_ggtt_flush(dev_priv);
}
 
void i915_gem_restore_gtt_mappings(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_i915_gem_object *obj;
	struct i915_address_space *vm;
 
	i915_check_and_clear_faults(dev);
 
	/* First fill our portion of the GTT with scratch pages */
	dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
				       dev_priv->gtt.base.start,
				       dev_priv->gtt.base.total,
				       true);
 
	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
		struct i915_vma *vma = i915_gem_obj_to_vma(obj,
							   &dev_priv->gtt.base);
		if (!vma)
			continue;
 
		i915_gem_clflush_object(obj, obj->pin_display);
		/* The bind_vma code tries to be smart about tracking mappings.
		 * Unfortunately above, we've just wiped out the mappings
		 * without telling our object about it. So we need to fake it.
		 */
		vma->bound &= ~GLOBAL_BIND;
		vma->bind_vma(vma, obj->cache_level, GLOBAL_BIND);
	}
 
 
	if (INTEL_INFO(dev)->gen >= 8) {
		if (IS_CHERRYVIEW(dev))
			chv_setup_private_ppat(dev_priv);
		else
			bdw_setup_private_ppat(dev_priv);
 
		return;
	}
 
	list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
		/* TODO: Perhaps it shouldn't be gen6 specific */
		if (i915_is_ggtt(vm)) {
			if (dev_priv->mm.aliasing_ppgtt)
				gen6_write_pdes(dev_priv->mm.aliasing_ppgtt);
			continue;
		}
 
		gen6_write_pdes(container_of(vm, struct i915_hw_ppgtt, base));
	}
 
	i915_ggtt_flush(dev_priv);
}
 
int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
{
	if (obj->has_dma_mapping)
		return 0;
 
	if (!dma_map_sg(&obj->base.dev->pdev->dev,
			obj->pages->sgl, obj->pages->nents,
			PCI_DMA_BIDIRECTIONAL))
		return -ENOSPC;
 
	return 0;
}
 
static inline void gen8_set_pte(void __iomem *addr, gen8_gtt_pte_t pte)
{
#ifdef writeq
	writeq(pte, addr);
#else
	iowrite32((u32)pte, addr);
	iowrite32(pte >> 32, addr + 4);
#endif
}
 
static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
				     struct sg_table *st,
				     uint64_t start,
				     enum i915_cache_level level, u32 unused)
{
	struct drm_i915_private *dev_priv = vm->dev->dev_private;
	unsigned first_entry = start >> PAGE_SHIFT;
	gen8_gtt_pte_t __iomem *gtt_entries =
		(gen8_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
	int i = 0;
	struct sg_page_iter sg_iter;
	dma_addr_t addr = 0; /* shut up gcc */
 
	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
		addr = sg_dma_address(sg_iter.sg) +
			(sg_iter.sg_pgoffset << PAGE_SHIFT);
		gen8_set_pte(>t_entries[i],
			     gen8_pte_encode(addr, level, true));
		i++;
	}
 
	/*
	 * XXX: This serves as a posting read to make sure that the PTE has
	 * actually been updated. There is some concern that even though
	 * registers and PTEs are within the same BAR that they are potentially
	 * of NUMA access patterns. Therefore, even with the way we assume
	 * hardware should work, we must keep this posting read for paranoia.
	 */
	if (i != 0)
		WARN_ON(readq(>t_entries[i-1])
			!= gen8_pte_encode(addr, level, true));
 
	/* This next bit makes the above posting read even more important. We
	 * want to flush the TLBs only after we're certain all the PTE updates
	 * have finished.
	 */
	I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
	POSTING_READ(GFX_FLSH_CNTL_GEN6);
}
 
/*
 * Binds an object into the global gtt with the specified cache level. The object
 * will be accessible to the GPU via commands whose operands reference offsets
 * within the global GTT as well as accessible by the GPU through the GMADR
 * mapped BAR (dev_priv->mm.gtt->gtt).
 */
static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
				     struct sg_table *st,
				     uint64_t start,
				     enum i915_cache_level level, u32 flags)
{
	struct drm_i915_private *dev_priv = vm->dev->dev_private;
	unsigned first_entry = start >> PAGE_SHIFT;
	gen6_gtt_pte_t __iomem *gtt_entries =
		(gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
	int i = 0;
	struct sg_page_iter sg_iter;
	dma_addr_t addr = 0;
 
	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
		addr = sg_page_iter_dma_address(&sg_iter);
		iowrite32(vm->pte_encode(addr, level, true, flags), >t_entries[i]);
			i++;
		}
 
	/* XXX: This serves as a posting read to make sure that the PTE has
	 * actually been updated. There is some concern that even though
	 * registers and PTEs are within the same BAR that they are potentially
	 * of NUMA access patterns. Therefore, even with the way we assume
	 * hardware should work, we must keep this posting read for paranoia.
	 */
	if (i != 0) {
		unsigned long gtt = readl(>t_entries[i-1]);
		WARN_ON(gtt != vm->pte_encode(addr, level, true, flags));
	}
 
	/* This next bit makes the above posting read even more important. We
	 * want to flush the TLBs only after we're certain all the PTE updates
	 * have finished.
	 */
	I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
	POSTING_READ(GFX_FLSH_CNTL_GEN6);
}
 
static void gen8_ggtt_clear_range(struct i915_address_space *vm,
				  uint64_t start,
				  uint64_t length,
				  bool use_scratch)
{
	struct drm_i915_private *dev_priv = vm->dev->dev_private;
	unsigned first_entry = start >> PAGE_SHIFT;
	unsigned num_entries = length >> PAGE_SHIFT;
	gen8_gtt_pte_t scratch_pte, __iomem *gtt_base =
		(gen8_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
	const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
	int i;
 
	if (WARN(num_entries > max_entries,
		 "First entry = %d; Num entries = %d (max=%d)\n",
		 first_entry, num_entries, max_entries))
		num_entries = max_entries;
 
	scratch_pte = gen8_pte_encode(vm->scratch.addr,
				      I915_CACHE_LLC,
				      use_scratch);
	for (i = 0; i < num_entries; i++)
		gen8_set_pte(>t_base[i], scratch_pte);
	readl(gtt_base);
}
 
static void gen6_ggtt_clear_range(struct i915_address_space *vm,
				  uint64_t start,
				  uint64_t length,
				  bool use_scratch)
{
	struct drm_i915_private *dev_priv = vm->dev->dev_private;
	unsigned first_entry = start >> PAGE_SHIFT;
	unsigned num_entries = length >> PAGE_SHIFT;
	gen6_gtt_pte_t scratch_pte, __iomem *gtt_base =
		(gen6_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
	const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
	int i;
 
	if (WARN(num_entries > max_entries,
		 "First entry = %d; Num entries = %d (max=%d)\n",
		 first_entry, num_entries, max_entries))
        num_entries = max_entries;
 
	scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, use_scratch, 0);
 
	for (i = 0; i < num_entries; i++)
		iowrite32(scratch_pte, >t_base[i]);
	readl(gtt_base);
}
 
 
static void i915_ggtt_bind_vma(struct i915_vma *vma,
			       enum i915_cache_level cache_level,
			       u32 unused)
{
	const unsigned long entry = vma->node.start >> PAGE_SHIFT;
	unsigned int flags = (cache_level == I915_CACHE_NONE) ?
		AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
 
	BUG_ON(!i915_is_ggtt(vma->vm));
	intel_gtt_insert_sg_entries(vma->obj->pages, entry, flags);
	vma->bound = GLOBAL_BIND;
}
 
static void i915_ggtt_clear_range(struct i915_address_space *vm,
				  uint64_t start,
				  uint64_t length,
				  bool unused)
{
	unsigned first_entry = start >> PAGE_SHIFT;
	unsigned num_entries = length >> PAGE_SHIFT;
	intel_gtt_clear_range(first_entry, num_entries);
}
 
static void i915_ggtt_unbind_vma(struct i915_vma *vma)
{
	const unsigned int first = vma->node.start >> PAGE_SHIFT;
	const unsigned int size = vma->obj->base.size >> PAGE_SHIFT;
 
	BUG_ON(!i915_is_ggtt(vma->vm));
	vma->bound = 0;
	intel_gtt_clear_range(first, size);
}
 
static void ggtt_bind_vma(struct i915_vma *vma,
			  enum i915_cache_level cache_level,
			  u32 flags)
{
	struct drm_device *dev = vma->vm->dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_i915_gem_object *obj = vma->obj;
 
	/* Currently applicable only to VLV */
	if (obj->gt_ro)
		flags |= PTE_READ_ONLY;
 
	/* If there is no aliasing PPGTT, or the caller needs a global mapping,
	 * or we have a global mapping already but the cacheability flags have
	 * changed, set the global PTEs.
	 *
	 * If there is an aliasing PPGTT it is anecdotally faster, so use that
	 * instead if none of the above hold true.
	 *
	 * NB: A global mapping should only be needed for special regions like
	 * "gtt mappable", SNB errata, or if specified via special execbuf
	 * flags. At all other times, the GPU will use the aliasing PPGTT.
	 */
	if (!dev_priv->mm.aliasing_ppgtt || flags & GLOBAL_BIND) {
		if (!(vma->bound & GLOBAL_BIND) ||
		    (cache_level != obj->cache_level)) {
			vma->vm->insert_entries(vma->vm, obj->pages,
						vma->node.start,
						cache_level, flags);
			vma->bound |= GLOBAL_BIND;
		}
	}
 
	if (dev_priv->mm.aliasing_ppgtt &&
	    (!(vma->bound & LOCAL_BIND) ||
	     (cache_level != obj->cache_level))) {
		struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
		appgtt->base.insert_entries(&appgtt->base,
					    vma->obj->pages,
					    vma->node.start,
					    cache_level, flags);
		vma->bound |= LOCAL_BIND;
	}
}
 
static void ggtt_unbind_vma(struct i915_vma *vma)
{
	struct drm_device *dev = vma->vm->dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_i915_gem_object *obj = vma->obj;
 
	if (vma->bound & GLOBAL_BIND) {
		vma->vm->clear_range(vma->vm,
				     vma->node.start,
				     obj->base.size,
				       true);
		vma->bound &= ~GLOBAL_BIND;
	}
 
	if (vma->bound & LOCAL_BIND) {
		struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
		appgtt->base.clear_range(&appgtt->base,
					 vma->node.start,
					 obj->base.size,
					 true);
		vma->bound &= ~LOCAL_BIND;
	}
}
 
void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
{
	struct drm_device *dev = obj->base.dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
	bool interruptible;
 
	interruptible = do_idling(dev_priv);
 
	if (!obj->has_dma_mapping)
		dma_unmap_sg(&dev->pdev->dev,
			     obj->pages->sgl, obj->pages->nents,
			     PCI_DMA_BIDIRECTIONAL);
 
	undo_idling(dev_priv, interruptible);
}
 
static void i915_gtt_color_adjust(struct drm_mm_node *node,
				  unsigned long color,
				  unsigned long *start,
				  unsigned long *end)
{
	if (node->color != color)
		*start += 4096;
 
	if (!list_empty(&node->node_list)) {
		node = list_entry(node->node_list.next,
				  struct drm_mm_node,
				  node_list);
		if (node->allocated && node->color != color)
			*end -= 4096;
	}
}
 
static int i915_gem_setup_global_gtt(struct drm_device *dev,
			      unsigned long start,
			      unsigned long mappable_end,
			      unsigned long end)
{
	/* Let GEM Manage all of the aperture.
	 *
	 * However, leave one page at the end still bound to the scratch page.
	 * There are a number of places where the hardware apparently prefetches
	 * past the end of the object, and we've seen multiple hangs with the
	 * GPU head pointer stuck in a batchbuffer bound at the last page of the
	 * aperture.  One page should be enough to keep any prefetching inside
	 * of the aperture.
	 */
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct i915_address_space *ggtt_vm = &dev_priv->gtt.base;
	struct drm_mm_node *entry;
	struct drm_i915_gem_object *obj;
	unsigned long hole_start, hole_end;
	int ret;
 
	BUG_ON(mappable_end > end);
 
	/* Subtract the guard page ... */
	drm_mm_init(&ggtt_vm->mm, start, end - start - PAGE_SIZE);
	if (!HAS_LLC(dev))
		dev_priv->gtt.base.mm.color_adjust = i915_gtt_color_adjust;
 
	/* Mark any preallocated objects as occupied */
	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
		struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
 
		DRM_DEBUG_KMS("reserving preallocated space: %lx + %zx\n",
			      i915_gem_obj_ggtt_offset(obj), obj->base.size);
 
		WARN_ON(i915_gem_obj_ggtt_bound(obj));
		ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node);
		if (ret) {
			DRM_DEBUG_KMS("Reservation failed: %i\n", ret);
			return ret;
		}
		vma->bound |= GLOBAL_BIND;
	}
 
	dev_priv->gtt.base.start = start;
	dev_priv->gtt.base.total = end - start;
 
	/* Clear any non-preallocated blocks */
	drm_mm_for_each_hole(entry, &ggtt_vm->mm, hole_start, hole_end) {
		DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
			      hole_start, hole_end);
		ggtt_vm->clear_range(ggtt_vm, hole_start,
				     hole_end - hole_start, true);
	}
 
	/* And finally clear the reserved guard page */
	ggtt_vm->clear_range(ggtt_vm, end - PAGE_SIZE, PAGE_SIZE, true);
 
	if (USES_PPGTT(dev) && !USES_FULL_PPGTT(dev)) {
		struct i915_hw_ppgtt *ppgtt;
 
		ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
		if (!ppgtt)
			return -ENOMEM;
 
		ret = __hw_ppgtt_init(dev, ppgtt);
		if (ret != 0)
			return ret;
 
		dev_priv->mm.aliasing_ppgtt = ppgtt;
	}
 
	return 0;
}
 
void i915_gem_init_global_gtt(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	unsigned long gtt_size, mappable_size;
 
	gtt_size = dev_priv->gtt.base.total;
	mappable_size = dev_priv->gtt.mappable_end;
 
	i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
}
 
void i915_global_gtt_cleanup(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct i915_address_space *vm = &dev_priv->gtt.base;
 
	if (dev_priv->mm.aliasing_ppgtt) {
		struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
 
		ppgtt->base.cleanup(&ppgtt->base);
	}
 
	if (drm_mm_initialized(&vm->mm)) {
		drm_mm_takedown(&vm->mm);
		list_del(&vm->global_link);
	}
 
	vm->cleanup(vm);
}
 
static int setup_scratch_page(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct page *page;
	dma_addr_t dma_addr;
 
	page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
	if (page == NULL)
		return -ENOMEM;
 
	set_pages_uc(page, 1);
 
#ifdef CONFIG_INTEL_IOMMU
	dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE,
				PCI_DMA_BIDIRECTIONAL);
	if (pci_dma_mapping_error(dev->pdev, dma_addr))
		return -EINVAL;
#else
	dma_addr = page_to_phys(page);
#endif
	dev_priv->gtt.base.scratch.page = page;
	dev_priv->gtt.base.scratch.addr = dma_addr;
 
	return 0;
}
 
static void teardown_scratch_page(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct page *page = dev_priv->gtt.base.scratch.page;
 
	set_pages_wb(page, 1);
	pci_unmap_page(dev->pdev, dev_priv->gtt.base.scratch.addr,
		       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 
	__free_page(page);
}
 
static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
{
	snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
	snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
	return snb_gmch_ctl << 20;
}
 
static inline unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
{
	bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
	bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
	if (bdw_gmch_ctl)
		bdw_gmch_ctl = 1 << bdw_gmch_ctl;
 
#ifdef CONFIG_X86_32
	/* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * PAGE_SIZE */
	if (bdw_gmch_ctl > 4)
		bdw_gmch_ctl = 4;
#endif
 
	return bdw_gmch_ctl << 20;
}
 
static inline unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
{
	gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
	gmch_ctrl &= SNB_GMCH_GGMS_MASK;
 
	if (gmch_ctrl)
		return 1 << (20 + gmch_ctrl);
 
	return 0;
}
 
static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
{
	snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
	snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
	return snb_gmch_ctl << 25; /* 32 MB units */
}
 
static inline size_t gen8_get_stolen_size(u16 bdw_gmch_ctl)
{
	bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
	bdw_gmch_ctl &= BDW_GMCH_GMS_MASK;
	return bdw_gmch_ctl << 25; /* 32 MB units */
}
 
static size_t chv_get_stolen_size(u16 gmch_ctrl)
{
	gmch_ctrl >>= SNB_GMCH_GMS_SHIFT;
	gmch_ctrl &= SNB_GMCH_GMS_MASK;
 
	/*
	 * 0x0  to 0x10: 32MB increments starting at 0MB
	 * 0x11 to 0x16: 4MB increments starting at 8MB
	 * 0x17 to 0x1d: 4MB increments start at 36MB
	 */
	if (gmch_ctrl < 0x11)
		return gmch_ctrl << 25;
	else if (gmch_ctrl < 0x17)
		return (gmch_ctrl - 0x11 + 2) << 22;
	else
		return (gmch_ctrl - 0x17 + 9) << 22;
}
 
static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl)
{
	gen9_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
	gen9_gmch_ctl &= BDW_GMCH_GMS_MASK;
 
	if (gen9_gmch_ctl < 0xf0)
		return gen9_gmch_ctl << 25; /* 32 MB units */
	else
		/* 4MB increments starting at 0xf0 for 4MB */
		return (gen9_gmch_ctl - 0xf0 + 1) << 22;
}
 
static int ggtt_probe_common(struct drm_device *dev,
			     size_t gtt_size)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	phys_addr_t gtt_phys_addr;
	int ret;
 
	/* For Modern GENs the PTEs and register space are split in the BAR */
	gtt_phys_addr = pci_resource_start(dev->pdev, 0) +
		(pci_resource_len(dev->pdev, 0) / 2);
 
	dev_priv->gtt.gsm = ioremap_wc(gtt_phys_addr, gtt_size);
	if (!dev_priv->gtt.gsm) {
		DRM_ERROR("Failed to map the gtt page table\n");
		return -ENOMEM;
	}
 
	ret = setup_scratch_page(dev);
	if (ret) {
		DRM_ERROR("Scratch setup failed\n");
		/* iounmap will also get called at remove, but meh */
		iounmap(dev_priv->gtt.gsm);
	}
 
	return ret;
}
 
/* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
 * bits. When using advanced contexts each context stores its own PAT, but
 * writing this data shouldn't be harmful even in those cases. */
static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv)
{
	uint64_t pat;
 
	pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC)     | /* for normal objects, no eLLC */
	      GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */
	      GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */
	      GEN8_PPAT(3, GEN8_PPAT_UC)                     | /* Uncached objects, mostly for scanout */
	      GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) |
	      GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) |
	      GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) |
	      GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
 
	if (!USES_PPGTT(dev_priv->dev))
		/* Spec: "For GGTT, there is NO pat_sel[2:0] from the entry,
		 * so RTL will always use the value corresponding to
		 * pat_sel = 000".
		 * So let's disable cache for GGTT to avoid screen corruptions.
		 * MOCS still can be used though.
		 * - System agent ggtt writes (i.e. cpu gtt mmaps) already work
		 * before this patch, i.e. the same uncached + snooping access
		 * like on gen6/7 seems to be in effect.
		 * - So this just fixes blitter/render access. Again it looks
		 * like it's not just uncached access, but uncached + snooping.
		 * So we can still hold onto all our assumptions wrt cpu
		 * clflushing on LLC machines.
		 */
		pat = GEN8_PPAT(0, GEN8_PPAT_UC);
 
	/* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b
	 * write would work. */
	I915_WRITE(GEN8_PRIVATE_PAT, pat);
	I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32);
}
 
static void chv_setup_private_ppat(struct drm_i915_private *dev_priv)
{
	uint64_t pat;
 
	/*
	 * Map WB on BDW to snooped on CHV.
	 *
	 * Only the snoop bit has meaning for CHV, the rest is
	 * ignored.
	 *
	 * The hardware will never snoop for certain types of accesses:
	 * - CPU GTT (GMADR->GGTT->no snoop->memory)
	 * - PPGTT page tables
	 * - some other special cycles
	 *
	 * As with BDW, we also need to consider the following for GT accesses:
	 * "For GGTT, there is NO pat_sel[2:0] from the entry,
	 * so RTL will always use the value corresponding to
	 * pat_sel = 000".
	 * Which means we must set the snoop bit in PAT entry 0
	 * in order to keep the global status page working.
	 */
	pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) |
	      GEN8_PPAT(1, 0) |
	      GEN8_PPAT(2, 0) |
	      GEN8_PPAT(3, 0) |
	      GEN8_PPAT(4, CHV_PPAT_SNOOP) |
	      GEN8_PPAT(5, CHV_PPAT_SNOOP) |
	      GEN8_PPAT(6, CHV_PPAT_SNOOP) |
	      GEN8_PPAT(7, CHV_PPAT_SNOOP);
 
	I915_WRITE(GEN8_PRIVATE_PAT, pat);
	I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32);
}
 
static int gen8_gmch_probe(struct drm_device *dev,
			   size_t *gtt_total,
			   size_t *stolen,
			   phys_addr_t *mappable_base,
			   unsigned long *mappable_end)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	unsigned int gtt_size;
	u16 snb_gmch_ctl;
	int ret;
 
	/* TODO: We're not aware of mappable constraints on gen8 yet */
	*mappable_base = pci_resource_start(dev->pdev, 2);
	*mappable_end = pci_resource_len(dev->pdev, 2);
 
	if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(39)))
		pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(39));
 
	pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
 
	if (INTEL_INFO(dev)->gen >= 9) {
		*stolen = gen9_get_stolen_size(snb_gmch_ctl);
		gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl);
	} else if (IS_CHERRYVIEW(dev)) {
		*stolen = chv_get_stolen_size(snb_gmch_ctl);
		gtt_size = chv_get_total_gtt_size(snb_gmch_ctl);
	} else {
	*stolen = gen8_get_stolen_size(snb_gmch_ctl);
		gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl);
	}
 
	*gtt_total = (gtt_size / sizeof(gen8_gtt_pte_t)) << PAGE_SHIFT;
 
	if (IS_CHERRYVIEW(dev))
		chv_setup_private_ppat(dev_priv);
	else
		bdw_setup_private_ppat(dev_priv);
 
	ret = ggtt_probe_common(dev, gtt_size);
 
	dev_priv->gtt.base.clear_range = gen8_ggtt_clear_range;
	dev_priv->gtt.base.insert_entries = gen8_ggtt_insert_entries;
 
	return ret;
}
 
static int gen6_gmch_probe(struct drm_device *dev,
			   size_t *gtt_total,
			   size_t *stolen,
			   phys_addr_t *mappable_base,
			   unsigned long *mappable_end)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	unsigned int gtt_size;
	u16 snb_gmch_ctl;
	int ret;
 
	*mappable_base = pci_resource_start(dev->pdev, 2);
	*mappable_end = pci_resource_len(dev->pdev, 2);
 
	/* 64/512MB is the current min/max we actually know of, but this is just
	 * a coarse sanity check.
	 */
	if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) {
		DRM_ERROR("Unknown GMADR size (%lx)\n",
			  dev_priv->gtt.mappable_end);
		return -ENXIO;
		}
 
	if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40)))
		pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40));
	pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
 
	*stolen = gen6_get_stolen_size(snb_gmch_ctl);
 
	gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl);
	*gtt_total = (gtt_size / sizeof(gen6_gtt_pte_t)) << PAGE_SHIFT;
 
	ret = ggtt_probe_common(dev, gtt_size);
 
	dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range;
	dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries;
 
	return ret;
}
 
static void gen6_gmch_remove(struct i915_address_space *vm)
{
 
	struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base);
 
 
 
 
 
	iounmap(gtt->gsm);
	teardown_scratch_page(vm->dev);
}
 
static int i915_gmch_probe(struct drm_device *dev,
			   size_t *gtt_total,
			   size_t *stolen,
			   phys_addr_t *mappable_base,
			   unsigned long *mappable_end)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int ret;
 
	ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL);
	if (!ret) {
		DRM_ERROR("failed to set up gmch\n");
		return -EIO;
	}
 
	intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end);
 
	dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev);
	dev_priv->gtt.base.clear_range = i915_ggtt_clear_range;
 
	if (unlikely(dev_priv->gtt.do_idle_maps))
		DRM_INFO("applying Ironlake quirks for intel_iommu\n");
 
	return 0;
}
 
static void i915_gmch_remove(struct i915_address_space *vm)
{
//	intel_gmch_remove();
}
 
int i915_gem_gtt_init(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct i915_gtt *gtt = &dev_priv->gtt;
	int ret;
 
	if (INTEL_INFO(dev)->gen <= 5) {
		gtt->gtt_probe = i915_gmch_probe;
		gtt->base.cleanup = i915_gmch_remove;
	} else if (INTEL_INFO(dev)->gen < 8) {
		gtt->gtt_probe = gen6_gmch_probe;
		gtt->base.cleanup = gen6_gmch_remove;
		if (IS_HASWELL(dev) && dev_priv->ellc_size)
			gtt->base.pte_encode = iris_pte_encode;
		else if (IS_HASWELL(dev))
			gtt->base.pte_encode = hsw_pte_encode;
		else if (IS_VALLEYVIEW(dev))
			gtt->base.pte_encode = byt_pte_encode;
		else if (INTEL_INFO(dev)->gen >= 7)
			gtt->base.pte_encode = ivb_pte_encode;
		else
			gtt->base.pte_encode = snb_pte_encode;
	} else {
		dev_priv->gtt.gtt_probe = gen8_gmch_probe;
		dev_priv->gtt.base.cleanup = gen6_gmch_remove;
	}
 
	ret = gtt->gtt_probe(dev, >t->base.total, >t->stolen_size,
			     >t->mappable_base, >t->mappable_end);
	if (ret)
		return ret;
 
	gtt->base.dev = dev;
 
	/* GMADR is the PCI mmio aperture into the global GTT. */
	DRM_INFO("Memory usable by graphics device = %zdM\n",
		 gtt->base.total >> 20);
	DRM_DEBUG_DRIVER("GMADR size = %ldM\n", gtt->mappable_end >> 20);
	DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20);
#ifdef CONFIG_INTEL_IOMMU
	if (intel_iommu_gfx_mapped)
		DRM_INFO("VT-d active for gfx access\n");
#endif
	/*
	 * i915.enable_ppgtt is read-only, so do an early pass to validate the
	 * user's requested state against the hardware/driver capabilities.  We
	 * do this now so that we can print out any log messages once rather
	 * than every time we check intel_enable_ppgtt().
	 */
	i915.enable_ppgtt = sanitize_enable_ppgtt(dev, i915.enable_ppgtt);
	DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt);
 
	return 0;
}
 
static struct i915_vma *__i915_gem_vma_create(struct drm_i915_gem_object *obj,
					      struct i915_address_space *vm)
{
	struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
	if (vma == NULL)
		return ERR_PTR(-ENOMEM);
 
	INIT_LIST_HEAD(&vma->vma_link);
	INIT_LIST_HEAD(&vma->mm_list);
	INIT_LIST_HEAD(&vma->exec_list);
	vma->vm = vm;
	vma->obj = obj;
 
	switch (INTEL_INFO(vm->dev)->gen) {
	case 9:
	case 8:
	case 7:
	case 6:
		if (i915_is_ggtt(vm)) {
			vma->unbind_vma = ggtt_unbind_vma;
			vma->bind_vma = ggtt_bind_vma;
		} else {
			vma->unbind_vma = ppgtt_unbind_vma;
			vma->bind_vma = ppgtt_bind_vma;
		}
		break;
	case 5:
	case 4:
	case 3:
	case 2:
		BUG_ON(!i915_is_ggtt(vm));
		vma->unbind_vma = i915_ggtt_unbind_vma;
		vma->bind_vma = i915_ggtt_bind_vma;
		break;
	default:
		BUG();
	}
 
	/* Keep GGTT vmas first to make debug easier */
	if (i915_is_ggtt(vm))
		list_add(&vma->vma_link, &obj->vma_list);
	else {
		list_add_tail(&vma->vma_link, &obj->vma_list);
		i915_ppgtt_get(i915_vm_to_ppgtt(vm));
	}
 
	return vma;
}
 
struct i915_vma *
i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
				  struct i915_address_space *vm)
{
	struct i915_vma *vma;
 
	vma = i915_gem_obj_to_vma(obj, vm);
	if (!vma)
		vma = __i915_gem_vma_create(obj, vm);
 
	return vma;
}
 
struct scatterlist *sg_next(struct scatterlist *sg)
{
    if (sg_is_last(sg))
        return NULL;
 
    sg++;
    if (unlikely(sg_is_chain(sg)))
            sg = sg_chain_ptr(sg);
 
    return sg;
}
 
 
void __sg_free_table(struct sg_table *table, unsigned int max_ents,
             bool skip_first_chunk, sg_free_fn *free_fn)
{
    struct scatterlist *sgl, *next;
 
    if (unlikely(!table->sgl))
        return;
 
    sgl = table->sgl;
    while (table->orig_nents) {
        unsigned int alloc_size = table->orig_nents;
        unsigned int sg_size;
 
        /*
         * If we have more than max_ents segments left,
         * then assign 'next' to the sg table after the current one.
         * sg_size is then one less than alloc size, since the last
         * element is the chain pointer.
         */
        if (alloc_size > max_ents) {
            next = sg_chain_ptr(&sgl[max_ents - 1]);
            alloc_size = max_ents;
            sg_size = alloc_size - 1;
        } else {
            sg_size = alloc_size;
            next = NULL;
        }
 
        table->orig_nents -= sg_size;
        if (!skip_first_chunk) {
            kfree(sgl);
            skip_first_chunk = false;
        }
        sgl = next;
    }
 
    table->sgl = NULL;
}
void sg_free_table(struct sg_table *table)
{
    __sg_free_table(table, SG_MAX_SINGLE_ALLOC, false, NULL);
}
 
int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask)
{
    struct scatterlist *sg, *prv;
    unsigned int left;
    unsigned int max_ents = SG_MAX_SINGLE_ALLOC;
 
#ifndef ARCH_HAS_SG_CHAIN
    BUG_ON(nents > max_ents);
#endif
 
    memset(table, 0, sizeof(*table));
 
    left = nents;
    prv = NULL;
    do {
        unsigned int sg_size, alloc_size = left;
 
        if (alloc_size > max_ents) {
                alloc_size = max_ents;
                sg_size = alloc_size - 1;
        } else
                sg_size = alloc_size;
 
        left -= sg_size;
 
        sg = kmalloc(alloc_size * sizeof(struct scatterlist), gfp_mask);
        if (unlikely(!sg)) {
                /*
                 * Adjust entry count to reflect that the last
                 * entry of the previous table won't be used for
                 * linkage.  Without this, sg_kfree() may get
                 * confused.
                 */
                if (prv)
                        table->nents = ++table->orig_nents;
 
                goto err;
        }
 
        sg_init_table(sg, alloc_size);
        table->nents = table->orig_nents += sg_size;
 
        /*
         * If this is the first mapping, assign the sg table header.
         * If this is not the first mapping, chain previous part.
         */
        if (prv)
                sg_chain(prv, max_ents, sg);
        else
                table->sgl = sg;
 
        /*
         * If no more entries after this one, mark the end
         */
        if (!left)
                sg_mark_end(&sg[sg_size - 1]);
 
        prv = sg;
    } while (left);
 
    return 0;
 
err:
    __sg_free_table(table, SG_MAX_SINGLE_ALLOC, false, NULL);
 
    return -ENOMEM;
}
 
 
void sg_init_table(struct scatterlist *sgl, unsigned int nents)
{
    memset(sgl, 0, sizeof(*sgl) * nents);
#ifdef CONFIG_DEBUG_SG
    {
            unsigned int i;
            for (i = 0; i < nents; i++)
                    sgl[i].sg_magic = SG_MAGIC;
    }
#endif
    sg_mark_end(&sgl[nents - 1]);
}
 
 
void __sg_page_iter_start(struct sg_page_iter *piter,
              struct scatterlist *sglist, unsigned int nents,
              unsigned long pgoffset)
{
    piter->__pg_advance = 0;
    piter->__nents = nents;
 
    piter->sg = sglist;
    piter->sg_pgoffset = pgoffset;
}
 
static int sg_page_count(struct scatterlist *sg)
{
    return PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT;
}
 
bool __sg_page_iter_next(struct sg_page_iter *piter)
{
    if (!piter->__nents || !piter->sg)
        return false;
 
    piter->sg_pgoffset += piter->__pg_advance;
    piter->__pg_advance = 1;
 
    while (piter->sg_pgoffset >= sg_page_count(piter->sg)) {
        piter->sg_pgoffset -= sg_page_count(piter->sg);
        piter->sg = sg_next(piter->sg);
        if (!--piter->__nents || !piter->sg)
            return false;
    }
 
    return true;
}
EXPORT_SYMBOL(__sg_page_iter_next);
>
>
>
>

Rev 5060	Rev 5354
1	/*	1	/*
2	* Copyright © 2010 Daniel Vetter	2	* Copyright © 2010 Daniel Vetter
3	* Copyright © 2011-2014 Intel Corporation	3	* Copyright © 2011-2014 Intel Corporation
4	*	4	*
5	* Permission is hereby granted, free of charge, to any person obtaining a	5	* Permission is hereby granted, free of charge, to any person obtaining a
6	* copy of this software and associated documentation files (the "Software"),	6	* copy of this software and associated documentation files (the "Software"),
7	* to deal in the Software without restriction, including without limitation	7	* to deal in the Software without restriction, including without limitation
8	* the rights to use, copy, modify, merge, publish, distribute, sublicense,	8	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
9	* and/or sell copies of the Software, and to permit persons to whom the	9	* and/or sell copies of the Software, and to permit persons to whom the
10	* Software is furnished to do so, subject to the following conditions:	10	* Software is furnished to do so, subject to the following conditions:
11	*	11	*
12	* The above copyright notice and this permission notice (including the next	12	* The above copyright notice and this permission notice (including the next
13	* paragraph) shall be included in all copies or substantial portions of the	13	* paragraph) shall be included in all copies or substantial portions of the
14	* Software.	14	* Software.
15	*	15	*
16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR	16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,	17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL	18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER	19	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING	20	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS	21	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
22	* IN THE SOFTWARE.	22	* IN THE SOFTWARE.
23	*	23	*
24	*/	24	*/
25		-
26		25
27	#define AGP_NORMAL_MEMORY 0	-
28		-
29	#define AGP_USER_TYPES (1 << 16)	-
30	#define AGP_USER_MEMORY (AGP_USER_TYPES)	-
31	#define AGP_USER_CACHED_MEMORY (AGP_USER_TYPES + 1)	-
32		26	#include
33	#include	27	#include
34	#include	28	#include
35	#include "i915_drv.h"	29	#include "i915_drv.h"
36	#include "i915_trace.h"	30	#include "i915_trace.h"
37	#include "intel_drv.h"	31	#include "intel_drv.h"
-		32
-		33	#include
38		34
39	static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv);	35	static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv);
40	static void chv_setup_private_ppat(struct drm_i915_private *dev_priv);	36	static void chv_setup_private_ppat(struct drm_i915_private *dev_priv);
41		37
42	bool intel_enable_ppgtt(struct drm_device *dev, bool full)	38	static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)
43	{	39	{
44	if (i915.enable_ppgtt == 0)	40	bool has_aliasing_ppgtt;
45	return false;	41	bool has_full_ppgtt;
-		42
46		43	has_aliasing_ppgtt = INTEL_INFO(dev)->gen >= 6;
-		44	has_full_ppgtt = INTEL_INFO(dev)->gen >= 7;
47	if (i915.enable_ppgtt == 1 && full)	-
48	return false;	45	if (IS_GEN8(dev))
49		-
50	return true;	46	has_full_ppgtt = false; /* XXX why? */
-		47
51	}	48	/*
-		49	* We don't allow disabling PPGTT for gen9+ as it's a requirement for
52		50	* execlists, the sole mechanism available to submit work.
53	static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)	51	*/
54	{	52	if (INTEL_INFO(dev)->gen < 9 &&
55	if (enable_ppgtt == 0 \|\| !HAS_ALIASING_PPGTT(dev))	53	(enable_ppgtt == 0 \|\| !has_aliasing_ppgtt))
56	return 0;	54	return 0;
57		55
58	if (enable_ppgtt == 1)	56	if (enable_ppgtt == 1)
59	return 1;	57	return 1;
60		58
61	if (enable_ppgtt == 2 && HAS_PPGTT(dev))	59	if (enable_ppgtt == 2 && has_full_ppgtt)
62	return 2;	60	return 2;
63		61
64	#ifdef CONFIG_INTEL_IOMMU	62	#ifdef CONFIG_INTEL_IOMMU
65	/* Disable ppgtt on SNB if VT-d is on. */	63	/* Disable ppgtt on SNB if VT-d is on. */
66	if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) {	64	if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) {
67	DRM_INFO("Disabling PPGTT because VT-d is on\n");	65	DRM_INFO("Disabling PPGTT because VT-d is on\n");
68	return 0;	66	return 0;
69	}	67	}
70	#endif	68	#endif
71		69
72	/* Early VLV doesn't have this */	70	/* Early VLV doesn't have this */
73	if (IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) &&	71	if (IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) &&
74	dev->pdev->revision < 0xb) {	72	dev->pdev->revision < 0xb) {
75	DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n");	73	DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n");
76	return 0;	74	return 0;
77	}	75	}
78		76
79	return HAS_ALIASING_PPGTT(dev) ? 1 : 0;	77	return has_aliasing_ppgtt ? 1 : 0;
80	}	78	}
81		79
82		80
83	static void ppgtt_bind_vma(struct i915_vma *vma,	81	static void ppgtt_bind_vma(struct i915_vma *vma,
84	enum i915_cache_level cache_level,	82	enum i915_cache_level cache_level,
85	u32 flags);	83	u32 flags);
86	static void ppgtt_unbind_vma(struct i915_vma *vma);	84	static void ppgtt_unbind_vma(struct i915_vma *vma);
87	static int gen8_ppgtt_enable(struct i915_hw_ppgtt *ppgtt);	-
88		85
89	static inline gen8_gtt_pte_t gen8_pte_encode(dma_addr_t addr,	86	static inline gen8_gtt_pte_t gen8_pte_encode(dma_addr_t addr,
90	enum i915_cache_level level,	87	enum i915_cache_level level,
91	bool valid)	88	bool valid)
92	{	89	{
93	gen8_gtt_pte_t pte = valid ? _PAGE_PRESENT \| _PAGE_RW : 0;	90	gen8_gtt_pte_t pte = valid ? _PAGE_PRESENT \| _PAGE_RW : 0;
94	pte \|= addr;	91	pte \|= addr;
95		92
96	switch (level) {	93	switch (level) {
97	case I915_CACHE_NONE:	94	case I915_CACHE_NONE:
98	pte \|= PPAT_UNCACHED_INDEX;	95	pte \|= PPAT_UNCACHED_INDEX;
99	break;	96	break;
100	case I915_CACHE_WT:	97	case I915_CACHE_WT:
101	pte \|= PPAT_DISPLAY_ELLC_INDEX;	98	pte \|= PPAT_DISPLAY_ELLC_INDEX;
102	break;	99	break;
103	default:	100	default:
104	pte \|= PPAT_CACHED_INDEX;	101	pte \|= PPAT_CACHED_INDEX;
105	break;	102	break;
106	}	103	}
107		104
108	return pte;	105	return pte;
109	}	106	}
110		107
111	static inline gen8_ppgtt_pde_t gen8_pde_encode(struct drm_device *dev,	108	static inline gen8_ppgtt_pde_t gen8_pde_encode(struct drm_device *dev,
112	dma_addr_t addr,	109	dma_addr_t addr,
113	enum i915_cache_level level)	110	enum i915_cache_level level)
114	{	111	{
115	gen8_ppgtt_pde_t pde = _PAGE_PRESENT \| _PAGE_RW;	112	gen8_ppgtt_pde_t pde = _PAGE_PRESENT \| _PAGE_RW;
116	pde \|= addr;	113	pde \|= addr;
117	if (level != I915_CACHE_NONE)	114	if (level != I915_CACHE_NONE)
118	pde \|= PPAT_CACHED_PDE_INDEX;	115	pde \|= PPAT_CACHED_PDE_INDEX;
119	else	116	else
120	pde \|= PPAT_UNCACHED_INDEX;	117	pde \|= PPAT_UNCACHED_INDEX;
121	return pde;	118	return pde;
122	}	119	}
123		120
124	static gen6_gtt_pte_t snb_pte_encode(dma_addr_t addr,	121	static gen6_gtt_pte_t snb_pte_encode(dma_addr_t addr,
125	enum i915_cache_level level,	122	enum i915_cache_level level,
126	bool valid, u32 unused)	123	bool valid, u32 unused)
127	{	124	{
128	gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;	125	gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
129	pte \|= GEN6_PTE_ADDR_ENCODE(addr);	126	pte \|= GEN6_PTE_ADDR_ENCODE(addr);
130		127
131	switch (level) {	128	switch (level) {
132	case I915_CACHE_L3_LLC:	129	case I915_CACHE_L3_LLC:
133	case I915_CACHE_LLC:	130	case I915_CACHE_LLC:
134	pte \|= GEN6_PTE_CACHE_LLC;	131	pte \|= GEN6_PTE_CACHE_LLC;
135	break;	132	break;
136	case I915_CACHE_NONE:	133	case I915_CACHE_NONE:
137	pte \|= GEN6_PTE_UNCACHED;	134	pte \|= GEN6_PTE_UNCACHED;
138	break;	135	break;
139	default:	136	default:
140	WARN_ON(1);	137	WARN_ON(1);
141	}	138	}
142		139
143	return pte;	140	return pte;
144	}	141	}
145		142
146	static gen6_gtt_pte_t ivb_pte_encode(dma_addr_t addr,	143	static gen6_gtt_pte_t ivb_pte_encode(dma_addr_t addr,
147	enum i915_cache_level level,	144	enum i915_cache_level level,
148	bool valid, u32 unused)	145	bool valid, u32 unused)
149	{	146	{
150	gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;	147	gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
151	pte \|= GEN6_PTE_ADDR_ENCODE(addr);	148	pte \|= GEN6_PTE_ADDR_ENCODE(addr);
152		149
153	switch (level) {	150	switch (level) {
154	case I915_CACHE_L3_LLC:	151	case I915_CACHE_L3_LLC:
155	pte \|= GEN7_PTE_CACHE_L3_LLC;	152	pte \|= GEN7_PTE_CACHE_L3_LLC;
156	break;	153	break;
157	case I915_CACHE_LLC:	154	case I915_CACHE_LLC:
158	pte \|= GEN6_PTE_CACHE_LLC;	155	pte \|= GEN6_PTE_CACHE_LLC;
159	break;	156	break;
160	case I915_CACHE_NONE:	157	case I915_CACHE_NONE:
161	pte \|= GEN6_PTE_UNCACHED;	158	pte \|= GEN6_PTE_UNCACHED;
162	break;	159	break;
163	default:	160	default:
164	WARN_ON(1);	161	WARN_ON(1);
165	}	162	}
166		163
167	return pte;	164	return pte;
168	}	165	}
169		166
170	static gen6_gtt_pte_t byt_pte_encode(dma_addr_t addr,	167	static gen6_gtt_pte_t byt_pte_encode(dma_addr_t addr,
171	enum i915_cache_level level,	168	enum i915_cache_level level,
172	bool valid, u32 flags)	169	bool valid, u32 flags)
173	{	170	{
174	gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;	171	gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
175	pte \|= GEN6_PTE_ADDR_ENCODE(addr);	172	pte \|= GEN6_PTE_ADDR_ENCODE(addr);
176		-
177	/* Mark the page as writeable. Other platforms don't have a	-
178	* setting for read-only/writable, so this matches that behavior.	-
179	*/	173
180	if (!(flags & PTE_READ_ONLY))	174	if (!(flags & PTE_READ_ONLY))
181	pte \|= BYT_PTE_WRITEABLE;	175	pte \|= BYT_PTE_WRITEABLE;
182		176
183	if (level != I915_CACHE_NONE)	177	if (level != I915_CACHE_NONE)
184	pte \|= BYT_PTE_SNOOPED_BY_CPU_CACHES;	178	pte \|= BYT_PTE_SNOOPED_BY_CPU_CACHES;
185		179
186	return pte;	180	return pte;
187	}	181	}
188		182
189	static gen6_gtt_pte_t hsw_pte_encode(dma_addr_t addr,	183	static gen6_gtt_pte_t hsw_pte_encode(dma_addr_t addr,
190	enum i915_cache_level level,	184	enum i915_cache_level level,
191	bool valid, u32 unused)	185	bool valid, u32 unused)
192	{	186	{
193	gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;	187	gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
194	pte \|= HSW_PTE_ADDR_ENCODE(addr);	188	pte \|= HSW_PTE_ADDR_ENCODE(addr);
195		189
196	if (level != I915_CACHE_NONE)	190	if (level != I915_CACHE_NONE)
197	pte \|= HSW_WB_LLC_AGE3;	191	pte \|= HSW_WB_LLC_AGE3;
198		192
199	return pte;	193	return pte;
200	}	194	}
201		195
202	static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,	196	static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,
203	enum i915_cache_level level,	197	enum i915_cache_level level,
204	bool valid, u32 unused)	198	bool valid, u32 unused)
205	{	199	{
206	gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;	200	gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
207	pte \|= HSW_PTE_ADDR_ENCODE(addr);	201	pte \|= HSW_PTE_ADDR_ENCODE(addr);
208		202
209	switch (level) {	203	switch (level) {
210	case I915_CACHE_NONE:	204	case I915_CACHE_NONE:
211	break;	205	break;
212	case I915_CACHE_WT:	206	case I915_CACHE_WT:
213	pte \|= HSW_WT_ELLC_LLC_AGE3;	207	pte \|= HSW_WT_ELLC_LLC_AGE3;
214	break;	208	break;
215	default:	209	default:
216	pte \|= HSW_WB_ELLC_LLC_AGE3;	210	pte \|= HSW_WB_ELLC_LLC_AGE3;
217	break;	211	break;
218	}	212	}
219		213
220	return pte;	214	return pte;
221	}	215	}
222		216
223	/* Broadwell Page Directory Pointer Descriptors */	217	/* Broadwell Page Directory Pointer Descriptors */
224	static int gen8_write_pdp(struct intel_engine_cs *ring, unsigned entry,	218	static int gen8_write_pdp(struct intel_engine_cs *ring, unsigned entry,
225	uint64_t val, bool synchronous)	219	uint64_t val)
226	{	220	{
227	struct drm_i915_private *dev_priv = ring->dev->dev_private;	-
228	int ret;	221	int ret;
229		222
230	BUG_ON(entry >= 4);	223	BUG_ON(entry >= 4);
231		-
232	if (synchronous) {	-
233	I915_WRITE(GEN8_RING_PDP_UDW(ring, entry), val >> 32);	-
234	I915_WRITE(GEN8_RING_PDP_LDW(ring, entry), (u32)val);	-
235	return 0;	-
236	}	-
237		224
238	ret = intel_ring_begin(ring, 6);	225	ret = intel_ring_begin(ring, 6);
239	if (ret)	226	if (ret)
240	return ret;	227	return ret;
241		228
242	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));	229	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
243	intel_ring_emit(ring, GEN8_RING_PDP_UDW(ring, entry));	230	intel_ring_emit(ring, GEN8_RING_PDP_UDW(ring, entry));
244	intel_ring_emit(ring, (u32)(val >> 32));	231	intel_ring_emit(ring, (u32)(val >> 32));
245	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));	232	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
246	intel_ring_emit(ring, GEN8_RING_PDP_LDW(ring, entry));	233	intel_ring_emit(ring, GEN8_RING_PDP_LDW(ring, entry));
247	intel_ring_emit(ring, (u32)(val));	234	intel_ring_emit(ring, (u32)(val));
248	intel_ring_advance(ring);	235	intel_ring_advance(ring);
249		236
250	return 0;	237	return 0;
251	}	238	}
252		239
253	static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,	240	static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,
254	struct intel_engine_cs *ring,	241	struct intel_engine_cs *ring)
255	bool synchronous)	-
256	{	242	{
257	int i, ret;	243	int i, ret;
258		244
259	/* bit of a hack to find the actual last used pd */	245	/* bit of a hack to find the actual last used pd */
260	int used_pd = ppgtt->num_pd_entries / GEN8_PDES_PER_PAGE;	246	int used_pd = ppgtt->num_pd_entries / GEN8_PDES_PER_PAGE;
261		247
262	for (i = used_pd - 1; i >= 0; i--) {	248	for (i = used_pd - 1; i >= 0; i--) {
263	dma_addr_t addr = ppgtt->pd_dma_addr[i];	249	dma_addr_t addr = ppgtt->pd_dma_addr[i];
264	ret = gen8_write_pdp(ring, i, addr, synchronous);	250	ret = gen8_write_pdp(ring, i, addr);
265	if (ret)	251	if (ret)
266	return ret;	252	return ret;
267	}	253	}
268		254
269	return 0;	255	return 0;
270	}	256	}
271		257
272	static void gen8_ppgtt_clear_range(struct i915_address_space *vm,	258	static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
273	uint64_t start,	259	uint64_t start,
274	uint64_t length,	260	uint64_t length,
275	bool use_scratch)	261	bool use_scratch)
276	{	262	{
277	struct i915_hw_ppgtt *ppgtt =	263	struct i915_hw_ppgtt *ppgtt =
278	container_of(vm, struct i915_hw_ppgtt, base);	264	container_of(vm, struct i915_hw_ppgtt, base);
279	gen8_gtt_pte_t *pt_vaddr, scratch_pte;	265	gen8_gtt_pte_t *pt_vaddr, scratch_pte;
280	unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;	266	unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;
281	unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;	267	unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
282	unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;	268	unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
283	unsigned num_entries = length >> PAGE_SHIFT;	269	unsigned num_entries = length >> PAGE_SHIFT;
284	unsigned last_pte, i;	270	unsigned last_pte, i;
285		-
286	pt_vaddr = (gen8_gtt_pte_t*)AllocKernelSpace(4096);	-
287	if(pt_vaddr == NULL)	-
288	return;	-
289		271
290	scratch_pte = gen8_pte_encode(ppgtt->base.scratch.addr,	272	scratch_pte = gen8_pte_encode(ppgtt->base.scratch.addr,
291	I915_CACHE_LLC, use_scratch);	273	I915_CACHE_LLC, use_scratch);
292		274
293	while (num_entries) {	275	while (num_entries) {
294	struct page *page_table = ppgtt->gen8_pt_pages[pdpe][pde];	276	struct page *page_table = ppgtt->gen8_pt_pages[pdpe][pde];
295		277
296	last_pte = pte + num_entries;	278	last_pte = pte + num_entries;
297	if (last_pte > GEN8_PTES_PER_PAGE)	279	if (last_pte > GEN8_PTES_PER_PAGE)
298	last_pte = GEN8_PTES_PER_PAGE;	280	last_pte = GEN8_PTES_PER_PAGE;
299		281
300	MapPage(pt_vaddr,(addr_t)page_table, PG_SW);	282	pt_vaddr = kmap_atomic(page_table);
301		283
302	for (i = pte; i < last_pte; i++) {	284	for (i = pte; i < last_pte; i++) {
303	pt_vaddr[i] = scratch_pte;	285	pt_vaddr[i] = scratch_pte;
304	num_entries--;	286	num_entries--;
305	}	287	}
306		288
307	if (!HAS_LLC(ppgtt->base.dev))	289	if (!HAS_LLC(ppgtt->base.dev))
308	drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);	290	drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
-		291	kunmap_atomic(pt_vaddr);
309		292
310	pte = 0;	293	pte = 0;
311	if (++pde == GEN8_PDES_PER_PAGE) {	294	if (++pde == GEN8_PDES_PER_PAGE) {
312	pdpe++;	295	pdpe++;
313	pde = 0;	296	pde = 0;
314	}	297	}
315	}	298	}
316	FreeKernelSpace(pt_vaddr);	-
317	}	299	}
318		300
319	static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,	301	static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
320	struct sg_table *pages,	302	struct sg_table *pages,
321	uint64_t start,	303	uint64_t start,
322	enum i915_cache_level cache_level, u32 unused)	304	enum i915_cache_level cache_level, u32 unused)
323	{	305	{
324	struct i915_hw_ppgtt *ppgtt =	306	struct i915_hw_ppgtt *ppgtt =
325	container_of(vm, struct i915_hw_ppgtt, base);	307	container_of(vm, struct i915_hw_ppgtt, base);
326	gen8_gtt_pte_t *pt_vaddr;	308	gen8_gtt_pte_t *pt_vaddr;
327	unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;	309	unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;
328	unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;	310	unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
329	unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;	311	unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
330	struct sg_page_iter sg_iter;	312	struct sg_page_iter sg_iter;
331		-
332	pt_vaddr = AllocKernelSpace(4096);	313
333	if(pt_vaddr == NULL)	-
334	return;	-
335		-
336	MapPage(pt_vaddr,(addr_t)(ppgtt->gen8_pt_pages[pdpe][pde]), 3);	314	pt_vaddr = NULL;
337		315
338	for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {	316	for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
339	if (WARN_ON(pdpe >= GEN8_LEGACY_PDPS))	317	if (WARN_ON(pdpe >= GEN8_LEGACY_PDPS))
340	break;	318	break;
-		319
-		320	if (pt_vaddr == NULL)
-		321	pt_vaddr = kmap_atomic(ppgtt->gen8_pt_pages[pdpe][pde]);
341		322
342	pt_vaddr[pte] =	323	pt_vaddr[pte] =
343	gen8_pte_encode(sg_page_iter_dma_address(&sg_iter),	324	gen8_pte_encode(sg_page_iter_dma_address(&sg_iter),
344	cache_level, true);	325	cache_level, true);
345	if (++pte == GEN8_PTES_PER_PAGE) {	326	if (++pte == GEN8_PTES_PER_PAGE) {
346	if (!HAS_LLC(ppgtt->base.dev))	327	if (!HAS_LLC(ppgtt->base.dev))
347	drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);	328	drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
-		329	kunmap_atomic(pt_vaddr);
-		330	pt_vaddr = NULL;
348	if (++pde == GEN8_PDES_PER_PAGE) {	331	if (++pde == GEN8_PDES_PER_PAGE) {
349	pdpe++;	332	pdpe++;
350	pde = 0;	333	pde = 0;
351	}	334	}
352	pte = 0;	335	pte = 0;
353	MapPage(pt_vaddr,(addr_t)(ppgtt->gen8_pt_pages[pdpe][pde]), 3);	-
354	}	336	}
355	}	337	}
-		338	if (pt_vaddr) {
-		339	if (!HAS_LLC(ppgtt->base.dev))
-		340	drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
356	FreeKernelSpace(pt_vaddr);	341	kunmap_atomic(pt_vaddr);
-		342	}
357	}	343	}
358		344
359	static void gen8_free_page_tables(struct page **pt_pages)	345	static void gen8_free_page_tables(struct page **pt_pages)
360	{	346	{
361	int i;	347	int i;
362		348
363	if (pt_pages == NULL)	349	if (pt_pages == NULL)
364	return;	350	return;
365		351
366	// for (i = 0; i < GEN8_PDES_PER_PAGE; i++)	352	// for (i = 0; i < GEN8_PDES_PER_PAGE; i++)
367	// if (pt_pages[i])	353	// if (pt_pages[i])
368	// __free_pages(pt_pages[i], 0);	354	// __free_pages(pt_pages[i], 0);
369	}	355	}
370		356
371	static void gen8_ppgtt_free(const struct i915_hw_ppgtt *ppgtt)	357	static void gen8_ppgtt_free(const struct i915_hw_ppgtt *ppgtt)
372	{	358	{
373	int i;	359	int i;
374		360
375	for (i = 0; i < ppgtt->num_pd_pages; i++) {	361	for (i = 0; i < ppgtt->num_pd_pages; i++) {
376	gen8_free_page_tables(ppgtt->gen8_pt_pages[i]);	362	gen8_free_page_tables(ppgtt->gen8_pt_pages[i]);
377	kfree(ppgtt->gen8_pt_pages[i]);	363	kfree(ppgtt->gen8_pt_pages[i]);
378	kfree(ppgtt->gen8_pt_dma_addr[i]);	364	kfree(ppgtt->gen8_pt_dma_addr[i]);
379	}	365	}
380		366
381	// __free_pages(ppgtt->pd_pages, get_order(ppgtt->num_pd_pages << PAGE_SHIFT));	367	// __free_pages(ppgtt->pd_pages, get_order(ppgtt->num_pd_pages << PAGE_SHIFT));
382	}	368	}
383		369
384	static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)	370	static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)
385	{	371	{
386	struct pci_dev *hwdev = ppgtt->base.dev->pdev;	372	struct pci_dev *hwdev = ppgtt->base.dev->pdev;
387	int i, j;	373	int i, j;
388		374
389	for (i = 0; i < ppgtt->num_pd_pages; i++) {	375	for (i = 0; i < ppgtt->num_pd_pages; i++) {
390	/* TODO: In the future we'll support sparse mappings, so this	376	/* TODO: In the future we'll support sparse mappings, so this
391	* will have to change. */	377	* will have to change. */
392	if (!ppgtt->pd_dma_addr[i])	378	if (!ppgtt->pd_dma_addr[i])
393	continue;	379	continue;
394		380
395	pci_unmap_page(hwdev, ppgtt->pd_dma_addr[i], PAGE_SIZE,	381	pci_unmap_page(hwdev, ppgtt->pd_dma_addr[i], PAGE_SIZE,
396	PCI_DMA_BIDIRECTIONAL);	382	PCI_DMA_BIDIRECTIONAL);
397		383
398	for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {	384	for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
399	dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j];	385	dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j];
400	if (addr)	386	if (addr)
401	pci_unmap_page(hwdev, addr, PAGE_SIZE,	387	pci_unmap_page(hwdev, addr, PAGE_SIZE,
402	PCI_DMA_BIDIRECTIONAL);	388	PCI_DMA_BIDIRECTIONAL);
403	}	389	}
404	}	390	}
405	}	391	}
406		392
407	static void gen8_ppgtt_cleanup(struct i915_address_space *vm)	393	static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
408	{	394	{
409	struct i915_hw_ppgtt *ppgtt =	395	struct i915_hw_ppgtt *ppgtt =
410	container_of(vm, struct i915_hw_ppgtt, base);	396	container_of(vm, struct i915_hw_ppgtt, base);
411		-
412	list_del(&vm->global_link);	-
413	drm_mm_takedown(&vm->mm);	-
414		397
415	gen8_ppgtt_unmap_pages(ppgtt);	398	gen8_ppgtt_unmap_pages(ppgtt);
416	gen8_ppgtt_free(ppgtt);	399	gen8_ppgtt_free(ppgtt);
417	}	400	}
418		401
419	static struct page **__gen8_alloc_page_tables(void)	402	static struct page **__gen8_alloc_page_tables(void)
420	{	403	{
421	struct page **pt_pages;	404	struct page **pt_pages;
422	int i;	405	int i;
423		406
424	pt_pages = kcalloc(GEN8_PDES_PER_PAGE, sizeof(struct page *), GFP_KERNEL);	407	pt_pages = kcalloc(GEN8_PDES_PER_PAGE, sizeof(struct page *), GFP_KERNEL);
425	if (!pt_pages)	408	if (!pt_pages)
426	return ERR_PTR(-ENOMEM);	409	return ERR_PTR(-ENOMEM);
427		410
428	for (i = 0; i < GEN8_PDES_PER_PAGE; i++) {	411	for (i = 0; i < GEN8_PDES_PER_PAGE; i++) {
429	pt_pages[i] = alloc_page(GFP_KERNEL);	412	pt_pages[i] = alloc_page(GFP_KERNEL);
430	if (!pt_pages[i])	413	if (!pt_pages[i])
431	goto bail;	414	goto bail;
432	}	415	}
433		416
434	return pt_pages;	417	return pt_pages;
435		418
436	bail:	419	bail:
437	gen8_free_page_tables(pt_pages);	420	gen8_free_page_tables(pt_pages);
438	kfree(pt_pages);	421	kfree(pt_pages);
439	return ERR_PTR(-ENOMEM);	422	return ERR_PTR(-ENOMEM);
440	}	423	}
441		424
442	static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt,	425	static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt,
443	const int max_pdp)	426	const int max_pdp)
444	{	427	{
445	struct page **pt_pages[GEN8_LEGACY_PDPS];	428	struct page **pt_pages[GEN8_LEGACY_PDPS];
446	int i, ret;	429	int i, ret;
447		430
448	for (i = 0; i < max_pdp; i++) {	431	for (i = 0; i < max_pdp; i++) {
449	pt_pages[i] = __gen8_alloc_page_tables();	432	pt_pages[i] = __gen8_alloc_page_tables();
450	if (IS_ERR(pt_pages[i])) {	433	if (IS_ERR(pt_pages[i])) {
451	ret = PTR_ERR(pt_pages[i]);	434	ret = PTR_ERR(pt_pages[i]);
452	goto unwind_out;	435	goto unwind_out;
453	}	436	}
454	}	437	}
455		438
456	/* NB: Avoid touching gen8_pt_pages until last to keep the allocation,	439	/* NB: Avoid touching gen8_pt_pages until last to keep the allocation,
457	* "atomic" - for cleanup purposes.	440	* "atomic" - for cleanup purposes.
458	*/	441	*/
459	for (i = 0; i < max_pdp; i++)	442	for (i = 0; i < max_pdp; i++)
460	ppgtt->gen8_pt_pages[i] = pt_pages[i];	443	ppgtt->gen8_pt_pages[i] = pt_pages[i];
461		444
462	return 0;	445	return 0;
463		446
464	unwind_out:	447	unwind_out:
465	while (i--) {	448	while (i--) {
466	gen8_free_page_tables(pt_pages[i]);	449	gen8_free_page_tables(pt_pages[i]);
467	kfree(pt_pages[i]);	450	kfree(pt_pages[i]);
468	}	451	}
469		452
470	return ret;	453	return ret;
471	}	454	}
472		455
473	static int gen8_ppgtt_allocate_dma(struct i915_hw_ppgtt *ppgtt)	456	static int gen8_ppgtt_allocate_dma(struct i915_hw_ppgtt *ppgtt)
474	{	457	{
475	int i;	458	int i;
476		459
477	for (i = 0; i < ppgtt->num_pd_pages; i++) {	460	for (i = 0; i < ppgtt->num_pd_pages; i++) {
478	ppgtt->gen8_pt_dma_addr[i] = kcalloc(GEN8_PDES_PER_PAGE,	461	ppgtt->gen8_pt_dma_addr[i] = kcalloc(GEN8_PDES_PER_PAGE,
479	sizeof(dma_addr_t),	462	sizeof(dma_addr_t),
480	GFP_KERNEL);	463	GFP_KERNEL);
481	if (!ppgtt->gen8_pt_dma_addr[i])	464	if (!ppgtt->gen8_pt_dma_addr[i])
482	return -ENOMEM;	465	return -ENOMEM;
483	}	466	}
484		467

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem_gtt.c – Rev 5060 → 5354