22,7 → 22,6 |
* |
*/ |
|
#define iowrite32(v, addr) writel((v), (addr)) |
|
#define AGP_NORMAL_MEMORY 0 |
|
36,7 → 35,7 |
#include "i915_trace.h" |
#include "intel_drv.h" |
|
typedef uint32_t gtt_pte_t; |
typedef uint32_t gen6_gtt_pte_t; |
|
/* PPGTT stuff */ |
#define GEN6_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0xff0)) |
52,11 → 51,11 |
#define GEN6_PTE_CACHE_LLC_MLC (3 << 1) |
#define GEN6_PTE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) |
|
static inline gtt_pte_t gen6_pte_encode(struct drm_device *dev, |
static inline gen6_gtt_pte_t gen6_pte_encode(struct drm_device *dev, |
dma_addr_t addr, |
enum i915_cache_level level) |
{ |
gtt_pte_t pte = GEN6_PTE_VALID; |
gen6_gtt_pte_t pte = GEN6_PTE_VALID; |
pte |= GEN6_PTE_ADDR_ENCODE(addr); |
|
switch (level) { |
80,18 → 79,85 |
BUG(); |
} |
|
|
return pte; |
} |
|
static int gen6_ppgtt_enable(struct drm_device *dev) |
{ |
drm_i915_private_t *dev_priv = dev->dev_private; |
uint32_t pd_offset; |
struct intel_ring_buffer *ring; |
struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; |
gen6_gtt_pte_t __iomem *pd_addr; |
uint32_t pd_entry; |
int i; |
|
pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm + |
ppgtt->pd_offset / sizeof(gen6_gtt_pte_t); |
for (i = 0; i < ppgtt->num_pd_entries; i++) { |
dma_addr_t pt_addr; |
|
pt_addr = ppgtt->pt_dma_addr[i]; |
pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr); |
pd_entry |= GEN6_PDE_VALID; |
|
writel(pd_entry, pd_addr + i); |
} |
readl(pd_addr); |
|
pd_offset = ppgtt->pd_offset; |
pd_offset /= 64; /* in cachelines, */ |
pd_offset <<= 16; |
|
if (INTEL_INFO(dev)->gen == 6) { |
uint32_t ecochk, gab_ctl, ecobits; |
|
ecobits = I915_READ(GAC_ECO_BITS); |
I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT | |
ECOBITS_PPGTT_CACHE64B); |
|
gab_ctl = I915_READ(GAB_CTL); |
I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT); |
|
ecochk = I915_READ(GAM_ECOCHK); |
I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | |
ECOCHK_PPGTT_CACHE64B); |
I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); |
} else if (INTEL_INFO(dev)->gen >= 7) { |
uint32_t ecochk, ecobits; |
|
ecobits = I915_READ(GAC_ECO_BITS); |
I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B); |
|
ecochk = I915_READ(GAM_ECOCHK); |
if (IS_HASWELL(dev)) { |
ecochk |= ECOCHK_PPGTT_WB_HSW; |
} else { |
ecochk |= ECOCHK_PPGTT_LLC_IVB; |
ecochk &= ~ECOCHK_PPGTT_GFDT_IVB; |
} |
I915_WRITE(GAM_ECOCHK, ecochk); |
/* GFX_MODE is per-ring on gen7+ */ |
} |
|
for_each_ring(ring, dev_priv, i) { |
if (INTEL_INFO(dev)->gen >= 7) |
I915_WRITE(RING_MODE_GEN7(ring), |
_MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); |
|
I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); |
I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset); |
} |
return 0; |
} |
|
/* PPGTT support for Sandybdrige/Gen6 and later */ |
static void gen6_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt, |
unsigned first_entry, |
unsigned num_entries) |
{ |
gtt_pte_t *pt_vaddr; |
gtt_pte_t scratch_pte; |
unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES; |
gen6_gtt_pte_t *pt_vaddr, scratch_pte; |
unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES; |
unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES; |
unsigned last_pte, i; |
|
109,7 → 175,7 |
if (last_pte > I915_PPGTT_PT_ENTRIES) |
last_pte = I915_PPGTT_PT_ENTRIES; |
|
MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pd]), 3); |
MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3); |
|
for (i = first_pte; i < last_pte; i++) |
pt_vaddr[i] = scratch_pte; |
116,7 → 182,7 |
|
num_entries -= last_pte - first_pte; |
first_pte = 0; |
act_pd++; |
act_pt++; |
}; |
|
FreeKernelSpace(pt_vaddr); |
127,18 → 193,12 |
unsigned first_entry, |
enum i915_cache_level cache_level) |
{ |
gtt_pte_t *pt_vaddr; |
unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES; |
unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES; |
unsigned i, j, m, segment_len; |
gen6_gtt_pte_t *pt_vaddr; |
unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES; |
unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES; |
struct sg_page_iter sg_iter; |
dma_addr_t page_addr; |
struct scatterlist *sg; |
|
/* init sg walking */ |
sg = pages->sgl; |
i = 0; |
segment_len = sg_dma_len(sg) >> PAGE_SHIFT; |
m = 0; |
|
pt_vaddr = AllocKernelSpace(4096); |
|
145,28 → 205,20 |
if(pt_vaddr == NULL) |
return; |
|
while (i < pages->nents) { |
MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pd]), 3); |
MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3); |
for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) { |
dma_addr_t page_addr; |
|
for (j = first_pte; j < I915_PPGTT_PT_ENTRIES; j++) { |
page_addr = sg_dma_address(sg) + (m << PAGE_SHIFT); |
pt_vaddr[j] = gen6_pte_encode(ppgtt->dev, page_addr, |
page_addr = sg_page_iter_dma_address(&sg_iter); |
pt_vaddr[act_pte] = gen6_pte_encode(ppgtt->dev, page_addr, |
cache_level); |
if (++act_pte == I915_PPGTT_PT_ENTRIES) { |
act_pt++; |
MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3); |
act_pte = 0; |
|
/* grab the next page */ |
if (++m == segment_len) { |
if (++i == pages->nents) |
break; |
|
sg = sg_next(sg); |
segment_len = sg_dma_len(sg) >> PAGE_SHIFT; |
m = 0; |
} |
} |
|
first_pte = 0; |
act_pd++; |
} |
FreeKernelSpace(pt_vaddr); |
} |
|
199,10 → 251,10 |
/* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024 |
* entries. For aliasing ppgtt support we just steal them at the end for |
* now. */ |
first_pd_entry_in_global_pt = |
gtt_total_entries(dev_priv->gtt) - I915_PPGTT_PD_ENTRIES; |
first_pd_entry_in_global_pt = gtt_total_entries(dev_priv->gtt); |
|
ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES; |
ppgtt->enable = gen6_ppgtt_enable; |
ppgtt->clear_range = gen6_ppgtt_clear_range; |
ppgtt->insert_entries = gen6_ppgtt_insert_entries; |
ppgtt->cleanup = gen6_ppgtt_cleanup; |
231,12 → 283,10 |
ppgtt->pt_dma_addr[i] = pt_addr; |
} |
|
ppgtt->scratch_page_dma_addr = dev_priv->gtt.scratch_page_dma; |
|
ppgtt->clear_range(ppgtt, 0, |
ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES); |
|
ppgtt->pd_offset = (first_pd_entry_in_global_pt)*sizeof(gtt_pte_t); |
ppgtt->pd_offset = first_pd_entry_in_global_pt * sizeof(gen6_gtt_pte_t); |
|
return 0; |
|
268,8 → 318,13 |
return -ENOMEM; |
|
ppgtt->dev = dev; |
ppgtt->scratch_page_dma_addr = dev_priv->gtt.scratch_page_dma; |
|
if (INTEL_INFO(dev)->gen < 8) |
ret = gen6_ppgtt_init(ppgtt); |
else |
BUG(); |
|
if (ret) |
kfree(ppgtt); |
else |
287,6 → 342,7 |
return; |
|
ppgtt->cleanup(ppgtt); |
dev_priv->mm.aliasing_ppgtt = NULL; |
} |
|
void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt, |
306,64 → 362,6 |
obj->base.size >> PAGE_SHIFT); |
} |
|
void i915_gem_init_ppgtt(struct drm_device *dev) |
{ |
drm_i915_private_t *dev_priv = dev->dev_private; |
uint32_t pd_offset; |
struct intel_ring_buffer *ring; |
struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; |
gtt_pte_t __iomem *pd_addr; |
uint32_t pd_entry; |
int i; |
|
if (!dev_priv->mm.aliasing_ppgtt) |
return; |
|
|
pd_addr = (gtt_pte_t __iomem*)dev_priv->gtt.gsm + ppgtt->pd_offset/sizeof(gtt_pte_t); |
for (i = 0; i < ppgtt->num_pd_entries; i++) { |
dma_addr_t pt_addr; |
|
pt_addr = ppgtt->pt_dma_addr[i]; |
pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr); |
pd_entry |= GEN6_PDE_VALID; |
|
writel(pd_entry, pd_addr + i); |
} |
readl(pd_addr); |
|
pd_offset = ppgtt->pd_offset; |
pd_offset /= 64; /* in cachelines, */ |
pd_offset <<= 16; |
|
if (INTEL_INFO(dev)->gen == 6) { |
uint32_t ecochk, gab_ctl, ecobits; |
|
ecobits = I915_READ(GAC_ECO_BITS); |
I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B); |
|
gab_ctl = I915_READ(GAB_CTL); |
I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT); |
|
ecochk = I915_READ(GAM_ECOCHK); |
I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | |
ECOCHK_PPGTT_CACHE64B); |
I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); |
} else if (INTEL_INFO(dev)->gen >= 7) { |
I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B); |
/* GFX_MODE is per-ring on gen7+ */ |
} |
|
for_each_ring(ring, dev_priv, i) { |
if (INTEL_INFO(dev)->gen >= 7) |
I915_WRITE(RING_MODE_GEN7(ring), |
_MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); |
|
I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); |
I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset); |
} |
} |
|
extern int intel_iommu_gfx_mapped; |
/* Certain Gen5 chipsets require require idling the GPU before |
* unmapping anything from the GTT when VT-d is enabled. |
444,22 → 442,17 |
enum i915_cache_level level) |
{ |
struct drm_i915_private *dev_priv = dev->dev_private; |
struct scatterlist *sg = st->sgl; |
gtt_pte_t __iomem *gtt_entries = |
(gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry; |
int unused, i = 0; |
unsigned int len, m = 0; |
gen6_gtt_pte_t __iomem *gtt_entries = |
(gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry; |
int i = 0; |
struct sg_page_iter sg_iter; |
dma_addr_t addr; |
|
for_each_sg(st->sgl, sg, st->nents, unused) { |
len = sg_dma_len(sg) >> PAGE_SHIFT; |
for (m = 0; m < len; m++) { |
addr = sg_dma_address(sg) + (m << PAGE_SHIFT); |
iowrite32(gen6_pte_encode(dev, addr, level), |
>t_entries[i]); |
for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) { |
addr = sg_page_iter_dma_address(&sg_iter); |
iowrite32(gen6_pte_encode(dev, addr, level), >t_entries[i]); |
i++; |
} |
} |
|
/* XXX: This serves as a posting read to make sure that the PTE has |
* actually been updated. There is some concern that even though |
484,14 → 477,15 |
unsigned int num_entries) |
{ |
struct drm_i915_private *dev_priv = dev->dev_private; |
gtt_pte_t scratch_pte; |
gtt_pte_t __iomem *gtt_base = (gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry; |
gen6_gtt_pte_t scratch_pte, __iomem *gtt_base = |
(gen6_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry; |
const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry; |
int i; |
|
if (WARN(num_entries > max_entries, |
"First entry = %d; Num entries = %d (max=%d)\n", |
first_entry, num_entries, max_entries)) |
// if (WARN(num_entries > max_entries, |
// "First entry = %d; Num entries = %d (max=%d)\n", |
// first_entry, num_entries, max_entries)) |
if (num_entries > max_entries) |
num_entries = max_entries; |
|
scratch_pte = gen6_pte_encode(dev, dev_priv->gtt.scratch_page_dma, |
657,13 → 651,18 |
gtt_size = dev_priv->gtt.total; |
mappable_size = dev_priv->gtt.mappable_end; |
|
#if 0 |
if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) { |
int ret; |
|
if (INTEL_INFO(dev)->gen <= 7) { |
/* PPGTT pdes are stolen from global gtt ptes, so shrink the |
* aperture accordingly when using aliasing ppgtt. */ |
gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE; |
gtt_size -= LFB_SIZE; |
} |
|
// gtt_size -= LFB_SIZE; |
|
i915_gem_setup_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size); |
|
ret = i915_gem_init_aliasing_ppgtt(dev); |
674,6 → 673,8 |
drm_mm_takedown(&dev_priv->mm.gtt_space); |
gtt_size += I915_PPGTT_PD_ENTRIES*PAGE_SIZE; |
} |
#endif |
|
i915_gem_setup_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size); |
} |
|
727,15 → 728,6 |
return snb_gmch_ctl << 25; /* 32 MB units */ |
} |
|
static inline size_t gen7_get_stolen_size(u16 snb_gmch_ctl) |
{ |
static const int stolen_decoder[] = { |
0, 0, 0, 0, 0, 32, 48, 64, 128, 256, 96, 160, 224, 352}; |
snb_gmch_ctl >>= IVB_GMCH_GMS_SHIFT; |
snb_gmch_ctl &= IVB_GMCH_GMS_MASK; |
return stolen_decoder[snb_gmch_ctl] << 20; |
} |
|
static int gen6_gmch_probe(struct drm_device *dev, |
size_t *gtt_total, |
size_t *stolen, |
765,15 → 757,13 |
pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); |
gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl); |
|
if (IS_GEN7(dev)) |
*stolen = gen7_get_stolen_size(snb_gmch_ctl); |
else |
*stolen = gen6_get_stolen_size(snb_gmch_ctl); |
*gtt_total = (gtt_size / sizeof(gen6_gtt_pte_t)) << PAGE_SHIFT; |
|
*gtt_total = (gtt_size / sizeof(gtt_pte_t)) << PAGE_SHIFT; |
/* For Modern GENs the PTEs and register space are split in the BAR */ |
gtt_bus_addr = pci_resource_start(dev->pdev, 0) + |
(pci_resource_len(dev->pdev, 0) / 2); |
|
/* For GEN6+ the PTEs for the ggtt live at 2MB + BAR0 */ |
gtt_bus_addr = pci_resource_start(dev->pdev, 0) + (2<<20); |
dev_priv->gtt.gsm = ioremap_wc(gtt_bus_addr, gtt_size); |
if (!dev_priv->gtt.gsm) { |
DRM_ERROR("Failed to map the gtt page table\n"); |
830,7 → 820,6 |
{ |
struct drm_i915_private *dev_priv = dev->dev_private; |
struct i915_gtt *gtt = &dev_priv->gtt; |
unsigned long gtt_size; |
int ret; |
|
if (INTEL_INFO(dev)->gen <= 5) { |
848,8 → 837,6 |
if (ret) |
return ret; |
|
gtt_size = (dev_priv->gtt.total >> PAGE_SHIFT) * sizeof(gtt_pte_t); |
|
/* GMADR is the PCI mmio aperture into the global GTT. */ |
DRM_INFO("Memory usable by graphics device = %zdM\n", |
dev_priv->gtt.total >> 20); |
997,3 → 984,40 |
sg_mark_end(&sgl[nents - 1]); |
} |
|
|
void __sg_page_iter_start(struct sg_page_iter *piter, |
struct scatterlist *sglist, unsigned int nents, |
unsigned long pgoffset) |
{ |
piter->__pg_advance = 0; |
piter->__nents = nents; |
|
piter->sg = sglist; |
piter->sg_pgoffset = pgoffset; |
} |
|
static int sg_page_count(struct scatterlist *sg) |
{ |
return PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT; |
} |
|
bool __sg_page_iter_next(struct sg_page_iter *piter) |
{ |
if (!piter->__nents || !piter->sg) |
return false; |
|
piter->sg_pgoffset += piter->__pg_advance; |
piter->__pg_advance = 1; |
|
while (piter->sg_pgoffset >= sg_page_count(piter->sg)) { |
piter->sg_pgoffset -= sg_page_count(piter->sg); |
piter->sg = sg_next(piter->sg); |
if (!--piter->__nents || !piter->sg) |
return false; |
} |
|
return true; |
} |
EXPORT_SYMBOL(__sg_page_iter_next); |
|
|