33,6 → 33,7 |
#include <linux/shmem_fs.h> |
#include <linux/slab.h> |
//#include <linux/swap.h> |
#include <linux/scatterlist.h> |
#include <linux/pci.h> |
|
extern int x86_clflush_size; |
447,10 → 448,9 |
int obj_do_bit17_swizzling, page_do_bit17_swizzling; |
int prefaulted = 0; |
int needs_clflush = 0; |
struct scatterlist *sg; |
int i; |
struct sg_page_iter sg_iter; |
|
user_data = (char __user *) (uintptr_t) args->data_ptr; |
user_data = to_user_ptr(args->data_ptr); |
remain = args->size; |
|
obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); |
477,12 → 477,10 |
|
offset = args->offset; |
|
for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) { |
struct page *page; |
for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, |
offset >> PAGE_SHIFT) { |
struct page *page = sg_page_iter_page(&sg_iter); |
|
if (i < offset >> PAGE_SHIFT) |
continue; |
|
if (remain <= 0) |
break; |
|
496,7 → 494,6 |
if ((shmem_page_offset + page_length) > PAGE_SIZE) |
page_length = PAGE_SIZE - shmem_page_offset; |
|
page = sg_page(sg); |
page_do_bit17_swizzling = obj_do_bit17_swizzling && |
(page_to_phys(page) & (1 << 17)) != 0; |
|
558,7 → 555,7 |
return 0; |
|
if (!access_ok(VERIFY_WRITE, |
(char __user *)(uintptr_t)args->data_ptr, |
to_user_ptr(args->data_ptr), |
args->size)) |
return -EFAULT; |
|
777,10 → 774,9 |
int hit_slowpath = 0; |
int needs_clflush_after = 0; |
int needs_clflush_before = 0; |
int i; |
struct scatterlist *sg; |
struct sg_page_iter sg_iter; |
|
user_data = (char __user *) (uintptr_t) args->data_ptr; |
user_data = to_user_ptr(args->data_ptr); |
remain = args->size; |
|
obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); |
813,13 → 809,11 |
offset = args->offset; |
obj->dirty = 1; |
|
for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) { |
struct page *page; |
for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, |
offset >> PAGE_SHIFT) { |
struct page *page = sg_page_iter_page(&sg_iter); |
int partial_cacheline_write; |
|
if (i < offset >> PAGE_SHIFT) |
continue; |
|
if (remain <= 0) |
break; |
|
841,7 → 835,6 |
((shmem_page_offset | page_length) |
& (x86_clflush_size - 1)); |
|
page = sg_page(sg); |
page_do_bit17_swizzling = obj_do_bit17_swizzling && |
(page_to_phys(page) & (1 << 17)) != 0; |
|
1102,8 → 1095,6 |
case -ERESTARTSYS: /* Signal */ |
return (int)end; |
case 0: /* Timeout */ |
if (timeout) |
set_normalized_timespec(timeout, 0, 0); |
return -ETIME; |
default: /* Completed */ |
WARN_ON(end < 0); /* We're not aware of other errors */ |
1590,9 → 1581,8 |
static void |
i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) |
{ |
int page_count = obj->base.size / PAGE_SIZE; |
struct scatterlist *sg; |
int ret, i; |
struct sg_page_iter sg_iter; |
int ret; |
|
BUG_ON(obj->madv == __I915_MADV_PURGED); |
|
1609,8 → 1599,8 |
if (obj->madv == I915_MADV_DONTNEED) |
obj->dirty = 0; |
|
for_each_sg(obj->pages->sgl, sg, page_count, i) { |
struct page *page = sg_page(sg); |
for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) { |
struct page *page = sg_page_iter_page(&sg_iter); |
|
page_cache_release(page); |
} |
1661,10 → 1651,11 |
{ |
struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
int page_count, i; |
struct address_space *mapping; |
struct sg_table *st; |
struct scatterlist *sg; |
struct sg_page_iter sg_iter; |
struct page *page; |
unsigned long last_pfn = 0; /* suppress gcc warning */ |
gfp_t gfp; |
|
/* Assert that the object is not currently in any GPU domain. As it |
1682,6 → 1673,7 |
if (sg_alloc_table(st, page_count, GFP_KERNEL)) { |
sg_free_table(st); |
kfree(st); |
FAIL(); |
return -ENOMEM; |
} |
|
1690,7 → 1682,9 |
* |
* Fail silently without starting the shrinker |
*/ |
for_each_sg(st->sgl, sg, page_count, i) { |
sg = st->sgl; |
st->nents = 0; |
for (i = 0; i < page_count; i++) { |
page = shmem_read_mapping_page_gfp(obj->base.filp, i, gfp); |
if (IS_ERR(page)) { |
dbgprintf("%s invalid page %p\n", __FUNCTION__, page); |
1697,20 → 1691,30 |
goto err_pages; |
|
} |
|
if (!i || page_to_pfn(page) != last_pfn + 1) { |
if (i) |
sg = sg_next(sg); |
st->nents++; |
sg_set_page(sg, page, PAGE_SIZE, 0); |
} else { |
sg->length += PAGE_SIZE; |
} |
last_pfn = page_to_pfn(page); |
} |
|
sg_mark_end(sg); |
obj->pages = st; |
|
// DRM_DEBUG_KMS("%s alloc %d pages\n", __FUNCTION__, page_count); |
|
return 0; |
|
err_pages: |
for_each_sg(st->sgl, sg, i, page_count) |
page_cache_release(sg_page(sg)); |
sg_mark_end(sg); |
for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) |
page_cache_release(sg_page_iter_page(&sg_iter)); |
sg_free_table(st); |
kfree(st); |
FAIL(); |
return PTR_ERR(page); |
} |
|
1997,7 → 2001,7 |
} |
} |
|
static void i915_gem_reset_fences(struct drm_device *dev) |
void i915_gem_restore_fences(struct drm_device *dev) |
{ |
struct drm_i915_private *dev_priv = dev->dev_private; |
int i; |
2004,18 → 2008,8 |
|
for (i = 0; i < dev_priv->num_fence_regs; i++) { |
struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; |
|
i915_gem_write_fence(dev, i, NULL); |
|
if (reg->obj) |
i915_gem_object_fence_lost(reg->obj); |
|
reg->pin_count = 0; |
reg->obj = NULL; |
INIT_LIST_HEAD(®->lru_list); |
i915_gem_write_fence(dev, i, reg->obj); |
} |
|
INIT_LIST_HEAD(&dev_priv->mm.fence_list); |
} |
|
void i915_gem_reset(struct drm_device *dev) |
2038,8 → 2032,7 |
obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; |
} |
|
/* The fence registers are invalidated so clear them out */ |
i915_gem_reset_fences(dev); |
i915_gem_restore_fences(dev); |
} |
|
/** |
2510,17 → 2503,36 |
return fence - dev_priv->fence_regs; |
} |
|
static void i915_gem_write_fence__ipi(void *data) |
{ |
asm volatile("wbinvd"); |
|
} |
|
static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, |
struct drm_i915_fence_reg *fence, |
bool enable) |
{ |
struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
int reg = fence_number(dev_priv, fence); |
struct drm_device *dev = obj->base.dev; |
struct drm_i915_private *dev_priv = dev->dev_private; |
int fence_reg = fence_number(dev_priv, fence); |
|
i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL); |
/* In order to fully serialize access to the fenced region and |
* the update to the fence register we need to take extreme |
* measures on SNB+. In theory, the write to the fence register |
* flushes all memory transactions before, and coupled with the |
* mb() placed around the register write we serialise all memory |
* operations with respect to the changes in the tiler. Yet, on |
* SNB+ we need to take a step further and emit an explicit wbinvd() |
* on each processor in order to manually flush all memory |
* transactions before updating the fence register. |
*/ |
if (HAS_LLC(obj->base.dev)) |
on_each_cpu(i915_gem_write_fence__ipi, NULL, 1); |
i915_gem_write_fence(dev, fence_reg, enable ? obj : NULL); |
|
if (enable) { |
obj->fence_reg = reg; |
obj->fence_reg = fence_reg; |
fence->obj = obj; |
list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list); |
} else { |
2549,6 → 2561,7 |
i915_gem_object_put_fence(struct drm_i915_gem_object *obj) |
{ |
struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
struct drm_i915_fence_reg *fence; |
int ret; |
|
ret = i915_gem_object_wait_fence(obj); |
2558,10 → 2571,10 |
if (obj->fence_reg == I915_FENCE_REG_NONE) |
return 0; |
|
i915_gem_object_update_fence(obj, |
&dev_priv->fence_regs[obj->fence_reg], |
false); |
fence = &dev_priv->fence_regs[obj->fence_reg]; |
|
i915_gem_object_fence_lost(obj); |
i915_gem_object_update_fence(obj, fence, false); |
|
return 0; |
} |
2774,6 → 2787,7 |
if (obj->base.size > |
(map_and_fenceable ? dev_priv->gtt.mappable_end : dev_priv->gtt.total)) { |
DRM_ERROR("Attempting to bind an object larger than the aperture\n"); |
FAIL(); |
return -E2BIG; |
} |
|
3633,12 → 3647,18 |
struct address_space *mapping; |
gfp_t mask; |
|
obj = i915_gem_object_alloc(dev); |
|
obj = kzalloc(sizeof(*obj), GFP_KERNEL); |
if (obj == NULL) |
{ |
FAIL(); |
return NULL; |
}; |
|
if (drm_gem_object_init(dev, &obj->base, size) != 0) { |
kfree(obj); |
FAIL(); |
return NULL; |
} |
|
3746,8 → 3766,6 |
if (!drm_core_check_feature(dev, DRIVER_MODESET)) |
i915_gem_evict_everything(dev); |
|
i915_gem_reset_fences(dev); |
|
/* Hack! Don't let anybody do execbuf while we don't control the chip. |
* We need to replace this with a semaphore, or something. |
* And not confound mm.suspended! |
3887,6 → 3905,12 |
if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1)) |
I915_WRITE(0x9008, I915_READ(0x9008) | 0xf0000); |
|
if (HAS_PCH_NOP(dev)) { |
u32 temp = I915_READ(GEN7_MSG_CTL); |
temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK); |
I915_WRITE(GEN7_MSG_CTL, temp); |
} |
|
i915_gem_l3_remap(dev); |
|
i915_gem_init_swizzling(dev); |
3900,7 → 3924,13 |
* contexts before PPGTT. |
*/ |
i915_gem_context_init(dev); |
i915_gem_init_ppgtt(dev); |
if (dev_priv->mm.aliasing_ppgtt) { |
ret = dev_priv->mm.aliasing_ppgtt->enable(dev); |
if (ret) { |
i915_gem_cleanup_aliasing_ppgtt(dev); |
DRM_INFO("PPGTT enable failed. This is not fatal, but unexpected\n"); |
} |
} |
|
return 0; |
} |
3913,7 → 3943,16 |
int ret; |
|
mutex_lock(&dev->struct_mutex); |
|
if (IS_VALLEYVIEW(dev)) { |
/* VLVA0 (potential hack), BIOS isn't actually waking us */ |
I915_WRITE(VLV_GTLC_WAKE_CTRL, 1); |
if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) & 1) == 1, 10)) |
DRM_DEBUG_DRIVER("allow wake ack timed out\n"); |
} |
|
i915_gem_init_global_gtt(dev); |
|
ret = i915_gem_init_hw(dev); |
mutex_unlock(&dev->struct_mutex); |
if (ret) { |
3921,6 → 3960,7 |
return ret; |
} |
|
|
return 0; |
} |
|
4038,13 → 4078,16 |
|
dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; |
|
if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) |
if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev)) |
dev_priv->num_fence_regs = 32; |
else if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) |
dev_priv->num_fence_regs = 16; |
else |
dev_priv->num_fence_regs = 8; |
|
/* Initialize fence registers to zero */ |
i915_gem_reset_fences(dev); |
INIT_LIST_HEAD(&dev_priv->mm.fence_list); |
i915_gem_restore_fences(dev); |
|
i915_gem_detect_bit_6_swizzle(dev); |
|