Subversion Repositories Kolibri OS

Compare Revisions

Regard whitespace Rev 3243 → Rev 3242

/drivers/video/drm/i915/i915_gem.c
30,6 → 30,7
#include "i915_drv.h"
#include "i915_trace.h"
#include "intel_drv.h"
//#include <linux/shmem_fs.h>
#include <linux/slab.h>
//#include <linux/swap.h>
#include <linux/pci.h>
52,6 → 53,21
 
#define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
 
static inline long IS_ERR(const void *ptr)
{
return IS_ERR_VALUE((unsigned long)ptr);
}
 
static inline void *ERR_PTR(long error)
{
return (void *) error;
}
 
static inline long PTR_ERR(const void *ptr)
{
return (long) ptr;
}
 
void
drm_gem_object_free(struct kref *kref)
{
905,12 → 921,12
* domain anymore. */
if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
i915_gem_clflush_object(obj);
i915_gem_chipset_flush(dev);
intel_gtt_chipset_flush();
}
}
 
if (needs_clflush_after)
i915_gem_chipset_flush(dev);
intel_gtt_chipset_flush();
 
return ret;
}
1373,8 → 1389,6
static void
i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
{
int page_count = obj->base.size / PAGE_SIZE;
struct scatterlist *sg;
int ret, i;
 
BUG_ON(obj->madv == __I915_MADV_PURGED);
1392,18 → 1406,12
if (obj->madv == I915_MADV_DONTNEED)
obj->dirty = 0;
 
for_each_sg(obj->pages->sgl, sg, page_count, i) {
struct page *page = sg_page(sg);
for (i = 0; i < obj->pages.nents; i++)
FreePage(obj->pages.page[i]);
 
 
 
page_cache_release(page);
}
//DRM_DEBUG_KMS("%s release %d pages\n", __FUNCTION__, page_count);
DRM_DEBUG_KMS("%s release %d pages\n", __FUNCTION__, obj->pages.nents);
obj->dirty = 0;
 
sg_free_table(obj->pages);
kfree(obj->pages);
kfree(obj->pages.page);
}
 
static int
1411,7 → 1419,10
{
const struct drm_i915_gem_object_ops *ops = obj->ops;
 
if (obj->pages == NULL)
// printf("page %x pin count %d\n",
// obj->pages.page, obj->pages_pin_count );
 
if (obj->pages.page == NULL)
return 0;
 
BUG_ON(obj->gtt_space);
1419,14 → 1430,10
if (obj->pages_pin_count)
return -EBUSY;
 
/* ->put_pages might need to allocate memory for the bit17 swizzle
* array, hence protect them from being reaped by removing them from gtt
* lists early. */
list_del(&obj->gtt_list);
 
ops->put_pages(obj);
obj->pages = NULL;
obj->pages.page = NULL;
 
list_del(&obj->gtt_list);
if (i915_gem_object_is_purgeable(obj))
i915_gem_object_truncate(obj);
 
1443,55 → 1450,43
static int
i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
{
dma_addr_t page;
int page_count, i;
struct sg_table *st;
struct scatterlist *sg;
struct page *page;
gfp_t gfp;
 
/* Assert that the object is not currently in any GPU domain. As it
* wasn't in the GTT, there shouldn't be any way it could have been in
* a GPU cache
/* Get the list of pages out of our struct file. They'll be pinned
* at this point until we release them.
*/
BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
 
st = kmalloc(sizeof(*st), GFP_KERNEL);
if (st == NULL)
return -ENOMEM;
 
page_count = obj->base.size / PAGE_SIZE;
if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
sg_free_table(st);
kfree(st);
BUG_ON(obj->pages.page != NULL);
obj->pages.page = malloc(page_count * sizeof(dma_addr_t));
if (obj->pages.page == NULL)
return -ENOMEM;
}
 
/* Get the list of pages out of our struct file. They'll be pinned
* at this point until we release them.
*
* Fail silently without starting the shrinker
*/
for_each_sg(st->sgl, sg, page_count, i) {
for (i = 0; i < page_count; i++) {
page = AllocPage(); // oh-oh
if ( page == 0 )
goto err_pages;
 
sg_set_page(sg, page, PAGE_SIZE, 0);
}
obj->pages.page[i] = page;
};
DRM_DEBUG_KMS("%s alloc %d pages\n", __FUNCTION__, page_count);
obj->pages.nents = page_count;
 
obj->pages = st;
 
// DRM_DEBUG_KMS("%s alloc %d pages\n", __FUNCTION__, page_count);
// if (obj->tiling_mode != I915_TILING_NONE)
// i915_gem_object_do_bit_17_swizzle(obj);
 
return 0;
 
err_pages:
for_each_sg(st->sgl, sg, i, page_count)
page_cache_release(sg_page(sg));
sg_free_table(st);
kfree(st);
return PTR_ERR(page);
while (i--)
FreePage(obj->pages.page[i]);
 
free(obj->pages.page);
obj->pages.page = NULL;
obj->pages.nents = 0;
 
return -ENOMEM;
}
 
/* Ensure that the associated pages are gathered from the backing storage
1508,7 → 1503,7
const struct drm_i915_gem_object_ops *ops = obj->ops;
int ret;
 
if (obj->pages)
if (obj->pages.page)
return 0;
 
BUG_ON(obj->pages_pin_count);
1523,11 → 1518,11
 
void
i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
struct intel_ring_buffer *ring)
struct intel_ring_buffer *ring,
u32 seqno)
{
struct drm_device *dev = obj->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
u32 seqno = intel_ring_get_seqno(ring);
 
BUG_ON(ring == NULL);
obj->ring = ring;
1588,56 → 1583,28
WARN_ON(i915_verify_lists(dev));
}
 
static int
i915_gem_handle_seqno_wrap(struct drm_device *dev)
static u32
i915_gem_get_seqno(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_ring_buffer *ring;
int ret, i, j;
drm_i915_private_t *dev_priv = dev->dev_private;
u32 seqno = dev_priv->next_seqno;
 
/* The hardware uses various monotonic 32-bit counters, if we
* detect that they will wraparound we need to idle the GPU
* and reset those counters.
*/
ret = 0;
for_each_ring(ring, dev_priv, i) {
for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
ret |= ring->sync_seqno[j] != 0;
}
if (ret == 0)
return ret;
/* reserve 0 for non-seqno */
if (++dev_priv->next_seqno == 0)
dev_priv->next_seqno = 1;
 
ret = i915_gpu_idle(dev);
if (ret)
return ret;
 
i915_gem_retire_requests(dev);
for_each_ring(ring, dev_priv, i) {
for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
ring->sync_seqno[j] = 0;
return seqno;
}
 
return 0;
}
 
int
i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
u32
i915_gem_next_request_seqno(struct intel_ring_buffer *ring)
{
struct drm_i915_private *dev_priv = dev->dev_private;
if (ring->outstanding_lazy_request == 0)
ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev);
 
/* reserve 0 for non-seqno */
if (dev_priv->next_seqno == 0) {
int ret = i915_gem_handle_seqno_wrap(dev);
if (ret)
return ret;
 
dev_priv->next_seqno = 1;
return ring->outstanding_lazy_request;
}
 
*seqno = dev_priv->next_seqno++;
return 0;
}
 
int
i915_add_request(struct intel_ring_buffer *ring,
struct drm_file *file,
1646,6 → 1613,7
drm_i915_private_t *dev_priv = ring->dev->dev_private;
struct drm_i915_gem_request *request;
u32 request_ring_position;
u32 seqno;
int was_empty;
int ret;
 
1664,6 → 1632,7
if (request == NULL)
return -ENOMEM;
 
seqno = i915_gem_next_request_seqno(ring);
 
/* Record the position of the start of the request so that
* should we detect the updated seqno part-way through the
1672,13 → 1641,15
*/
request_ring_position = intel_ring_get_tail(ring);
 
ret = ring->add_request(ring);
ret = ring->add_request(ring, &seqno);
if (ret) {
kfree(request);
return ret;
}
 
request->seqno = intel_ring_get_seqno(ring);
trace_i915_gem_request_add(ring, seqno);
 
request->seqno = seqno;
request->ring = ring;
request->tail = request_ring_position;
request->emitted_jiffies = GetTimerTicks();
1703,7 → 1674,7
}
 
if (out_seqno)
*out_seqno = request->seqno;
*out_seqno = seqno;
return 0;
}
 
1788,6 → 1759,7
i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
{
uint32_t seqno;
int i;
 
if (list_empty(&ring->request_list))
return;
1796,6 → 1768,10
 
seqno = ring->get_seqno(ring, true);
 
for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++)
if (seqno >= ring->sync_seqno[i])
ring->sync_seqno[i] = 0;
 
while (!list_empty(&ring->request_list)) {
struct drm_i915_gem_request *request;
 
1915,28 → 1891,6
return 0;
}
 
/**
* i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
* @DRM_IOCTL_ARGS: standard ioctl arguments
*
* Returns 0 if successful, else an error is returned with the remaining time in
* the timeout parameter.
* -ETIME: object is still busy after timeout
* -ERESTARTSYS: signal interrupted the wait
* -ENONENT: object doesn't exist
* Also possible, but rare:
* -EAGAIN: GPU wedged
* -ENOMEM: damn
* -ENODEV: Internal IRQ fail
* -E?: The add request failed
*
* The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
* non-zero timeout parameter the wait ioctl will wait for the given number of
* nanoseconds on an object becoming unbusy. Since the wait itself does so
* without holding struct_mutex the object may become re-busied before this
* function completes. A similar but shorter * race condition exists in the busy
* ioctl
*/
 
 
 
1946,11 → 1900,6
 
 
 
 
 
 
 
 
/**
* i915_gem_object_sync - sync an object to a ring.
*
1989,11 → 1938,7
 
ret = to->sync_to(to, from, seqno);
if (!ret)
/* We use last_read_seqno because sync_to()
* might have just caused seqno wrap under
* the radar.
*/
from->sync_seqno[idx] = obj->last_read_seqno;
from->sync_seqno[idx] = seqno;
 
return ret;
}
2037,7 → 1982,7
if (obj->pin_count)
return -EBUSY;
 
BUG_ON(obj->pages == NULL);
BUG_ON(obj->pages.page == NULL);
 
ret = i915_gem_object_finish_gpu(obj);
if (ret)
2076,6 → 2021,14
return 0;
}
 
static int i915_ring_idle(struct intel_ring_buffer *ring)
{
if (list_empty(&ring->active_list))
return 0;
 
return i915_wait_seqno(ring, i915_gem_next_request_seqno(ring));
}
 
int i915_gpu_idle(struct drm_device *dev)
{
drm_i915_private_t *dev_priv = dev->dev_private;
2088,7 → 2041,7
if (ret)
return ret;
 
ret = intel_ring_idle(ring);
ret = i915_ring_idle(ring);
if (ret)
return ret;
}
2478,7 → 2431,7
{
struct drm_device *dev = obj->base.dev;
drm_i915_private_t *dev_priv = dev->dev_private;
struct drm_mm_node *node;
struct drm_mm_node *free_space;
u32 size, fence_size, fence_alignment, unfenced_alignment;
bool mappable, fenceable;
int ret;
2522,50 → 2475,66
if (ret)
return ret;
 
i915_gem_object_pin_pages(obj);
 
node = kzalloc(sizeof(*node), GFP_KERNEL);
if (node == NULL) {
i915_gem_object_unpin_pages(obj);
return -ENOMEM;
}
 
search_free:
if (map_and_fenceable)
ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space, node,
free_space =
drm_mm_search_free_in_range_color(&dev_priv->mm.gtt_space,
size, alignment, obj->cache_level,
0, dev_priv->mm.gtt_mappable_end);
0, dev_priv->mm.gtt_mappable_end,
false);
else
ret = drm_mm_insert_node_generic(&dev_priv->mm.gtt_space, node,
size, alignment, obj->cache_level);
if (ret) {
free_space = drm_mm_search_free_color(&dev_priv->mm.gtt_space,
size, alignment, obj->cache_level,
false);
 
i915_gem_object_unpin_pages(obj);
kfree(node);
if (free_space != NULL) {
if (map_and_fenceable)
obj->gtt_space =
drm_mm_get_block_range_generic(free_space,
size, alignment, obj->cache_level,
0, dev_priv->mm.gtt_mappable_end,
false);
else
obj->gtt_space =
drm_mm_get_block_generic(free_space,
size, alignment, obj->cache_level,
false);
}
if (obj->gtt_space == NULL) {
ret = 1; //i915_gem_evict_something(dev, size, alignment,
// map_and_fenceable);
if (ret)
return ret;
 
goto search_free;
}
if (WARN_ON(!i915_gem_valid_gtt_space(dev, node, obj->cache_level))) {
i915_gem_object_unpin_pages(obj);
drm_mm_put_block(node);
if (WARN_ON(!i915_gem_valid_gtt_space(dev,
obj->gtt_space,
obj->cache_level))) {
drm_mm_put_block(obj->gtt_space);
obj->gtt_space = NULL;
return -EINVAL;
}
 
 
ret = i915_gem_gtt_prepare_object(obj);
if (ret) {
i915_gem_object_unpin_pages(obj);
drm_mm_put_block(node);
drm_mm_put_block(obj->gtt_space);
obj->gtt_space = NULL;
return ret;
}
 
if (!dev_priv->mm.aliasing_ppgtt)
i915_gem_gtt_bind_object(obj, obj->cache_level);
 
list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list);
list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
 
obj->gtt_space = node;
obj->gtt_offset = node->start;
obj->gtt_offset = obj->gtt_space->start;
 
fenceable =
node->size == fence_size &&
(node->start & (fence_alignment - 1)) == 0;
obj->gtt_space->size == fence_size &&
(obj->gtt_space->start & (fence_alignment - 1)) == 0;
 
mappable =
obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
2572,7 → 2541,6
 
obj->map_and_fenceable = mappable && fenceable;
 
i915_gem_object_unpin_pages(obj);
trace_i915_gem_object_bind(obj, map_and_fenceable);
i915_gem_verify_gtt(dev);
return 0;
2585,7 → 2553,7
* to GPU, and we can ignore the cache flush because it'll happen
* again at bind time.
*/
if (obj->pages == NULL)
if (obj->pages.page == NULL)
return;
 
/* If the GPU is snooping the contents of the CPU cache,
2598,7 → 2566,7
*/
if (obj->cache_level != I915_CACHE_NONE)
return;
#if 0
 
if(obj->mapped != NULL)
{
uint8_t *page_virtual;
2645,8 → 2613,6
"mfence");
}
}
#endif
 
}
 
/** Flushes the GTT write domain for the object if it's dirty. */
2686,7 → 2652,7
return;
 
i915_gem_clflush_object(obj);
i915_gem_chipset_flush(obj->base.dev);
intel_gtt_chipset_flush();
old_write_domain = obj->base.write_domain;
obj->base.write_domain = 0;
 
3023,16 → 2989,11
#endif
 
if (obj->gtt_space == NULL) {
struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
 
ret = i915_gem_object_bind_to_gtt(obj, alignment,
map_and_fenceable,
nonblocking);
if (ret)
return ret;
 
if (!dev_priv->mm.aliasing_ppgtt)
i915_gem_gtt_bind_object(obj, obj->cache_level);
}
 
if (!obj->has_global_gtt_mapping && map_and_fenceable)
3086,15 → 3047,14
goto out;
}
 
if (obj->user_pin_count == 0) {
obj->user_pin_count++;
obj->pin_filp = file;
if (obj->user_pin_count == 1) {
ret = i915_gem_object_pin(obj, args->alignment, true, false);
if (ret)
goto out;
}
 
obj->user_pin_count++;
obj->pin_filp = file;
 
/* XXX - flush the CPU caches for pinned objects
* as the X server doesn't manage domains yet
*/
3335,7 → 3295,7
i915_gem_object_put_pages(obj);
// i915_gem_object_free_mmap_offset(obj);
 
BUG_ON(obj->pages);
BUG_ON(obj->pages.page);
 
// if (obj->base.import_attach)
// drm_prime_gem_destroy(&obj->base, NULL);
3398,7 → 3358,7
if (!IS_IVYBRIDGE(dev))
return;
 
if (!dev_priv->l3_parity.remap_info)
if (!dev_priv->mm.l3_remap_info)
return;
 
misccpctl = I915_READ(GEN7_MISCCPCTL);
3407,12 → 3367,12
 
for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
if (remap && remap != dev_priv->mm.l3_remap_info[i/4])
DRM_DEBUG("0x%x was already programmed to %x\n",
GEN7_L3LOG_BASE + i, remap);
if (remap && !dev_priv->l3_parity.remap_info[i/4])
if (remap && !dev_priv->mm.l3_remap_info[i/4])
DRM_DEBUG_DRIVER("Clearing remapped register\n");
I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->mm.l3_remap_info[i/4]);
}
 
/* Make sure all the writes land before disabling dop clock gating */
3442,6 → 3402,68
I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
}
 
void i915_gem_init_ppgtt(struct drm_device *dev)
{
drm_i915_private_t *dev_priv = dev->dev_private;
uint32_t pd_offset;
struct intel_ring_buffer *ring;
struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
uint32_t __iomem *pd_addr;
uint32_t pd_entry;
int i;
 
if (!dev_priv->mm.aliasing_ppgtt)
return;
 
 
pd_addr = dev_priv->mm.gtt->gtt + ppgtt->pd_offset/sizeof(uint32_t);
for (i = 0; i < ppgtt->num_pd_entries; i++) {
dma_addr_t pt_addr;
 
if (dev_priv->mm.gtt->needs_dmar)
pt_addr = ppgtt->pt_dma_addr[i];
else
pt_addr = ppgtt->pt_pages[i];
 
pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
pd_entry |= GEN6_PDE_VALID;
 
writel(pd_entry, pd_addr + i);
}
readl(pd_addr);
 
pd_offset = ppgtt->pd_offset;
pd_offset /= 64; /* in cachelines, */
pd_offset <<= 16;
 
if (INTEL_INFO(dev)->gen == 6) {
uint32_t ecochk, gab_ctl, ecobits;
 
ecobits = I915_READ(GAC_ECO_BITS);
I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
 
gab_ctl = I915_READ(GAB_CTL);
I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
 
ecochk = I915_READ(GAM_ECOCHK);
I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT |
ECOCHK_PPGTT_CACHE64B);
I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
} else if (INTEL_INFO(dev)->gen >= 7) {
I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B);
/* GFX_MODE is per-ring on gen7+ */
}
 
for_each_ring(ring, dev_priv, i) {
if (INTEL_INFO(dev)->gen >= 7)
I915_WRITE(RING_MODE_GEN7(ring),
_MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
 
I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
}
}
 
static bool
intel_enable_blt(struct drm_device *dev)
{
3464,7 → 3486,7
drm_i915_private_t *dev_priv = dev->dev_private;
int ret;
 
if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
if (!intel_enable_gtt())
return -EIO;
 
if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))