Subversion Repositories Kolibri OS

#include "drmP.h"

#include "drm.h"

#include "i915_drm.h"

#include "i915_drv.h"

#include "intel_drv.h"

//#include

#undef mb

#undef rmb

#undef wmb

#define mb() asm volatile("mfence")

#define rmb() asm volatile ("lfence")

#define wmb() asm volatile ("sfence")

typedef struct

{

    struct drm_i915_gem_object *batch;

    struct list_head  objects;

    u32    exec_start;

    u32    exec_len;

}batchbuffer_t;

struct change_domains {

    uint32_t invalidate_domains;

    uint32_t flush_domains;

    uint32_t flush_rings;

    uint32_t flips;

};

/*

 * Set the next domain for the specified object. This

 * may not actually perform the necessary flushing/invaliding though,

 * as that may want to be batched with other set_domain operations

 *

 * This is (we hope) the only really tricky part of gem. The goal

 * is fairly simple -- track which caches hold bits of the object

 * and make sure they remain coherent. A few concrete examples may

 * help to explain how it works. For shorthand, we use the notation

 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the

 * a pair of read and write domain masks.

 *

 * Case 1: the batch buffer

 *

 *  1. Allocated

 *  2. Written by CPU

 *  3. Mapped to GTT

 *  4. Read by GPU

 *  5. Unmapped from GTT

 *  6. Freed

 *

 *  Let's take these a step at a time

 *

 *  1. Allocated

 *      Pages allocated from the kernel may still have

 *      cache contents, so we set them to (CPU, CPU) always.

 *  2. Written by CPU (using pwrite)

 *      The pwrite function calls set_domain (CPU, CPU) and

 *      this function does nothing (as nothing changes)

 *  3. Mapped by GTT

 *      This function asserts that the object is not

 *      currently in any GPU-based read or write domains

 *  4. Read by GPU

 *      i915_gem_execbuffer calls set_domain (COMMAND, 0).

 *      As write_domain is zero, this function adds in the

 *      current read domains (CPU+COMMAND, 0).

 *      flush_domains is set to CPU.

 *      invalidate_domains is set to COMMAND

 *      clflush is run to get data out of the CPU caches

 *      then i915_dev_set_domain calls i915_gem_flush to

 *      emit an MI_FLUSH and drm_agp_chipset_flush

 *  5. Unmapped from GTT

 *      i915_gem_object_unbind calls set_domain (CPU, CPU)

 *      flush_domains and invalidate_domains end up both zero

 *      so no flushing/invalidating happens

 *  6. Freed

 *      yay, done

 *

 * Case 2: The shared render buffer

 *

 *  1. Allocated

 *  2. Mapped to GTT

 *  3. Read/written by GPU

 *  4. set_domain to (CPU,CPU)

 *  5. Read/written by CPU

 *  6. Read/written by GPU

 *

 *  1. Allocated

 *      Same as last example, (CPU, CPU)

 *  2. Mapped to GTT

 *      Nothing changes (assertions find that it is not in the GPU)

 *  3. Read/written by GPU

 *      execbuffer calls set_domain (RENDER, RENDER)

 *      flush_domains gets CPU

 *      invalidate_domains gets GPU

 *      clflush (obj)

 *      MI_FLUSH and drm_agp_chipset_flush

 *  4. set_domain (CPU, CPU)

 *      flush_domains gets GPU

 *      invalidate_domains gets CPU

 *      wait_rendering (obj) to make sure all drawing is complete.

 *      This will include an MI_FLUSH to get the data from GPU

 *      to memory

 *      clflush (obj) to invalidate the CPU cache

 *      Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)

 *  5. Read/written by CPU

 *      cache lines are loaded and dirtied

 *  6. Read written by GPU

 *      Same as last GPU access

 *

 * Case 3: The constant buffer

 *

 *  1. Allocated

 *  2. Written by CPU

 *  3. Read by GPU

 *  4. Updated (written) by CPU again

 *  5. Read by GPU

 *

 *  1. Allocated

 *      (CPU, CPU)

 *  2. Written by CPU

 *      (CPU, CPU)

 *  3. Read by GPU

 *      (CPU+RENDER, 0)

 *      flush_domains = CPU

 *      invalidate_domains = RENDER

 *      clflush (obj)

 *      MI_FLUSH

 *      drm_agp_chipset_flush

 *  4. Updated (written) by CPU again

 *      (CPU, CPU)

 *      flush_domains = 0 (no previous write domain)

 *      invalidate_domains = 0 (no new read domains)

 *  5. Read by GPU

 *      (CPU+RENDER, 0)

 *      flush_domains = CPU

 *      invalidate_domains = RENDER

 *      clflush (obj)

 *      MI_FLUSH

 *      drm_agp_chipset_flush

 */

static void

i915_gem_object_set_to_gpu_domain(struct drm_i915_gem_object *obj,

                  struct intel_ring_buffer *ring,

                  struct change_domains *cd)

{

    uint32_t invalidate_domains = 0, flush_domains = 0;

    /*

     * If the object isn't moving to a new write domain,

     * let the object stay in multiple read domains

     */

    if (obj->base.pending_write_domain == 0)

        obj->base.pending_read_domains |= obj->base.read_domains;

    /*

     * Flush the current write domain if

     * the new read domains don't match. Invalidate

     * any read domains which differ from the old

     * write domain

     */

    if (obj->base.write_domain &&

        (((obj->base.write_domain != obj->base.pending_read_domains ||

           obj->ring != ring)) ||

         (obj->fenced_gpu_access && !obj->pending_fenced_gpu_access))) {

        flush_domains |= obj->base.write_domain;

        invalidate_domains |=

            obj->base.pending_read_domains & ~obj->base.write_domain;

    }

    /*

     * Invalidate any read caches which may have

     * stale data. That is, any new read domains.

     */

    invalidate_domains |= obj->base.pending_read_domains & ~obj->base.read_domains;

    if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)

        i915_gem_clflush_object(obj);

    if (obj->base.pending_write_domain)

        cd->flips |= atomic_read(&obj->pending_flip);

    /* The actual obj->write_domain will be updated with

     * pending_write_domain after we emit the accumulated flush for all

     * of our domain changes in execbuffers (which clears objects'

     * write_domains).  So if we have a current write domain that we

     * aren't changing, set pending_write_domain to that.

     */

    if (flush_domains == 0 && obj->base.pending_write_domain == 0)

        obj->base.pending_write_domain = obj->base.write_domain;

    cd->invalidate_domains |= invalidate_domains;

    cd->flush_domains |= flush_domains;

    if (flush_domains & I915_GEM_GPU_DOMAINS)

        cd->flush_rings |= obj->ring->id;

    if (invalidate_domains & I915_GEM_GPU_DOMAINS)

        cd->flush_rings |= ring->id;

}

static int

i915_gem_execbuffer_flush(struct drm_device *dev,

              uint32_t invalidate_domains,

              uint32_t flush_domains,

              uint32_t flush_rings)

{

    drm_i915_private_t *dev_priv = dev->dev_private;

    int i, ret;

    if (flush_domains & I915_GEM_DOMAIN_CPU)

        intel_gtt_chipset_flush();

    if (flush_domains & I915_GEM_DOMAIN_GTT)

        wmb();

    if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {

        for (i = 0; i < I915_NUM_RINGS; i++)

            if (flush_rings & (1 << i)) {

                ret = i915_gem_flush_ring(&dev_priv->ring[i],

                              invalidate_domains,

                              flush_domains);

                if (ret)

                    return ret;

            }

    }

    return 0;

}

static int

i915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring,

                struct list_head *objects)

{

    struct drm_i915_gem_object *obj;

    struct change_domains cd;

    int ret;

    memset(&cd, 0, sizeof(cd));

    list_for_each_entry(obj, objects, exec_list)

        i915_gem_object_set_to_gpu_domain(obj, ring, &cd);

    if (cd.invalidate_domains | cd.flush_domains) {

        ret = i915_gem_execbuffer_flush(ring->dev,

                        cd.invalidate_domains,

                        cd.flush_domains,

                        cd.flush_rings);

        if (ret)

            return ret;

    }

//    if (cd.flips) {

//        ret = i915_gem_execbuffer_wait_for_flips(ring, cd.flips);

//        if (ret)

//            return ret;

//    }

//    list_for_each_entry(obj, objects, exec_list) {

//        ret = i915_gem_execbuffer_sync_rings(obj, ring);

//        if (ret)

//            return ret;

//    }

    return 0;

}

static void

i915_gem_execbuffer_move_to_active(struct list_head *objects,

                   struct intel_ring_buffer *ring,

                   u32 seqno)

{

    struct drm_i915_gem_object *obj;

    list_for_each_entry(obj, objects, exec_list) {

          u32 old_read = obj->base.read_domains;

          u32 old_write = obj->base.write_domain;

        obj->base.read_domains = obj->base.pending_read_domains;

        obj->base.write_domain = obj->base.pending_write_domain;

        obj->fenced_gpu_access = obj->pending_fenced_gpu_access;

        i915_gem_object_move_to_active(obj, ring, seqno);

        if (obj->base.write_domain) {

            obj->dirty = 1;

            obj->pending_gpu_write = true;

            list_move_tail(&obj->gpu_write_list,

                       &ring->gpu_write_list);

//            intel_mark_busy(ring->dev, obj);

        }

//        trace_i915_gem_object_change_domain(obj, old_read, old_write);

    }

}

static void

i915_gem_execbuffer_retire_commands(struct drm_device *dev,

                    struct intel_ring_buffer *ring)

{

    struct drm_i915_gem_request *request;

    u32 invalidate;

    /*

     * Ensure that the commands in the batch buffer are

     * finished before the interrupt fires.

     *

     * The sampler always gets flushed on i965 (sigh).

     */

    invalidate = I915_GEM_DOMAIN_COMMAND;

    if (INTEL_INFO(dev)->gen >= 4)

        invalidate |= I915_GEM_DOMAIN_SAMPLER;

    if (ring->flush(ring, invalidate, 0)) {

        i915_gem_next_request_seqno(ring);

        return;

    }

    /* Add a breadcrumb for the completion of the batch buffer */

    request = kzalloc(sizeof(*request), GFP_KERNEL);

    if (request == NULL || i915_add_request(ring, NULL, request)) {

        i915_gem_next_request_seqno(ring);

        kfree(request);

    }

}

int exec_batch(struct drm_device *dev, struct intel_ring_buffer *ring,

               batchbuffer_t *exec)

{

    drm_i915_private_t *dev_priv = dev->dev_private;

    struct drm_i915_gem_object *obj;

    u32 seqno;

    int i;

    int ret;

    ring = &dev_priv->ring[RCS];

    mutex_lock(&dev->struct_mutex);

    list_for_each_entry(obj, &exec->objects, exec_list)

    {

        obj->base.pending_read_domains = 0;

        obj->base.pending_write_domain = 0;

    };

    exec->batch->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;

    ret = i915_gem_execbuffer_move_to_gpu(ring, &exec->objects);

    if (ret)

        goto err;

    seqno = i915_gem_next_request_seqno(ring);

//    for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++) {

//        if (seqno < ring->sync_seqno[i]) {

            /* The GPU can not handle its semaphore value wrapping,

             * so every billion or so execbuffers, we need to stall

             * the GPU in order to reset the counters.

             */

//            ret = i915_gpu_idle(dev);

//            if (ret)

//                goto err;

//            BUG_ON(ring->sync_seqno[i]);

//        }

//    };

    ret = ring->dispatch_execbuffer(ring, exec->exec_start, exec->exec_len);

    if (ret)

        goto err;

    i915_gem_execbuffer_move_to_active(&exec->objects, ring, seqno);

    i915_gem_execbuffer_retire_commands(dev, ring);

err:

    mutex_unlock(&dev->struct_mutex);

    return ret;

};