Subversion Repositories Kolibri OS

#include "radeon.h"

void msleep(unsigned int msecs)

{

    msecs /= 10;

    if(!msecs) msecs = 1;

     __asm__ __volatile__ (

     "call *__imp__Delay"

     ::"b" (msecs));

     __asm__ __volatile__ (

     "":::"ebx");

};

static void delay_loop(unsigned long loops)

{

        asm volatile(

                "       jz 3f           \n"

                "       jmp 1f          \n"

                ".align 16              \n"

                "       jnz 2b          \n"

        );

{

inline void __const_udelay(unsigned long xloops)

        int d0;

        asm("mull %%edx"

                : "=d" (xloops), "=&a" (d0)

                : "1" (xloops), ""

}

void __udelay(unsigned long usecs)

{

        __const_udelay(usecs * 0x000010c7); /* 2**32 / 1000000 (rounded up) */

}

unsigned int _sw_hweight32(unsigned int w)

{

#ifdef CONFIG_ARCH_HAS_FAST_MULTIPLIER

        w -= (w >> 1) & 0x55555555;

        w =  (w + (w >> 4)) & 0x0f0f0f0f;

        return (w * 0x01010101) >> 24;

        unsigned int res = w - ((w >> 1) & 0x55555555);

        res = (res & 0x33333333) + ((res >> 2) & 0x33333333);

        res = (res + (res >> 4)) & 0x0F0F0F0F;

        res = res + (res >> 8);

        return (res + (res >> 16)) & 0x000000FF;

#endif

}

void cpu_detect1()

{

    u32 junk, tfms, cap0, misc;

    int i;

    cpuid(0x00000001, &tfms, &misc, &junk, &cap0);

    if (cap0 & (1<<19))

    {

        x86_clflush_size = ((misc >> 8) & 0xff) * 8;

    }

#if 0

    cpuid(0x80000002, (unsigned int*)&cpuinfo.model_name[0], (unsigned int*)&cpuinfo.model_name[4],

          (unsigned int*)&cpuinfo.model_name[8], (unsigned int*)&cpuinfo.model_name[12]);

    cpuid(0x80000003, (unsigned int*)&cpuinfo.model_name[16], (unsigned int*)&cpuinfo.model_name[20],

          (unsigned int*)&cpuinfo.model_name[24], (unsigned int*)&cpuinfo.model_name[28]);

    cpuid(0x80000004, (unsigned int*)&cpuinfo.model_name[32], (unsigned int*)&cpuinfo.model_name[36],

          (unsigned int*)&cpuinfo.model_name[40], (unsigned int*)&cpuinfo.model_name[44]);

    printf("\n%s\n\n",cpuinfo.model_name);

    cpuinfo.def_mtrr = read_msr(MSR_MTRRdefType);

    cpuinfo.mtrr_cap = read_msr(IA32_MTRRCAP);

    printf("MSR_MTRRdefType %016llx\n\n", cpuinfo.def_mtrr);

    cpuinfo.var_mtrr_count = (u8_t)cpuinfo.mtrr_cap;

    for(i = 0; i < cpuinfo.var_mtrr_count; i++)

    {

        u64_t mtrr_base;

        u64_t mtrr_mask;

        cpuinfo.var_mtrr[i].base = read_msr(MTRRphysBase_MSR(i));

        cpuinfo.var_mtrr[i].mask = read_msr(MTRRphysMask_MSR(i));

        printf("MTRR_%d base: %016llx mask: %016llx\n", i,

               cpuinfo.var_mtrr[i].base,

               cpuinfo.var_mtrr[i].mask);

    };

    unsigned int cr0, cr3, cr4, eflags;

    eflags = safe_cli();

    /* Enter the no-fill (CD=1, NW=0) cache mode and flush caches. */

    cr0 = read_cr0() | (1<<30);

    write_cr0(cr0);

    wbinvd();

    cr4 = read_cr4();

    write_cr4(cr4 & ~(1<<7));

    cr3 = read_cr3();

    write_cr3(cr3);

    /* Save MTRR state */

    rdmsr(MSR_MTRRdefType, deftype_lo, deftype_hi);

    /* Disable MTRRs, and set the default type to uncached */

    native_write_msr(MSR_MTRRdefType, deftype_lo & ~0xcff, deftype_hi);

    wbinvd();

    i = 0;

    set_mtrr(i++,0,0x80000000>>12,MTRR_WB);

    set_mtrr(i++,0x80000000>>12,0x40000000>>12,MTRR_WB);

    set_mtrr(i++,0xC0000000>>12,0x20000000>>12,MTRR_WB);

    set_mtrr(i++,0xdb800000>>12,0x00800000>>12,MTRR_UC);

    set_mtrr(i++,0xdc000000>>12,0x04000000>>12,MTRR_UC);

    set_mtrr(i++,0xE0000000>>12,0x10000000>>12,MTRR_WC);

    for(; i < cpuinfo.var_mtrr_count; i++)

        set_mtrr(i,0,0,0);

    write_cr3(cr3);

    /* Intel (P6) standard MTRRs */

    native_write_msr(MSR_MTRRdefType, deftype_lo, deftype_hi);

    /* Enable caches */

    write_cr0(read_cr0() & ~(1<<30));

    /* Restore value of CR4 */

    write_cr4(cr4);

    safe_sti(eflags);

    printf("\nnew MTRR map\n\n");

    for(i = 0; i < cpuinfo.var_mtrr_count; i++)

    {

        u64_t mtrr_base;

        u64_t mtrr_mask;

        cpuinfo.var_mtrr[i].base = read_msr(MTRRphysBase_MSR(i));

        cpuinfo.var_mtrr[i].mask = read_msr(MTRRphysMask_MSR(i));

        printf("MTRR_%d base: %016llx mask: %016llx\n", i,

               cpuinfo.var_mtrr[i].base,

               cpuinfo.var_mtrr[i].mask);

    };

#endif

    tsc_khz = (unsigned int)(GetCpuFreq()/1000);

}

static atomic_t fence_context_counter = ATOMIC_INIT(0);

/**

 * fence_context_alloc - allocate an array of fence contexts

 * @num:        [in]    amount of contexts to allocate

 *

 * This function will return the first index of the number of fences allocated.

        return atomic_add_return(num, &fence_context_counter) - num;

}

EXPORT_SYMBOL(fence_context_alloc);

int fence_signal(struct fence *fence)

{

        unsigned long flags;

        if (!fence)

                return -EINVAL;

//        if (!ktime_to_ns(fence->timestamp)) {

//                fence->timestamp = ktime_get();

//                smp_mb__before_atomic();

//        }

        if (test_and_set_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags))

                return -EINVAL;

//        trace_fence_signaled(fence);

        if (test_bit(FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags)) {

                struct fence_cb *cur, *tmp;

                spin_lock_irqsave(fence->lock, flags);

                list_for_each_entry_safe(cur, tmp, &fence->cb_list, node) {

                        list_del_init(&cur->node);

                        cur->func(fence, cur);

                }

                spin_unlock_irqrestore(fence->lock, flags);

        }

        return 0;

}

EXPORT_SYMBOL(fence_signal);

int fence_signal_locked(struct fence *fence)

{

        struct fence_cb *cur, *tmp;

        int ret = 0;

        if (WARN_ON(!fence))

                return -EINVAL;

//        if (!ktime_to_ns(fence->timestamp)) {

//                fence->timestamp = ktime_get();

//                smp_mb__before_atomic();

//        }

        if (test_and_set_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {

                ret = -EINVAL;

                /*

                 * we might have raced with the unlocked fence_signal,

                 * still run through all callbacks

                 */

        }// else

//                trace_fence_signaled(fence);

        list_for_each_entry_safe(cur, tmp, &fence->cb_list, node) {

                list_del_init(&cur->node);

                cur->func(fence, cur);

        }

        return ret;

}

EXPORT_SYMBOL(fence_signal_locked);

void fence_enable_sw_signaling(struct fence *fence)

{

        unsigned long flags;

        if (!test_and_set_bit(FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags) &&

            !test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {

//                trace_fence_enable_signal(fence);

                spin_lock_irqsave(fence->lock, flags);

                if (!fence->ops->enable_signaling(fence))

                        fence_signal_locked(fence);

                spin_unlock_irqrestore(fence->lock, flags);

        }

}

EXPORT_SYMBOL(fence_enable_sw_signaling);

signed long

fence_wait_timeout(struct fence *fence, bool intr, signed long timeout)

{

        signed long ret;

        if (WARN_ON(timeout < 0))

                return -EINVAL;

//        trace_fence_wait_start(fence);

        ret = fence->ops->wait(fence, intr, timeout);

//        trace_fence_wait_end(fence);

        return ret;

}

EXPORT_SYMBOL(fence_wait_timeout);

void fence_release(struct kref *kref)

{

        struct fence *fence =

                        container_of(kref, struct fence, refcount);

//        trace_fence_destroy(fence);

        BUG_ON(!list_empty(&fence->cb_list));

        if (fence->ops->release)

                fence->ops->release(fence);

        else

                fence_free(fence);

}

EXPORT_SYMBOL(fence_release);

void fence_free(struct fence *fence)

{

        kfree_rcu(fence, rcu);

}

EXPORT_SYMBOL(fence_free);

reservation_object_add_shared_inplace(struct reservation_object *obj,

                                      struct reservation_object_list *fobj,

                                      struct fence *fence)

{

        u32 i;

        fence_get(fence);

//        preempt_disable();

        write_seqcount_begin(&obj->seq);

        for (i = 0; i < fobj->shared_count; ++i) {

                struct fence *old_fence;

                old_fence = rcu_dereference_protected(fobj->shared[i],

                                                reservation_object_held(obj));

                if (old_fence->context == fence->context) {

                        /* memory barrier is added by write_seqcount_begin */

                        RCU_INIT_POINTER(fobj->shared[i], fence);

                        write_seqcount_end(&obj->seq);

                        preempt_enable();

                        fence_put(old_fence);

                        return;

                }

        }

        /*

         * memory barrier is added by write_seqcount_begin,

         * fobj->shared_count is protected by this lock too

         */

        RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence);

        fobj->shared_count++;

        write_seqcount_end(&obj->seq);

//        preempt_enable();

}

static void

reservation_object_add_shared_replace(struct reservation_object *obj,

                                      struct reservation_object_list *old,

                                      struct reservation_object_list *fobj,

                                      struct fence *fence)

{

        unsigned i;

        struct fence *old_fence = NULL;

        fence_get(fence);

        if (!old) {

                RCU_INIT_POINTER(fobj->shared[0], fence);

                fobj->shared_count = 1;

                goto done;

        }

        /*

         * no need to bump fence refcounts, rcu_read access

         * requires the use of kref_get_unless_zero, and the

         * references from the old struct are carried over to

         * the new.

         */

        fobj->shared_count = old->shared_count;

        for (i = 0; i < old->shared_count; ++i) {

                struct fence *check;

                check = rcu_dereference_protected(old->shared[i],

                                                reservation_object_held(obj));

                if (!old_fence && check->context == fence->context) {

                        old_fence = check;

                        RCU_INIT_POINTER(fobj->shared[i], fence);

                } else

                        RCU_INIT_POINTER(fobj->shared[i], check);

        }

        if (!old_fence) {

                RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence);

                fobj->shared_count++;

        }

done:

//        preempt_disable();

        write_seqcount_begin(&obj->seq);

        /*

         * RCU_INIT_POINTER can be used here,

         * seqcount provides the necessary barriers

         */

        RCU_INIT_POINTER(obj->fence, fobj);

        write_seqcount_end(&obj->seq);

//        preempt_enable();

        if (old)

                kfree_rcu(old, rcu);

        if (old_fence)

                fence_put(old_fence);

}

int reservation_object_reserve_shared(struct reservation_object *obj)

{

        struct reservation_object_list *fobj, *old;

        u32 max;

        old = reservation_object_get_list(obj);

        if (old && old->shared_max) {

                if (old->shared_count < old->shared_max) {

                        /* perform an in-place update */

                        kfree(obj->staged);

                        obj->staged = NULL;

                        return 0;

                } else

                        max = old->shared_max * 2;

        } else

                max = 4;

        /*

         * resize obj->staged or allocate if it doesn't exist,

         * noop if already correct size

         */

        fobj = krealloc(obj->staged, offsetof(typeof(*fobj), shared[max]),

                        GFP_KERNEL);

        if (!fobj)

                return -ENOMEM;

        obj->staged = fobj;

        fobj->shared_max = max;

        return 0;

}

EXPORT_SYMBOL(reservation_object_reserve_shared);

void reservation_object_add_shared_fence(struct reservation_object *obj,

                                         struct fence *fence)

{

        struct reservation_object_list *old, *fobj = obj->staged;

        old = reservation_object_get_list(obj);

        obj->staged = NULL;

        if (!fobj) {

                BUG_ON(old->shared_count >= old->shared_max);

                reservation_object_add_shared_inplace(obj, old, fence);

        } else

                reservation_object_add_shared_replace(obj, old, fobj, fence);

}

EXPORT_SYMBOL(reservation_object_add_shared_fence);

void reservation_object_add_excl_fence(struct reservation_object *obj,

                                       struct fence *fence)

{

        struct fence *old_fence = reservation_object_get_excl(obj);

        struct reservation_object_list *old;

        u32 i = 0;

        old = reservation_object_get_list(obj);

        if (old)

                i = old->shared_count;

        if (fence)

                fence_get(fence);

//        preempt_disable();

        write_seqcount_begin(&obj->seq);

        /* write_seqcount_begin provides the necessary memory barrier */

        RCU_INIT_POINTER(obj->fence_excl, fence);

        if (old)

                old->shared_count = 0;

        write_seqcount_end(&obj->seq);

//        preempt_enable();

        /* inplace update, no shared fences */

        while (i--)

                fence_put(rcu_dereference_protected(old->shared[i],

                                                reservation_object_held(obj)));

        if (old_fence)

                fence_put(old_fence);

}

EXPORT_SYMBOL(reservation_object_add_excl_fence);

void

fence_init(struct fence *fence, const struct fence_ops *ops,

             spinlock_t *lock, unsigned context, unsigned seqno)

{

        BUG_ON(!lock);

        BUG_ON(!ops || !ops->wait || !ops->enable_signaling ||

               !ops->get_driver_name || !ops->get_timeline_name);

        kref_init(&fence->refcount);

        fence->ops = ops;

        INIT_LIST_HEAD(&fence->cb_list);

        fence->lock = lock;

        fence->context = context;

        fence->seqno = seqno;

        fence->flags = 0UL;

//        trace_fence_init(fence);

}

EXPORT_SYMBOL(fence_init);

#include 

struct rcu_ctrlblk {

        struct rcu_head *rcucblist;     /* List of pending callbacks (CBs). */

        struct rcu_head **donetail;     /* ->next pointer of last "done" CB. */

        struct rcu_head **curtail;      /* ->next pointer of last CB. */

//        RCU_TRACE(long qlen);           /* Number of pending CBs. */

//        RCU_TRACE(unsigned long gp_start); /* Start time for stalls. */

//        RCU_TRACE(unsigned long ticks_this_gp); /* Statistic for stalls. */

//        RCU_TRACE(unsigned long jiffies_stall); /* Jiffies at next stall. */

//        RCU_TRACE(const char *name);    /* Name of RCU type. */

};

/* Definition for rcupdate control block. */

static struct rcu_ctrlblk rcu_sched_ctrlblk = {

        .donetail       = &rcu_sched_ctrlblk.rcucblist,

        .curtail        = &rcu_sched_ctrlblk.rcucblist,

//        RCU_TRACE(.name = "rcu_sched")

};

static void __call_rcu(struct rcu_head *head,

                       void (*func)(struct rcu_head *rcu),

                       struct rcu_ctrlblk *rcp)

{

        unsigned long flags;

//        debug_rcu_head_queue(head);

        head->func = func;

        head->next = NULL;

        local_irq_save(flags);

        *rcp->curtail = head;

        rcp->curtail = &head->next;

//        RCU_TRACE(rcp->qlen++);

        local_irq_restore(flags);

}

/*

 * Post an RCU callback to be invoked after the end of an RCU-sched grace

 * period.  But since we have but one CPU, that would be after any

 * quiescent state.

 */

void call_rcu_sched(struct rcu_head *head, void (*func)(struct rcu_head *rcu))

{

        __call_rcu(head, func, &rcu_sched_ctrlblk);