Subversion Repositories Kolibri OS

#include 

#include 

#include 

#include 

#include "radeon.h"

int x86_clflush_size;

unsigned int tsc_khz;

struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)

{

    struct file *filep;

    int count;

    filep = __builtin_malloc(sizeof(*filep));

    if(unlikely(filep == NULL))

        return ERR_PTR(-ENOMEM);

    count = size / PAGE_SIZE;

    filep->pages = kzalloc(sizeof(struct page *) * count, 0);

    if(unlikely(filep->pages == NULL))

    {

        kfree(filep);

        return ERR_PTR(-ENOMEM);

    };

    filep->count     = count;

    filep->allocated = 0;

    filep->vma       = NULL;

//    printf("%s file %p pages %p count %d\n",

//              __FUNCTION__,filep, filep->pages, count);

    return filep;

}

static void *check_bytes8(const u8 *start, u8 value, unsigned int bytes)

{

        while (bytes) {

                if (*start != value)

                        return (void *)start;

                start++;

                bytes--;

        }

        return NULL;

}

/**

 * memchr_inv - Find an unmatching character in an area of memory.

 * @start: The memory area

 * @c: Find a character other than c

 * @bytes: The size of the area.

 *

 * returns the address of the first character other than @c, or %NULL

 * if the whole buffer contains just @c.

 */

void *memchr_inv(const void *start, int c, size_t bytes)

{

        u8 value = c;

        u64 value64;

        unsigned int words, prefix;

        if (bytes <= 16)

                return check_bytes8(start, value, bytes);

        value64 = value;

#if defined(ARCH_HAS_FAST_MULTIPLIER) && BITS_PER_LONG == 64

        value64 *= 0x0101010101010101;

#elif defined(ARCH_HAS_FAST_MULTIPLIER)

        value64 *= 0x01010101;

        value64 |= value64 << 32;

#else

        value64 |= value64 << 8;

        value64 |= value64 << 16;

        value64 |= value64 << 32;

#endif

        prefix = (unsigned long)start % 8;

        if (prefix) {

                u8 *r;

                prefix = 8 - prefix;

                r = check_bytes8(start, value, prefix);

                if (r)

                        return r;

                start += prefix;

                bytes -= prefix;

        }

        words = bytes / 8;

        while (words) {

                if (*(u64 *)start != value64)

                        return check_bytes8(start, value, 8);

                start += 8;

                words--;

        }

        return check_bytes8(start, value, bytes % 8);

}

#define _U  0x01    /* upper */

#define _L  0x02    /* lower */

#define _D  0x04    /* digit */

#define _C  0x08    /* cntrl */

#define _P  0x10    /* punct */

#define _S  0x20    /* white space (space/lf/tab) */

#define _X  0x40    /* hex digit */

#define _SP 0x80    /* hard space (0x20) */

extern const unsigned char _ctype[];

#define __ismask(x) (_ctype[(int)(unsigned char)(x)])

#define isalnum(c)  ((__ismask(c)&(_U|_L|_D)) != 0)

#define isalpha(c)  ((__ismask(c)&(_U|_L)) != 0)

#define iscntrl(c)  ((__ismask(c)&(_C)) != 0)

#define isdigit(c)  ((__ismask(c)&(_D)) != 0)

#define isgraph(c)  ((__ismask(c)&(_P|_U|_L|_D)) != 0)

#define islower(c)  ((__ismask(c)&(_L)) != 0)

#define isprint(c)  ((__ismask(c)&(_P|_U|_L|_D|_SP)) != 0)

#define ispunct(c)  ((__ismask(c)&(_P)) != 0)

/* Note: isspace() must return false for %NUL-terminator */

#define isspace(c)  ((__ismask(c)&(_S)) != 0)

#define isupper(c)  ((__ismask(c)&(_U)) != 0)

#define isxdigit(c) ((__ismask(c)&(_D|_X)) != 0)

#define isascii(c) (((unsigned char)(c))<=0x7f)

#define toascii(c) (((unsigned char)(c))&0x7f)

static inline unsigned char __tolower(unsigned char c)

{

    if (isupper(c))

        c -= 'A'-'a';

    return c;

}

static inline unsigned char __toupper(unsigned char c)

{

    if (islower(c))

        c -= 'a'-'A';

    return c;

}

#define tolower(c) __tolower(c)

#define toupper(c) __toupper(c)

/*

 * Fast implementation of tolower() for internal usage. Do not use in your

 * code.

 */

static inline char _tolower(const char c)

{

    return c | 0x20;

}

#define KMAP_MAX    256

static struct mutex kmap_mutex;

static struct page* kmap_table[KMAP_MAX];

static int kmap_av;

static int kmap_first;

static void* kmap_base;

int kmap_init()

{

    kmap_base = AllocKernelSpace(KMAP_MAX*4096);

    if(kmap_base == NULL)

        return -1;

    kmap_av = KMAP_MAX;

    MutexInit(&kmap_mutex);

    return 0;

};

void *kmap(struct page *page)

{

    void *vaddr = NULL;

    int i;

    do

    {

        MutexLock(&kmap_mutex);

        if(kmap_av != 0)

        {

            for(i = kmap_first; i < KMAP_MAX; i++)

            {

                if(kmap_table[i] == NULL)

                {

                    kmap_av--;

                    kmap_first = i;

                    kmap_table[i] = page;

                    vaddr = kmap_base + (i<<12);

                    MapPage(vaddr,(addr_t)page,3);

                    break;

                };

            };

        };

        MutexUnlock(&kmap_mutex);

    }while(vaddr == NULL);

    return vaddr;

};

void *kmap_atomic(struct page *page) __attribute__ ((alias ("kmap")));

void kunmap(struct page *page)

{

    void *vaddr;

    int   i;

    MutexLock(&kmap_mutex);

    for(i = 0; i < KMAP_MAX; i++)

    {

        if(kmap_table[i] == page)

        {

            kmap_av++;

            if(i < kmap_first)

                kmap_first = i;

            kmap_table[i] = NULL;

            vaddr = kmap_base + (i<<12);

            MapPage(vaddr,0,0);

            break;

        };

    };

    MutexUnlock(&kmap_mutex);

};

void kunmap_atomic(void *vaddr)

{

    int i;

    MapPage(vaddr,0,0);

    i = (vaddr - kmap_base) >> 12;

    MutexLock(&kmap_mutex);

    kmap_av++;

    if(i < kmap_first)

        kmap_first = i;

    kmap_table[i] = NULL;

    MutexUnlock(&kmap_mutex);

}

void msleep(unsigned int msecs)

{

    msecs /= 10;

    if(!msecs) msecs = 1;

     __asm__ __volatile__ (

     "call *__imp__Delay"

     ::"b" (msecs));

     __asm__ __volatile__ (

     "":::"ebx");

};

/* simple loop based delay: */

static void delay_loop(unsigned long loops)

{

        asm volatile(

                "       test %0,%0      \n"

                "       jz 3f           \n"

                "       jmp 1f          \n"

                ".align 16              \n"

                "1:     jmp 2f          \n"

                ".align 16              \n"

                "2:     dec %0          \n"

                "       jnz 2b          \n"

                "3:     dec %0          \n"

                : /* we don't need output */

                :"a" (loops)

        );

}

static void (*delay_fn)(unsigned long) = delay_loop;

void __delay(unsigned long loops)

{

        delay_fn(loops);

}

inline void __const_udelay(unsigned long xloops)

{

        int d0;

        xloops *= 4;

        asm("mull %%edx"

                : "=d" (xloops), "=&a" (d0)

                : "1" (xloops), ""

                (loops_per_jiffy * (HZ/4)));

        __delay(++xloops);

}

void __udelay(unsigned long usecs)

{

        __const_udelay(usecs * 0x000010c7); /* 2**32 / 1000000 (rounded up) */

}

unsigned int _sw_hweight32(unsigned int w)

{

#ifdef CONFIG_ARCH_HAS_FAST_MULTIPLIER

        w -= (w >> 1) & 0x55555555;

        w =  (w & 0x33333333) + ((w >> 2) & 0x33333333);

        w =  (w + (w >> 4)) & 0x0f0f0f0f;

        return (w * 0x01010101) >> 24;

#else

        unsigned int res = w - ((w >> 1) & 0x55555555);

        res = (res & 0x33333333) + ((res >> 2) & 0x33333333);

        res = (res + (res >> 4)) & 0x0F0F0F0F;

        res = res + (res >> 8);

        return (res + (res >> 16)) & 0x000000FF;

#endif

}

EXPORT_SYMBOL(_sw_hweight32);

void usleep_range(unsigned long min, unsigned long max)

{

    udelay(max);

}

EXPORT_SYMBOL(usleep_range);

void *kmemdup(const void *src, size_t len, gfp_t gfp)

{

    void *p;

    p = kmalloc(len, gfp);

    if (p)

        memcpy(p, src, len);

    return p;

}

void cpu_detect1()

{

    u32 junk, tfms, cap0, misc;

    int i;

    cpuid(0x00000001, &tfms, &misc, &junk, &cap0);

    if (cap0 & (1<<19))

    {

        x86_clflush_size = ((misc >> 8) & 0xff) * 8;

    }

#if 0

    cpuid(0x80000002, (unsigned int*)&cpuinfo.model_name[0], (unsigned int*)&cpuinfo.model_name[4],

          (unsigned int*)&cpuinfo.model_name[8], (unsigned int*)&cpuinfo.model_name[12]);

    cpuid(0x80000003, (unsigned int*)&cpuinfo.model_name[16], (unsigned int*)&cpuinfo.model_name[20],

          (unsigned int*)&cpuinfo.model_name[24], (unsigned int*)&cpuinfo.model_name[28]);

    cpuid(0x80000004, (unsigned int*)&cpuinfo.model_name[32], (unsigned int*)&cpuinfo.model_name[36],

          (unsigned int*)&cpuinfo.model_name[40], (unsigned int*)&cpuinfo.model_name[44]);

    printf("\n%s\n\n",cpuinfo.model_name);

    cpuinfo.def_mtrr = read_msr(MSR_MTRRdefType);

    cpuinfo.mtrr_cap = read_msr(IA32_MTRRCAP);

    printf("MSR_MTRRdefType %016llx\n\n", cpuinfo.def_mtrr);

    cpuinfo.var_mtrr_count = (u8_t)cpuinfo.mtrr_cap;

    for(i = 0; i < cpuinfo.var_mtrr_count; i++)

    {

        u64_t mtrr_base;

        u64_t mtrr_mask;

        cpuinfo.var_mtrr[i].base = read_msr(MTRRphysBase_MSR(i));

        cpuinfo.var_mtrr[i].mask = read_msr(MTRRphysMask_MSR(i));

        printf("MTRR_%d base: %016llx mask: %016llx\n", i,

               cpuinfo.var_mtrr[i].base,

               cpuinfo.var_mtrr[i].mask);

    };

    unsigned int cr0, cr3, cr4, eflags;

    eflags = safe_cli();

    /* Enter the no-fill (CD=1, NW=0) cache mode and flush caches. */

    cr0 = read_cr0() | (1<<30);

    write_cr0(cr0);

    wbinvd();

    cr4 = read_cr4();

    write_cr4(cr4 & ~(1<<7));

    cr3 = read_cr3();

    write_cr3(cr3);

    /* Save MTRR state */

    rdmsr(MSR_MTRRdefType, deftype_lo, deftype_hi);

    /* Disable MTRRs, and set the default type to uncached */

    native_write_msr(MSR_MTRRdefType, deftype_lo & ~0xcff, deftype_hi);

    wbinvd();

    i = 0;

    set_mtrr(i++,0,0x80000000>>12,MTRR_WB);

    set_mtrr(i++,0x80000000>>12,0x40000000>>12,MTRR_WB);

    set_mtrr(i++,0xC0000000>>12,0x20000000>>12,MTRR_WB);

    set_mtrr(i++,0xdb800000>>12,0x00800000>>12,MTRR_UC);

    set_mtrr(i++,0xdc000000>>12,0x04000000>>12,MTRR_UC);

    set_mtrr(i++,0xE0000000>>12,0x10000000>>12,MTRR_WC);

    for(; i < cpuinfo.var_mtrr_count; i++)

        set_mtrr(i,0,0,0);

    write_cr3(cr3);

    /* Intel (P6) standard MTRRs */

    native_write_msr(MSR_MTRRdefType, deftype_lo, deftype_hi);

    /* Enable caches */

    write_cr0(read_cr0() & ~(1<<30));

    /* Restore value of CR4 */

    write_cr4(cr4);

    safe_sti(eflags);

    printf("\nnew MTRR map\n\n");

    for(i = 0; i < cpuinfo.var_mtrr_count; i++)

    {

        u64_t mtrr_base;

        u64_t mtrr_mask;

        cpuinfo.var_mtrr[i].base = read_msr(MTRRphysBase_MSR(i));

        cpuinfo.var_mtrr[i].mask = read_msr(MTRRphysMask_MSR(i));

        printf("MTRR_%d base: %016llx mask: %016llx\n", i,

               cpuinfo.var_mtrr[i].base,

               cpuinfo.var_mtrr[i].mask);

    };

#endif

    tsc_khz = (unsigned int)(GetCpuFreq()/1000);

}

static atomic_t fence_context_counter = ATOMIC_INIT(0);

/**

 * fence_context_alloc - allocate an array of fence contexts

 * @num:        [in]    amount of contexts to allocate

 *

 * This function will return the first index of the number of fences allocated.

 * The fence context is used for setting fence->context to a unique number.

 */

unsigned fence_context_alloc(unsigned num)

{

        BUG_ON(!num);

        return atomic_add_return(num, &fence_context_counter) - num;

}

EXPORT_SYMBOL(fence_context_alloc);

int fence_signal(struct fence *fence)

{

        unsigned long flags;

        if (!fence)

                return -EINVAL;

//        if (!ktime_to_ns(fence->timestamp)) {

//                fence->timestamp = ktime_get();

//                smp_mb__before_atomic();

//        }

        if (test_and_set_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags))

                return -EINVAL;

//        trace_fence_signaled(fence);

        if (test_bit(FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags)) {

                struct fence_cb *cur, *tmp;

                spin_lock_irqsave(fence->lock, flags);

                list_for_each_entry_safe(cur, tmp, &fence->cb_list, node) {

                        list_del_init(&cur->node);

                        cur->func(fence, cur);

                }

                spin_unlock_irqrestore(fence->lock, flags);

        }

        return 0;

}

EXPORT_SYMBOL(fence_signal);

int fence_signal_locked(struct fence *fence)

{

        struct fence_cb *cur, *tmp;

        int ret = 0;

        if (WARN_ON(!fence))

                return -EINVAL;

//        if (!ktime_to_ns(fence->timestamp)) {

//                fence->timestamp = ktime_get();

//                smp_mb__before_atomic();

//        }

        if (test_and_set_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {

                ret = -EINVAL;

                /*

                 * we might have raced with the unlocked fence_signal,

                 * still run through all callbacks

                 */

        }// else

//                trace_fence_signaled(fence);

        list_for_each_entry_safe(cur, tmp, &fence->cb_list, node) {

                list_del_init(&cur->node);

                cur->func(fence, cur);

        }

        return ret;

}

EXPORT_SYMBOL(fence_signal_locked);

void fence_enable_sw_signaling(struct fence *fence)

{

        unsigned long flags;

        if (!test_and_set_bit(FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags) &&

            !test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {

//                trace_fence_enable_signal(fence);

                spin_lock_irqsave(fence->lock, flags);

                if (!fence->ops->enable_signaling(fence))

                        fence_signal_locked(fence);

                spin_unlock_irqrestore(fence->lock, flags);

        }

}

EXPORT_SYMBOL(fence_enable_sw_signaling);

signed long

fence_wait_timeout(struct fence *fence, bool intr, signed long timeout)

{

        signed long ret;

        if (WARN_ON(timeout < 0))

                return -EINVAL;

//        trace_fence_wait_start(fence);

        ret = fence->ops->wait(fence, intr, timeout);

//        trace_fence_wait_end(fence);

        return ret;

}

EXPORT_SYMBOL(fence_wait_timeout);

void fence_release(struct kref *kref)

{

        struct fence *fence =

                        container_of(kref, struct fence, refcount);

//        trace_fence_destroy(fence);

        BUG_ON(!list_empty(&fence->cb_list));

        if (fence->ops->release)

                fence->ops->release(fence);

        else

                fence_free(fence);

}

EXPORT_SYMBOL(fence_release);

void fence_free(struct fence *fence)

{

        kfree_rcu(fence, rcu);

}

EXPORT_SYMBOL(fence_free);

reservation_object_add_shared_inplace(struct reservation_object *obj,

                                      struct reservation_object_list *fobj,

                                      struct fence *fence)

{

        u32 i;

        fence_get(fence);

//        preempt_disable();

        write_seqcount_begin(&obj->seq);

        for (i = 0; i < fobj->shared_count; ++i) {

                struct fence *old_fence;

                old_fence = rcu_dereference_protected(fobj->shared[i],

                                                reservation_object_held(obj));

                if (old_fence->context == fence->context) {

                        /* memory barrier is added by write_seqcount_begin */

                        RCU_INIT_POINTER(fobj->shared[i], fence);

                        write_seqcount_end(&obj->seq);

                        preempt_enable();

                        fence_put(old_fence);

                        return;

                }

        }

        /*

         * memory barrier is added by write_seqcount_begin,

         * fobj->shared_count is protected by this lock too

         */

        RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence);

        fobj->shared_count++;

        write_seqcount_end(&obj->seq);

//        preempt_enable();

}

static void

reservation_object_add_shared_replace(struct reservation_object *obj,

                                      struct reservation_object_list *old,

                                      struct reservation_object_list *fobj,

                                      struct fence *fence)

{

        unsigned i;

        struct fence *old_fence = NULL;

        fence_get(fence);

        if (!old) {

                RCU_INIT_POINTER(fobj->shared[0], fence);

                fobj->shared_count = 1;

                goto done;

        }

        /*

         * no need to bump fence refcounts, rcu_read access

         * requires the use of kref_get_unless_zero, and the

         * references from the old struct are carried over to

         * the new.

         */

        fobj->shared_count = old->shared_count;

        for (i = 0; i < old->shared_count; ++i) {

                struct fence *check;

                check = rcu_dereference_protected(old->shared[i],

                                                reservation_object_held(obj));

                if (!old_fence && check->context == fence->context) {

                        old_fence = check;

                        RCU_INIT_POINTER(fobj->shared[i], fence);

                } else

                        RCU_INIT_POINTER(fobj->shared[i], check);

        }

        if (!old_fence) {

                RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence);

                fobj->shared_count++;

        }

done:

//        preempt_disable();

        write_seqcount_begin(&obj->seq);

        /*

         * RCU_INIT_POINTER can be used here,

         * seqcount provides the necessary barriers

         */

        RCU_INIT_POINTER(obj->fence, fobj);

        write_seqcount_end(&obj->seq);

//        preempt_enable();

        if (old)

                kfree_rcu(old, rcu);

        if (old_fence)

                fence_put(old_fence);

}

int reservation_object_reserve_shared(struct reservation_object *obj)

{

        struct reservation_object_list *fobj, *old;

        u32 max;

        old = reservation_object_get_list(obj);

        if (old && old->shared_max) {

                if (old->shared_count < old->shared_max) {

                        /* perform an in-place update */

                        kfree(obj->staged);

                        obj->staged = NULL;

                        return 0;

                } else

                        max = old->shared_max * 2;

        } else

                max = 4;

        /*

         * resize obj->staged or allocate if it doesn't exist,

         * noop if already correct size

         */

        fobj = krealloc(obj->staged, offsetof(typeof(*fobj), shared[max]),

                        GFP_KERNEL);

        if (!fobj)

                return -ENOMEM;

        obj->staged = fobj;

        fobj->shared_max = max;

        return 0;

}

EXPORT_SYMBOL(reservation_object_reserve_shared);

void reservation_object_add_shared_fence(struct reservation_object *obj,

                                         struct fence *fence)

{

        struct reservation_object_list *old, *fobj = obj->staged;

        old = reservation_object_get_list(obj);

        obj->staged = NULL;

        if (!fobj) {

                BUG_ON(old->shared_count >= old->shared_max);

                reservation_object_add_shared_inplace(obj, old, fence);

        } else

                reservation_object_add_shared_replace(obj, old, fobj, fence);

}

EXPORT_SYMBOL(reservation_object_add_shared_fence);

void reservation_object_add_excl_fence(struct reservation_object *obj,

                                       struct fence *fence)

{

        struct fence *old_fence = reservation_object_get_excl(obj);

        struct reservation_object_list *old;

        u32 i = 0;

        old = reservation_object_get_list(obj);

        if (old)

                i = old->shared_count;

        if (fence)

                fence_get(fence);

//        preempt_disable();

        write_seqcount_begin(&obj->seq);

        /* write_seqcount_begin provides the necessary memory barrier */

        RCU_INIT_POINTER(obj->fence_excl, fence);

        if (old)

                old->shared_count = 0;

        write_seqcount_end(&obj->seq);

//        preempt_enable();

        /* inplace update, no shared fences */

        while (i--)

                fence_put(rcu_dereference_protected(old->shared[i],

                                                reservation_object_held(obj)));

        if (old_fence)

                fence_put(old_fence);

}

EXPORT_SYMBOL(reservation_object_add_excl_fence);

void

fence_init(struct fence *fence, const struct fence_ops *ops,

             spinlock_t *lock, unsigned context, unsigned seqno)

{

        BUG_ON(!lock);

        BUG_ON(!ops || !ops->wait || !ops->enable_signaling ||

               !ops->get_driver_name || !ops->get_timeline_name);

        kref_init(&fence->refcount);

        fence->ops = ops;

        INIT_LIST_HEAD(&fence->cb_list);

        fence->lock = lock;

        fence->context = context;

        fence->seqno = seqno;

        fence->flags = 0UL;

//        trace_fence_init(fence);

}

EXPORT_SYMBOL(fence_init);

#include 

struct rcu_ctrlblk {

        struct rcu_head *rcucblist;     /* List of pending callbacks (CBs). */

        struct rcu_head **donetail;     /* ->next pointer of last "done" CB. */

        struct rcu_head **curtail;      /* ->next pointer of last CB. */

//        RCU_TRACE(long qlen);           /* Number of pending CBs. */

//        RCU_TRACE(unsigned long gp_start); /* Start time for stalls. */

//        RCU_TRACE(unsigned long ticks_this_gp); /* Statistic for stalls. */

//        RCU_TRACE(unsigned long jiffies_stall); /* Jiffies at next stall. */

//        RCU_TRACE(const char *name);    /* Name of RCU type. */

};

/* Definition for rcupdate control block. */

static struct rcu_ctrlblk rcu_sched_ctrlblk = {

        .donetail       = &rcu_sched_ctrlblk.rcucblist,

        .curtail        = &rcu_sched_ctrlblk.rcucblist,

//        RCU_TRACE(.name = "rcu_sched")

};

static void __call_rcu(struct rcu_head *head,

                       void (*func)(struct rcu_head *rcu),

                       struct rcu_ctrlblk *rcp)

{

        unsigned long flags;

//        debug_rcu_head_queue(head);

        head->func = func;

        head->next = NULL;

        local_irq_save(flags);

        *rcp->curtail = head;

        rcp->curtail = &head->next;

//        RCU_TRACE(rcp->qlen++);

        local_irq_restore(flags);

}

/*

 * Post an RCU callback to be invoked after the end of an RCU-sched grace

 * period.  But since we have but one CPU, that would be after any

 * quiescent state.

 */

void call_rcu_sched(struct rcu_head *head, void (*func)(struct rcu_head *rcu))

{

        __call_rcu(head, func, &rcu_sched_ctrlblk);

}

fb_get_options(const char *name, char **option)

{

    return 1;

}

ktime_t ktime_get(void)

{

    ktime_t t;

    t.tv64 = GetClockNs();

    return t;

}

void radeon_cursor_reset(struct drm_crtc *crtc)

{

}

/* Greatest common divisor */

unsigned long gcd(unsigned long a, unsigned long b)

{

        unsigned long r;

        if (a < b)

                swap(a, b);

        if (!b)

                return a;

        while ((r = a % b) != 0) {

                a = b;

                b = r;

        }

        return b;

}

void vfree(const void *addr)

{

    KernelFree(addr);

}