Subversion Repositories Kolibri OS

Rev

Rev 5078 | Rev 6104 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
5078 serge 1
#include 
2
#include 
3
#include 
4
#include 
5271 serge 5
#include "radeon.h"
5078 serge 6
 
7
int x86_clflush_size;
8
unsigned int tsc_khz;
9
 
10
struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
11
{
12
    struct file *filep;
13
    int count;
14
 
5271 serge 15
    filep = __builtin_malloc(sizeof(*filep));
5078 serge 16
 
17
    if(unlikely(filep == NULL))
18
        return ERR_PTR(-ENOMEM);
19
 
20
    count = size / PAGE_SIZE;
21
 
22
    filep->pages = kzalloc(sizeof(struct page *) * count, 0);
23
    if(unlikely(filep->pages == NULL))
24
    {
25
        kfree(filep);
26
        return ERR_PTR(-ENOMEM);
27
    };
28
 
29
    filep->count     = count;
30
    filep->allocated = 0;
31
    filep->vma       = NULL;
32
 
33
//    printf("%s file %p pages %p count %d\n",
34
//              __FUNCTION__,filep, filep->pages, count);
35
 
36
    return filep;
37
}
38
 
39
static void *check_bytes8(const u8 *start, u8 value, unsigned int bytes)
40
{
41
        while (bytes) {
42
                if (*start != value)
43
                        return (void *)start;
44
                start++;
45
                bytes--;
46
        }
47
        return NULL;
48
}
49
 
50
/**
51
 * memchr_inv - Find an unmatching character in an area of memory.
52
 * @start: The memory area
53
 * @c: Find a character other than c
54
 * @bytes: The size of the area.
55
 *
56
 * returns the address of the first character other than @c, or %NULL
57
 * if the whole buffer contains just @c.
58
 */
59
void *memchr_inv(const void *start, int c, size_t bytes)
60
{
61
        u8 value = c;
62
        u64 value64;
63
        unsigned int words, prefix;
64
 
65
        if (bytes <= 16)
66
                return check_bytes8(start, value, bytes);
67
 
68
        value64 = value;
69
#if defined(ARCH_HAS_FAST_MULTIPLIER) && BITS_PER_LONG == 64
70
        value64 *= 0x0101010101010101;
71
#elif defined(ARCH_HAS_FAST_MULTIPLIER)
72
        value64 *= 0x01010101;
73
        value64 |= value64 << 32;
74
#else
75
        value64 |= value64 << 8;
76
        value64 |= value64 << 16;
77
        value64 |= value64 << 32;
78
#endif
79
 
80
        prefix = (unsigned long)start % 8;
81
        if (prefix) {
82
                u8 *r;
83
 
84
                prefix = 8 - prefix;
85
                r = check_bytes8(start, value, prefix);
86
                if (r)
87
                        return r;
88
                start += prefix;
89
                bytes -= prefix;
90
        }
91
 
92
        words = bytes / 8;
93
 
94
        while (words) {
95
                if (*(u64 *)start != value64)
96
                        return check_bytes8(start, value, 8);
97
                start += 8;
98
                words--;
99
        }
100
 
101
        return check_bytes8(start, value, bytes % 8);
102
}
103
 
104
 
105
 
106
#define _U  0x01    /* upper */
107
#define _L  0x02    /* lower */
108
#define _D  0x04    /* digit */
109
#define _C  0x08    /* cntrl */
110
#define _P  0x10    /* punct */
111
#define _S  0x20    /* white space (space/lf/tab) */
112
#define _X  0x40    /* hex digit */
113
#define _SP 0x80    /* hard space (0x20) */
114
 
115
extern const unsigned char _ctype[];
116
 
117
#define __ismask(x) (_ctype[(int)(unsigned char)(x)])
118
 
119
#define isalnum(c)  ((__ismask(c)&(_U|_L|_D)) != 0)
120
#define isalpha(c)  ((__ismask(c)&(_U|_L)) != 0)
121
#define iscntrl(c)  ((__ismask(c)&(_C)) != 0)
122
#define isdigit(c)  ((__ismask(c)&(_D)) != 0)
123
#define isgraph(c)  ((__ismask(c)&(_P|_U|_L|_D)) != 0)
124
#define islower(c)  ((__ismask(c)&(_L)) != 0)
125
#define isprint(c)  ((__ismask(c)&(_P|_U|_L|_D|_SP)) != 0)
126
#define ispunct(c)  ((__ismask(c)&(_P)) != 0)
127
/* Note: isspace() must return false for %NUL-terminator */
128
#define isspace(c)  ((__ismask(c)&(_S)) != 0)
129
#define isupper(c)  ((__ismask(c)&(_U)) != 0)
130
#define isxdigit(c) ((__ismask(c)&(_D|_X)) != 0)
131
 
132
#define isascii(c) (((unsigned char)(c))<=0x7f)
133
#define toascii(c) (((unsigned char)(c))&0x7f)
134
 
135
static inline unsigned char __tolower(unsigned char c)
136
{
137
    if (isupper(c))
138
        c -= 'A'-'a';
139
    return c;
140
}
141
 
142
static inline unsigned char __toupper(unsigned char c)
143
{
144
    if (islower(c))
145
        c -= 'a'-'A';
146
    return c;
147
}
148
 
149
#define tolower(c) __tolower(c)
150
#define toupper(c) __toupper(c)
151
 
152
/*
153
 * Fast implementation of tolower() for internal usage. Do not use in your
154
 * code.
155
 */
156
static inline char _tolower(const char c)
157
{
158
    return c | 0x20;
159
}
160
 
161
 
162
//const char hex_asc[] = "0123456789abcdef";
163
 
164
/**
165
 * hex_to_bin - convert a hex digit to its real value
166
 * @ch: ascii character represents hex digit
167
 *
168
 * hex_to_bin() converts one hex digit to its actual value or -1 in case of bad
169
 * input.
170
 */
171
int hex_to_bin(char ch)
172
{
173
    if ((ch >= '0') && (ch <= '9'))
174
        return ch - '0';
175
    ch = tolower(ch);
176
    if ((ch >= 'a') && (ch <= 'f'))
177
        return ch - 'a' + 10;
178
    return -1;
179
}
180
EXPORT_SYMBOL(hex_to_bin);
181
 
182
/**
183
 * hex2bin - convert an ascii hexadecimal string to its binary representation
184
 * @dst: binary result
185
 * @src: ascii hexadecimal string
186
 * @count: result length
187
 *
188
 * Return 0 on success, -1 in case of bad input.
189
 */
190
int hex2bin(u8 *dst, const char *src, size_t count)
191
{
192
    while (count--) {
193
        int hi = hex_to_bin(*src++);
194
        int lo = hex_to_bin(*src++);
195
 
196
        if ((hi < 0) || (lo < 0))
197
            return -1;
198
 
199
        *dst++ = (hi << 4) | lo;
200
    }
201
    return 0;
202
}
203
EXPORT_SYMBOL(hex2bin);
204
 
205
/**
206
 * hex_dump_to_buffer - convert a blob of data to "hex ASCII" in memory
207
 * @buf: data blob to dump
208
 * @len: number of bytes in the @buf
209
 * @rowsize: number of bytes to print per line; must be 16 or 32
210
 * @groupsize: number of bytes to print at a time (1, 2, 4, 8; default = 1)
211
 * @linebuf: where to put the converted data
212
 * @linebuflen: total size of @linebuf, including space for terminating NUL
213
 * @ascii: include ASCII after the hex output
214
 *
215
 * hex_dump_to_buffer() works on one "line" of output at a time, i.e.,
216
 * 16 or 32 bytes of input data converted to hex + ASCII output.
217
 *
218
 * Given a buffer of u8 data, hex_dump_to_buffer() converts the input data
219
 * to a hex + ASCII dump at the supplied memory location.
220
 * The converted output is always NUL-terminated.
221
 *
222
 * E.g.:
223
 *   hex_dump_to_buffer(frame->data, frame->len, 16, 1,
224
 *          linebuf, sizeof(linebuf), true);
225
 *
226
 * example output buffer:
227
 * 40 41 42 43 44 45 46 47 48 49 4a 4b 4c 4d 4e 4f  @ABCDEFGHIJKLMNO
228
 */
229
void hex_dump_to_buffer(const void *buf, size_t len, int rowsize,
230
            int groupsize, char *linebuf, size_t linebuflen,
231
            bool ascii)
232
{
233
    const u8 *ptr = buf;
234
    u8 ch;
235
    int j, lx = 0;
236
    int ascii_column;
237
 
238
    if (rowsize != 16 && rowsize != 32)
239
        rowsize = 16;
240
 
241
    if (!len)
242
        goto nil;
243
    if (len > rowsize)      /* limit to one line at a time */
244
        len = rowsize;
245
    if ((len % groupsize) != 0) /* no mixed size output */
246
        groupsize = 1;
247
 
248
    switch (groupsize) {
249
    case 8: {
250
        const u64 *ptr8 = buf;
251
        int ngroups = len / groupsize;
252
 
253
        for (j = 0; j < ngroups; j++)
254
            lx += scnprintf(linebuf + lx, linebuflen - lx,
255
                    "%s%16.16llx", j ? " " : "",
256
                    (unsigned long long)*(ptr8 + j));
257
        ascii_column = 17 * ngroups + 2;
258
        break;
259
    }
260
 
261
    case 4: {
262
        const u32 *ptr4 = buf;
263
        int ngroups = len / groupsize;
264
 
265
        for (j = 0; j < ngroups; j++)
266
            lx += scnprintf(linebuf + lx, linebuflen - lx,
267
                    "%s%8.8x", j ? " " : "", *(ptr4 + j));
268
        ascii_column = 9 * ngroups + 2;
269
        break;
270
    }
271
 
272
    case 2: {
273
        const u16 *ptr2 = buf;
274
        int ngroups = len / groupsize;
275
 
276
        for (j = 0; j < ngroups; j++)
277
            lx += scnprintf(linebuf + lx, linebuflen - lx,
278
                    "%s%4.4x", j ? " " : "", *(ptr2 + j));
279
        ascii_column = 5 * ngroups + 2;
280
        break;
281
    }
282
 
283
    default:
284
        for (j = 0; (j < len) && (lx + 3) <= linebuflen; j++) {
285
            ch = ptr[j];
286
            linebuf[lx++] = hex_asc_hi(ch);
287
            linebuf[lx++] = hex_asc_lo(ch);
288
            linebuf[lx++] = ' ';
289
        }
290
        if (j)
291
            lx--;
292
 
293
        ascii_column = 3 * rowsize + 2;
294
        break;
295
    }
296
    if (!ascii)
297
        goto nil;
298
 
299
    while (lx < (linebuflen - 1) && lx < (ascii_column - 1))
300
        linebuf[lx++] = ' ';
301
    for (j = 0; (j < len) && (lx + 2) < linebuflen; j++) {
302
        ch = ptr[j];
303
        linebuf[lx++] = (isascii(ch) && isprint(ch)) ? ch : '.';
304
    }
305
nil:
306
    linebuf[lx++] = '\0';
307
}
308
 
309
/**
310
 * print_hex_dump - print a text hex dump to syslog for a binary blob of data
311
 * @level: kernel log level (e.g. KERN_DEBUG)
312
 * @prefix_str: string to prefix each line with;
313
 *  caller supplies trailing spaces for alignment if desired
314
 * @prefix_type: controls whether prefix of an offset, address, or none
315
 *  is printed (%DUMP_PREFIX_OFFSET, %DUMP_PREFIX_ADDRESS, %DUMP_PREFIX_NONE)
316
 * @rowsize: number of bytes to print per line; must be 16 or 32
317
 * @groupsize: number of bytes to print at a time (1, 2, 4, 8; default = 1)
318
 * @buf: data blob to dump
319
 * @len: number of bytes in the @buf
320
 * @ascii: include ASCII after the hex output
321
 *
322
 * Given a buffer of u8 data, print_hex_dump() prints a hex + ASCII dump
323
 * to the kernel log at the specified kernel log level, with an optional
324
 * leading prefix.
325
 *
326
 * print_hex_dump() works on one "line" of output at a time, i.e.,
327
 * 16 or 32 bytes of input data converted to hex + ASCII output.
328
 * print_hex_dump() iterates over the entire input @buf, breaking it into
329
 * "line size" chunks to format and print.
330
 *
331
 * E.g.:
332
 *   print_hex_dump(KERN_DEBUG, "raw data: ", DUMP_PREFIX_ADDRESS,
333
 *          16, 1, frame->data, frame->len, true);
334
 *
335
 * Example output using %DUMP_PREFIX_OFFSET and 1-byte mode:
336
 * 0009ab42: 40 41 42 43 44 45 46 47 48 49 4a 4b 4c 4d 4e 4f  @ABCDEFGHIJKLMNO
337
 * Example output using %DUMP_PREFIX_ADDRESS and 4-byte mode:
338
 * ffffffff88089af0: 73727170 77767574 7b7a7978 7f7e7d7c  pqrstuvwxyz{|}~.
339
 */
340
void print_hex_dump(const char *level, const char *prefix_str, int prefix_type,
341
            int rowsize, int groupsize,
342
            const void *buf, size_t len, bool ascii)
343
{
344
    const u8 *ptr = buf;
345
    int i, linelen, remaining = len;
346
    unsigned char linebuf[32 * 3 + 2 + 32 + 1];
347
 
348
    if (rowsize != 16 && rowsize != 32)
349
        rowsize = 16;
350
 
351
    for (i = 0; i < len; i += rowsize) {
352
        linelen = min(remaining, rowsize);
353
        remaining -= rowsize;
354
 
355
        hex_dump_to_buffer(ptr + i, linelen, rowsize, groupsize,
356
                   linebuf, sizeof(linebuf), ascii);
357
 
358
        switch (prefix_type) {
359
        case DUMP_PREFIX_ADDRESS:
360
            printk("%s%s%p: %s\n",
361
                   level, prefix_str, ptr + i, linebuf);
362
            break;
363
        case DUMP_PREFIX_OFFSET:
364
            printk("%s%s%.8x: %s\n", level, prefix_str, i, linebuf);
365
            break;
366
        default:
367
            printk("%s%s%s\n", level, prefix_str, linebuf);
368
            break;
369
        }
370
    }
371
}
372
 
373
void print_hex_dump_bytes(const char *prefix_str, int prefix_type,
374
                          const void *buf, size_t len)
375
{
376
    print_hex_dump(KERN_DEBUG, prefix_str, prefix_type, 16, 1,
377
                       buf, len, true);
378
}
379
 
5271 serge 380
void msleep(unsigned int msecs)
381
{
382
    msecs /= 10;
383
    if(!msecs) msecs = 1;
5078 serge 384
 
5271 serge 385
     __asm__ __volatile__ (
386
     "call *__imp__Delay"
387
     ::"b" (msecs));
388
     __asm__ __volatile__ (
389
     "":::"ebx");
390
 
391
};
392
 
393
 
394
/* simple loop based delay: */
395
static void delay_loop(unsigned long loops)
5078 serge 396
{
5271 serge 397
        asm volatile(
398
                "       test %0,%0      \n"
399
                "       jz 3f           \n"
400
                "       jmp 1f          \n"
401
 
402
                ".align 16              \n"
403
                "1:     jmp 2f          \n"
404
 
405
                ".align 16              \n"
406
                "2:     dec %0          \n"
407
                "       jnz 2b          \n"
408
                "3:     dec %0          \n"
409
 
410
                : /* we don't need output */
411
                :"a" (loops)
412
        );
5078 serge 413
}
414
 
5271 serge 415
 
416
static void (*delay_fn)(unsigned long) = delay_loop;
417
 
418
void __delay(unsigned long loops)
5078 serge 419
{
5271 serge 420
        delay_fn(loops);
5078 serge 421
}
422
 
5271 serge 423
 
424
inline void __const_udelay(unsigned long xloops)
5078 serge 425
{
5271 serge 426
        int d0;
5078 serge 427
 
5271 serge 428
        xloops *= 4;
429
        asm("mull %%edx"
430
                : "=d" (xloops), "=&a" (d0)
431
                : "1" (xloops), ""
432
                (loops_per_jiffy * (HZ/4)));
5078 serge 433
 
5271 serge 434
        __delay(++xloops);
435
}
5078 serge 436
 
5271 serge 437
void __udelay(unsigned long usecs)
438
{
439
        __const_udelay(usecs * 0x000010c7); /* 2**32 / 1000000 (rounded up) */
5078 serge 440
}
441
 
5271 serge 442
unsigned int _sw_hweight32(unsigned int w)
443
{
444
#ifdef CONFIG_ARCH_HAS_FAST_MULTIPLIER
445
        w -= (w >> 1) & 0x55555555;
446
        w =  (w & 0x33333333) + ((w >> 2) & 0x33333333);
447
        w =  (w + (w >> 4)) & 0x0f0f0f0f;
448
        return (w * 0x01010101) >> 24;
449
#else
450
        unsigned int res = w - ((w >> 1) & 0x55555555);
451
        res = (res & 0x33333333) + ((res >> 2) & 0x33333333);
452
        res = (res + (res >> 4)) & 0x0F0F0F0F;
453
        res = res + (res >> 8);
454
        return (res + (res >> 16)) & 0x000000FF;
455
#endif
456
}
457
EXPORT_SYMBOL(_sw_hweight32);
5078 serge 458
 
459
 
5271 serge 460
void usleep_range(unsigned long min, unsigned long max)
461
{
462
    udelay(max);
463
}
464
EXPORT_SYMBOL(usleep_range);
465
 
466
 
5078 serge 467
void *kmemdup(const void *src, size_t len, gfp_t gfp)
468
{
469
    void *p;
470
 
471
    p = kmalloc(len, gfp);
472
    if (p)
473
        memcpy(p, src, len);
474
    return p;
475
}
476
 
5271 serge 477
void cpu_detect1()
478
{
5078 serge 479
 
5271 serge 480
    u32 junk, tfms, cap0, misc;
481
    int i;
482
 
483
    cpuid(0x00000001, &tfms, &misc, &junk, &cap0);
484
 
485
    if (cap0 & (1<<19))
486
    {
487
        x86_clflush_size = ((misc >> 8) & 0xff) * 8;
488
    }
489
 
490
#if 0
491
    cpuid(0x80000002, (unsigned int*)&cpuinfo.model_name[0], (unsigned int*)&cpuinfo.model_name[4],
492
          (unsigned int*)&cpuinfo.model_name[8], (unsigned int*)&cpuinfo.model_name[12]);
493
    cpuid(0x80000003, (unsigned int*)&cpuinfo.model_name[16], (unsigned int*)&cpuinfo.model_name[20],
494
          (unsigned int*)&cpuinfo.model_name[24], (unsigned int*)&cpuinfo.model_name[28]);
495
    cpuid(0x80000004, (unsigned int*)&cpuinfo.model_name[32], (unsigned int*)&cpuinfo.model_name[36],
496
          (unsigned int*)&cpuinfo.model_name[40], (unsigned int*)&cpuinfo.model_name[44]);
497
 
498
    printf("\n%s\n\n",cpuinfo.model_name);
499
 
500
    cpuinfo.def_mtrr = read_msr(MSR_MTRRdefType);
501
    cpuinfo.mtrr_cap = read_msr(IA32_MTRRCAP);
502
 
503
    printf("MSR_MTRRdefType %016llx\n\n", cpuinfo.def_mtrr);
504
 
505
    cpuinfo.var_mtrr_count = (u8_t)cpuinfo.mtrr_cap;
506
 
507
    for(i = 0; i < cpuinfo.var_mtrr_count; i++)
508
    {
509
        u64_t mtrr_base;
510
        u64_t mtrr_mask;
511
 
512
        cpuinfo.var_mtrr[i].base = read_msr(MTRRphysBase_MSR(i));
513
        cpuinfo.var_mtrr[i].mask = read_msr(MTRRphysMask_MSR(i));
514
 
515
        printf("MTRR_%d base: %016llx mask: %016llx\n", i,
516
               cpuinfo.var_mtrr[i].base,
517
               cpuinfo.var_mtrr[i].mask);
518
    };
519
 
520
    unsigned int cr0, cr3, cr4, eflags;
521
 
522
    eflags = safe_cli();
523
 
524
    /* Enter the no-fill (CD=1, NW=0) cache mode and flush caches. */
525
    cr0 = read_cr0() | (1<<30);
526
    write_cr0(cr0);
527
    wbinvd();
528
 
529
    cr4 = read_cr4();
530
    write_cr4(cr4 & ~(1<<7));
531
 
532
    cr3 = read_cr3();
533
    write_cr3(cr3);
534
 
535
    /* Save MTRR state */
536
    rdmsr(MSR_MTRRdefType, deftype_lo, deftype_hi);
537
 
538
    /* Disable MTRRs, and set the default type to uncached */
539
    native_write_msr(MSR_MTRRdefType, deftype_lo & ~0xcff, deftype_hi);
540
    wbinvd();
541
 
542
    i = 0;
543
    set_mtrr(i++,0,0x80000000>>12,MTRR_WB);
544
    set_mtrr(i++,0x80000000>>12,0x40000000>>12,MTRR_WB);
545
    set_mtrr(i++,0xC0000000>>12,0x20000000>>12,MTRR_WB);
546
    set_mtrr(i++,0xdb800000>>12,0x00800000>>12,MTRR_UC);
547
    set_mtrr(i++,0xdc000000>>12,0x04000000>>12,MTRR_UC);
548
    set_mtrr(i++,0xE0000000>>12,0x10000000>>12,MTRR_WC);
549
 
550
    for(; i < cpuinfo.var_mtrr_count; i++)
551
        set_mtrr(i,0,0,0);
552
 
553
    write_cr3(cr3);
554
 
555
    /* Intel (P6) standard MTRRs */
556
    native_write_msr(MSR_MTRRdefType, deftype_lo, deftype_hi);
557
 
558
    /* Enable caches */
559
    write_cr0(read_cr0() & ~(1<<30));
560
 
561
    /* Restore value of CR4 */
562
    write_cr4(cr4);
563
 
564
    safe_sti(eflags);
565
 
566
    printf("\nnew MTRR map\n\n");
567
 
568
    for(i = 0; i < cpuinfo.var_mtrr_count; i++)
569
    {
570
        u64_t mtrr_base;
571
        u64_t mtrr_mask;
572
 
573
        cpuinfo.var_mtrr[i].base = read_msr(MTRRphysBase_MSR(i));
574
        cpuinfo.var_mtrr[i].mask = read_msr(MTRRphysMask_MSR(i));
575
 
576
        printf("MTRR_%d base: %016llx mask: %016llx\n", i,
577
               cpuinfo.var_mtrr[i].base,
578
               cpuinfo.var_mtrr[i].mask);
579
    };
580
#endif
581
 
582
    tsc_khz = (unsigned int)(GetCpuFreq()/1000);
583
}
584
 
585
 
586
static atomic_t fence_context_counter = ATOMIC_INIT(0);
587
 
588
/**
589
 * fence_context_alloc - allocate an array of fence contexts
590
 * @num:        [in]    amount of contexts to allocate
591
 *
592
 * This function will return the first index of the number of fences allocated.
593
 * The fence context is used for setting fence->context to a unique number.
594
 */
595
unsigned fence_context_alloc(unsigned num)
5078 serge 596
{
5271 serge 597
        BUG_ON(!num);
598
        return atomic_add_return(num, &fence_context_counter) - num;
599
}
600
EXPORT_SYMBOL(fence_context_alloc);
5078 serge 601
 
5271 serge 602
 
603
int fence_signal(struct fence *fence)
604
{
605
        unsigned long flags;
606
 
607
        if (!fence)
608
                return -EINVAL;
609
 
610
//        if (!ktime_to_ns(fence->timestamp)) {
611
//                fence->timestamp = ktime_get();
612
//                smp_mb__before_atomic();
613
//        }
614
 
615
        if (test_and_set_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags))
616
                return -EINVAL;
617
 
618
//        trace_fence_signaled(fence);
619
 
620
        if (test_bit(FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags)) {
621
                struct fence_cb *cur, *tmp;
622
 
623
                spin_lock_irqsave(fence->lock, flags);
624
                list_for_each_entry_safe(cur, tmp, &fence->cb_list, node) {
625
                        list_del_init(&cur->node);
626
                        cur->func(fence, cur);
627
                }
628
                spin_unlock_irqrestore(fence->lock, flags);
5078 serge 629
        }
5271 serge 630
        return 0;
631
}
632
EXPORT_SYMBOL(fence_signal);
5078 serge 633
 
5271 serge 634
int fence_signal_locked(struct fence *fence)
635
{
636
        struct fence_cb *cur, *tmp;
637
        int ret = 0;
638
 
639
        if (WARN_ON(!fence))
640
                return -EINVAL;
641
 
642
//        if (!ktime_to_ns(fence->timestamp)) {
643
//                fence->timestamp = ktime_get();
644
//                smp_mb__before_atomic();
645
//        }
646
 
647
        if (test_and_set_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
648
                ret = -EINVAL;
649
 
650
                /*
651
                 * we might have raced with the unlocked fence_signal,
652
                 * still run through all callbacks
653
                 */
654
        }// else
655
//                trace_fence_signaled(fence);
656
 
657
        list_for_each_entry_safe(cur, tmp, &fence->cb_list, node) {
658
                list_del_init(&cur->node);
659
                cur->func(fence, cur);
660
        }
661
        return ret;
5078 serge 662
}
5271 serge 663
EXPORT_SYMBOL(fence_signal_locked);
5078 serge 664
 
5271 serge 665
 
666
void fence_enable_sw_signaling(struct fence *fence)
667
{
668
        unsigned long flags;
669
 
670
        if (!test_and_set_bit(FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags) &&
671
            !test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
672
//                trace_fence_enable_signal(fence);
673
 
674
                spin_lock_irqsave(fence->lock, flags);
675
 
676
                if (!fence->ops->enable_signaling(fence))
677
                        fence_signal_locked(fence);
678
 
679
                spin_unlock_irqrestore(fence->lock, flags);
680
        }
681
}
682
EXPORT_SYMBOL(fence_enable_sw_signaling);
683
 
684
 
685
 
686
signed long
687
fence_wait_timeout(struct fence *fence, bool intr, signed long timeout)
688
{
689
        signed long ret;
690
 
691
        if (WARN_ON(timeout < 0))
692
                return -EINVAL;
693
 
694
//        trace_fence_wait_start(fence);
695
        ret = fence->ops->wait(fence, intr, timeout);
696
//        trace_fence_wait_end(fence);
697
        return ret;
698
}
699
EXPORT_SYMBOL(fence_wait_timeout);
700
 
701
void fence_release(struct kref *kref)
702
{
703
        struct fence *fence =
704
                        container_of(kref, struct fence, refcount);
705
 
706
//        trace_fence_destroy(fence);
707
 
708
        BUG_ON(!list_empty(&fence->cb_list));
709
 
710
        if (fence->ops->release)
711
                fence->ops->release(fence);
712
        else
713
                fence_free(fence);
714
}
715
EXPORT_SYMBOL(fence_release);
716
 
717
void fence_free(struct fence *fence)
718
{
719
        kfree_rcu(fence, rcu);
720
}
721
EXPORT_SYMBOL(fence_free);
722
 
723
 
724
reservation_object_add_shared_inplace(struct reservation_object *obj,
725
                                      struct reservation_object_list *fobj,
726
                                      struct fence *fence)
727
{
728
        u32 i;
729
 
730
        fence_get(fence);
731
 
732
//        preempt_disable();
733
        write_seqcount_begin(&obj->seq);
734
 
735
        for (i = 0; i < fobj->shared_count; ++i) {
736
                struct fence *old_fence;
737
 
738
                old_fence = rcu_dereference_protected(fobj->shared[i],
739
                                                reservation_object_held(obj));
740
 
741
                if (old_fence->context == fence->context) {
742
                        /* memory barrier is added by write_seqcount_begin */
743
                        RCU_INIT_POINTER(fobj->shared[i], fence);
744
                        write_seqcount_end(&obj->seq);
745
                        preempt_enable();
746
 
747
                        fence_put(old_fence);
748
                        return;
749
                }
750
        }
751
 
752
        /*
753
         * memory barrier is added by write_seqcount_begin,
754
         * fobj->shared_count is protected by this lock too
755
         */
756
        RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence);
757
        fobj->shared_count++;
758
 
759
        write_seqcount_end(&obj->seq);
760
//        preempt_enable();
761
}
762
 
763
 
764
 
765
static void
766
reservation_object_add_shared_replace(struct reservation_object *obj,
767
                                      struct reservation_object_list *old,
768
                                      struct reservation_object_list *fobj,
769
                                      struct fence *fence)
770
{
771
        unsigned i;
772
        struct fence *old_fence = NULL;
773
 
774
        fence_get(fence);
775
 
776
        if (!old) {
777
                RCU_INIT_POINTER(fobj->shared[0], fence);
778
                fobj->shared_count = 1;
779
                goto done;
780
        }
781
 
782
        /*
783
         * no need to bump fence refcounts, rcu_read access
784
         * requires the use of kref_get_unless_zero, and the
785
         * references from the old struct are carried over to
786
         * the new.
787
         */
788
        fobj->shared_count = old->shared_count;
789
 
790
        for (i = 0; i < old->shared_count; ++i) {
791
                struct fence *check;
792
 
793
                check = rcu_dereference_protected(old->shared[i],
794
                                                reservation_object_held(obj));
795
 
796
                if (!old_fence && check->context == fence->context) {
797
                        old_fence = check;
798
                        RCU_INIT_POINTER(fobj->shared[i], fence);
799
                } else
800
                        RCU_INIT_POINTER(fobj->shared[i], check);
801
        }
802
        if (!old_fence) {
803
                RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence);
804
                fobj->shared_count++;
805
        }
806
 
807
done:
808
//        preempt_disable();
809
        write_seqcount_begin(&obj->seq);
810
        /*
811
         * RCU_INIT_POINTER can be used here,
812
         * seqcount provides the necessary barriers
813
         */
814
        RCU_INIT_POINTER(obj->fence, fobj);
815
        write_seqcount_end(&obj->seq);
816
//        preempt_enable();
817
 
818
        if (old)
819
                kfree_rcu(old, rcu);
820
 
821
        if (old_fence)
822
                fence_put(old_fence);
823
}
824
 
825
 
826
int reservation_object_reserve_shared(struct reservation_object *obj)
827
{
828
        struct reservation_object_list *fobj, *old;
829
        u32 max;
830
 
831
        old = reservation_object_get_list(obj);
832
 
833
        if (old && old->shared_max) {
834
                if (old->shared_count < old->shared_max) {
835
                        /* perform an in-place update */
836
                        kfree(obj->staged);
837
                        obj->staged = NULL;
838
                        return 0;
839
                } else
840
                        max = old->shared_max * 2;
841
        } else
842
                max = 4;
843
 
844
        /*
845
         * resize obj->staged or allocate if it doesn't exist,
846
         * noop if already correct size
847
         */
848
        fobj = krealloc(obj->staged, offsetof(typeof(*fobj), shared[max]),
849
                        GFP_KERNEL);
850
        if (!fobj)
851
                return -ENOMEM;
852
 
853
        obj->staged = fobj;
854
        fobj->shared_max = max;
855
        return 0;
856
}
857
EXPORT_SYMBOL(reservation_object_reserve_shared);
858
 
859
void reservation_object_add_shared_fence(struct reservation_object *obj,
860
                                         struct fence *fence)
861
{
862
        struct reservation_object_list *old, *fobj = obj->staged;
863
 
864
        old = reservation_object_get_list(obj);
865
        obj->staged = NULL;
866
 
867
        if (!fobj) {
868
                BUG_ON(old->shared_count >= old->shared_max);
869
                reservation_object_add_shared_inplace(obj, old, fence);
870
        } else
871
                reservation_object_add_shared_replace(obj, old, fobj, fence);
872
}
873
EXPORT_SYMBOL(reservation_object_add_shared_fence);
874
 
875
 
876
void reservation_object_add_excl_fence(struct reservation_object *obj,
877
                                       struct fence *fence)
878
{
879
        struct fence *old_fence = reservation_object_get_excl(obj);
880
        struct reservation_object_list *old;
881
        u32 i = 0;
882
 
883
        old = reservation_object_get_list(obj);
884
        if (old)
885
                i = old->shared_count;
886
 
887
        if (fence)
888
                fence_get(fence);
889
 
890
//        preempt_disable();
891
        write_seqcount_begin(&obj->seq);
892
        /* write_seqcount_begin provides the necessary memory barrier */
893
        RCU_INIT_POINTER(obj->fence_excl, fence);
894
        if (old)
895
                old->shared_count = 0;
896
        write_seqcount_end(&obj->seq);
897
//        preempt_enable();
898
 
899
        /* inplace update, no shared fences */
900
        while (i--)
901
                fence_put(rcu_dereference_protected(old->shared[i],
902
                                                reservation_object_held(obj)));
903
 
904
        if (old_fence)
905
                fence_put(old_fence);
906
}
907
EXPORT_SYMBOL(reservation_object_add_excl_fence);
908
 
909
void
910
fence_init(struct fence *fence, const struct fence_ops *ops,
911
             spinlock_t *lock, unsigned context, unsigned seqno)
912
{
913
        BUG_ON(!lock);
914
        BUG_ON(!ops || !ops->wait || !ops->enable_signaling ||
915
               !ops->get_driver_name || !ops->get_timeline_name);
916
 
917
        kref_init(&fence->refcount);
918
        fence->ops = ops;
919
        INIT_LIST_HEAD(&fence->cb_list);
920
        fence->lock = lock;
921
        fence->context = context;
922
        fence->seqno = seqno;
923
        fence->flags = 0UL;
924
 
925
//        trace_fence_init(fence);
926
}
927
EXPORT_SYMBOL(fence_init);
928
 
929
 
930
#include 
931
 
932
struct rcu_ctrlblk {
933
        struct rcu_head *rcucblist;     /* List of pending callbacks (CBs). */
934
        struct rcu_head **donetail;     /* ->next pointer of last "done" CB. */
935
        struct rcu_head **curtail;      /* ->next pointer of last CB. */
936
//        RCU_TRACE(long qlen);           /* Number of pending CBs. */
937
//        RCU_TRACE(unsigned long gp_start); /* Start time for stalls. */
938
//        RCU_TRACE(unsigned long ticks_this_gp); /* Statistic for stalls. */
939
//        RCU_TRACE(unsigned long jiffies_stall); /* Jiffies at next stall. */
940
//        RCU_TRACE(const char *name);    /* Name of RCU type. */
941
};
942
 
943
/* Definition for rcupdate control block. */
944
static struct rcu_ctrlblk rcu_sched_ctrlblk = {
945
        .donetail       = &rcu_sched_ctrlblk.rcucblist,
946
        .curtail        = &rcu_sched_ctrlblk.rcucblist,
947
//        RCU_TRACE(.name = "rcu_sched")
948
};
949
 
950
static void __call_rcu(struct rcu_head *head,
951
                       void (*func)(struct rcu_head *rcu),
952
                       struct rcu_ctrlblk *rcp)
953
{
954
        unsigned long flags;
955
 
956
//        debug_rcu_head_queue(head);
957
        head->func = func;
958
        head->next = NULL;
959
 
960
        local_irq_save(flags);
961
        *rcp->curtail = head;
962
        rcp->curtail = &head->next;
963
//        RCU_TRACE(rcp->qlen++);
964
        local_irq_restore(flags);
965
}
966
 
967
/*
968
 * Post an RCU callback to be invoked after the end of an RCU-sched grace
969
 * period.  But since we have but one CPU, that would be after any
970
 * quiescent state.
971
 */
972
void call_rcu_sched(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
973
{
974
        __call_rcu(head, func, &rcu_sched_ctrlblk);
975
}
976
 
977