Rev 1408 | Go to most recent revision | Show entire file | Regard whitespace | Details | Blame | Last modification | View Log | RSS feed
| Rev 1408 | Rev 3031 | ||
|---|---|---|---|
| Line 13... | Line 13... | ||
| 13 | #endif |
13 | #endif |
| Line 14... | Line 14... | ||
| 14 | 14 | ||
| 15 | #include |
15 | #include |
| Line -... | Line 16... | ||
| - | 16 | #include |
|
| - | 17 | ||
| 16 | #include |
18 | #define BIT_64(n) (U64_C(1) << (n)) |
| 17 | 19 | ||
| 18 | /* |
20 | /* |
| 19 | * These have to be done with inline assembly: that way the bit-setting |
21 | * These have to be done with inline assembly: that way the bit-setting |
| 20 | * is guaranteed to be atomic. All bit operations return 0 if the bit |
22 | * is guaranteed to be atomic. All bit operations return 0 if the bit |
| Line 260... | Line 262... | ||
| 260 | * @addr: Address to count from |
262 | * @addr: Address to count from |
| 261 | * |
263 | * |
| 262 | * This operation is non-atomic and can be reordered. |
264 | * This operation is non-atomic and can be reordered. |
| 263 | * If two examples of this operation race, one can appear to succeed |
265 | * If two examples of this operation race, one can appear to succeed |
| 264 | * but actually fail. You must protect multiple accesses with a lock. |
266 | * but actually fail. You must protect multiple accesses with a lock. |
| - | 267 | * |
|
| - | 268 | * Note: the operation is performed atomically with respect to |
|
| - | 269 | * the local CPU, but not other CPUs. Portable code should not |
|
| - | 270 | * rely on this behaviour. |
|
| - | 271 | * KVM relies on this behaviour on x86 for modifying memory that is also |
|
| - | 272 | * accessed from a hypervisor on the same CPU if running in a VM: don't change |
|
| - | 273 | * this without also updating arch/x86/kernel/kvm.c |
|
| 265 | */ |
274 | */ |
| 266 | static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr) |
275 | static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr) |
| 267 | { |
276 | { |
| 268 | int oldbit; |
277 | int oldbit; |
| Line 307... | Line 316... | ||
| 307 | } |
316 | } |
| Line 308... | Line 317... | ||
| 308 | 317 | ||
| 309 | static __always_inline int constant_test_bit(unsigned int nr, const volatile unsigned long *addr) |
318 | static __always_inline int constant_test_bit(unsigned int nr, const volatile unsigned long *addr) |
| 310 | { |
319 | { |
| 311 | return ((1UL << (nr % BITS_PER_LONG)) & |
320 | return ((1UL << (nr % BITS_PER_LONG)) & |
| 312 | (((unsigned long *)addr)[nr / BITS_PER_LONG])) != 0; |
321 | (addr[nr / BITS_PER_LONG])) != 0; |
| Line 313... | Line 322... | ||
| 313 | } |
322 | } |
| 314 | 323 | ||
| 315 | static inline int variable_test_bit(int nr, volatile const unsigned long *addr) |
324 | static inline int variable_test_bit(int nr, volatile const unsigned long *addr) |
| Line 344... | Line 353... | ||
| 344 | * |
353 | * |
| 345 | * Undefined if no bit exists, so code should check against 0 first. |
354 | * Undefined if no bit exists, so code should check against 0 first. |
| 346 | */ |
355 | */ |
| 347 | static inline unsigned long __ffs(unsigned long word) |
356 | static inline unsigned long __ffs(unsigned long word) |
| 348 | { |
357 | { |
| 349 | asm("bsf %1,%0" |
358 | asm("rep; bsf %1,%0" |
| 350 | : "=r" (word) |
359 | : "=r" (word) |
| 351 | : "rm" (word)); |
360 | : "rm" (word)); |
| 352 | return word; |
361 | return word; |
| 353 | } |
362 | } |
| Line 358... | Line 367... | ||
| 358 | * |
367 | * |
| 359 | * Undefined if no zero exists, so code should check against ~0UL first. |
368 | * Undefined if no zero exists, so code should check against ~0UL first. |
| 360 | */ |
369 | */ |
| 361 | static inline unsigned long ffz(unsigned long word) |
370 | static inline unsigned long ffz(unsigned long word) |
| 362 | { |
371 | { |
| 363 | asm("bsf %1,%0" |
372 | asm("rep; bsf %1,%0" |
| 364 | : "=r" (word) |
373 | : "=r" (word) |
| 365 | : "r" (~word)); |
374 | : "r" (~word)); |
| 366 | return word; |
375 | return word; |
| 367 | } |
376 | } |
| Line 378... | Line 387... | ||
| 378 | : "=r" (word) |
387 | : "=r" (word) |
| 379 | : "rm" (word)); |
388 | : "rm" (word)); |
| 380 | return word; |
389 | return word; |
| 381 | } |
390 | } |
| Line -... | Line 391... | ||
| - | 391 | ||
| - | 392 | #undef ADDR |
|
| 382 | 393 | ||
| 383 | #ifdef __KERNEL__ |
394 | #ifdef __KERNEL__ |
| 384 | /** |
395 | /** |
| 385 | * ffs - find first set bit in word |
396 | * ffs - find first set bit in word |
| 386 | * @x: the word to search |
397 | * @x: the word to search |
| Line 396... | Line 407... | ||
| 396 | { |
407 | { |
| 397 | int r; |
408 | int r; |
| 398 | #ifdef CONFIG_X86_CMOV |
409 | #ifdef CONFIG_X86_CMOV |
| 399 | asm("bsfl %1,%0\n\t" |
410 | asm("bsfl %1,%0\n\t" |
| 400 | "cmovzl %2,%0" |
411 | "cmovzl %2,%0" |
| 401 | : "=r" (r) : "rm" (x), "r" (-1)); |
412 | : "=&r" (r) : "rm" (x), "r" (-1)); |
| 402 | #else |
413 | #else |
| 403 | asm("bsfl %1,%0\n\t" |
414 | asm("bsfl %1,%0\n\t" |
| 404 | "jnz 1f\n\t" |
415 | "jnz 1f\n\t" |
| 405 | "movl $-1,%0\n" |
416 | "movl $-1,%0\n" |
| 406 | "1:" : "=r" (r) : "rm" (x)); |
417 | "1:" : "=r" (r) : "rm" (x)); |