Rev 5345 | Go to most recent revision | Details | Last modification | View Log | RSS feed
Rev | Author | Line No. | Line |
---|---|---|---|
5272 | serge | 1 | /* rwsem.h: R/W semaphores implemented using XADD/CMPXCHG for i486+ |
2 | * |
||
3 | * Written by David Howells (dhowells@redhat.com). |
||
4 | * |
||
5 | * Derived from asm-x86/semaphore.h |
||
6 | * |
||
7 | * |
||
8 | * The MSW of the count is the negated number of active writers and waiting |
||
9 | * lockers, and the LSW is the total number of active locks |
||
10 | * |
||
11 | * The lock count is initialized to 0 (no active and no waiting lockers). |
||
12 | * |
||
13 | * When a writer subtracts WRITE_BIAS, it'll get 0xffff0001 for the case of an |
||
14 | * uncontended lock. This can be determined because XADD returns the old value. |
||
15 | * Readers increment by 1 and see a positive value when uncontended, negative |
||
16 | * if there are writers (and maybe) readers waiting (in which case it goes to |
||
17 | * sleep). |
||
18 | * |
||
19 | * The value of WAITING_BIAS supports up to 32766 waiting processes. This can |
||
20 | * be extended to 65534 by manually checking the whole MSW rather than relying |
||
21 | * on the S flag. |
||
22 | * |
||
23 | * The value of ACTIVE_BIAS supports up to 65535 active processes. |
||
24 | * |
||
25 | * This should be totally fair - if anything is waiting, a process that wants a |
||
26 | * lock will go to the back of the queue. When the currently active lock is |
||
27 | * released, if there's a writer at the front of the queue, then that and only |
||
28 | * that will be woken up; if there's a bunch of consequtive readers at the |
||
29 | * front, then they'll all be woken up, but no other readers will be. |
||
30 | */ |
||
31 | |||
32 | #ifndef _ASM_X86_RWSEM_H |
||
33 | #define _ASM_X86_RWSEM_H |
||
34 | |||
35 | #ifndef _LINUX_RWSEM_H |
||
36 | #error "please don't include asm/rwsem.h directly, use linux/rwsem.h instead" |
||
37 | #endif |
||
38 | |||
39 | #ifdef __KERNEL__ |
||
40 | #include |
||
41 | |||
42 | /* |
||
43 | * The bias values and the counter type limits the number of |
||
44 | * potential readers/writers to 32767 for 32 bits and 2147483647 |
||
45 | * for 64 bits. |
||
46 | */ |
||
47 | |||
48 | #ifdef CONFIG_X86_64 |
||
49 | # define RWSEM_ACTIVE_MASK 0xffffffffL |
||
50 | #else |
||
51 | # define RWSEM_ACTIVE_MASK 0x0000ffffL |
||
52 | #endif |
||
53 | |||
54 | #define RWSEM_UNLOCKED_VALUE 0x00000000L |
||
55 | #define RWSEM_ACTIVE_BIAS 0x00000001L |
||
56 | #define RWSEM_WAITING_BIAS (-RWSEM_ACTIVE_MASK-1) |
||
57 | #define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS |
||
58 | #define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) |
||
59 | |||
60 | /* |
||
61 | * lock for reading |
||
62 | */ |
||
63 | static inline void __down_read(struct rw_semaphore *sem) |
||
64 | { |
||
65 | asm volatile("# beginning down_read\n\t" |
||
66 | LOCK_PREFIX _ASM_INC "(%1)\n\t" |
||
67 | /* adds 0x00000001 */ |
||
68 | " jns 1f\n" |
||
69 | " call call_rwsem_down_read_failed\n" |
||
70 | "1:\n\t" |
||
71 | "# ending down_read\n\t" |
||
72 | : "+m" (sem->count) |
||
73 | : "a" (sem) |
||
74 | : "memory", "cc"); |
||
75 | } |
||
76 | |||
77 | /* |
||
78 | * trylock for reading -- returns 1 if successful, 0 if contention |
||
79 | */ |
||
80 | static inline int __down_read_trylock(struct rw_semaphore *sem) |
||
81 | { |
||
82 | long result, tmp; |
||
83 | asm volatile("# beginning __down_read_trylock\n\t" |
||
84 | " mov %0,%1\n\t" |
||
85 | "1:\n\t" |
||
86 | " mov %1,%2\n\t" |
||
87 | " add %3,%2\n\t" |
||
88 | " jle 2f\n\t" |
||
89 | LOCK_PREFIX " cmpxchg %2,%0\n\t" |
||
90 | " jnz 1b\n\t" |
||
91 | "2:\n\t" |
||
92 | "# ending __down_read_trylock\n\t" |
||
93 | : "+m" (sem->count), "=&a" (result), "=&r" (tmp) |
||
94 | : "i" (RWSEM_ACTIVE_READ_BIAS) |
||
95 | : "memory", "cc"); |
||
96 | return result >= 0 ? 1 : 0; |
||
97 | } |
||
98 | |||
99 | /* |
||
100 | * lock for writing |
||
101 | */ |
||
102 | static inline void __down_write_nested(struct rw_semaphore *sem, int subclass) |
||
103 | { |
||
104 | long tmp; |
||
105 | asm volatile("# beginning down_write\n\t" |
||
106 | LOCK_PREFIX " xadd %1,(%2)\n\t" |
||
107 | /* adds 0xffff0001, returns the old value */ |
||
108 | " test " __ASM_SEL(%w1,%k1) "," __ASM_SEL(%w1,%k1) "\n\t" |
||
109 | /* was the active mask 0 before? */ |
||
110 | " jz 1f\n" |
||
111 | " call call_rwsem_down_write_failed\n" |
||
112 | "1:\n" |
||
113 | "# ending down_write" |
||
114 | : "+m" (sem->count), "=d" (tmp) |
||
115 | : "a" (sem), "1" (RWSEM_ACTIVE_WRITE_BIAS) |
||
116 | : "memory", "cc"); |
||
117 | } |
||
118 | |||
119 | static inline void __down_write(struct rw_semaphore *sem) |
||
120 | { |
||
121 | __down_write_nested(sem, 0); |
||
122 | } |
||
123 | |||
124 | /* |
||
125 | * trylock for writing -- returns 1 if successful, 0 if contention |
||
126 | */ |
||
127 | static inline int __down_write_trylock(struct rw_semaphore *sem) |
||
128 | { |
||
129 | long result, tmp; |
||
130 | asm volatile("# beginning __down_write_trylock\n\t" |
||
131 | " mov %0,%1\n\t" |
||
132 | "1:\n\t" |
||
133 | " test " __ASM_SEL(%w1,%k1) "," __ASM_SEL(%w1,%k1) "\n\t" |
||
134 | /* was the active mask 0 before? */ |
||
135 | " jnz 2f\n\t" |
||
136 | " mov %1,%2\n\t" |
||
137 | " add %3,%2\n\t" |
||
138 | LOCK_PREFIX " cmpxchg %2,%0\n\t" |
||
139 | " jnz 1b\n\t" |
||
140 | "2:\n\t" |
||
141 | " sete %b1\n\t" |
||
142 | " movzbl %b1, %k1\n\t" |
||
143 | "# ending __down_write_trylock\n\t" |
||
144 | : "+m" (sem->count), "=&a" (result), "=&r" (tmp) |
||
145 | : "er" (RWSEM_ACTIVE_WRITE_BIAS) |
||
146 | : "memory", "cc"); |
||
147 | return result; |
||
148 | } |
||
149 | |||
150 | /* |
||
151 | * unlock after reading |
||
152 | */ |
||
153 | static inline void __up_read(struct rw_semaphore *sem) |
||
154 | { |
||
155 | long tmp; |
||
156 | asm volatile("# beginning __up_read\n\t" |
||
157 | LOCK_PREFIX " xadd %1,(%2)\n\t" |
||
158 | /* subtracts 1, returns the old value */ |
||
159 | " jns 1f\n\t" |
||
160 | " call call_rwsem_wake\n" /* expects old value in %edx */ |
||
161 | "1:\n" |
||
162 | "# ending __up_read\n" |
||
163 | : "+m" (sem->count), "=d" (tmp) |
||
164 | : "a" (sem), "1" (-RWSEM_ACTIVE_READ_BIAS) |
||
165 | : "memory", "cc"); |
||
166 | } |
||
167 | |||
168 | /* |
||
169 | * unlock after writing |
||
170 | */ |
||
171 | static inline void __up_write(struct rw_semaphore *sem) |
||
172 | { |
||
173 | long tmp; |
||
174 | asm volatile("# beginning __up_write\n\t" |
||
175 | LOCK_PREFIX " xadd %1,(%2)\n\t" |
||
176 | /* subtracts 0xffff0001, returns the old value */ |
||
177 | " jns 1f\n\t" |
||
178 | " call call_rwsem_wake\n" /* expects old value in %edx */ |
||
179 | "1:\n\t" |
||
180 | "# ending __up_write\n" |
||
181 | : "+m" (sem->count), "=d" (tmp) |
||
182 | : "a" (sem), "1" (-RWSEM_ACTIVE_WRITE_BIAS) |
||
183 | : "memory", "cc"); |
||
184 | } |
||
185 | |||
186 | /* |
||
187 | * downgrade write lock to read lock |
||
188 | */ |
||
189 | static inline void __downgrade_write(struct rw_semaphore *sem) |
||
190 | { |
||
191 | asm volatile("# beginning __downgrade_write\n\t" |
||
192 | LOCK_PREFIX _ASM_ADD "%2,(%1)\n\t" |
||
193 | /* |
||
194 | * transitions 0xZZZZ0001 -> 0xYYYY0001 (i386) |
||
195 | * 0xZZZZZZZZ00000001 -> 0xYYYYYYYY00000001 (x86_64) |
||
196 | */ |
||
197 | " jns 1f\n\t" |
||
198 | " call call_rwsem_downgrade_wake\n" |
||
199 | "1:\n\t" |
||
200 | "# ending __downgrade_write\n" |
||
201 | : "+m" (sem->count) |
||
202 | : "a" (sem), "er" (-RWSEM_WAITING_BIAS) |
||
203 | : "memory", "cc"); |
||
204 | } |
||
205 | |||
206 | /* |
||
207 | * implement atomic add functionality |
||
208 | */ |
||
209 | static inline void rwsem_atomic_add(long delta, struct rw_semaphore *sem) |
||
210 | { |
||
211 | asm volatile(LOCK_PREFIX _ASM_ADD "%1,%0" |
||
212 | : "+m" (sem->count) |
||
213 | : "er" (delta)); |
||
214 | } |
||
215 | |||
216 | /* |
||
217 | * implement exchange and add functionality |
||
218 | */ |
||
219 | static inline long rwsem_atomic_update(long delta, struct rw_semaphore *sem) |
||
220 | { |
||
221 | return delta + xadd(&sem->count, delta); |
||
222 | } |
||
223 | |||
224 | #endif /* __KERNEL__ */ |
||
225 | #endif /* _ASM_X86_RWSEM_H */ |