Rev 5270 | Rev 6936 | Go to most recent revision | Show entire file | Regard whitespace | Details | Blame | Last modification | View Log | RSS feed
| Rev 5270 | Rev 6082 | ||
|---|---|---|---|
| Line 27... | Line 27... | ||
| 27 | * However, if the list being initialized is visible to readers, you |
27 | * However, if the list being initialized is visible to readers, you |
| 28 | * need to keep the compiler from being too mischievous. |
28 | * need to keep the compiler from being too mischievous. |
| 29 | */ |
29 | */ |
| 30 | static inline void INIT_LIST_HEAD_RCU(struct list_head *list) |
30 | static inline void INIT_LIST_HEAD_RCU(struct list_head *list) |
| 31 | { |
31 | { |
| 32 | ACCESS_ONCE(list->next) = list; |
32 | WRITE_ONCE(list->next, list); |
| 33 | ACCESS_ONCE(list->prev) = list; |
33 | WRITE_ONCE(list->prev, list); |
| 34 | } |
34 | } |
| Line 35... | Line 35... | ||
| 35 | 35 | ||
| 36 | /* |
36 | /* |
| 37 | * return the ->next pointer of a list_head in an rcu safe |
37 | * return the ->next pointer of a list_head in an rcu safe |
| Line 245... | Line 245... | ||
| 245 | * |
245 | * |
| 246 | * This primitive may safely run concurrently with the _rcu list-mutation |
246 | * This primitive may safely run concurrently with the _rcu list-mutation |
| 247 | * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock(). |
247 | * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock(). |
| 248 | */ |
248 | */ |
| 249 | #define list_entry_rcu(ptr, type, member) \ |
249 | #define list_entry_rcu(ptr, type, member) \ |
| 250 | ({ \ |
- | |
| 251 | typeof(*ptr) __rcu *__ptr = (typeof(*ptr) __rcu __force *)ptr; \ |
- | |
| 252 | container_of((typeof(ptr))rcu_dereference_raw(__ptr), type, member); \ |
250 | container_of(lockless_dereference(ptr), type, member) |
| 253 | }) |
- | |
| Line 254... | Line 251... | ||
| 254 | 251 | ||
| 255 | /** |
252 | /** |
| 256 | * Where are list_empty_rcu() and list_first_entry_rcu()? |
253 | * Where are list_empty_rcu() and list_first_entry_rcu()? |
| 257 | * |
254 | * |
| Line 286... | Line 283... | ||
| 286 | * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock(). |
283 | * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock(). |
| 287 | */ |
284 | */ |
| 288 | #define list_first_or_null_rcu(ptr, type, member) \ |
285 | #define list_first_or_null_rcu(ptr, type, member) \ |
| 289 | ({ \ |
286 | ({ \ |
| 290 | struct list_head *__ptr = (ptr); \ |
287 | struct list_head *__ptr = (ptr); \ |
| 291 | struct list_head *__next = ACCESS_ONCE(__ptr->next); \ |
288 | struct list_head *__next = READ_ONCE(__ptr->next); \ |
| 292 | likely(__ptr != __next) ? list_entry_rcu(__next, type, member) : NULL; \ |
289 | likely(__ptr != __next) ? list_entry_rcu(__next, type, member) : NULL; \ |
| 293 | }) |
290 | }) |
| Line 294... | Line 291... | ||
| 294 | 291 | ||
| 295 | /** |
292 | /** |
| Line 522... | Line 519... | ||
| 522 | * hlist_for_each_entry_continue_rcu - iterate over a hlist continuing after current point |
519 | * hlist_for_each_entry_continue_rcu - iterate over a hlist continuing after current point |
| 523 | * @pos: the type * to use as a loop cursor. |
520 | * @pos: the type * to use as a loop cursor. |
| 524 | * @member: the name of the hlist_node within the struct. |
521 | * @member: the name of the hlist_node within the struct. |
| 525 | */ |
522 | */ |
| 526 | #define hlist_for_each_entry_continue_rcu(pos, member) \ |
523 | #define hlist_for_each_entry_continue_rcu(pos, member) \ |
| 527 | for (pos = hlist_entry_safe(rcu_dereference((pos)->member.next),\ |
524 | for (pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu( \ |
| 528 | typeof(*(pos)), member); \ |
525 | &(pos)->member)), typeof(*(pos)), member); \ |
| 529 | pos; \ |
526 | pos; \ |
| 530 | pos = hlist_entry_safe(rcu_dereference((pos)->member.next),\ |
527 | pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu( \ |
| 531 | typeof(*(pos)), member)) |
528 | &(pos)->member)), typeof(*(pos)), member)) |
| Line 532... | Line 529... | ||
| 532 | 529 | ||
| 533 | /** |
530 | /** |
| 534 | * hlist_for_each_entry_continue_rcu_bh - iterate over a hlist continuing after current point |
531 | * hlist_for_each_entry_continue_rcu_bh - iterate over a hlist continuing after current point |
| 535 | * @pos: the type * to use as a loop cursor. |
532 | * @pos: the type * to use as a loop cursor. |
| 536 | * @member: the name of the hlist_node within the struct. |
533 | * @member: the name of the hlist_node within the struct. |
| 537 | */ |
534 | */ |
| 538 | #define hlist_for_each_entry_continue_rcu_bh(pos, member) \ |
535 | #define hlist_for_each_entry_continue_rcu_bh(pos, member) \ |
| 539 | for (pos = hlist_entry_safe(rcu_dereference_bh((pos)->member.next),\ |
536 | for (pos = hlist_entry_safe(rcu_dereference_bh(hlist_next_rcu( \ |
| 540 | typeof(*(pos)), member); \ |
537 | &(pos)->member)), typeof(*(pos)), member); \ |
| 541 | pos; \ |
538 | pos; \ |
| 542 | pos = hlist_entry_safe(rcu_dereference_bh((pos)->member.next),\ |
539 | pos = hlist_entry_safe(rcu_dereference_bh(hlist_next_rcu( \ |
| Line 543... | Line 540... | ||
| 543 | typeof(*(pos)), member)) |
540 | &(pos)->member)), typeof(*(pos)), member)) |
| 544 | 541 | ||
| 545 | /** |
542 | /** |
| 546 | * hlist_for_each_entry_from_rcu - iterate over a hlist continuing from current point |
543 | * hlist_for_each_entry_from_rcu - iterate over a hlist continuing from current point |
| 547 | * @pos: the type * to use as a loop cursor. |
544 | * @pos: the type * to use as a loop cursor. |
| 548 | * @member: the name of the hlist_node within the struct. |
545 | * @member: the name of the hlist_node within the struct. |
| 549 | */ |
546 | */ |
| 550 | #define hlist_for_each_entry_from_rcu(pos, member) \ |
547 | #define hlist_for_each_entry_from_rcu(pos, member) \ |
| 551 | for (; pos; \ |
548 | for (; pos; \ |
| Line 552... | Line 549... | ||
| 552 | pos = hlist_entry_safe(rcu_dereference((pos)->member.next),\ |
549 | pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu( \ |
| 553 | typeof(*(pos)), member)) |
550 | &(pos)->member)), typeof(*(pos)), member)) |