458,46 → 458,10 |
* If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an |
* RCU-sched read-side critical section. In absence of |
* CONFIG_DEBUG_LOCK_ALLOC, this assumes we are in an RCU-sched read-side |
* critical section unless it can prove otherwise. Note that disabling |
* of preemption (including disabling irqs) counts as an RCU-sched |
* read-side critical section. This is useful for debug checks in functions |
* that required that they be called within an RCU-sched read-side |
* critical section. |
* |
* Check debug_lockdep_rcu_enabled() to prevent false positives during boot |
* and while lockdep is disabled. |
* |
* Note that if the CPU is in the idle loop from an RCU point of |
* view (ie: that we are in the section between rcu_idle_enter() and |
* rcu_idle_exit()) then rcu_read_lock_held() returns false even if the CPU |
* did an rcu_read_lock(). The reason for this is that RCU ignores CPUs |
* that are in such a section, considering these as in extended quiescent |
* state, so such a CPU is effectively never in an RCU read-side critical |
* section regardless of what RCU primitives it invokes. This state of |
* affairs is required --- we need to keep an RCU-free window in idle |
* where the CPU may possibly enter into low power mode. This way we can |
* notice an extended quiescent state to other CPUs that started a grace |
* period. Otherwise we would delay any grace period as long as we run in |
* the idle task. |
* |
* Similarly, we avoid claiming an SRCU read lock held if the current |
* CPU is offline. |
* critical section unless it can prove otherwise. |
*/ |
#ifdef CONFIG_PREEMPT_COUNT |
static inline int rcu_read_lock_sched_held(void) |
{ |
int lockdep_opinion = 0; |
|
if (!debug_lockdep_rcu_enabled()) |
return 1; |
if (!rcu_is_watching()) |
return 0; |
if (!rcu_lockdep_current_cpu_online()) |
return 0; |
if (debug_locks) |
lockdep_opinion = lock_is_held(&rcu_sched_lock_map); |
return lockdep_opinion || preempt_count() != 0 || irqs_disabled(); |
} |
int rcu_read_lock_sched_held(void); |
#else /* #ifdef CONFIG_PREEMPT_COUNT */ |
static inline int rcu_read_lock_sched_held(void) |
{ |
537,14 → 501,14 |
#ifdef CONFIG_PROVE_RCU |
|
/** |
* rcu_lockdep_assert - emit lockdep splat if specified condition not met |
* RCU_LOCKDEP_WARN - emit lockdep splat if specified condition is met |
* @c: condition to check |
* @s: informative message |
*/ |
#define rcu_lockdep_assert(c, s) \ |
#define RCU_LOCKDEP_WARN(c, s) \ |
do { \ |
static bool __section(.data.unlikely) __warned; \ |
if (debug_lockdep_rcu_enabled() && !__warned && !(c)) { \ |
if (debug_lockdep_rcu_enabled() && !__warned && (c)) { \ |
__warned = true; \ |
lockdep_rcu_suspicious(__FILE__, __LINE__, s); \ |
} \ |
553,7 → 517,7 |
#if defined(CONFIG_PROVE_RCU) && !defined(CONFIG_PREEMPT_RCU) |
static inline void rcu_preempt_sleep_check(void) |
{ |
rcu_lockdep_assert(!lock_is_held(&rcu_lock_map), |
RCU_LOCKDEP_WARN(lock_is_held(&rcu_lock_map), |
"Illegal context switch in RCU read-side critical section"); |
} |
#else /* #ifdef CONFIG_PROVE_RCU */ |
565,15 → 529,15 |
#define rcu_sleep_check() \ |
do { \ |
rcu_preempt_sleep_check(); \ |
rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map), \ |
RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map), \ |
"Illegal context switch in RCU-bh read-side critical section"); \ |
rcu_lockdep_assert(!lock_is_held(&rcu_sched_lock_map), \ |
RCU_LOCKDEP_WARN(lock_is_held(&rcu_sched_lock_map), \ |
"Illegal context switch in RCU-sched read-side critical section"); \ |
} while (0) |
|
#else /* #ifdef CONFIG_PROVE_RCU */ |
|
#define rcu_lockdep_assert(c, s) do { } while (0) |
#define RCU_LOCKDEP_WARN(c, s) do { } while (0) |
#define rcu_sleep_check() do { } while (0) |
|
#endif /* #else #ifdef CONFIG_PROVE_RCU */ |
604,13 → 568,13 |
({ \ |
/* Dependency order vs. p above. */ \ |
typeof(*p) *________p1 = (typeof(*p) *__force)lockless_dereference(p); \ |
rcu_lockdep_assert(c, "suspicious rcu_dereference_check() usage"); \ |
RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_check() usage"); \ |
rcu_dereference_sparse(p, space); \ |
((typeof(*p) __force __kernel *)(________p1)); \ |
}) |
#define __rcu_dereference_protected(p, c, space) \ |
({ \ |
rcu_lockdep_assert(c, "suspicious rcu_dereference_protected() usage"); \ |
RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_protected() usage"); \ |
rcu_dereference_sparse(p, space); \ |
((typeof(*p) __force __kernel *)(p)); \ |
}) |
834,7 → 798,7 |
__rcu_read_lock(); |
__acquire(RCU); |
rcu_lock_acquire(&rcu_lock_map); |
rcu_lockdep_assert(rcu_is_watching(), |
RCU_LOCKDEP_WARN(!rcu_is_watching(), |
"rcu_read_lock() used illegally while idle"); |
} |
|
885,7 → 849,7 |
*/ |
static inline void rcu_read_unlock(void) |
{ |
rcu_lockdep_assert(rcu_is_watching(), |
RCU_LOCKDEP_WARN(!rcu_is_watching(), |
"rcu_read_unlock() used illegally while idle"); |
__release(RCU); |
__rcu_read_unlock(); |
914,7 → 878,7 |
local_bh_disable(); |
__acquire(RCU_BH); |
rcu_lock_acquire(&rcu_bh_lock_map); |
rcu_lockdep_assert(rcu_is_watching(), |
RCU_LOCKDEP_WARN(!rcu_is_watching(), |
"rcu_read_lock_bh() used illegally while idle"); |
} |
|
925,7 → 889,7 |
*/ |
static inline void rcu_read_unlock_bh(void) |
{ |
rcu_lockdep_assert(rcu_is_watching(), |
RCU_LOCKDEP_WARN(!rcu_is_watching(), |
"rcu_read_unlock_bh() used illegally while idle"); |
rcu_lock_release(&rcu_bh_lock_map); |
__release(RCU_BH); |
950,7 → 914,7 |
preempt_disable(); |
__acquire(RCU_SCHED); |
rcu_lock_acquire(&rcu_sched_lock_map); |
rcu_lockdep_assert(rcu_is_watching(), |
RCU_LOCKDEP_WARN(!rcu_is_watching(), |
"rcu_read_lock_sched() used illegally while idle"); |
} |
|
968,7 → 932,7 |
*/ |
static inline void rcu_read_unlock_sched(void) |
{ |
rcu_lockdep_assert(rcu_is_watching(), |
RCU_LOCKDEP_WARN(!rcu_is_watching(), |
"rcu_read_unlock_sched() used illegally while idle"); |
rcu_lock_release(&rcu_sched_lock_map); |
__release(RCU_SCHED); |