Rev 5272 | Show entire file | Regard whitespace | Details | Blame | Last modification | View Log | RSS feed
Rev 5272 | Rev 6082 | ||
---|---|---|---|
Line 8... | Line 8... | ||
8 | 8 | ||
9 | #include |
9 | #include |
Line 10... | Line 10... | ||
10 | #include |
10 | #include |
11 | 11 | ||
- | 12 | /* |
|
- | 13 | * We put the hardirq and softirq counter into the preemption |
|
- | 14 | * counter. The bitmask has the following meaning: |
|
- | 15 | * |
|
- | 16 | * - bits 0-7 are the preemption count (max preemption depth: 256) |
|
- | 17 | * - bits 8-15 are the softirq count (max # of softirqs: 256) |
|
- | 18 | * |
|
12 | /* |
19 | * The hardirq count could in theory be the same as the number of |
- | 20 | * interrupts in the system, but we run all interrupt handlers with |
|
- | 21 | * interrupts disabled, so we cannot have nesting interrupts. Though |
|
- | 22 | * there are a few palaeontologic drivers which reenable interrupts in |
|
- | 23 | * the handler, so we need more than one bit here. |
|
- | 24 | * |
|
- | 25 | * PREEMPT_MASK: 0x000000ff |
|
- | 26 | * SOFTIRQ_MASK: 0x0000ff00 |
|
- | 27 | * HARDIRQ_MASK: 0x000f0000 |
|
13 | * We use the MSB mostly because its available; see |
28 | * NMI_MASK: 0x00100000 |
- | 29 | * PREEMPT_NEED_RESCHED: 0x80000000 |
|
- | 30 | */ |
|
- | 31 | #define PREEMPT_BITS 8 |
|
- | 32 | #define SOFTIRQ_BITS 8 |
|
- | 33 | #define HARDIRQ_BITS 4 |
|
- | 34 | #define NMI_BITS 1 |
|
- | 35 | ||
- | 36 | #define PREEMPT_SHIFT 0 |
|
- | 37 | #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) |
|
- | 38 | #define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) |
|
- | 39 | #define NMI_SHIFT (HARDIRQ_SHIFT + HARDIRQ_BITS) |
|
- | 40 | ||
- | 41 | #define __IRQ_MASK(x) ((1UL << (x))-1) |
|
- | 42 | ||
- | 43 | #define PREEMPT_MASK (__IRQ_MASK(PREEMPT_BITS) << PREEMPT_SHIFT) |
|
- | 44 | #define SOFTIRQ_MASK (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT) |
|
- | 45 | #define HARDIRQ_MASK (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT) |
|
- | 46 | #define NMI_MASK (__IRQ_MASK(NMI_BITS) << NMI_SHIFT) |
|
- | 47 | ||
- | 48 | #define PREEMPT_OFFSET (1UL << PREEMPT_SHIFT) |
|
- | 49 | #define SOFTIRQ_OFFSET (1UL << SOFTIRQ_SHIFT) |
|
- | 50 | #define HARDIRQ_OFFSET (1UL << HARDIRQ_SHIFT) |
|
- | 51 | #define NMI_OFFSET (1UL << NMI_SHIFT) |
|
- | 52 | ||
- | 53 | #define SOFTIRQ_DISABLE_OFFSET (2 * SOFTIRQ_OFFSET) |
|
14 | * the other bits -- can't include that header due to inclusion hell. |
54 | |
Line -... | Line 55... | ||
- | 55 | /* We use the MSB mostly because its available */ |
|
15 | */ |
56 | #define PREEMPT_NEED_RESCHED 0x80000000 |
Line -... | Line 57... | ||
- | 57 | ||
- | 58 | /* preempt_count() and related functions, depends on PREEMPT_NEED_RESCHED */ |
|
- | 59 | #include |
|
- | 60 | ||
- | 61 | #define hardirq_count() (preempt_count() & HARDIRQ_MASK) |
|
- | 62 | #define softirq_count() (preempt_count() & SOFTIRQ_MASK) |
|
- | 63 | #define irq_count() (preempt_count() & (HARDIRQ_MASK | SOFTIRQ_MASK \ |
|
- | 64 | | NMI_MASK)) |
|
- | 65 | ||
- | 66 | /* |
|
- | 67 | * Are we doing bottom half or hardware interrupt processing? |
|
- | 68 | * Are we in a softirq context? Interrupt context? |
|
- | 69 | * in_softirq - Are we currently processing softirq or have bh disabled? |
|
- | 70 | * in_serving_softirq - Are we currently processing softirq? |
|
- | 71 | */ |
|
- | 72 | #define in_irq() (hardirq_count()) |
|
- | 73 | #define in_softirq() (softirq_count()) |
|
- | 74 | #define in_interrupt() (irq_count()) |
|
- | 75 | #define in_serving_softirq() (softirq_count() & SOFTIRQ_OFFSET) |
|
- | 76 | ||
- | 77 | /* |
|
- | 78 | * Are we in NMI context? |
|
- | 79 | */ |
|
- | 80 | #define in_nmi() (preempt_count() & NMI_MASK) |
|
- | 81 | ||
- | 82 | /* |
|
- | 83 | * The preempt_count offset after preempt_disable(); |
|
- | 84 | */ |
|
- | 85 | #if defined(CONFIG_PREEMPT_COUNT) |
|
- | 86 | # define PREEMPT_DISABLE_OFFSET PREEMPT_OFFSET |
|
- | 87 | #else |
|
- | 88 | # define PREEMPT_DISABLE_OFFSET 0 |
|
- | 89 | #endif |
|
- | 90 | ||
- | 91 | /* |
|
- | 92 | * The preempt_count offset after spin_lock() |
|
- | 93 | */ |
|
- | 94 | #define PREEMPT_LOCK_OFFSET PREEMPT_DISABLE_OFFSET |
|
- | 95 | ||
- | 96 | /* |
|
- | 97 | * The preempt_count offset needed for things like: |
|
- | 98 | * |
|
- | 99 | * spin_lock_bh() |
|
- | 100 | * |
|
- | 101 | * Which need to disable both preemption (CONFIG_PREEMPT_COUNT) and |
|
- | 102 | * softirqs, such that unlock sequences of: |
|
- | 103 | * |
|
- | 104 | * spin_unlock(); |
|
- | 105 | * local_bh_enable(); |
|
- | 106 | * |
|
- | 107 | * Work as expected. |
|
- | 108 | */ |
|
- | 109 | #define SOFTIRQ_LOCK_OFFSET (SOFTIRQ_DISABLE_OFFSET + PREEMPT_LOCK_OFFSET) |
|
- | 110 | ||
- | 111 | /* |
|
- | 112 | * Are we running in atomic context? WARNING: this macro cannot |
|
- | 113 | * always detect atomic context; in particular, it cannot know about |
|
- | 114 | * held spinlocks in non-preemptible kernels. Thus it should not be |
|
- | 115 | * used in the general case to determine whether sleeping is possible. |
|
- | 116 | * Do not use in_atomic() in driver code. |
|
- | 117 | */ |
|
- | 118 | #define in_atomic() (preempt_count() != 0) |
|
- | 119 | ||
- | 120 | /* |
|
- | 121 | * Check whether we were atomic before we did preempt_disable(): |
|
16 | #define PREEMPT_NEED_RESCHED 0x80000000 |
122 | * (used by the scheduler) |
17 | 123 | */ |
|
18 | #include |
124 | #define in_atomic_preempt_off() (preempt_count() != PREEMPT_DISABLE_OFFSET) |
- | 125 | ||
19 | 126 | #if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_PREEMPT_TRACER) |
|
20 | #if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_PREEMPT_TRACER) |
127 | extern void preempt_count_add(int val); |
21 | extern void preempt_count_add(int val); |
128 | extern void preempt_count_sub(int val); |
22 | extern void preempt_count_sub(int val); |
129 | #define preempt_count_dec_and_test() \ |
23 | #define preempt_count_dec_and_test() ({ preempt_count_sub(1); should_resched(); }) |
130 | ({ preempt_count_sub(1); should_resched(0); }) |
24 | #else |
131 | #else |
Line 47... | Line 154... | ||
47 | preempt_count_dec(); \ |
154 | preempt_count_dec(); \ |
48 | } while (0) |
155 | } while (0) |
Line 49... | Line 156... | ||
49 | 156 | ||
Line -... | Line 157... | ||
- | 157 | #define preempt_enable_no_resched() sched_preempt_enable_no_resched() |
|
- | 158 | ||
50 | #define preempt_enable_no_resched() sched_preempt_enable_no_resched() |
159 | #define preemptible() (preempt_count() == 0 && !irqs_disabled()) |
51 | 160 | ||
52 | #ifdef CONFIG_PREEMPT |
161 | #ifdef CONFIG_PREEMPT |
53 | #define preempt_enable() \ |
162 | #define preempt_enable() \ |
54 | do { \ |
163 | do { \ |
55 | barrier(); \ |
164 | barrier(); \ |
56 | if (unlikely(preempt_count_dec_and_test())) \ |
165 | if (unlikely(preempt_count_dec_and_test())) \ |
Line -... | Line 166... | ||
- | 166 | __preempt_schedule(); \ |
|
- | 167 | } while (0) |
|
- | 168 | ||
- | 169 | #define preempt_enable_notrace() \ |
|
- | 170 | do { \ |
|
- | 171 | barrier(); \ |
|
- | 172 | if (unlikely(__preempt_count_dec_and_test())) \ |
|
57 | __preempt_schedule(); \ |
173 | __preempt_schedule_notrace(); \ |
58 | } while (0) |
174 | } while (0) |
59 | 175 | ||
60 | #define preempt_check_resched() \ |
176 | #define preempt_check_resched() \ |
61 | do { \ |
177 | do { \ |
Line 62... | Line 178... | ||
62 | if (should_resched()) \ |
178 | if (should_resched(0)) \ |
63 | __preempt_schedule(); \ |
179 | __preempt_schedule(); \ |
64 | } while (0) |
180 | } while (0) |
65 | 181 | ||
66 | #else |
182 | #else /* !CONFIG_PREEMPT */ |
67 | #define preempt_enable() \ |
183 | #define preempt_enable() \ |
68 | do { \ |
- | |
69 | barrier(); \ |
- | |
70 | preempt_count_dec(); \ |
- | |
71 | } while (0) |
- | |
72 | #define preempt_check_resched() do { } while (0) |
- | |
73 | #endif |
- | |
74 | - | ||
75 | #define preempt_disable_notrace() \ |
- | |
Line 76... | Line 184... | ||
76 | do { \ |
184 | do { \ |
77 | __preempt_count_inc(); \ |
185 | barrier(); \ |
78 | barrier(); \ |
186 | preempt_count_dec(); \ |
79 | } while (0) |
187 | } while (0) |
80 | 188 | ||
Line 81... | Line -... | ||
81 | #define preempt_enable_no_resched_notrace() \ |
- | |
82 | do { \ |
- | |
83 | barrier(); \ |
- | |
84 | __preempt_count_dec(); \ |
189 | #define preempt_enable_notrace() \ |
85 | } while (0) |
190 | do { \ |
Line 86... | Line 191... | ||
86 | 191 | barrier(); \ |
|
87 | #ifdef CONFIG_PREEMPT |
192 | __preempt_count_dec(); \ |
- | 193 | } while (0) |
|
88 | 194 | ||
89 | #ifndef CONFIG_CONTEXT_TRACKING |
- | |
90 | #define __preempt_schedule_context() __preempt_schedule() |
- | |
91 | #endif |
195 | #define preempt_check_resched() do { } while (0) |
92 | 196 | #endif /* CONFIG_PREEMPT */ |
|
93 | #define preempt_enable_notrace() \ |
197 | |
94 | do { \ |
198 | #define preempt_disable_notrace() \ |
95 | barrier(); \ |
199 | do { \ |
96 | if (unlikely(__preempt_count_dec_and_test())) \ |
200 | __preempt_count_inc(); \ |
97 | __preempt_schedule_context(); \ |
201 | barrier(); \ |
98 | } while (0) |
- | |
Line 99... | Line 202... | ||
99 | #else |
202 | } while (0) |
Line 100... | Line 203... | ||
100 | #define preempt_enable_notrace() \ |
203 | |
101 | do { \ |
204 | #define preempt_enable_no_resched_notrace() \ |
Line 119... | Line 222... | ||
119 | #define preempt_check_resched() do { } while (0) |
222 | #define preempt_check_resched() do { } while (0) |
Line 120... | Line 223... | ||
120 | 223 | ||
121 | #define preempt_disable_notrace() barrier() |
224 | #define preempt_disable_notrace() barrier() |
122 | #define preempt_enable_no_resched_notrace() barrier() |
225 | #define preempt_enable_no_resched_notrace() barrier() |
- | 226 | #define preempt_enable_notrace() barrier() |
|
Line 123... | Line 227... | ||
123 | #define preempt_enable_notrace() barrier() |
227 | #define preemptible() 0 |
Line 124... | Line 228... | ||
124 | 228 | ||
125 | #endif /* CONFIG_PREEMPT_COUNT */ |
229 | #endif /* CONFIG_PREEMPT_COUNT */ |
Line 178... | Line 282... | ||
178 | struct preempt_notifier { |
282 | struct preempt_notifier { |
179 | struct hlist_node link; |
283 | struct hlist_node link; |
180 | struct preempt_ops *ops; |
284 | struct preempt_ops *ops; |
181 | }; |
285 | }; |
Line -... | Line 286... | ||
- | 286 | ||
- | 287 | void preempt_notifier_inc(void); |
|
182 | 288 | void preempt_notifier_dec(void); |
|
183 | void preempt_notifier_register(struct preempt_notifier *notifier); |
289 | void preempt_notifier_register(struct preempt_notifier *notifier); |
Line 184... | Line 290... | ||
184 | void preempt_notifier_unregister(struct preempt_notifier *notifier); |
290 | void preempt_notifier_unregister(struct preempt_notifier *notifier); |
185 | 291 |