Rev 6082 | Rev 6936 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed
Rev | Author | Line No. | Line |
---|---|---|---|
5270 | serge | 1 | #ifndef __LINUX_GFP_H |
2 | #define __LINUX_GFP_H |
||
3 | |||
4 | #include |
||
5 | #include |
||
6 | #include |
||
7 | #include |
||
8 | |||
9 | struct vm_area_struct; |
||
10 | |||
11 | /* Plain integer GFP bitmasks. Do not use this directly. */ |
||
12 | #define ___GFP_DMA 0x01u |
||
13 | #define ___GFP_HIGHMEM 0x02u |
||
14 | #define ___GFP_DMA32 0x04u |
||
15 | #define ___GFP_MOVABLE 0x08u |
||
6082 | serge | 16 | #define ___GFP_RECLAIMABLE 0x10u |
5270 | serge | 17 | #define ___GFP_HIGH 0x20u |
18 | #define ___GFP_IO 0x40u |
||
19 | #define ___GFP_FS 0x80u |
||
20 | #define ___GFP_COLD 0x100u |
||
21 | #define ___GFP_NOWARN 0x200u |
||
22 | #define ___GFP_REPEAT 0x400u |
||
23 | #define ___GFP_NOFAIL 0x800u |
||
24 | #define ___GFP_NORETRY 0x1000u |
||
25 | #define ___GFP_MEMALLOC 0x2000u |
||
26 | #define ___GFP_COMP 0x4000u |
||
27 | #define ___GFP_ZERO 0x8000u |
||
28 | #define ___GFP_NOMEMALLOC 0x10000u |
||
29 | #define ___GFP_HARDWALL 0x20000u |
||
30 | #define ___GFP_THISNODE 0x40000u |
||
6082 | serge | 31 | #define ___GFP_ATOMIC 0x80000u |
32 | #define ___GFP_NOACCOUNT 0x100000u |
||
5270 | serge | 33 | #define ___GFP_NOTRACK 0x200000u |
6082 | serge | 34 | #define ___GFP_DIRECT_RECLAIM 0x400000u |
5270 | serge | 35 | #define ___GFP_OTHER_NODE 0x800000u |
36 | #define ___GFP_WRITE 0x1000000u |
||
6082 | serge | 37 | #define ___GFP_KSWAPD_RECLAIM 0x2000000u |
5270 | serge | 38 | /* If the above are modified, __GFP_BITS_SHIFT may need updating */ |
39 | |||
40 | /* |
||
6082 | serge | 41 | * Physical address zone modifiers (see linux/mmzone.h - low four bits) |
5270 | serge | 42 | * |
43 | * Do not put any conditional on these. If necessary modify the definitions |
||
44 | * without the underscores and use them consistently. The definitions here may |
||
45 | * be used in bit comparisons. |
||
46 | */ |
||
47 | #define __GFP_DMA ((__force gfp_t)___GFP_DMA) |
||
48 | #define __GFP_HIGHMEM ((__force gfp_t)___GFP_HIGHMEM) |
||
49 | #define __GFP_DMA32 ((__force gfp_t)___GFP_DMA32) |
||
50 | #define __GFP_MOVABLE ((__force gfp_t)___GFP_MOVABLE) /* Page is movable */ |
||
6082 | serge | 51 | #define __GFP_MOVABLE ((__force gfp_t)___GFP_MOVABLE) /* ZONE_MOVABLE allowed */ |
5270 | serge | 52 | #define GFP_ZONEMASK (__GFP_DMA|__GFP_HIGHMEM|__GFP_DMA32|__GFP_MOVABLE) |
6082 | serge | 53 | |
5270 | serge | 54 | /* |
6082 | serge | 55 | * Page mobility and placement hints |
5270 | serge | 56 | * |
6082 | serge | 57 | * These flags provide hints about how mobile the page is. Pages with similar |
58 | * mobility are placed within the same pageblocks to minimise problems due |
||
59 | * to external fragmentation. |
||
5270 | serge | 60 | * |
6082 | serge | 61 | * __GFP_MOVABLE (also a zone modifier) indicates that the page can be |
62 | * moved by page migration during memory compaction or can be reclaimed. |
||
5270 | serge | 63 | * |
6082 | serge | 64 | * __GFP_RECLAIMABLE is used for slab allocations that specify |
65 | * SLAB_RECLAIM_ACCOUNT and whose pages can be freed via shrinkers. |
||
5270 | serge | 66 | * |
6082 | serge | 67 | * __GFP_WRITE indicates the caller intends to dirty the page. Where possible, |
68 | * these pages will be spread between local zones to avoid all the dirty |
||
69 | * pages being in one zone (fair zone allocation policy). |
||
70 | * |
||
71 | * __GFP_HARDWALL enforces the cpuset memory allocation policy. |
||
72 | * |
||
73 | * __GFP_THISNODE forces the allocation to be satisified from the requested |
||
74 | * node with no fallbacks or placement policy enforcements. |
||
5270 | serge | 75 | */ |
6082 | serge | 76 | #define __GFP_RECLAIMABLE ((__force gfp_t)___GFP_RECLAIMABLE) |
77 | #define __GFP_WRITE ((__force gfp_t)___GFP_WRITE) |
||
78 | #define __GFP_HARDWALL ((__force gfp_t)___GFP_HARDWALL) |
||
79 | #define __GFP_THISNODE ((__force gfp_t)___GFP_THISNODE) |
||
5270 | serge | 80 | |
6082 | serge | 81 | /* |
82 | * Watermark modifiers -- controls access to emergency reserves |
||
83 | * |
||
84 | * __GFP_HIGH indicates that the caller is high-priority and that granting |
||
85 | * the request is necessary before the system can make forward progress. |
||
86 | * For example, creating an IO context to clean pages. |
||
87 | * |
||
88 | * __GFP_ATOMIC indicates that the caller cannot reclaim or sleep and is |
||
89 | * high priority. Users are typically interrupt handlers. This may be |
||
90 | * used in conjunction with __GFP_HIGH |
||
91 | * |
||
92 | * __GFP_MEMALLOC allows access to all memory. This should only be used when |
||
93 | * the caller guarantees the allocation will allow more memory to be freed |
||
94 | * very shortly e.g. process exiting or swapping. Users either should |
||
95 | * be the MM or co-ordinating closely with the VM (e.g. swap over NFS). |
||
96 | * |
||
97 | * __GFP_NOMEMALLOC is used to explicitly forbid access to emergency reserves. |
||
98 | * This takes precedence over the __GFP_MEMALLOC flag if both are set. |
||
99 | * |
||
100 | * __GFP_NOACCOUNT ignores the accounting for kmemcg limit enforcement. |
||
101 | */ |
||
102 | #define __GFP_ATOMIC ((__force gfp_t)___GFP_ATOMIC) |
||
103 | #define __GFP_HIGH ((__force gfp_t)___GFP_HIGH) |
||
104 | #define __GFP_MEMALLOC ((__force gfp_t)___GFP_MEMALLOC) |
||
105 | #define __GFP_NOMEMALLOC ((__force gfp_t)___GFP_NOMEMALLOC) |
||
106 | #define __GFP_NOACCOUNT ((__force gfp_t)___GFP_NOACCOUNT) |
||
5270 | serge | 107 | |
108 | /* |
||
6082 | serge | 109 | * Reclaim modifiers |
110 | * |
||
111 | * __GFP_IO can start physical IO. |
||
112 | * |
||
113 | * __GFP_FS can call down to the low-level FS. Clearing the flag avoids the |
||
114 | * allocator recursing into the filesystem which might already be holding |
||
115 | * locks. |
||
116 | * |
||
117 | * __GFP_DIRECT_RECLAIM indicates that the caller may enter direct reclaim. |
||
118 | * This flag can be cleared to avoid unnecessary delays when a fallback |
||
119 | * option is available. |
||
120 | * |
||
121 | * __GFP_KSWAPD_RECLAIM indicates that the caller wants to wake kswapd when |
||
122 | * the low watermark is reached and have it reclaim pages until the high |
||
123 | * watermark is reached. A caller may wish to clear this flag when fallback |
||
124 | * options are available and the reclaim is likely to disrupt the system. The |
||
125 | * canonical example is THP allocation where a fallback is cheap but |
||
126 | * reclaim/compaction may cause indirect stalls. |
||
127 | * |
||
128 | * __GFP_RECLAIM is shorthand to allow/forbid both direct and kswapd reclaim. |
||
129 | * |
||
130 | * __GFP_REPEAT: Try hard to allocate the memory, but the allocation attempt |
||
131 | * _might_ fail. This depends upon the particular VM implementation. |
||
132 | * |
||
133 | * __GFP_NOFAIL: The VM implementation _must_ retry infinitely: the caller |
||
134 | * cannot handle allocation failures. New users should be evaluated carefully |
||
135 | * (and the flag should be used only when there is no reasonable failure |
||
136 | * policy) but it is definitely preferable to use the flag rather than |
||
137 | * opencode endless loop around allocator. |
||
138 | * |
||
139 | * __GFP_NORETRY: The VM implementation must not retry indefinitely and will |
||
140 | * return NULL when direct reclaim and memory compaction have failed to allow |
||
141 | * the allocation to succeed. The OOM killer is not called with the current |
||
142 | * implementation. |
||
5270 | serge | 143 | */ |
6082 | serge | 144 | #define __GFP_IO ((__force gfp_t)___GFP_IO) |
145 | #define __GFP_FS ((__force gfp_t)___GFP_FS) |
||
146 | #define __GFP_DIRECT_RECLAIM ((__force gfp_t)___GFP_DIRECT_RECLAIM) /* Caller can reclaim */ |
||
147 | #define __GFP_KSWAPD_RECLAIM ((__force gfp_t)___GFP_KSWAPD_RECLAIM) /* kswapd can wake */ |
||
148 | #define __GFP_RECLAIM ((__force gfp_t)(___GFP_DIRECT_RECLAIM|___GFP_KSWAPD_RECLAIM)) |
||
149 | #define __GFP_REPEAT ((__force gfp_t)___GFP_REPEAT) |
||
150 | #define __GFP_NOFAIL ((__force gfp_t)___GFP_NOFAIL) |
||
151 | #define __GFP_NORETRY ((__force gfp_t)___GFP_NORETRY) |
||
152 | |||
153 | /* |
||
154 | * Action modifiers |
||
155 | * |
||
156 | * __GFP_COLD indicates that the caller does not expect to be used in the near |
||
157 | * future. Where possible, a cache-cold page will be returned. |
||
158 | * |
||
159 | * __GFP_NOWARN suppresses allocation failure reports. |
||
160 | * |
||
161 | * __GFP_COMP address compound page metadata. |
||
162 | * |
||
163 | * __GFP_ZERO returns a zeroed page on success. |
||
164 | * |
||
165 | * __GFP_NOTRACK avoids tracking with kmemcheck. |
||
166 | * |
||
167 | * __GFP_NOTRACK_FALSE_POSITIVE is an alias of __GFP_NOTRACK. It's a means of |
||
168 | * distinguishing in the source between false positives and allocations that |
||
169 | * cannot be supported (e.g. page tables). |
||
170 | * |
||
171 | * __GFP_OTHER_NODE is for allocations that are on a remote node but that |
||
172 | * should not be accounted for as a remote allocation in vmstat. A |
||
173 | * typical user would be khugepaged collapsing a huge page on a remote |
||
174 | * node. |
||
175 | */ |
||
176 | #define __GFP_COLD ((__force gfp_t)___GFP_COLD) |
||
177 | #define __GFP_NOWARN ((__force gfp_t)___GFP_NOWARN) |
||
178 | #define __GFP_COMP ((__force gfp_t)___GFP_COMP) |
||
179 | #define __GFP_ZERO ((__force gfp_t)___GFP_ZERO) |
||
180 | #define __GFP_NOTRACK ((__force gfp_t)___GFP_NOTRACK) |
||
5270 | serge | 181 | #define __GFP_NOTRACK_FALSE_POSITIVE (__GFP_NOTRACK) |
6082 | serge | 182 | #define __GFP_OTHER_NODE ((__force gfp_t)___GFP_OTHER_NODE) |
5270 | serge | 183 | |
6082 | serge | 184 | /* Room for N __GFP_FOO bits */ |
185 | #define __GFP_BITS_SHIFT 26 |
||
5270 | serge | 186 | #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1)) |
187 | |||
6082 | serge | 188 | /* |
189 | * Useful GFP flag combinations that are commonly used. It is recommended |
||
190 | * that subsystems start with one of these combinations and then set/clear |
||
191 | * __GFP_FOO flags as necessary. |
||
192 | * |
||
193 | * GFP_ATOMIC users can not sleep and need the allocation to succeed. A lower |
||
194 | * watermark is applied to allow access to "atomic reserves" |
||
195 | * |
||
196 | * GFP_KERNEL is typical for kernel-internal allocations. The caller requires |
||
197 | * ZONE_NORMAL or a lower zone for direct access but can direct reclaim. |
||
198 | * |
||
199 | * GFP_NOWAIT is for kernel allocations that should not stall for direct |
||
200 | * reclaim, start physical IO or use any filesystem callback. |
||
201 | * |
||
202 | * GFP_NOIO will use direct reclaim to discard clean pages or slab pages |
||
203 | * that do not require the starting of any physical IO. |
||
204 | * |
||
205 | * GFP_NOFS will use direct reclaim but will not use any filesystem interfaces. |
||
206 | * |
||
207 | * GFP_USER is for userspace allocations that also need to be directly |
||
208 | * accessibly by the kernel or hardware. It is typically used by hardware |
||
209 | * for buffers that are mapped to userspace (e.g. graphics) that hardware |
||
210 | * still must DMA to. cpuset limits are enforced for these allocations. |
||
211 | * |
||
212 | * GFP_DMA exists for historical reasons and should be avoided where possible. |
||
213 | * The flags indicates that the caller requires that the lowest zone be |
||
214 | * used (ZONE_DMA or 16M on x86-64). Ideally, this would be removed but |
||
215 | * it would require careful auditing as some users really require it and |
||
216 | * others use the flag to avoid lowmem reserves in ZONE_DMA and treat the |
||
217 | * lowest zone as a type of emergency reserve. |
||
218 | * |
||
219 | * GFP_DMA32 is similar to GFP_DMA except that the caller requires a 32-bit |
||
220 | * address. |
||
221 | * |
||
222 | * GFP_HIGHUSER is for userspace allocations that may be mapped to userspace, |
||
223 | * do not need to be directly accessible by the kernel but that cannot |
||
224 | * move once in use. An example may be a hardware allocation that maps |
||
225 | * data directly into userspace but has no addressing limitations. |
||
226 | * |
||
227 | * GFP_HIGHUSER_MOVABLE is for userspace allocations that the kernel does not |
||
228 | * need direct access to but can use kmap() when access is required. They |
||
229 | * are expected to be movable via page reclaim or page migration. Typically, |
||
230 | * pages on the LRU would also be allocated with GFP_HIGHUSER_MOVABLE. |
||
231 | * |
||
232 | * GFP_TRANSHUGE is used for THP allocations. They are compound allocations |
||
233 | * that will fail quickly if memory is not available and will not wake |
||
234 | * kswapd on failure. |
||
235 | */ |
||
236 | #define GFP_ATOMIC (__GFP_HIGH|__GFP_ATOMIC|__GFP_KSWAPD_RECLAIM) |
||
237 | #define GFP_KERNEL (__GFP_RECLAIM | __GFP_IO | __GFP_FS) |
||
238 | #define GFP_NOWAIT (__GFP_KSWAPD_RECLAIM) |
||
239 | #define GFP_NOIO (__GFP_RECLAIM) |
||
240 | #define GFP_NOFS (__GFP_RECLAIM | __GFP_IO) |
||
241 | #define GFP_TEMPORARY (__GFP_RECLAIM | __GFP_IO | __GFP_FS | \ |
||
5270 | serge | 242 | __GFP_RECLAIMABLE) |
6082 | serge | 243 | #define GFP_USER (__GFP_RECLAIM | __GFP_IO | __GFP_FS | __GFP_HARDWALL) |
244 | #define GFP_DMA __GFP_DMA |
||
245 | #define GFP_DMA32 __GFP_DMA32 |
||
5270 | serge | 246 | #define GFP_HIGHUSER (GFP_USER | __GFP_HIGHMEM) |
247 | #define GFP_HIGHUSER_MOVABLE (GFP_HIGHUSER | __GFP_MOVABLE) |
||
6082 | serge | 248 | #define GFP_TRANSHUGE ((GFP_HIGHUSER_MOVABLE | __GFP_COMP | \ |
249 | __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN) & \ |
||
250 | ~__GFP_KSWAPD_RECLAIM) |
||
5270 | serge | 251 | |
6082 | serge | 252 | /* Convert GFP flags to their corresponding migrate type */ |
5270 | serge | 253 | #define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE|__GFP_MOVABLE) |
6082 | serge | 254 | #define GFP_MOVABLE_SHIFT 3 |
5270 | serge | 255 | |
6082 | serge | 256 | #undef GFP_MOVABLE_MASK |
257 | #undef GFP_MOVABLE_SHIFT |
||
5270 | serge | 258 | |
6082 | serge | 259 | static inline bool gfpflags_allow_blocking(const gfp_t gfp_flags) |
260 | { |
||
6102 | serge | 261 | return (bool __force)(gfp_flags & __GFP_DIRECT_RECLAIM); |
6082 | serge | 262 | } |
5270 | serge | 263 | |
264 | #ifdef CONFIG_HIGHMEM |
||
265 | #define OPT_ZONE_HIGHMEM ZONE_HIGHMEM |
||
266 | #else |
||
267 | #define OPT_ZONE_HIGHMEM ZONE_NORMAL |
||
268 | #endif |
||
269 | |||
270 | #ifdef CONFIG_ZONE_DMA |
||
271 | #define OPT_ZONE_DMA ZONE_DMA |
||
272 | #else |
||
273 | #define OPT_ZONE_DMA ZONE_NORMAL |
||
274 | #endif |
||
275 | |||
276 | #ifdef CONFIG_ZONE_DMA32 |
||
277 | #define OPT_ZONE_DMA32 ZONE_DMA32 |
||
278 | #else |
||
279 | #define OPT_ZONE_DMA32 ZONE_NORMAL |
||
280 | #endif |
||
281 | |||
282 | /* |
||
283 | * GFP_ZONE_TABLE is a word size bitstring that is used for looking up the |
||
284 | * zone to use given the lowest 4 bits of gfp_t. Entries are ZONE_SHIFT long |
||
285 | * and there are 16 of them to cover all possible combinations of |
||
286 | * __GFP_DMA, __GFP_DMA32, __GFP_MOVABLE and __GFP_HIGHMEM. |
||
287 | * |
||
288 | * The zone fallback order is MOVABLE=>HIGHMEM=>NORMAL=>DMA32=>DMA. |
||
289 | * But GFP_MOVABLE is not only a zone specifier but also an allocation |
||
290 | * policy. Therefore __GFP_MOVABLE plus another zone selector is valid. |
||
291 | * Only 1 bit of the lowest 3 bits (DMA,DMA32,HIGHMEM) can be set to "1". |
||
292 | * |
||
293 | * bit result |
||
294 | * ================= |
||
295 | * 0x0 => NORMAL |
||
296 | * 0x1 => DMA or NORMAL |
||
297 | * 0x2 => HIGHMEM or NORMAL |
||
298 | * 0x3 => BAD (DMA+HIGHMEM) |
||
299 | * 0x4 => DMA32 or DMA or NORMAL |
||
300 | * 0x5 => BAD (DMA+DMA32) |
||
301 | * 0x6 => BAD (HIGHMEM+DMA32) |
||
302 | * 0x7 => BAD (HIGHMEM+DMA32+DMA) |
||
303 | * 0x8 => NORMAL (MOVABLE+0) |
||
304 | * 0x9 => DMA or NORMAL (MOVABLE+DMA) |
||
305 | * 0xa => MOVABLE (Movable is valid only if HIGHMEM is set too) |
||
306 | * 0xb => BAD (MOVABLE+HIGHMEM+DMA) |
||
307 | * 0xc => DMA32 (MOVABLE+DMA32) |
||
308 | * 0xd => BAD (MOVABLE+DMA32+DMA) |
||
309 | * 0xe => BAD (MOVABLE+DMA32+HIGHMEM) |
||
310 | * 0xf => BAD (MOVABLE+DMA32+HIGHMEM+DMA) |
||
311 | * |
||
312 | * ZONES_SHIFT must be <= 2 on 32 bit platforms. |
||
313 | */ |
||
314 | |||
315 | #if 16 * ZONES_SHIFT > BITS_PER_LONG |
||
316 | #error ZONES_SHIFT too large to create GFP_ZONE_TABLE integer |
||
317 | #endif |
||
318 | |||
319 | #define GFP_ZONE_TABLE ( \ |
||
320 | (ZONE_NORMAL << 0 * ZONES_SHIFT) \ |
||
321 | | (OPT_ZONE_DMA << ___GFP_DMA * ZONES_SHIFT) \ |
||
322 | | (OPT_ZONE_HIGHMEM << ___GFP_HIGHMEM * ZONES_SHIFT) \ |
||
323 | | (OPT_ZONE_DMA32 << ___GFP_DMA32 * ZONES_SHIFT) \ |
||
324 | | (ZONE_NORMAL << ___GFP_MOVABLE * ZONES_SHIFT) \ |
||
325 | | (OPT_ZONE_DMA << (___GFP_MOVABLE | ___GFP_DMA) * ZONES_SHIFT) \ |
||
326 | | (ZONE_MOVABLE << (___GFP_MOVABLE | ___GFP_HIGHMEM) * ZONES_SHIFT) \ |
||
327 | | (OPT_ZONE_DMA32 << (___GFP_MOVABLE | ___GFP_DMA32) * ZONES_SHIFT) \ |
||
328 | ) |
||
329 | |||
330 | /* |
||
331 | * GFP_ZONE_BAD is a bitmap for all combinations of __GFP_DMA, __GFP_DMA32 |
||
332 | * __GFP_HIGHMEM and __GFP_MOVABLE that are not permitted. One flag per |
||
333 | * entry starting with bit 0. Bit is set if the combination is not |
||
334 | * allowed. |
||
335 | */ |
||
336 | #define GFP_ZONE_BAD ( \ |
||
337 | 1 << (___GFP_DMA | ___GFP_HIGHMEM) \ |
||
338 | | 1 << (___GFP_DMA | ___GFP_DMA32) \ |
||
339 | | 1 << (___GFP_DMA32 | ___GFP_HIGHMEM) \ |
||
340 | | 1 << (___GFP_DMA | ___GFP_DMA32 | ___GFP_HIGHMEM) \ |
||
341 | | 1 << (___GFP_MOVABLE | ___GFP_HIGHMEM | ___GFP_DMA) \ |
||
342 | | 1 << (___GFP_MOVABLE | ___GFP_DMA32 | ___GFP_DMA) \ |
||
343 | | 1 << (___GFP_MOVABLE | ___GFP_DMA32 | ___GFP_HIGHMEM) \ |
||
344 | | 1 << (___GFP_MOVABLE | ___GFP_DMA32 | ___GFP_DMA | ___GFP_HIGHMEM) \ |
||
345 | ) |
||
346 | |||
347 | |||
348 | #endif /* __LINUX_GFP_H */><>><>><>><>><>><>><>><>><>><>><>><>><>><>><>><>=>><> |