Rev 5346 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed
Rev | Author | Line No. | Line |
---|---|---|---|
4075 | Serge | 1 | /* |
2 | * Copyright (c) Red Hat Inc. |
||
3 | |||
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
||
5 | * copy of this software and associated documentation files (the "Software"), |
||
6 | * to deal in the Software without restriction, including without limitation |
||
7 | * the rights to use, copy, modify, merge, publish, distribute, sub license, |
||
8 | * and/or sell copies of the Software, and to permit persons to whom the |
||
9 | * Software is furnished to do so, subject to the following conditions: |
||
10 | * |
||
11 | * The above copyright notice and this permission notice (including the |
||
12 | * next paragraph) shall be included in all copies or substantial portions |
||
13 | * of the Software. |
||
14 | * |
||
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
||
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
||
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL |
||
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
||
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
||
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER |
||
21 | * DEALINGS IN THE SOFTWARE. |
||
22 | * |
||
23 | * Authors: Dave Airlie |
||
24 | * Jerome Glisse |
||
25 | * Pauli Nieminen |
||
26 | */ |
||
27 | |||
28 | /* simple list based uncached page pool |
||
29 | * - Pool collects resently freed pages for reuse |
||
30 | * - Use page->lru to keep a free list |
||
31 | * - doesn't track currently in use pages |
||
32 | */ |
||
33 | |||
34 | #define pr_fmt(fmt) "[TTM] " fmt |
||
35 | |||
36 | #include |
||
37 | #include |
||
38 | //#include |
||
39 | //#include |
||
40 | #include |
||
41 | #include |
||
42 | #include |
||
43 | #include |
||
5078 | serge | 44 | //#include |
4075 | Serge | 45 | |
5346 | serge | 46 | #include |
4075 | Serge | 47 | |
48 | #include |
||
49 | #include |
||
50 | |||
51 | #ifdef TTM_HAS_AGP |
||
52 | #include |
||
53 | #endif |
||
54 | |||
55 | #define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(struct page *)) |
||
56 | #define SMALL_ALLOCATION 16 |
||
57 | #define FREE_ALL_PAGES (~0U) |
||
58 | /* times are in msecs */ |
||
59 | #define PAGE_FREE_INTERVAL 1000 |
||
60 | |||
61 | /** |
||
62 | * struct ttm_page_pool - Pool to reuse recently allocated uc/wc pages. |
||
63 | * |
||
64 | * @lock: Protects the shared pool from concurrnet access. Must be used with |
||
65 | * irqsave/irqrestore variants because pool allocator maybe called from |
||
66 | * delayed work. |
||
67 | * @fill_lock: Prevent concurrent calls to fill. |
||
68 | * @list: Pool of free uc/wc pages for fast reuse. |
||
69 | * @gfp_flags: Flags to pass for alloc_page. |
||
70 | * @npages: Number of pages in pool. |
||
71 | */ |
||
72 | struct ttm_page_pool { |
||
6104 | serge | 73 | spinlock_t lock; |
74 | bool fill_lock; |
||
4075 | Serge | 75 | struct list_head list; |
6104 | serge | 76 | gfp_t gfp_flags; |
77 | unsigned npages; |
||
78 | char *name; |
||
4075 | Serge | 79 | unsigned long nfrees; |
80 | unsigned long nrefills; |
||
81 | }; |
||
82 | |||
83 | /** |
||
84 | * Limits for the pool. They are handled without locks because only place where |
||
85 | * they may change is in sysfs store. They won't have immediate effect anyway |
||
86 | * so forcing serialization to access them is pointless. |
||
87 | */ |
||
88 | |||
89 | struct ttm_pool_opts { |
||
90 | unsigned alloc_size; |
||
91 | unsigned max_size; |
||
92 | unsigned small; |
||
93 | }; |
||
94 | |||
95 | #define NUM_POOLS 4 |
||
96 | |||
97 | /** |
||
98 | * struct ttm_pool_manager - Holds memory pools for fst allocation |
||
99 | * |
||
100 | * Manager is read only object for pool code so it doesn't need locking. |
||
101 | * |
||
102 | * @free_interval: minimum number of jiffies between freeing pages from pool. |
||
103 | * @page_alloc_inited: reference counting for pool allocation. |
||
104 | * @work: Work that is used to shrink the pool. Work is only run when there is |
||
105 | * some pages to free. |
||
106 | * @small_allocation: Limit in number of pages what is small allocation. |
||
107 | * |
||
108 | * @pools: All pool objects in use. |
||
109 | **/ |
||
110 | struct ttm_pool_manager { |
||
111 | struct kobject kobj; |
||
112 | struct ttm_pool_opts options; |
||
113 | |||
114 | union { |
||
115 | struct ttm_page_pool pools[NUM_POOLS]; |
||
116 | struct { |
||
117 | struct ttm_page_pool wc_pool; |
||
118 | struct ttm_page_pool uc_pool; |
||
119 | struct ttm_page_pool wc_pool_dma32; |
||
120 | struct ttm_page_pool uc_pool_dma32; |
||
121 | } ; |
||
122 | }; |
||
123 | }; |
||
124 | |||
125 | |||
126 | static struct ttm_pool_manager *_manager; |
||
127 | |||
128 | |||
129 | /** |
||
130 | * Select the right pool or requested caching state and ttm flags. */ |
||
131 | static struct ttm_page_pool *ttm_get_pool(int flags, |
||
132 | enum ttm_caching_state cstate) |
||
133 | { |
||
134 | int pool_index; |
||
135 | |||
136 | if (cstate == tt_cached) |
||
137 | return NULL; |
||
138 | |||
139 | if (cstate == tt_wc) |
||
140 | pool_index = 0x0; |
||
141 | else |
||
142 | pool_index = 0x1; |
||
143 | |||
144 | if (flags & TTM_PAGE_FLAG_DMA32) |
||
145 | pool_index |= 0x2; |
||
146 | |||
147 | return &_manager->pools[pool_index]; |
||
148 | } |
||
149 | |||
150 | /* set memory back to wb and free the pages. */ |
||
151 | static void ttm_pages_put(struct page *pages[], unsigned npages) |
||
152 | { |
||
153 | unsigned i; |
||
154 | for (i = 0; i < npages; ++i) |
||
155 | __free_page(pages[i]); |
||
156 | } |
||
157 | |||
158 | static void ttm_pool_update_free_locked(struct ttm_page_pool *pool, |
||
159 | unsigned freed_pages) |
||
160 | { |
||
161 | pool->npages -= freed_pages; |
||
162 | pool->nfrees += freed_pages; |
||
163 | } |
||
164 | |||
165 | |||
166 | |||
167 | |||
168 | /* Put all pages in pages list to correct pool to wait for reuse */ |
||
169 | static void ttm_put_pages(struct page **pages, unsigned npages, int flags, |
||
170 | enum ttm_caching_state cstate) |
||
171 | { |
||
172 | unsigned long irq_flags; |
||
5078 | serge | 173 | struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); |
4075 | Serge | 174 | unsigned i; |
175 | |||
5078 | serge | 176 | if (1) { |
4075 | Serge | 177 | /* No pool for this memory type so free the pages */ |
178 | for (i = 0; i < npages; i++) { |
||
179 | if (pages[i]) { |
||
180 | __free_page(pages[i]); |
||
181 | pages[i] = NULL; |
||
182 | } |
||
183 | } |
||
184 | return; |
||
185 | } |
||
186 | |||
187 | } |
||
188 | |||
189 | /* |
||
190 | * On success pages list will hold count number of correctly |
||
191 | * cached pages. |
||
192 | */ |
||
193 | static int ttm_get_pages(struct page **pages, unsigned npages, int flags, |
||
194 | enum ttm_caching_state cstate) |
||
195 | { |
||
5078 | serge | 196 | struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); |
4075 | Serge | 197 | struct list_head plist; |
198 | struct page *p = NULL; |
||
5078 | serge | 199 | gfp_t gfp_flags = 0; |
4075 | Serge | 200 | unsigned count; |
201 | int r; |
||
202 | |||
5078 | serge | 203 | |
4075 | Serge | 204 | /* No pool for cached pages */ |
5078 | serge | 205 | if (1) { |
4075 | Serge | 206 | |
207 | for (r = 0; r < npages; ++r) { |
||
208 | p = alloc_page(gfp_flags); |
||
209 | if (!p) { |
||
210 | |||
211 | return -ENOMEM; |
||
212 | } |
||
213 | |||
214 | pages[r] = p; |
||
215 | } |
||
216 | return 0; |
||
217 | } |
||
218 | |||
219 | |||
220 | |||
221 | return 0; |
||
222 | } |
||
223 | |||
5078 | serge | 224 | static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, gfp_t flags, |
4075 | Serge | 225 | char *name) |
226 | { |
||
227 | spin_lock_init(&pool->lock); |
||
228 | pool->fill_lock = false; |
||
229 | INIT_LIST_HEAD(&pool->list); |
||
230 | pool->npages = pool->nfrees = 0; |
||
231 | pool->gfp_flags = flags; |
||
232 | pool->name = name; |
||
233 | } |
||
234 | |||
235 | int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) |
||
236 | { |
||
237 | int ret; |
||
238 | |||
239 | WARN_ON(_manager); |
||
240 | |||
241 | _manager = kzalloc(sizeof(*_manager), GFP_KERNEL); |
||
242 | |||
243 | _manager->options.max_size = max_pages; |
||
244 | _manager->options.small = SMALL_ALLOCATION; |
||
245 | _manager->options.alloc_size = NUM_PAGES_TO_ALLOC; |
||
246 | |||
247 | return 0; |
||
248 | } |
||
249 | |||
250 | void ttm_page_alloc_fini(void) |
||
251 | { |
||
252 | int i; |
||
253 | |||
254 | _manager = NULL; |
||
255 | } |
||
256 | |||
257 | int ttm_pool_populate(struct ttm_tt *ttm) |
||
258 | { |
||
259 | struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; |
||
260 | unsigned i; |
||
261 | int ret; |
||
262 | |||
263 | if (ttm->state != tt_unpopulated) |
||
264 | return 0; |
||
265 | |||
266 | for (i = 0; i < ttm->num_pages; ++i) { |
||
267 | ret = ttm_get_pages(&ttm->pages[i], 1, |
||
268 | ttm->page_flags, |
||
269 | ttm->caching_state); |
||
270 | if (ret != 0) { |
||
271 | ttm_pool_unpopulate(ttm); |
||
272 | return -ENOMEM; |
||
273 | } |
||
274 | } |
||
275 | ttm->state = tt_unbound; |
||
276 | return 0; |
||
277 | } |
||
278 | EXPORT_SYMBOL(ttm_pool_populate); |
||
279 | |||
280 | void ttm_pool_unpopulate(struct ttm_tt *ttm) |
||
281 | { |
||
282 | unsigned i; |
||
283 | |||
284 | for (i = 0; i < ttm->num_pages; ++i) { |
||
285 | if (ttm->pages[i]) { |
||
286 | ttm_put_pages(&ttm->pages[i], 1, |
||
287 | ttm->page_flags, |
||
288 | ttm->caching_state); |
||
289 | } |
||
290 | } |
||
291 | ttm->state = tt_unpopulated; |
||
292 | } |
||
293 | EXPORT_SYMBOL(ttm_pool_unpopulate);>>>>> |
||
294 |