Rev 4569 | Rev 5271 | Go to most recent revision | Only display areas with differences | Regard whitespace | Details | Blame | Last modification | View Log | RSS feed
Rev 4569 | Rev 5078 | ||
---|---|---|---|
1 | /************************************************************************** |
1 | /************************************************************************** |
2 | * |
2 | * |
3 | * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA |
3 | * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA |
4 | * All Rights Reserved. |
4 | * All Rights Reserved. |
5 | * |
5 | * |
6 | * Permission is hereby granted, free of charge, to any person obtaining a |
6 | * Permission is hereby granted, free of charge, to any person obtaining a |
7 | * copy of this software and associated documentation files (the |
7 | * copy of this software and associated documentation files (the |
8 | * "Software"), to deal in the Software without restriction, including |
8 | * "Software"), to deal in the Software without restriction, including |
9 | * without limitation the rights to use, copy, modify, merge, publish, |
9 | * without limitation the rights to use, copy, modify, merge, publish, |
10 | * distribute, sub license, and/or sell copies of the Software, and to |
10 | * distribute, sub license, and/or sell copies of the Software, and to |
11 | * permit persons to whom the Software is furnished to do so, subject to |
11 | * permit persons to whom the Software is furnished to do so, subject to |
12 | * the following conditions: |
12 | * the following conditions: |
13 | * |
13 | * |
14 | * The above copyright notice and this permission notice (including the |
14 | * The above copyright notice and this permission notice (including the |
15 | * next paragraph) shall be included in all copies or substantial portions |
15 | * next paragraph) shall be included in all copies or substantial portions |
16 | * of the Software. |
16 | * of the Software. |
17 | * |
17 | * |
18 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
18 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
19 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
19 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
20 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL |
20 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL |
21 | * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, |
21 | * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, |
22 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR |
22 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR |
23 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE |
23 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE |
24 | * USE OR OTHER DEALINGS IN THE SOFTWARE. |
24 | * USE OR OTHER DEALINGS IN THE SOFTWARE. |
25 | * |
25 | * |
26 | **************************************************************************/ |
26 | **************************************************************************/ |
27 | /* |
27 | /* |
28 | * Authors: Thomas Hellstrom |
28 | * Authors: Thomas Hellstrom |
29 | */ |
29 | */ |
30 | 30 | ||
31 | #define pr_fmt(fmt) "[TTM] " fmt |
31 | #define pr_fmt(fmt) "[TTM] " fmt |
32 | 32 | ||
33 | #include |
33 | #include |
34 | 34 | ||
35 | #include |
35 | #include |
36 | //#include |
36 | //#include |
37 | //#include |
37 | //#include |
38 | #include |
38 | #include |
39 | //#include |
39 | //#include |
40 | //#include |
40 | //#include |
41 | #include |
41 | #include |
42 | #include |
42 | #include |
43 | //#include |
43 | //#include |
44 | #include |
44 | #include |
45 | #include |
45 | #include |
46 | #include |
46 | #include |
47 | #include |
47 | #include |
48 | #include |
48 | #include |
49 | 49 | ||
50 | /** |
50 | /** |
51 | * Allocates storage for pointers to the pages that back the ttm. |
51 | * Allocates storage for pointers to the pages that back the ttm. |
52 | */ |
52 | */ |
53 | static void ttm_tt_alloc_page_directory(struct ttm_tt *ttm) |
53 | static void ttm_tt_alloc_page_directory(struct ttm_tt *ttm) |
54 | { |
54 | { |
55 | ttm->pages = drm_calloc_large(ttm->num_pages, sizeof(void*)); |
55 | ttm->pages = drm_calloc_large(ttm->num_pages, sizeof(void*)); |
56 | } |
56 | } |
57 | 57 | ||
58 | static void ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt *ttm) |
58 | static void ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt *ttm) |
59 | { |
59 | { |
60 | ttm->ttm.pages = drm_calloc_large(ttm->ttm.num_pages, sizeof(void*)); |
60 | ttm->ttm.pages = drm_calloc_large(ttm->ttm.num_pages, |
61 | ttm->dma_address = drm_calloc_large(ttm->ttm.num_pages, |
61 | sizeof(*ttm->ttm.pages) + |
62 | sizeof(*ttm->dma_address)); |
62 | sizeof(*ttm->dma_address) + |
- | 63 | sizeof(*ttm->cpu_address)); |
|
- | 64 | ttm->cpu_address = (void *) (ttm->ttm.pages + ttm->ttm.num_pages); |
|
- | 65 | ttm->dma_address = (void *) (ttm->cpu_address + ttm->ttm.num_pages); |
|
63 | } |
66 | } |
64 | 67 | ||
65 | #ifdef CONFIG_X86 |
68 | #ifdef CONFIG_X86 |
66 | static inline int ttm_tt_set_page_caching(struct page *p, |
69 | static inline int ttm_tt_set_page_caching(struct page *p, |
67 | enum ttm_caching_state c_old, |
70 | enum ttm_caching_state c_old, |
68 | enum ttm_caching_state c_new) |
71 | enum ttm_caching_state c_new) |
69 | { |
72 | { |
70 | int ret = 0; |
73 | int ret = 0; |
71 | 74 | ||
72 | if (PageHighMem(p)) |
75 | if (PageHighMem(p)) |
73 | return 0; |
76 | return 0; |
74 | 77 | ||
75 | if (c_old != tt_cached) { |
78 | if (c_old != tt_cached) { |
76 | /* p isn't in the default caching state, set it to |
79 | /* p isn't in the default caching state, set it to |
77 | * writeback first to free its current memtype. */ |
80 | * writeback first to free its current memtype. */ |
78 | 81 | ||
79 | ret = set_pages_wb(p, 1); |
82 | ret = set_pages_wb(p, 1); |
80 | if (ret) |
83 | if (ret) |
81 | return ret; |
84 | return ret; |
82 | } |
85 | } |
83 | 86 | ||
84 | if (c_new == tt_wc) |
87 | if (c_new == tt_wc) |
85 | ret = set_memory_wc((unsigned long) page_address(p), 1); |
88 | ret = set_memory_wc((unsigned long) page_address(p), 1); |
86 | else if (c_new == tt_uncached) |
89 | else if (c_new == tt_uncached) |
87 | ret = set_pages_uc(p, 1); |
90 | ret = set_pages_uc(p, 1); |
88 | 91 | ||
89 | return ret; |
92 | return ret; |
90 | } |
93 | } |
91 | #else /* CONFIG_X86 */ |
94 | #else /* CONFIG_X86 */ |
92 | static inline int ttm_tt_set_page_caching(struct page *p, |
95 | static inline int ttm_tt_set_page_caching(struct page *p, |
93 | enum ttm_caching_state c_old, |
96 | enum ttm_caching_state c_old, |
94 | enum ttm_caching_state c_new) |
97 | enum ttm_caching_state c_new) |
95 | { |
98 | { |
96 | return 0; |
99 | return 0; |
97 | } |
100 | } |
98 | #endif /* CONFIG_X86 */ |
101 | #endif /* CONFIG_X86 */ |
99 | 102 | ||
100 | /* |
103 | /* |
101 | * Change caching policy for the linear kernel map |
104 | * Change caching policy for the linear kernel map |
102 | * for range of pages in a ttm. |
105 | * for range of pages in a ttm. |
103 | */ |
106 | */ |
104 | 107 | ||
105 | static int ttm_tt_set_caching(struct ttm_tt *ttm, |
108 | static int ttm_tt_set_caching(struct ttm_tt *ttm, |
106 | enum ttm_caching_state c_state) |
109 | enum ttm_caching_state c_state) |
107 | { |
110 | { |
108 | int i, j; |
111 | int i, j; |
109 | struct page *cur_page; |
112 | struct page *cur_page; |
110 | int ret; |
113 | int ret; |
111 | 114 | ||
112 | if (ttm->caching_state == c_state) |
115 | if (ttm->caching_state == c_state) |
113 | return 0; |
116 | return 0; |
114 | 117 | ||
115 | if (ttm->state == tt_unpopulated) { |
118 | if (ttm->state == tt_unpopulated) { |
116 | /* Change caching but don't populate */ |
119 | /* Change caching but don't populate */ |
117 | ttm->caching_state = c_state; |
120 | ttm->caching_state = c_state; |
118 | return 0; |
121 | return 0; |
119 | } |
122 | } |
120 | 123 | ||
121 | // if (ttm->caching_state == tt_cached) |
124 | if (ttm->caching_state == tt_cached) |
122 | // drm_clflush_pages(ttm->pages, ttm->num_pages); |
125 | drm_clflush_pages(ttm->pages, ttm->num_pages); |
123 | 126 | ||
124 | for (i = 0; i < ttm->num_pages; ++i) { |
127 | for (i = 0; i < ttm->num_pages; ++i) { |
125 | cur_page = ttm->pages[i]; |
128 | cur_page = ttm->pages[i]; |
126 | if (likely(cur_page != NULL)) { |
129 | if (likely(cur_page != NULL)) { |
127 | ret = ttm_tt_set_page_caching(cur_page, |
130 | ret = ttm_tt_set_page_caching(cur_page, |
128 | ttm->caching_state, |
131 | ttm->caching_state, |
129 | c_state); |
132 | c_state); |
130 | if (unlikely(ret != 0)) |
133 | if (unlikely(ret != 0)) |
131 | goto out_err; |
134 | goto out_err; |
132 | } |
135 | } |
133 | } |
136 | } |
134 | 137 | ||
135 | ttm->caching_state = c_state; |
138 | ttm->caching_state = c_state; |
136 | 139 | ||
137 | return 0; |
140 | return 0; |
138 | 141 | ||
139 | out_err: |
142 | out_err: |
140 | for (j = 0; j < i; ++j) { |
143 | for (j = 0; j < i; ++j) { |
141 | cur_page = ttm->pages[j]; |
144 | cur_page = ttm->pages[j]; |
142 | if (likely(cur_page != NULL)) { |
145 | if (likely(cur_page != NULL)) { |
143 | (void)ttm_tt_set_page_caching(cur_page, c_state, |
146 | (void)ttm_tt_set_page_caching(cur_page, c_state, |
144 | ttm->caching_state); |
147 | ttm->caching_state); |
145 | } |
148 | } |
146 | } |
149 | } |
147 | 150 | ||
148 | return ret; |
151 | return ret; |
149 | } |
152 | } |
150 | 153 | ||
151 | int ttm_tt_set_placement_caching(struct ttm_tt *ttm, uint32_t placement) |
154 | int ttm_tt_set_placement_caching(struct ttm_tt *ttm, uint32_t placement) |
152 | { |
155 | { |
153 | enum ttm_caching_state state; |
156 | enum ttm_caching_state state; |
154 | 157 | ||
155 | if (placement & TTM_PL_FLAG_WC) |
158 | if (placement & TTM_PL_FLAG_WC) |
156 | state = tt_wc; |
159 | state = tt_wc; |
157 | else if (placement & TTM_PL_FLAG_UNCACHED) |
160 | else if (placement & TTM_PL_FLAG_UNCACHED) |
158 | state = tt_uncached; |
161 | state = tt_uncached; |
159 | else |
162 | else |
160 | state = tt_cached; |
163 | state = tt_cached; |
161 | 164 | ||
162 | return ttm_tt_set_caching(ttm, state); |
165 | return ttm_tt_set_caching(ttm, state); |
163 | } |
166 | } |
164 | EXPORT_SYMBOL(ttm_tt_set_placement_caching); |
167 | EXPORT_SYMBOL(ttm_tt_set_placement_caching); |
165 | 168 | ||
166 | void ttm_tt_destroy(struct ttm_tt *ttm) |
169 | void ttm_tt_destroy(struct ttm_tt *ttm) |
167 | { |
170 | { |
168 | if (unlikely(ttm == NULL)) |
171 | if (unlikely(ttm == NULL)) |
169 | return; |
172 | return; |
170 | 173 | ||
171 | if (ttm->state == tt_bound) { |
174 | if (ttm->state == tt_bound) { |
172 | ttm_tt_unbind(ttm); |
175 | ttm_tt_unbind(ttm); |
173 | } |
176 | } |
174 | 177 | ||
175 | // if (ttm->state == tt_unbound) |
178 | if (ttm->state == tt_unbound) |
176 | // ttm_tt_unpopulate(ttm); |
179 | ttm_tt_unpopulate(ttm); |
177 | 180 | ||
178 | // if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP) && |
181 | // if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP) && |
179 | // ttm->swap_storage) |
182 | // ttm->swap_storage) |
180 | // fput(ttm->swap_storage); |
183 | // fput(ttm->swap_storage); |
181 | 184 | ||
182 | ttm->swap_storage = NULL; |
185 | ttm->swap_storage = NULL; |
183 | ttm->func->destroy(ttm); |
186 | ttm->func->destroy(ttm); |
184 | } |
187 | } |
185 | 188 | ||
186 | int ttm_tt_init(struct ttm_tt *ttm, struct ttm_bo_device *bdev, |
189 | int ttm_tt_init(struct ttm_tt *ttm, struct ttm_bo_device *bdev, |
187 | unsigned long size, uint32_t page_flags, |
190 | unsigned long size, uint32_t page_flags, |
188 | struct page *dummy_read_page) |
191 | struct page *dummy_read_page) |
189 | { |
192 | { |
190 | ttm->bdev = bdev; |
193 | ttm->bdev = bdev; |
191 | ttm->glob = bdev->glob; |
194 | ttm->glob = bdev->glob; |
192 | ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; |
195 | ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; |
193 | ttm->caching_state = tt_cached; |
196 | ttm->caching_state = tt_cached; |
194 | ttm->page_flags = page_flags; |
197 | ttm->page_flags = page_flags; |
195 | ttm->dummy_read_page = dummy_read_page; |
198 | ttm->dummy_read_page = dummy_read_page; |
196 | ttm->state = tt_unpopulated; |
199 | ttm->state = tt_unpopulated; |
197 | ttm->swap_storage = NULL; |
200 | ttm->swap_storage = NULL; |
198 | 201 | ||
199 | ttm_tt_alloc_page_directory(ttm); |
202 | ttm_tt_alloc_page_directory(ttm); |
200 | if (!ttm->pages) { |
203 | if (!ttm->pages) { |
201 | ttm_tt_destroy(ttm); |
204 | ttm_tt_destroy(ttm); |
202 | printf("Failed allocating page table\n"); |
205 | printf("Failed allocating page table\n"); |
203 | return -ENOMEM; |
206 | return -ENOMEM; |
204 | } |
207 | } |
205 | return 0; |
208 | return 0; |
206 | } |
209 | } |
207 | EXPORT_SYMBOL(ttm_tt_init); |
210 | EXPORT_SYMBOL(ttm_tt_init); |
208 | 211 | ||
209 | void ttm_tt_fini(struct ttm_tt *ttm) |
212 | void ttm_tt_fini(struct ttm_tt *ttm) |
210 | { |
213 | { |
211 | drm_free_large(ttm->pages); |
214 | drm_free_large(ttm->pages); |
212 | ttm->pages = NULL; |
215 | ttm->pages = NULL; |
213 | } |
216 | } |
214 | EXPORT_SYMBOL(ttm_tt_fini); |
217 | EXPORT_SYMBOL(ttm_tt_fini); |
215 | 218 | ||
216 | int ttm_dma_tt_init(struct ttm_dma_tt *ttm_dma, struct ttm_bo_device *bdev, |
219 | int ttm_dma_tt_init(struct ttm_dma_tt *ttm_dma, struct ttm_bo_device *bdev, |
217 | unsigned long size, uint32_t page_flags, |
220 | unsigned long size, uint32_t page_flags, |
218 | struct page *dummy_read_page) |
221 | struct page *dummy_read_page) |
219 | { |
222 | { |
220 | struct ttm_tt *ttm = &ttm_dma->ttm; |
223 | struct ttm_tt *ttm = &ttm_dma->ttm; |
221 | 224 | ||
222 | ttm->bdev = bdev; |
225 | ttm->bdev = bdev; |
223 | ttm->glob = bdev->glob; |
226 | ttm->glob = bdev->glob; |
224 | ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; |
227 | ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; |
225 | ttm->caching_state = tt_cached; |
228 | ttm->caching_state = tt_cached; |
226 | ttm->page_flags = page_flags; |
229 | ttm->page_flags = page_flags; |
227 | ttm->dummy_read_page = dummy_read_page; |
230 | ttm->dummy_read_page = dummy_read_page; |
228 | ttm->state = tt_unpopulated; |
231 | ttm->state = tt_unpopulated; |
229 | ttm->swap_storage = NULL; |
232 | ttm->swap_storage = NULL; |
230 | 233 | ||
231 | INIT_LIST_HEAD(&ttm_dma->pages_list); |
234 | INIT_LIST_HEAD(&ttm_dma->pages_list); |
232 | ttm_dma_tt_alloc_page_directory(ttm_dma); |
235 | ttm_dma_tt_alloc_page_directory(ttm_dma); |
233 | if (!ttm->pages || !ttm_dma->dma_address) { |
236 | if (!ttm->pages) { |
234 | ttm_tt_destroy(ttm); |
237 | ttm_tt_destroy(ttm); |
235 | printf("Failed allocating page table\n"); |
238 | printf("Failed allocating page table\n"); |
236 | return -ENOMEM; |
239 | return -ENOMEM; |
237 | } |
240 | } |
238 | return 0; |
241 | return 0; |
239 | } |
242 | } |
240 | EXPORT_SYMBOL(ttm_dma_tt_init); |
243 | EXPORT_SYMBOL(ttm_dma_tt_init); |
241 | 244 | ||
242 | void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma) |
245 | void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma) |
243 | { |
246 | { |
244 | struct ttm_tt *ttm = &ttm_dma->ttm; |
247 | struct ttm_tt *ttm = &ttm_dma->ttm; |
245 | 248 | ||
246 | drm_free_large(ttm->pages); |
249 | drm_free_large(ttm->pages); |
247 | ttm->pages = NULL; |
250 | ttm->pages = NULL; |
248 | drm_free_large(ttm_dma->dma_address); |
251 | ttm_dma->cpu_address = NULL; |
249 | ttm_dma->dma_address = NULL; |
252 | ttm_dma->dma_address = NULL; |
250 | } |
253 | } |
251 | EXPORT_SYMBOL(ttm_dma_tt_fini); |
254 | EXPORT_SYMBOL(ttm_dma_tt_fini); |
252 | 255 | ||
253 | void ttm_tt_unbind(struct ttm_tt *ttm) |
256 | void ttm_tt_unbind(struct ttm_tt *ttm) |
254 | { |
257 | { |
255 | int ret; |
258 | int ret; |
256 | 259 | ||
257 | if (ttm->state == tt_bound) { |
260 | if (ttm->state == tt_bound) { |
258 | ret = ttm->func->unbind(ttm); |
261 | ret = ttm->func->unbind(ttm); |
259 | BUG_ON(ret); |
262 | BUG_ON(ret); |
260 | ttm->state = tt_unbound; |
263 | ttm->state = tt_unbound; |
261 | } |
264 | } |
262 | } |
265 | } |
263 | - | ||
264 | #if 0 |
- | |
265 | 266 | ||
266 | int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) |
267 | int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) |
267 | { |
268 | { |
268 | int ret = 0; |
269 | int ret = 0; |
269 | 270 | ||
270 | if (!ttm) |
271 | if (!ttm) |
271 | return -EINVAL; |
272 | return -EINVAL; |
272 | 273 | ||
273 | if (ttm->state == tt_bound) |
274 | if (ttm->state == tt_bound) |
274 | return 0; |
275 | return 0; |
275 | 276 | ||
276 | ret = ttm->bdev->driver->ttm_tt_populate(ttm); |
277 | ret = ttm->bdev->driver->ttm_tt_populate(ttm); |
277 | if (ret) |
278 | if (ret) |
278 | return ret; |
279 | return ret; |
279 | 280 | ||
280 | ret = ttm->func->bind(ttm, bo_mem); |
281 | ret = ttm->func->bind(ttm, bo_mem); |
281 | if (unlikely(ret != 0)) |
282 | if (unlikely(ret != 0)) |
282 | return ret; |
283 | return ret; |
283 | 284 | ||
284 | ttm->state = tt_bound; |
285 | ttm->state = tt_bound; |
285 | 286 | ||
286 | return 0; |
287 | return 0; |
287 | } |
288 | } |
288 | EXPORT_SYMBOL(ttm_tt_bind); |
289 | EXPORT_SYMBOL(ttm_tt_bind); |
289 | #endif |
- | |
290 | 290 | ||
291 | /* |
291 | #if 0 |
292 | int ttm_tt_swapin(struct ttm_tt *ttm) |
292 | int ttm_tt_swapin(struct ttm_tt *ttm) |
293 | { |
293 | { |
294 | struct address_space *swap_space; |
294 | struct address_space *swap_space; |
295 | struct file *swap_storage; |
295 | struct file *swap_storage; |
296 | struct page *from_page; |
296 | struct page *from_page; |
297 | struct page *to_page; |
297 | struct page *to_page; |
298 | int i; |
298 | int i; |
299 | int ret = -ENOMEM; |
299 | int ret = -ENOMEM; |
300 | 300 | ||
301 | swap_storage = ttm->swap_storage; |
301 | swap_storage = ttm->swap_storage; |
302 | BUG_ON(swap_storage == NULL); |
302 | BUG_ON(swap_storage == NULL); |
303 | 303 | ||
304 | swap_space = file_inode(swap_storage)->i_mapping; |
304 | swap_space = file_inode(swap_storage)->i_mapping; |
305 | 305 | ||
306 | for (i = 0; i < ttm->num_pages; ++i) { |
306 | for (i = 0; i < ttm->num_pages; ++i) { |
307 | from_page = shmem_read_mapping_page(swap_space, i); |
307 | from_page = shmem_read_mapping_page(swap_space, i); |
308 | if (IS_ERR(from_page)) { |
308 | if (IS_ERR(from_page)) { |
309 | ret = PTR_ERR(from_page); |
309 | ret = PTR_ERR(from_page); |
310 | goto out_err; |
310 | goto out_err; |
311 | } |
311 | } |
312 | to_page = ttm->pages[i]; |
312 | to_page = ttm->pages[i]; |
313 | if (unlikely(to_page == NULL)) |
313 | if (unlikely(to_page == NULL)) |
314 | goto out_err; |
314 | goto out_err; |
315 | 315 | ||
316 | copy_highpage(to_page, from_page); |
316 | copy_highpage(to_page, from_page); |
317 | page_cache_release(from_page); |
317 | page_cache_release(from_page); |
318 | } |
318 | } |
319 | 319 | ||
320 | if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP)) |
320 | if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP)) |
321 | fput(swap_storage); |
321 | fput(swap_storage); |
322 | ttm->swap_storage = NULL; |
322 | ttm->swap_storage = NULL; |
323 | ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED; |
323 | ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED; |
324 | 324 | ||
325 | return 0; |
325 | return 0; |
326 | out_err: |
326 | out_err: |
327 | return ret; |
327 | return ret; |
328 | } |
328 | } |
329 | 329 | ||
330 | int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistent_swap_storage) |
330 | int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistent_swap_storage) |
331 | { |
331 | { |
332 | struct address_space *swap_space; |
332 | struct address_space *swap_space; |
333 | struct file *swap_storage; |
333 | struct file *swap_storage; |
334 | struct page *from_page; |
334 | struct page *from_page; |
335 | struct page *to_page; |
335 | struct page *to_page; |
336 | int i; |
336 | int i; |
337 | int ret = -ENOMEM; |
337 | int ret = -ENOMEM; |
338 | 338 | ||
339 | BUG_ON(ttm->state != tt_unbound && ttm->state != tt_unpopulated); |
339 | BUG_ON(ttm->state != tt_unbound && ttm->state != tt_unpopulated); |
340 | BUG_ON(ttm->caching_state != tt_cached); |
340 | BUG_ON(ttm->caching_state != tt_cached); |
341 | 341 | ||
342 | if (!persistent_swap_storage) { |
342 | if (!persistent_swap_storage) { |
343 | swap_storage = shmem_file_setup("ttm swap", |
343 | swap_storage = shmem_file_setup("ttm swap", |
344 | ttm->num_pages << PAGE_SHIFT, |
344 | ttm->num_pages << PAGE_SHIFT, |
345 | 0); |
345 | 0); |
346 | if (unlikely(IS_ERR(swap_storage))) { |
346 | if (unlikely(IS_ERR(swap_storage))) { |
347 | pr_err("Failed allocating swap storage\n"); |
347 | pr_err("Failed allocating swap storage\n"); |
348 | return PTR_ERR(swap_storage); |
348 | return PTR_ERR(swap_storage); |
349 | } |
349 | } |
350 | } else |
350 | } else |
351 | swap_storage = persistent_swap_storage; |
351 | swap_storage = persistent_swap_storage; |
352 | 352 | ||
353 | swap_space = file_inode(swap_storage)->i_mapping; |
353 | swap_space = file_inode(swap_storage)->i_mapping; |
354 | 354 | ||
355 | for (i = 0; i < ttm->num_pages; ++i) { |
355 | for (i = 0; i < ttm->num_pages; ++i) { |
356 | from_page = ttm->pages[i]; |
356 | from_page = ttm->pages[i]; |
357 | if (unlikely(from_page == NULL)) |
357 | if (unlikely(from_page == NULL)) |
358 | continue; |
358 | continue; |
359 | to_page = shmem_read_mapping_page(swap_space, i); |
359 | to_page = shmem_read_mapping_page(swap_space, i); |
360 | if (unlikely(IS_ERR(to_page))) { |
360 | if (unlikely(IS_ERR(to_page))) { |
361 | ret = PTR_ERR(to_page); |
361 | ret = PTR_ERR(to_page); |
362 | goto out_err; |
362 | goto out_err; |
363 | } |
363 | } |
364 | copy_highpage(to_page, from_page); |
364 | copy_highpage(to_page, from_page); |
365 | set_page_dirty(to_page); |
365 | set_page_dirty(to_page); |
366 | mark_page_accessed(to_page); |
366 | mark_page_accessed(to_page); |
367 | page_cache_release(to_page); |
367 | page_cache_release(to_page); |
368 | } |
368 | } |
369 | 369 | ||
370 | ttm_tt_unpopulate(ttm); |
370 | ttm_tt_unpopulate(ttm); |
371 | ttm->swap_storage = swap_storage; |
371 | ttm->swap_storage = swap_storage; |
372 | ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED; |
372 | ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED; |
373 | if (persistent_swap_storage) |
373 | if (persistent_swap_storage) |
374 | ttm->page_flags |= TTM_PAGE_FLAG_PERSISTENT_SWAP; |
374 | ttm->page_flags |= TTM_PAGE_FLAG_PERSISTENT_SWAP; |
375 | 375 | ||
376 | return 0; |
376 | return 0; |
377 | out_err: |
377 | out_err: |
378 | if (!persistent_swap_storage) |
378 | if (!persistent_swap_storage) |
379 | fput(swap_storage); |
379 | fput(swap_storage); |
380 | 380 | ||
381 | return ret; |
381 | return ret; |
382 | } |
382 | } |
- | 383 | #endif |
|
- | 384 | ||
383 | 385 | static void ttm_tt_clear_mapping(struct ttm_tt *ttm) |
|
- | 386 | { |
|
- | 387 | pgoff_t i; |
|
- | 388 | struct page **page = ttm->pages; |
|
- | 389 | ||
- | 390 | } |
|
- | 391 | ||
- | 392 | void ttm_tt_unpopulate(struct ttm_tt *ttm) |
|
- | 393 | { |
|
- | 394 | if (ttm->state == tt_unpopulated) |
|
- | 395 | return; |
|
- | 396 | ||
384 | */>><>>>> |
397 | ttm_tt_clear_mapping(ttm); |
- | 398 | ttm->bdev->driver->ttm_tt_unpopulate(ttm); |
|
- | 399 | }>><>>>> |