Rev 4569 | Rev 5271 | Go to most recent revision | Only display areas with differences | Regard whitespace | Details | Blame | Last modification | View Log | RSS feed
Rev 4569 | Rev 5078 | ||
---|---|---|---|
1 | /************************************************************************** |
1 | /************************************************************************** |
2 | * |
2 | * |
3 | * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA |
3 | * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA |
4 | * All Rights Reserved. |
4 | * All Rights Reserved. |
5 | * |
5 | * |
6 | * Permission is hereby granted, free of charge, to any person obtaining a |
6 | * Permission is hereby granted, free of charge, to any person obtaining a |
7 | * copy of this software and associated documentation files (the |
7 | * copy of this software and associated documentation files (the |
8 | * "Software"), to deal in the Software without restriction, including |
8 | * "Software"), to deal in the Software without restriction, including |
9 | * without limitation the rights to use, copy, modify, merge, publish, |
9 | * without limitation the rights to use, copy, modify, merge, publish, |
10 | * distribute, sub license, and/or sell copies of the Software, and to |
10 | * distribute, sub license, and/or sell copies of the Software, and to |
11 | * permit persons to whom the Software is furnished to do so, subject to |
11 | * permit persons to whom the Software is furnished to do so, subject to |
12 | * the following conditions: |
12 | * the following conditions: |
13 | * |
13 | * |
14 | * The above copyright notice and this permission notice (including the |
14 | * The above copyright notice and this permission notice (including the |
15 | * next paragraph) shall be included in all copies or substantial portions |
15 | * next paragraph) shall be included in all copies or substantial portions |
16 | * of the Software. |
16 | * of the Software. |
17 | * |
17 | * |
18 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
18 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
19 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
19 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
20 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL |
20 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL |
21 | * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, |
21 | * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, |
22 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR |
22 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR |
23 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE |
23 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE |
24 | * USE OR OTHER DEALINGS IN THE SOFTWARE. |
24 | * USE OR OTHER DEALINGS IN THE SOFTWARE. |
25 | * |
25 | * |
26 | **************************************************************************/ |
26 | **************************************************************************/ |
27 | /* |
27 | /* |
28 | * Authors: Thomas Hellstrom |
28 | * Authors: Thomas Hellstrom |
29 | */ |
29 | */ |
30 | 30 | ||
31 | #define pr_fmt(fmt) "[TTM] " fmt |
31 | #define pr_fmt(fmt) "[TTM] " fmt |
32 | 32 | ||
33 | #include |
33 | #include |
34 | #include |
34 | #include |
35 | #include |
35 | #include |
36 | #include |
36 | #include |
37 | #include |
37 | #include |
38 | #include |
38 | #include |
39 | #include |
39 | #include |
40 | #include |
40 | #include |
- | 41 | ||
- | 42 | #define pr_err(fmt, ...) \ |
|
- | 43 | printk(KERN_ERR pr_fmt(fmt), ##__VA_ARGS__) |
|
41 | 44 | ||
42 | #define TTM_ASSERT_LOCKED(param) |
45 | #define TTM_ASSERT_LOCKED(param) |
43 | #define TTM_DEBUG(fmt, arg...) |
46 | #define TTM_DEBUG(fmt, arg...) |
44 | #define TTM_BO_HASH_ORDER 13 |
47 | #define TTM_BO_HASH_ORDER 13 |
45 | - | ||
46 | #define pr_err(fmt, ...) \ |
- | |
- | 48 | ||
47 | printk(KERN_ERR pr_fmt(fmt), ##__VA_ARGS__) |
49 | |
48 | 50 | ||
- | 51 | static inline int ttm_mem_type_from_flags(uint32_t flags, uint32_t *mem_type) |
|
49 | int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible) |
52 | { |
- | 53 | int i; |
|
- | 54 | ||
50 | { |
55 | for (i = 0; i <= TTM_PL_PRIV5; i++) |
51 | 56 | if (flags & (1 << i)) { |
|
52 | mutex_lock(&man->io_reserve_mutex); |
- | |
53 | return 0; |
- | |
54 | } |
- | |
55 | - | ||
56 | void ttm_mem_io_unlock(struct ttm_mem_type_manager *man) |
57 | *mem_type = i; |
57 | { |
- | |
58 | if (likely(man->io_reserve_fastpath)) |
- | |
59 | return; |
58 | return 0; |
60 | 59 | } |
|
61 | mutex_unlock(&man->io_reserve_mutex); |
60 | return -EINVAL; |
62 | } |
61 | } |
63 | 62 | ||
64 | - | ||
65 | #if 0 |
- | |
66 | static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type) |
- | |
67 | { |
- | |
68 | struct ttm_mem_type_manager *man = &bdev->man[mem_type]; |
- | |
69 | - | ||
70 | pr_err(" has_type: %d\n", man->has_type); |
- | |
71 | pr_err(" use_type: %d\n", man->use_type); |
- | |
72 | pr_err(" flags: 0x%08X\n", man->flags); |
- | |
73 | pr_err(" gpu_offset: 0x%08lX\n", man->gpu_offset); |
- | |
74 | pr_err(" size: %llu\n", man->size); |
- | |
75 | pr_err(" available_caching: 0x%08X\n", man->available_caching); |
- | |
76 | pr_err(" default_caching: 0x%08X\n", man->default_caching); |
- | |
77 | if (mem_type != TTM_PL_SYSTEM) |
- | |
78 | (*man->func->debug)(man, TTM_PFX); |
- | |
79 | } |
- | |
80 | - | ||
81 | static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, |
- | |
82 | struct ttm_placement *placement) |
- | |
83 | { |
- | |
84 | int i, ret, mem_type; |
- | |
85 | - | ||
86 | pr_err("No space for %p (%lu pages, %luK, %luM)\n", |
- | |
87 | bo, bo->mem.num_pages, bo->mem.size >> 10, |
- | |
88 | bo->mem.size >> 20); |
- | |
89 | for (i = 0; i < placement->num_placement; i++) { |
- | |
90 | ret = ttm_mem_type_from_flags(placement->placement[i], |
- | |
91 | &mem_type); |
- | |
92 | if (ret) |
- | |
93 | return; |
- | |
94 | pr_err(" placement[%d]=0x%08X (%d)\n", |
- | |
95 | i, placement->placement[i], mem_type); |
- | |
96 | ttm_mem_type_debug(bo->bdev, mem_type); |
- | |
97 | } |
- | |
98 | } |
- | |
99 | - | ||
100 | static ssize_t ttm_bo_global_show(struct kobject *kobj, |
- | |
101 | struct attribute *attr, |
- | |
102 | char *buffer) |
- | |
103 | { |
- | |
104 | struct ttm_bo_global *glob = |
- | |
105 | container_of(kobj, struct ttm_bo_global, kobj); |
- | |
106 | - | ||
107 | return snprintf(buffer, PAGE_SIZE, "%lu\n", |
- | |
108 | (unsigned long) atomic_read(&glob->bo_count)); |
- | |
109 | } |
- | |
110 | - | ||
111 | static struct attribute *ttm_bo_global_attrs[] = { |
- | |
112 | &ttm_bo_count, |
- | |
113 | NULL |
- | |
114 | }; |
- | |
115 | - | ||
116 | static const struct sysfs_ops ttm_bo_global_ops = { |
- | |
117 | .show = &ttm_bo_global_show |
- | |
118 | }; |
- | |
119 | - | ||
120 | static struct kobj_type ttm_bo_glob_kobj_type = { |
63 | |
121 | .release = &ttm_bo_global_kobj_release, |
64 | |
122 | .sysfs_ops = &ttm_bo_global_ops, |
65 | |
123 | .default_attrs = ttm_bo_global_attrs |
66 | |
124 | }; |
67 | |
125 | #endif |
68 | |
126 | 69 | ||
127 | 70 | ||
128 | static inline uint32_t ttm_bo_type_flags(unsigned type) |
71 | static inline uint32_t ttm_bo_type_flags(unsigned type) |
129 | { |
72 | { |
130 | return 1 << (type); |
73 | return 1 << (type); |
131 | } |
74 | } |
132 | 75 | ||
133 | static void ttm_bo_release_list(struct kref *list_kref) |
76 | static void ttm_bo_release_list(struct kref *list_kref) |
134 | { |
77 | { |
135 | struct ttm_buffer_object *bo = |
78 | struct ttm_buffer_object *bo = |
136 | container_of(list_kref, struct ttm_buffer_object, list_kref); |
79 | container_of(list_kref, struct ttm_buffer_object, list_kref); |
137 | struct ttm_bo_device *bdev = bo->bdev; |
80 | struct ttm_bo_device *bdev = bo->bdev; |
138 | size_t acc_size = bo->acc_size; |
81 | size_t acc_size = bo->acc_size; |
139 | 82 | ||
140 | BUG_ON(atomic_read(&bo->list_kref.refcount)); |
83 | BUG_ON(atomic_read(&bo->list_kref.refcount)); |
141 | BUG_ON(atomic_read(&bo->kref.refcount)); |
84 | BUG_ON(atomic_read(&bo->kref.refcount)); |
142 | BUG_ON(atomic_read(&bo->cpu_writers)); |
85 | BUG_ON(atomic_read(&bo->cpu_writers)); |
143 | BUG_ON(bo->sync_obj != NULL); |
86 | BUG_ON(bo->sync_obj != NULL); |
144 | BUG_ON(bo->mem.mm_node != NULL); |
87 | BUG_ON(bo->mem.mm_node != NULL); |
145 | BUG_ON(!list_empty(&bo->lru)); |
88 | BUG_ON(!list_empty(&bo->lru)); |
146 | BUG_ON(!list_empty(&bo->ddestroy)); |
89 | BUG_ON(!list_empty(&bo->ddestroy)); |
147 | 90 | ||
148 | if (bo->ttm) |
91 | if (bo->ttm) |
149 | ttm_tt_destroy(bo->ttm); |
92 | ttm_tt_destroy(bo->ttm); |
150 | atomic_dec(&bo->glob->bo_count); |
93 | atomic_dec(&bo->glob->bo_count); |
- | 94 | if (bo->resv == &bo->ttm_resv) |
|
- | 95 | reservation_object_fini(&bo->ttm_resv); |
|
- | 96 | mutex_destroy(&bo->wu_mutex); |
|
151 | if (bo->destroy) |
97 | if (bo->destroy) |
152 | bo->destroy(bo); |
98 | bo->destroy(bo); |
153 | else { |
99 | else { |
154 | kfree(bo); |
100 | kfree(bo); |
155 | } |
101 | } |
156 | ttm_mem_global_free(bdev->glob->mem_glob, acc_size); |
- | |
157 | } |
102 | } |
158 | 103 | ||
159 | void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) |
104 | void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) |
160 | { |
105 | { |
161 | struct ttm_bo_device *bdev = bo->bdev; |
106 | struct ttm_bo_device *bdev = bo->bdev; |
162 | struct ttm_mem_type_manager *man; |
107 | struct ttm_mem_type_manager *man; |
163 | 108 | ||
164 | // BUG_ON(!ttm_bo_is_reserved(bo)); |
109 | lockdep_assert_held(&bo->resv->lock.base); |
165 | 110 | ||
166 | if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { |
111 | if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { |
167 | 112 | ||
168 | BUG_ON(!list_empty(&bo->lru)); |
113 | BUG_ON(!list_empty(&bo->lru)); |
169 | 114 | ||
170 | man = &bdev->man[bo->mem.mem_type]; |
115 | man = &bdev->man[bo->mem.mem_type]; |
171 | list_add_tail(&bo->lru, &man->lru); |
116 | list_add_tail(&bo->lru, &man->lru); |
172 | kref_get(&bo->list_kref); |
117 | kref_get(&bo->list_kref); |
173 | 118 | ||
174 | if (bo->ttm != NULL) { |
119 | if (bo->ttm != NULL) { |
175 | list_add_tail(&bo->swap, &bo->glob->swap_lru); |
120 | list_add_tail(&bo->swap, &bo->glob->swap_lru); |
176 | kref_get(&bo->list_kref); |
121 | kref_get(&bo->list_kref); |
177 | } |
122 | } |
178 | } |
123 | } |
179 | } |
124 | } |
180 | EXPORT_SYMBOL(ttm_bo_add_to_lru); |
125 | EXPORT_SYMBOL(ttm_bo_add_to_lru); |
181 | 126 | ||
182 | int ttm_bo_del_from_lru(struct ttm_buffer_object *bo) |
127 | int ttm_bo_del_from_lru(struct ttm_buffer_object *bo) |
183 | { |
128 | { |
184 | int put_count = 0; |
129 | int put_count = 0; |
185 | 130 | ||
186 | if (!list_empty(&bo->swap)) { |
131 | if (!list_empty(&bo->swap)) { |
187 | list_del_init(&bo->swap); |
132 | list_del_init(&bo->swap); |
188 | ++put_count; |
133 | ++put_count; |
189 | } |
134 | } |
190 | if (!list_empty(&bo->lru)) { |
135 | if (!list_empty(&bo->lru)) { |
191 | list_del_init(&bo->lru); |
136 | list_del_init(&bo->lru); |
192 | ++put_count; |
137 | ++put_count; |
193 | } |
138 | } |
194 | 139 | ||
195 | /* |
140 | /* |
196 | * TODO: Add a driver hook to delete from |
141 | * TODO: Add a driver hook to delete from |
197 | * driver-specific LRU's here. |
142 | * driver-specific LRU's here. |
198 | */ |
143 | */ |
199 | 144 | ||
200 | return put_count; |
145 | return put_count; |
201 | } |
146 | } |
202 | 147 | ||
203 | static void ttm_bo_ref_bug(struct kref *list_kref) |
148 | static void ttm_bo_ref_bug(struct kref *list_kref) |
204 | { |
149 | { |
205 | BUG(); |
150 | BUG(); |
206 | } |
151 | } |
207 | 152 | ||
208 | void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count, |
153 | void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count, |
209 | bool never_free) |
154 | bool never_free) |
210 | { |
155 | { |
211 | // kref_sub(&bo->list_kref, count, |
156 | // kref_sub(&bo->list_kref, count, |
212 | // (never_free) ? ttm_bo_ref_bug : ttm_bo_release_list); |
157 | // (never_free) ? ttm_bo_ref_bug : ttm_bo_release_list); |
213 | } |
158 | } |
214 | 159 | ||
215 | void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo) |
160 | void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo) |
216 | { |
161 | { |
217 | int put_count; |
162 | int put_count; |
218 | 163 | ||
219 | spin_lock(&bo->glob->lru_lock); |
164 | spin_lock(&bo->glob->lru_lock); |
220 | put_count = ttm_bo_del_from_lru(bo); |
165 | put_count = ttm_bo_del_from_lru(bo); |
221 | spin_unlock(&bo->glob->lru_lock); |
166 | spin_unlock(&bo->glob->lru_lock); |
222 | ttm_bo_list_ref_sub(bo, put_count, true); |
167 | ttm_bo_list_ref_sub(bo, put_count, true); |
223 | } |
168 | } |
224 | EXPORT_SYMBOL(ttm_bo_del_sub_from_lru); |
169 | EXPORT_SYMBOL(ttm_bo_del_sub_from_lru); |
225 | 170 | ||
226 | /* |
171 | /* |
227 | * Call bo->mutex locked. |
172 | * Call bo->mutex locked. |
228 | */ |
173 | */ |
229 | static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) |
174 | static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) |
230 | { |
175 | { |
231 | struct ttm_bo_device *bdev = bo->bdev; |
176 | struct ttm_bo_device *bdev = bo->bdev; |
232 | struct ttm_bo_global *glob = bo->glob; |
177 | struct ttm_bo_global *glob = bo->glob; |
233 | int ret = 0; |
178 | int ret = 0; |
234 | uint32_t page_flags = 0; |
179 | uint32_t page_flags = 0; |
235 | 180 | ||
236 | TTM_ASSERT_LOCKED(&bo->mutex); |
181 | TTM_ASSERT_LOCKED(&bo->mutex); |
237 | bo->ttm = NULL; |
182 | bo->ttm = NULL; |
238 | 183 | ||
239 | if (bdev->need_dma32) |
184 | if (bdev->need_dma32) |
240 | page_flags |= TTM_PAGE_FLAG_DMA32; |
185 | page_flags |= TTM_PAGE_FLAG_DMA32; |
241 | 186 | ||
242 | switch (bo->type) { |
187 | switch (bo->type) { |
243 | case ttm_bo_type_device: |
188 | case ttm_bo_type_device: |
244 | if (zero_alloc) |
189 | if (zero_alloc) |
245 | page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC; |
190 | page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC; |
246 | case ttm_bo_type_kernel: |
191 | case ttm_bo_type_kernel: |
247 | bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, |
192 | bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, |
248 | page_flags, glob->dummy_read_page); |
193 | page_flags, glob->dummy_read_page); |
249 | if (unlikely(bo->ttm == NULL)) |
194 | if (unlikely(bo->ttm == NULL)) |
250 | ret = -ENOMEM; |
195 | ret = -ENOMEM; |
251 | break; |
196 | break; |
252 | case ttm_bo_type_sg: |
197 | case ttm_bo_type_sg: |
253 | bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, |
198 | bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, |
254 | page_flags | TTM_PAGE_FLAG_SG, |
199 | page_flags | TTM_PAGE_FLAG_SG, |
255 | glob->dummy_read_page); |
200 | glob->dummy_read_page); |
256 | if (unlikely(bo->ttm == NULL)) { |
201 | if (unlikely(bo->ttm == NULL)) { |
257 | ret = -ENOMEM; |
202 | ret = -ENOMEM; |
258 | break; |
203 | break; |
259 | } |
204 | } |
260 | bo->ttm->sg = bo->sg; |
205 | bo->ttm->sg = bo->sg; |
261 | break; |
206 | break; |
262 | default: |
207 | default: |
263 | pr_err("Illegal buffer object type\n"); |
208 | pr_err("Illegal buffer object type\n"); |
264 | ret = -EINVAL; |
209 | ret = -EINVAL; |
265 | break; |
210 | break; |
266 | } |
211 | } |
267 | 212 | ||
268 | return ret; |
213 | return ret; |
269 | } |
214 | } |
270 | - | ||
271 | #if 0 |
215 | |
272 | static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, |
216 | static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, |
273 | struct ttm_mem_reg *mem, |
217 | struct ttm_mem_reg *mem, |
274 | bool evict, bool interruptible, |
218 | bool evict, bool interruptible, |
275 | bool no_wait_gpu) |
219 | bool no_wait_gpu) |
276 | { |
220 | { |
277 | struct ttm_bo_device *bdev = bo->bdev; |
221 | struct ttm_bo_device *bdev = bo->bdev; |
278 | bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); |
222 | bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); |
279 | bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem); |
223 | bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem); |
280 | struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type]; |
224 | struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type]; |
281 | struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type]; |
225 | struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type]; |
282 | int ret = 0; |
226 | int ret = 0; |
283 | 227 | ||
284 | if (old_is_pci || new_is_pci || |
228 | if (old_is_pci || new_is_pci || |
285 | ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) { |
229 | ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) { |
286 | ret = ttm_mem_io_lock(old_man, true); |
230 | ret = ttm_mem_io_lock(old_man, true); |
287 | if (unlikely(ret != 0)) |
231 | if (unlikely(ret != 0)) |
288 | goto out_err; |
232 | goto out_err; |
289 | ttm_bo_unmap_virtual_locked(bo); |
233 | ttm_bo_unmap_virtual_locked(bo); |
290 | ttm_mem_io_unlock(old_man); |
234 | ttm_mem_io_unlock(old_man); |
291 | } |
235 | } |
292 | 236 | ||
293 | /* |
237 | /* |
294 | * Create and bind a ttm if required. |
238 | * Create and bind a ttm if required. |
295 | */ |
239 | */ |
296 | 240 | ||
297 | if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) { |
241 | if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) { |
298 | if (bo->ttm == NULL) { |
242 | if (bo->ttm == NULL) { |
299 | bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED); |
243 | bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED); |
300 | ret = ttm_bo_add_ttm(bo, zero); |
244 | ret = ttm_bo_add_ttm(bo, zero); |
301 | if (ret) |
245 | if (ret) |
302 | goto out_err; |
246 | goto out_err; |
303 | } |
247 | } |
304 | 248 | ||
305 | ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); |
249 | ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); |
306 | if (ret) |
250 | if (ret) |
307 | goto out_err; |
251 | goto out_err; |
308 | 252 | ||
309 | if (mem->mem_type != TTM_PL_SYSTEM) { |
253 | if (mem->mem_type != TTM_PL_SYSTEM) { |
310 | ret = ttm_tt_bind(bo->ttm, mem); |
254 | ret = ttm_tt_bind(bo->ttm, mem); |
311 | if (ret) |
255 | if (ret) |
312 | goto out_err; |
256 | goto out_err; |
313 | } |
257 | } |
314 | 258 | ||
315 | if (bo->mem.mem_type == TTM_PL_SYSTEM) { |
259 | if (bo->mem.mem_type == TTM_PL_SYSTEM) { |
316 | if (bdev->driver->move_notify) |
260 | if (bdev->driver->move_notify) |
317 | bdev->driver->move_notify(bo, mem); |
261 | bdev->driver->move_notify(bo, mem); |
318 | bo->mem = *mem; |
262 | bo->mem = *mem; |
319 | mem->mm_node = NULL; |
263 | mem->mm_node = NULL; |
320 | goto moved; |
264 | goto moved; |
321 | } |
265 | } |
322 | } |
266 | } |
323 | 267 | ||
324 | if (bdev->driver->move_notify) |
268 | if (bdev->driver->move_notify) |
325 | bdev->driver->move_notify(bo, mem); |
269 | bdev->driver->move_notify(bo, mem); |
326 | 270 | ||
327 | if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && |
271 | if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && |
328 | !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) |
272 | !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) |
329 | ret = ttm_bo_move_ttm(bo, evict, no_wait_gpu, mem); |
273 | ret = ttm_bo_move_ttm(bo, evict, no_wait_gpu, mem); |
330 | else if (bdev->driver->move) |
274 | else if (bdev->driver->move) |
331 | ret = bdev->driver->move(bo, evict, interruptible, |
275 | ret = bdev->driver->move(bo, evict, interruptible, |
332 | no_wait_gpu, mem); |
276 | no_wait_gpu, mem); |
333 | else |
277 | else |
334 | ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, mem); |
278 | ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, mem); |
335 | 279 | ||
336 | if (ret) { |
280 | if (ret) { |
337 | if (bdev->driver->move_notify) { |
281 | if (bdev->driver->move_notify) { |
338 | struct ttm_mem_reg tmp_mem = *mem; |
282 | struct ttm_mem_reg tmp_mem = *mem; |
339 | *mem = bo->mem; |
283 | *mem = bo->mem; |
340 | bo->mem = tmp_mem; |
284 | bo->mem = tmp_mem; |
341 | bdev->driver->move_notify(bo, mem); |
285 | bdev->driver->move_notify(bo, mem); |
342 | bo->mem = *mem; |
286 | bo->mem = *mem; |
343 | *mem = tmp_mem; |
287 | *mem = tmp_mem; |
344 | } |
288 | } |
345 | 289 | ||
346 | goto out_err; |
290 | goto out_err; |
347 | } |
291 | } |
348 | 292 | ||
349 | moved: |
293 | moved: |
350 | if (bo->evicted) { |
294 | if (bo->evicted) { |
- | 295 | if (bdev->driver->invalidate_caches) { |
|
351 | ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement); |
296 | ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement); |
352 | if (ret) |
297 | if (ret) |
353 | pr_err("Can not flush read caches\n"); |
298 | pr_err("Can not flush read caches\n"); |
- | 299 | } |
|
354 | bo->evicted = false; |
300 | bo->evicted = false; |
355 | } |
301 | } |
356 | 302 | ||
357 | if (bo->mem.mm_node) { |
303 | if (bo->mem.mm_node) { |
358 | bo->offset = (bo->mem.start << PAGE_SHIFT) + |
304 | bo->offset = (bo->mem.start << PAGE_SHIFT) + |
359 | bdev->man[bo->mem.mem_type].gpu_offset; |
305 | bdev->man[bo->mem.mem_type].gpu_offset; |
360 | bo->cur_placement = bo->mem.placement; |
306 | bo->cur_placement = bo->mem.placement; |
361 | } else |
307 | } else |
362 | bo->offset = 0; |
308 | bo->offset = 0; |
363 | 309 | ||
364 | return 0; |
310 | return 0; |
365 | 311 | ||
366 | out_err: |
312 | out_err: |
367 | new_man = &bdev->man[bo->mem.mem_type]; |
313 | new_man = &bdev->man[bo->mem.mem_type]; |
368 | if ((new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm) { |
314 | if ((new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm) { |
369 | ttm_tt_unbind(bo->ttm); |
315 | ttm_tt_unbind(bo->ttm); |
370 | ttm_tt_destroy(bo->ttm); |
316 | ttm_tt_destroy(bo->ttm); |
371 | bo->ttm = NULL; |
317 | bo->ttm = NULL; |
372 | } |
318 | } |
373 | 319 | ||
374 | return ret; |
320 | return ret; |
375 | } |
321 | } |
376 | 322 | ||
377 | /** |
323 | /** |
378 | * Call bo::reserved. |
324 | * Call bo::reserved. |
379 | * Will release GPU memory type usage on destruction. |
325 | * Will release GPU memory type usage on destruction. |
380 | * This is the place to put in driver specific hooks to release |
326 | * This is the place to put in driver specific hooks to release |
381 | * driver private resources. |
327 | * driver private resources. |
382 | * Will release the bo::reserved lock. |
328 | * Will release the bo::reserved lock. |
383 | */ |
329 | */ |
384 | 330 | ||
385 | static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) |
331 | static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) |
386 | { |
332 | { |
387 | if (bo->bdev->driver->move_notify) |
333 | if (bo->bdev->driver->move_notify) |
388 | bo->bdev->driver->move_notify(bo, NULL); |
334 | bo->bdev->driver->move_notify(bo, NULL); |
389 | 335 | ||
390 | if (bo->ttm) { |
336 | if (bo->ttm) { |
391 | ttm_tt_unbind(bo->ttm); |
337 | ttm_tt_unbind(bo->ttm); |
392 | ttm_tt_destroy(bo->ttm); |
338 | ttm_tt_destroy(bo->ttm); |
393 | bo->ttm = NULL; |
339 | bo->ttm = NULL; |
394 | } |
340 | } |
395 | ttm_bo_mem_put(bo, &bo->mem); |
341 | ttm_bo_mem_put(bo, &bo->mem); |
396 | 342 | ||
397 | ww_mutex_unlock (&bo->resv->lock); |
343 | ww_mutex_unlock (&bo->resv->lock); |
398 | } |
344 | } |
399 | 345 | ||
400 | static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) |
346 | static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) |
401 | { |
347 | { |
402 | struct ttm_bo_device *bdev = bo->bdev; |
348 | struct ttm_bo_device *bdev = bo->bdev; |
403 | struct ttm_bo_global *glob = bo->glob; |
349 | struct ttm_bo_global *glob = bo->glob; |
404 | struct ttm_bo_driver *driver = bdev->driver; |
350 | struct ttm_bo_driver *driver = bdev->driver; |
405 | void *sync_obj = NULL; |
351 | void *sync_obj = NULL; |
406 | int put_count; |
352 | int put_count; |
407 | int ret; |
353 | int ret; |
408 | 354 | ||
409 | spin_lock(&glob->lru_lock); |
355 | spin_lock(&glob->lru_lock); |
410 | ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); |
356 | ret = __ttm_bo_reserve(bo, false, true, false, NULL); |
411 | 357 | ||
412 | spin_lock(&bdev->fence_lock); |
358 | spin_lock(&bdev->fence_lock); |
413 | (void) ttm_bo_wait(bo, false, false, true); |
359 | (void) ttm_bo_wait(bo, false, false, true); |
414 | if (!ret && !bo->sync_obj) { |
360 | if (!ret && !bo->sync_obj) { |
415 | spin_unlock(&bdev->fence_lock); |
361 | spin_unlock(&bdev->fence_lock); |
416 | put_count = ttm_bo_del_from_lru(bo); |
362 | put_count = ttm_bo_del_from_lru(bo); |
417 | 363 | ||
418 | spin_unlock(&glob->lru_lock); |
364 | spin_unlock(&glob->lru_lock); |
419 | ttm_bo_cleanup_memtype_use(bo); |
365 | ttm_bo_cleanup_memtype_use(bo); |
420 | 366 | ||
421 | ttm_bo_list_ref_sub(bo, put_count, true); |
367 | ttm_bo_list_ref_sub(bo, put_count, true); |
422 | 368 | ||
423 | return; |
369 | return; |
424 | } |
370 | } |
425 | if (bo->sync_obj) |
371 | if (bo->sync_obj) |
426 | sync_obj = driver->sync_obj_ref(bo->sync_obj); |
372 | sync_obj = driver->sync_obj_ref(bo->sync_obj); |
427 | spin_unlock(&bdev->fence_lock); |
373 | spin_unlock(&bdev->fence_lock); |
428 | 374 | ||
429 | if (!ret) { |
375 | if (!ret) { |
430 | 376 | ||
431 | /* |
377 | /* |
432 | * Make NO_EVICT bos immediately available to |
378 | * Make NO_EVICT bos immediately available to |
433 | * shrinkers, now that they are queued for |
379 | * shrinkers, now that they are queued for |
434 | * destruction. |
380 | * destruction. |
435 | */ |
381 | */ |
436 | if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) { |
382 | if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) { |
437 | bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT; |
383 | bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT; |
438 | ttm_bo_add_to_lru(bo); |
384 | ttm_bo_add_to_lru(bo); |
439 | } |
385 | } |
440 | 386 | ||
441 | ww_mutex_unlock(&bo->resv->lock); |
387 | __ttm_bo_unreserve(bo); |
442 | } |
388 | } |
443 | 389 | ||
444 | kref_get(&bo->list_kref); |
390 | kref_get(&bo->list_kref); |
445 | list_add_tail(&bo->ddestroy, &bdev->ddestroy); |
391 | list_add_tail(&bo->ddestroy, &bdev->ddestroy); |
446 | spin_unlock(&glob->lru_lock); |
392 | spin_unlock(&glob->lru_lock); |
447 | 393 | ||
448 | if (sync_obj) { |
394 | if (sync_obj) { |
449 | driver->sync_obj_flush(sync_obj); |
395 | driver->sync_obj_flush(sync_obj); |
450 | driver->sync_obj_unref(&sync_obj); |
396 | driver->sync_obj_unref(&sync_obj); |
451 | } |
397 | } |
452 | schedule_delayed_work(&bdev->wq, |
398 | // schedule_delayed_work(&bdev->wq, |
453 | ((HZ / 100) < 1) ? 1 : HZ / 100); |
399 | // ((HZ / 100) < 1) ? 1 : HZ / 100); |
454 | } |
400 | } |
455 | 401 | ||
456 | /** |
402 | /** |
457 | * function ttm_bo_cleanup_refs_and_unlock |
403 | * function ttm_bo_cleanup_refs_and_unlock |
458 | * If bo idle, remove from delayed- and lru lists, and unref. |
404 | * If bo idle, remove from delayed- and lru lists, and unref. |
459 | * If not idle, do nothing. |
405 | * If not idle, do nothing. |
460 | * |
406 | * |
461 | * Must be called with lru_lock and reservation held, this function |
407 | * Must be called with lru_lock and reservation held, this function |
462 | * will drop both before returning. |
408 | * will drop both before returning. |
463 | * |
409 | * |
464 | * @interruptible Any sleeps should occur interruptibly. |
410 | * @interruptible Any sleeps should occur interruptibly. |
465 | * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. |
411 | * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. |
466 | */ |
412 | */ |
467 | 413 | ||
468 | static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, |
414 | static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, |
469 | bool interruptible, |
415 | bool interruptible, |
470 | bool no_wait_gpu) |
416 | bool no_wait_gpu) |
471 | { |
417 | { |
472 | struct ttm_bo_device *bdev = bo->bdev; |
418 | struct ttm_bo_device *bdev = bo->bdev; |
473 | struct ttm_bo_driver *driver = bdev->driver; |
419 | struct ttm_bo_driver *driver = bdev->driver; |
474 | struct ttm_bo_global *glob = bo->glob; |
420 | struct ttm_bo_global *glob = bo->glob; |
475 | int put_count; |
421 | int put_count; |
476 | int ret; |
422 | int ret; |
477 | 423 | ||
478 | spin_lock(&bdev->fence_lock); |
424 | spin_lock(&bdev->fence_lock); |
479 | ret = ttm_bo_wait(bo, false, false, true); |
425 | ret = ttm_bo_wait(bo, false, false, true); |
480 | 426 | ||
481 | if (ret && !no_wait_gpu) { |
427 | if (ret && !no_wait_gpu) { |
482 | void *sync_obj; |
428 | void *sync_obj; |
483 | 429 | ||
484 | /* |
430 | /* |
485 | * Take a reference to the fence and unreserve, |
431 | * Take a reference to the fence and unreserve, |
486 | * at this point the buffer should be dead, so |
432 | * at this point the buffer should be dead, so |
487 | * no new sync objects can be attached. |
433 | * no new sync objects can be attached. |
488 | */ |
434 | */ |
489 | sync_obj = driver->sync_obj_ref(bo->sync_obj); |
435 | sync_obj = driver->sync_obj_ref(bo->sync_obj); |
490 | spin_unlock(&bdev->fence_lock); |
436 | spin_unlock(&bdev->fence_lock); |
491 | 437 | ||
492 | ww_mutex_unlock(&bo->resv->lock); |
438 | __ttm_bo_unreserve(bo); |
493 | spin_unlock(&glob->lru_lock); |
439 | spin_unlock(&glob->lru_lock); |
494 | 440 | ||
495 | ret = driver->sync_obj_wait(sync_obj, false, interruptible); |
441 | ret = driver->sync_obj_wait(sync_obj, false, interruptible); |
496 | driver->sync_obj_unref(&sync_obj); |
442 | driver->sync_obj_unref(&sync_obj); |
497 | if (ret) |
443 | if (ret) |
498 | return ret; |
444 | return ret; |
499 | 445 | ||
500 | /* |
446 | /* |
501 | * remove sync_obj with ttm_bo_wait, the wait should be |
447 | * remove sync_obj with ttm_bo_wait, the wait should be |
502 | * finished, and no new wait object should have been added. |
448 | * finished, and no new wait object should have been added. |
503 | */ |
449 | */ |
504 | spin_lock(&bdev->fence_lock); |
450 | spin_lock(&bdev->fence_lock); |
505 | ret = ttm_bo_wait(bo, false, false, true); |
451 | ret = ttm_bo_wait(bo, false, false, true); |
506 | WARN_ON(ret); |
452 | WARN_ON(ret); |
507 | spin_unlock(&bdev->fence_lock); |
453 | spin_unlock(&bdev->fence_lock); |
508 | if (ret) |
454 | if (ret) |
509 | return ret; |
455 | return ret; |
510 | 456 | ||
511 | spin_lock(&glob->lru_lock); |
457 | spin_lock(&glob->lru_lock); |
512 | ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); |
458 | ret = __ttm_bo_reserve(bo, false, true, false, NULL); |
513 | 459 | ||
514 | /* |
460 | /* |
515 | * We raced, and lost, someone else holds the reservation now, |
461 | * We raced, and lost, someone else holds the reservation now, |
516 | * and is probably busy in ttm_bo_cleanup_memtype_use. |
462 | * and is probably busy in ttm_bo_cleanup_memtype_use. |
517 | * |
463 | * |
518 | * Even if it's not the case, because we finished waiting any |
464 | * Even if it's not the case, because we finished waiting any |
519 | * delayed destruction would succeed, so just return success |
465 | * delayed destruction would succeed, so just return success |
520 | * here. |
466 | * here. |
521 | */ |
467 | */ |
522 | if (ret) { |
468 | if (ret) { |
523 | spin_unlock(&glob->lru_lock); |
469 | spin_unlock(&glob->lru_lock); |
524 | return 0; |
470 | return 0; |
525 | } |
471 | } |
526 | } else |
472 | } else |
527 | spin_unlock(&bdev->fence_lock); |
473 | spin_unlock(&bdev->fence_lock); |
528 | 474 | ||
529 | if (ret || unlikely(list_empty(&bo->ddestroy))) { |
475 | if (ret || unlikely(list_empty(&bo->ddestroy))) { |
530 | ww_mutex_unlock(&bo->resv->lock); |
476 | __ttm_bo_unreserve(bo); |
531 | spin_unlock(&glob->lru_lock); |
477 | spin_unlock(&glob->lru_lock); |
532 | return ret; |
478 | return ret; |
533 | } |
479 | } |
534 | 480 | ||
535 | put_count = ttm_bo_del_from_lru(bo); |
481 | put_count = ttm_bo_del_from_lru(bo); |
536 | list_del_init(&bo->ddestroy); |
482 | list_del_init(&bo->ddestroy); |
537 | ++put_count; |
483 | ++put_count; |
538 | 484 | ||
539 | spin_unlock(&glob->lru_lock); |
485 | spin_unlock(&glob->lru_lock); |
540 | ttm_bo_cleanup_memtype_use(bo); |
486 | ttm_bo_cleanup_memtype_use(bo); |
541 | 487 | ||
542 | ttm_bo_list_ref_sub(bo, put_count, true); |
488 | ttm_bo_list_ref_sub(bo, put_count, true); |
543 | 489 | ||
544 | return 0; |
490 | return 0; |
545 | } |
491 | } |
546 | 492 | ||
547 | /** |
493 | /** |
548 | * Traverse the delayed list, and call ttm_bo_cleanup_refs on all |
494 | * Traverse the delayed list, and call ttm_bo_cleanup_refs on all |
549 | * encountered buffers. |
495 | * encountered buffers. |
550 | */ |
496 | */ |
551 | 497 | ||
552 | static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) |
498 | static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) |
553 | { |
499 | { |
554 | struct ttm_bo_global *glob = bdev->glob; |
500 | struct ttm_bo_global *glob = bdev->glob; |
555 | struct ttm_buffer_object *entry = NULL; |
501 | struct ttm_buffer_object *entry = NULL; |
556 | int ret = 0; |
502 | int ret = 0; |
557 | 503 | ||
558 | spin_lock(&glob->lru_lock); |
504 | spin_lock(&glob->lru_lock); |
559 | if (list_empty(&bdev->ddestroy)) |
505 | if (list_empty(&bdev->ddestroy)) |
560 | goto out_unlock; |
506 | goto out_unlock; |
561 | 507 | ||
562 | entry = list_first_entry(&bdev->ddestroy, |
508 | entry = list_first_entry(&bdev->ddestroy, |
563 | struct ttm_buffer_object, ddestroy); |
509 | struct ttm_buffer_object, ddestroy); |
564 | kref_get(&entry->list_kref); |
510 | kref_get(&entry->list_kref); |
565 | 511 | ||
566 | for (;;) { |
512 | for (;;) { |
567 | struct ttm_buffer_object *nentry = NULL; |
513 | struct ttm_buffer_object *nentry = NULL; |
568 | 514 | ||
569 | if (entry->ddestroy.next != &bdev->ddestroy) { |
515 | if (entry->ddestroy.next != &bdev->ddestroy) { |
570 | nentry = list_first_entry(&entry->ddestroy, |
516 | nentry = list_first_entry(&entry->ddestroy, |
571 | struct ttm_buffer_object, ddestroy); |
517 | struct ttm_buffer_object, ddestroy); |
572 | kref_get(&nentry->list_kref); |
518 | kref_get(&nentry->list_kref); |
573 | } |
519 | } |
574 | 520 | ||
575 | ret = ttm_bo_reserve_nolru(entry, false, true, false, 0); |
521 | ret = __ttm_bo_reserve(entry, false, true, false, NULL); |
576 | if (remove_all && ret) { |
522 | if (remove_all && ret) { |
577 | spin_unlock(&glob->lru_lock); |
523 | spin_unlock(&glob->lru_lock); |
578 | ret = ttm_bo_reserve_nolru(entry, false, false, |
524 | ret = __ttm_bo_reserve(entry, false, false, |
579 | false, 0); |
525 | false, NULL); |
580 | spin_lock(&glob->lru_lock); |
526 | spin_lock(&glob->lru_lock); |
581 | } |
527 | } |
582 | 528 | ||
583 | if (!ret) |
529 | if (!ret) |
584 | ret = ttm_bo_cleanup_refs_and_unlock(entry, false, |
530 | ret = ttm_bo_cleanup_refs_and_unlock(entry, false, |
585 | !remove_all); |
531 | !remove_all); |
586 | else |
532 | else |
587 | spin_unlock(&glob->lru_lock); |
533 | spin_unlock(&glob->lru_lock); |
588 | 534 | ||
589 | kref_put(&entry->list_kref, ttm_bo_release_list); |
535 | kref_put(&entry->list_kref, ttm_bo_release_list); |
590 | entry = nentry; |
536 | entry = nentry; |
591 | 537 | ||
592 | if (ret || !entry) |
538 | if (ret || !entry) |
593 | goto out; |
539 | goto out; |
594 | 540 | ||
595 | spin_lock(&glob->lru_lock); |
541 | spin_lock(&glob->lru_lock); |
596 | if (list_empty(&entry->ddestroy)) |
542 | if (list_empty(&entry->ddestroy)) |
597 | break; |
543 | break; |
598 | } |
544 | } |
599 | 545 | ||
600 | out_unlock: |
546 | out_unlock: |
601 | spin_unlock(&glob->lru_lock); |
547 | spin_unlock(&glob->lru_lock); |
602 | out: |
548 | out: |
603 | if (entry) |
549 | if (entry) |
604 | kref_put(&entry->list_kref, ttm_bo_release_list); |
550 | kref_put(&entry->list_kref, ttm_bo_release_list); |
605 | return ret; |
551 | return ret; |
606 | } |
552 | } |
607 | 553 | ||
608 | static void ttm_bo_delayed_workqueue(struct work_struct *work) |
554 | static void ttm_bo_delayed_workqueue(struct work_struct *work) |
609 | { |
555 | { |
610 | struct ttm_bo_device *bdev = |
556 | struct ttm_bo_device *bdev = |
611 | container_of(work, struct ttm_bo_device, wq.work); |
557 | container_of(work, struct ttm_bo_device, wq.work); |
612 | 558 | ||
613 | if (ttm_bo_delayed_delete(bdev, false)) { |
559 | if (ttm_bo_delayed_delete(bdev, false)) { |
614 | schedule_delayed_work(&bdev->wq, |
560 | schedule_delayed_work(&bdev->wq, |
615 | ((HZ / 100) < 1) ? 1 : HZ / 100); |
561 | ((HZ / 100) < 1) ? 1 : HZ / 100); |
616 | } |
562 | } |
617 | } |
563 | } |
618 | #endif |
- | |
619 | 564 | ||
620 | static void ttm_bo_release(struct kref *kref) |
565 | static void ttm_bo_release(struct kref *kref) |
621 | { |
566 | { |
622 | struct ttm_buffer_object *bo = |
567 | struct ttm_buffer_object *bo = |
623 | container_of(kref, struct ttm_buffer_object, kref); |
568 | container_of(kref, struct ttm_buffer_object, kref); |
624 | struct ttm_bo_device *bdev = bo->bdev; |
569 | struct ttm_bo_device *bdev = bo->bdev; |
625 | struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; |
570 | struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; |
626 | 571 | ||
627 | drm_vma_offset_remove(&bdev->vma_manager, &bo->vma_node); |
572 | drm_vma_offset_remove(&bdev->vma_manager, &bo->vma_node); |
628 | ttm_mem_io_lock(man, false); |
573 | ttm_mem_io_lock(man, false); |
629 | // ttm_mem_io_free_vm(bo); |
574 | ttm_mem_io_free_vm(bo); |
630 | ttm_mem_io_unlock(man); |
575 | ttm_mem_io_unlock(man); |
631 | // ttm_bo_cleanup_refs_or_queue(bo); |
576 | ttm_bo_cleanup_refs_or_queue(bo); |
632 | // kref_put(&bo->list_kref, ttm_bo_release_list); |
577 | kref_put(&bo->list_kref, ttm_bo_release_list); |
633 | } |
578 | } |
634 | 579 | ||
635 | void ttm_bo_unref(struct ttm_buffer_object **p_bo) |
580 | void ttm_bo_unref(struct ttm_buffer_object **p_bo) |
636 | { |
581 | { |
637 | struct ttm_buffer_object *bo = *p_bo; |
582 | struct ttm_buffer_object *bo = *p_bo; |
638 | 583 | ||
639 | *p_bo = NULL; |
584 | *p_bo = NULL; |
640 | kref_put(&bo->kref, ttm_bo_release); |
585 | kref_put(&bo->kref, ttm_bo_release); |
641 | } |
586 | } |
642 | EXPORT_SYMBOL(ttm_bo_unref); |
587 | EXPORT_SYMBOL(ttm_bo_unref); |
643 | - | ||
644 | #if 0 |
- | |
645 | int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev) |
- | |
646 | { |
- | |
647 | return cancel_delayed_work_sync(&bdev->wq); |
- | |
648 | } |
- | |
649 | EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue); |
- | |
650 | - | ||
651 | void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) |
- | |
652 | { |
- | |
653 | if (resched) |
- | |
654 | schedule_delayed_work(&bdev->wq, |
- | |
655 | ((HZ / 100) < 1) ? 1 : HZ / 100); |
- | |
656 | } |
- | |
657 | EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue); |
- | |
658 | - | ||
659 | static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, |
- | |
660 | bool no_wait_gpu) |
- | |
661 | { |
- | |
662 | struct ttm_bo_device *bdev = bo->bdev; |
- | |
663 | struct ttm_mem_reg evict_mem; |
- | |
664 | struct ttm_placement placement; |
- | |
665 | int ret = 0; |
- | |
666 | - | ||
667 | spin_lock(&bdev->fence_lock); |
- | |
668 | ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); |
- | |
669 | spin_unlock(&bdev->fence_lock); |
- | |
670 | - | ||
671 | if (unlikely(ret != 0)) { |
- | |
672 | if (ret != -ERESTARTSYS) { |
- | |
673 | pr_err("Failed to expire sync object before buffer eviction\n"); |
- | |
674 | } |
- | |
675 | goto out; |
- | |
676 | } |
- | |
677 | - | ||
678 | // BUG_ON(!ttm_bo_is_reserved(bo)); |
- | |
679 | - | ||
680 | evict_mem = bo->mem; |
- | |
681 | evict_mem.mm_node = NULL; |
- | |
682 | evict_mem.bus.io_reserved_vm = false; |
- | |
683 | evict_mem.bus.io_reserved_count = 0; |
- | |
684 | - | ||
685 | placement.fpfn = 0; |
- | |
686 | placement.lpfn = 0; |
- | |
687 | placement.num_placement = 0; |
- | |
688 | placement.num_busy_placement = 0; |
- | |
689 | bdev->driver->evict_flags(bo, &placement); |
- | |
690 | ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible, |
- | |
691 | no_wait_gpu); |
- | |
692 | if (ret) { |
- | |
693 | if (ret != -ERESTARTSYS) { |
- | |
694 | pr_err("Failed to find memory space for buffer 0x%p eviction\n", |
- | |
695 | bo); |
- | |
696 | ttm_bo_mem_space_debug(bo, &placement); |
- | |
697 | } |
- | |
698 | goto out; |
- | |
699 | } |
- | |
700 | - | ||
701 | ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible, |
- | |
702 | no_wait_gpu); |
- | |
703 | if (ret) { |
- | |
704 | if (ret != -ERESTARTSYS) |
- | |
705 | pr_err("Buffer eviction failed\n"); |
- | |
706 | ttm_bo_mem_put(bo, &evict_mem); |
- | |
707 | goto out; |
- | |
708 | } |
- | |
709 | bo->evicted = true; |
- | |
710 | out: |
- | |
711 | return ret; |
- | |
712 | } |
- | |
713 | - | ||
714 | static int ttm_mem_evict_first(struct ttm_bo_device *bdev, |
- | |
715 | uint32_t mem_type, |
- | |
716 | bool interruptible, |
- | |
717 | bool no_wait_gpu) |
- | |
718 | { |
- | |
719 | struct ttm_bo_global *glob = bdev->glob; |
- | |
720 | struct ttm_mem_type_manager *man = &bdev->man[mem_type]; |
- | |
721 | struct ttm_buffer_object *bo; |
- | |
722 | int ret = -EBUSY, put_count; |
- | |
723 | - | ||
724 | spin_lock(&glob->lru_lock); |
- | |
725 | list_for_each_entry(bo, &man->lru, lru) { |
- | |
726 | ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); |
- | |
727 | if (!ret) |
- | |
728 | break; |
- | |
729 | } |
- | |
730 | - | ||
731 | if (ret) { |
- | |
732 | spin_unlock(&glob->lru_lock); |
- | |
733 | return ret; |
- | |
734 | } |
- | |
735 | - | ||
736 | kref_get(&bo->list_kref); |
- | |
737 | - | ||
738 | if (!list_empty(&bo->ddestroy)) { |
- | |
739 | ret = ttm_bo_cleanup_refs_and_unlock(bo, interruptible, |
- | |
740 | no_wait_gpu); |
- | |
741 | kref_put(&bo->list_kref, ttm_bo_release_list); |
- | |
742 | return ret; |
- | |
743 | } |
- | |
744 | - | ||
745 | put_count = ttm_bo_del_from_lru(bo); |
- | |
746 | spin_unlock(&glob->lru_lock); |
- | |
747 | - | ||
748 | BUG_ON(ret != 0); |
- | |
749 | - | ||
750 | ttm_bo_list_ref_sub(bo, put_count, true); |
- | |
751 | - | ||
752 | ret = ttm_bo_evict(bo, interruptible, no_wait_gpu); |
- | |
753 | ttm_bo_unreserve(bo); |
- | |
754 | - | ||
755 | kref_put(&bo->list_kref, ttm_bo_release_list); |
- | |
756 | return ret; |
- | |
757 | } |
- | |
758 | 588 | ||
759 | void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) |
589 | void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) |
760 | { |
590 | { |
761 | struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type]; |
591 | struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type]; |
762 | 592 | ||
763 | if (mem->mm_node) |
593 | if (mem->mm_node) |
764 | (*man->func->put_node)(man, mem); |
594 | (*man->func->put_node)(man, mem); |
765 | } |
595 | } |
766 | EXPORT_SYMBOL(ttm_bo_mem_put); |
596 | EXPORT_SYMBOL(ttm_bo_mem_put); |
767 | 597 | ||
768 | /** |
598 | /** |
769 | * Repeatedly evict memory from the LRU for @mem_type until we create enough |
599 | * Repeatedly evict memory from the LRU for @mem_type until we create enough |
770 | * space, or we've evicted everything and there isn't enough space. |
600 | * space, or we've evicted everything and there isn't enough space. |
771 | */ |
601 | */ |
772 | static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, |
602 | static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, |
773 | uint32_t mem_type, |
603 | uint32_t mem_type, |
774 | struct ttm_placement *placement, |
604 | struct ttm_placement *placement, |
775 | struct ttm_mem_reg *mem, |
605 | struct ttm_mem_reg *mem, |
776 | bool interruptible, |
606 | bool interruptible, |
777 | bool no_wait_gpu) |
607 | bool no_wait_gpu) |
778 | { |
608 | { |
779 | struct ttm_bo_device *bdev = bo->bdev; |
609 | struct ttm_bo_device *bdev = bo->bdev; |
780 | struct ttm_mem_type_manager *man = &bdev->man[mem_type]; |
610 | struct ttm_mem_type_manager *man = &bdev->man[mem_type]; |
781 | int ret; |
611 | int ret; |
782 | 612 | ||
783 | do { |
613 | do { |
784 | ret = (*man->func->get_node)(man, bo, placement, mem); |
614 | ret = (*man->func->get_node)(man, bo, placement, 0, mem); |
785 | if (unlikely(ret != 0)) |
615 | if (unlikely(ret != 0)) |
786 | return ret; |
616 | return ret; |
787 | if (mem->mm_node) |
617 | if (mem->mm_node) |
788 | break; |
618 | break; |
789 | ret = ttm_mem_evict_first(bdev, mem_type, |
619 | // ret = ttm_mem_evict_first(bdev, mem_type, |
790 | interruptible, no_wait_gpu); |
620 | // interruptible, no_wait_gpu); |
791 | if (unlikely(ret != 0)) |
621 | // if (unlikely(ret != 0)) |
792 | return ret; |
622 | // return ret; |
793 | } while (1); |
623 | } while (1); |
794 | if (mem->mm_node == NULL) |
624 | if (mem->mm_node == NULL) |
795 | return -ENOMEM; |
625 | return -ENOMEM; |
796 | mem->mem_type = mem_type; |
626 | mem->mem_type = mem_type; |
797 | return 0; |
627 | return 0; |
798 | } |
628 | } |
799 | 629 | ||
800 | static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man, |
630 | static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man, |
801 | uint32_t cur_placement, |
631 | uint32_t cur_placement, |
802 | uint32_t proposed_placement) |
632 | uint32_t proposed_placement) |
803 | { |
633 | { |
804 | uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING; |
634 | uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING; |
805 | uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING; |
635 | uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING; |
806 | 636 | ||
807 | /** |
637 | /** |
808 | * Keep current caching if possible. |
638 | * Keep current caching if possible. |
809 | */ |
639 | */ |
810 | 640 | ||
811 | if ((cur_placement & caching) != 0) |
641 | if ((cur_placement & caching) != 0) |
812 | result |= (cur_placement & caching); |
642 | result |= (cur_placement & caching); |
813 | else if ((man->default_caching & caching) != 0) |
643 | else if ((man->default_caching & caching) != 0) |
814 | result |= man->default_caching; |
644 | result |= man->default_caching; |
815 | else if ((TTM_PL_FLAG_CACHED & caching) != 0) |
645 | else if ((TTM_PL_FLAG_CACHED & caching) != 0) |
816 | result |= TTM_PL_FLAG_CACHED; |
646 | result |= TTM_PL_FLAG_CACHED; |
817 | else if ((TTM_PL_FLAG_WC & caching) != 0) |
647 | else if ((TTM_PL_FLAG_WC & caching) != 0) |
818 | result |= TTM_PL_FLAG_WC; |
648 | result |= TTM_PL_FLAG_WC; |
819 | else if ((TTM_PL_FLAG_UNCACHED & caching) != 0) |
649 | else if ((TTM_PL_FLAG_UNCACHED & caching) != 0) |
820 | result |= TTM_PL_FLAG_UNCACHED; |
650 | result |= TTM_PL_FLAG_UNCACHED; |
821 | 651 | ||
822 | return result; |
652 | return result; |
823 | } |
653 | } |
824 | 654 | ||
825 | static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, |
655 | static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, |
826 | uint32_t mem_type, |
656 | uint32_t mem_type, |
827 | uint32_t proposed_placement, |
657 | uint32_t proposed_placement, |
828 | uint32_t *masked_placement) |
658 | uint32_t *masked_placement) |
829 | { |
659 | { |
830 | uint32_t cur_flags = ttm_bo_type_flags(mem_type); |
660 | uint32_t cur_flags = ttm_bo_type_flags(mem_type); |
831 | 661 | ||
832 | if ((cur_flags & proposed_placement & TTM_PL_MASK_MEM) == 0) |
662 | if ((cur_flags & proposed_placement & TTM_PL_MASK_MEM) == 0) |
833 | return false; |
663 | return false; |
834 | 664 | ||
835 | if ((proposed_placement & man->available_caching) == 0) |
665 | if ((proposed_placement & man->available_caching) == 0) |
836 | return false; |
666 | return false; |
837 | 667 | ||
838 | cur_flags |= (proposed_placement & man->available_caching); |
668 | cur_flags |= (proposed_placement & man->available_caching); |
839 | 669 | ||
840 | *masked_placement = cur_flags; |
670 | *masked_placement = cur_flags; |
841 | return true; |
671 | return true; |
842 | } |
672 | } |
843 | 673 | ||
844 | /** |
674 | /** |
845 | * Creates space for memory region @mem according to its type. |
675 | * Creates space for memory region @mem according to its type. |
846 | * |
676 | * |
847 | * This function first searches for free space in compatible memory types in |
677 | * This function first searches for free space in compatible memory types in |
848 | * the priority order defined by the driver. If free space isn't found, then |
678 | * the priority order defined by the driver. If free space isn't found, then |
849 | * ttm_bo_mem_force_space is attempted in priority order to evict and find |
679 | * ttm_bo_mem_force_space is attempted in priority order to evict and find |
850 | * space. |
680 | * space. |
851 | */ |
681 | */ |
852 | int ttm_bo_mem_space(struct ttm_buffer_object *bo, |
682 | int ttm_bo_mem_space(struct ttm_buffer_object *bo, |
853 | struct ttm_placement *placement, |
683 | struct ttm_placement *placement, |
854 | struct ttm_mem_reg *mem, |
684 | struct ttm_mem_reg *mem, |
855 | bool interruptible, |
685 | bool interruptible, |
856 | bool no_wait_gpu) |
686 | bool no_wait_gpu) |
857 | { |
687 | { |
858 | struct ttm_bo_device *bdev = bo->bdev; |
688 | struct ttm_bo_device *bdev = bo->bdev; |
859 | struct ttm_mem_type_manager *man; |
689 | struct ttm_mem_type_manager *man; |
860 | uint32_t mem_type = TTM_PL_SYSTEM; |
690 | uint32_t mem_type = TTM_PL_SYSTEM; |
861 | uint32_t cur_flags = 0; |
691 | uint32_t cur_flags = 0; |
862 | bool type_found = false; |
692 | bool type_found = false; |
863 | bool type_ok = false; |
693 | bool type_ok = false; |
864 | bool has_erestartsys = false; |
694 | bool has_erestartsys = false; |
865 | int i, ret; |
695 | int i, ret; |
866 | 696 | ||
867 | mem->mm_node = NULL; |
697 | mem->mm_node = NULL; |
868 | for (i = 0; i < placement->num_placement; ++i) { |
698 | for (i = 0; i < placement->num_placement; ++i) { |
869 | ret = ttm_mem_type_from_flags(placement->placement[i], |
699 | ret = ttm_mem_type_from_flags(placement->placement[i], |
870 | &mem_type); |
700 | &mem_type); |
871 | if (ret) |
701 | if (ret) |
872 | return ret; |
702 | return ret; |
873 | man = &bdev->man[mem_type]; |
703 | man = &bdev->man[mem_type]; |
874 | 704 | ||
875 | type_ok = ttm_bo_mt_compatible(man, |
705 | type_ok = ttm_bo_mt_compatible(man, |
876 | mem_type, |
706 | mem_type, |
877 | placement->placement[i], |
707 | placement->placement[i], |
878 | &cur_flags); |
708 | &cur_flags); |
879 | 709 | ||
880 | if (!type_ok) |
710 | if (!type_ok) |
881 | continue; |
711 | continue; |
882 | 712 | ||
883 | cur_flags = ttm_bo_select_caching(man, bo->mem.placement, |
713 | cur_flags = ttm_bo_select_caching(man, bo->mem.placement, |
884 | cur_flags); |
714 | cur_flags); |
885 | /* |
715 | /* |
886 | * Use the access and other non-mapping-related flag bits from |
716 | * Use the access and other non-mapping-related flag bits from |
887 | * the memory placement flags to the current flags |
717 | * the memory placement flags to the current flags |
888 | */ |
718 | */ |
889 | ttm_flag_masked(&cur_flags, placement->placement[i], |
719 | ttm_flag_masked(&cur_flags, placement->placement[i], |
890 | ~TTM_PL_MASK_MEMTYPE); |
720 | ~TTM_PL_MASK_MEMTYPE); |
891 | 721 | ||
892 | if (mem_type == TTM_PL_SYSTEM) |
722 | if (mem_type == TTM_PL_SYSTEM) |
893 | break; |
723 | break; |
894 | 724 | ||
895 | if (man->has_type && man->use_type) { |
725 | if (man->has_type && man->use_type) { |
896 | type_found = true; |
726 | type_found = true; |
897 | ret = (*man->func->get_node)(man, bo, placement, mem); |
727 | ret = (*man->func->get_node)(man, bo, placement, |
- | 728 | cur_flags, mem); |
|
898 | if (unlikely(ret)) |
729 | if (unlikely(ret)) |
899 | return ret; |
730 | return ret; |
900 | } |
731 | } |
901 | if (mem->mm_node) |
732 | if (mem->mm_node) |
902 | break; |
733 | break; |
903 | } |
734 | } |
904 | 735 | ||
905 | if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) { |
736 | if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) { |
906 | mem->mem_type = mem_type; |
737 | mem->mem_type = mem_type; |
907 | mem->placement = cur_flags; |
738 | mem->placement = cur_flags; |
908 | return 0; |
739 | return 0; |
909 | } |
740 | } |
910 | 741 | ||
911 | if (!type_found) |
742 | if (!type_found) |
912 | return -EINVAL; |
743 | return -EINVAL; |
913 | 744 | ||
914 | for (i = 0; i < placement->num_busy_placement; ++i) { |
745 | for (i = 0; i < placement->num_busy_placement; ++i) { |
915 | ret = ttm_mem_type_from_flags(placement->busy_placement[i], |
746 | ret = ttm_mem_type_from_flags(placement->busy_placement[i], |
916 | &mem_type); |
747 | &mem_type); |
917 | if (ret) |
748 | if (ret) |
918 | return ret; |
749 | return ret; |
919 | man = &bdev->man[mem_type]; |
750 | man = &bdev->man[mem_type]; |
920 | if (!man->has_type) |
751 | if (!man->has_type) |
921 | continue; |
752 | continue; |
922 | if (!ttm_bo_mt_compatible(man, |
753 | if (!ttm_bo_mt_compatible(man, |
923 | mem_type, |
754 | mem_type, |
924 | placement->busy_placement[i], |
755 | placement->busy_placement[i], |
925 | &cur_flags)) |
756 | &cur_flags)) |
926 | continue; |
757 | continue; |
927 | 758 | ||
928 | cur_flags = ttm_bo_select_caching(man, bo->mem.placement, |
759 | cur_flags = ttm_bo_select_caching(man, bo->mem.placement, |
929 | cur_flags); |
760 | cur_flags); |
930 | /* |
761 | /* |
931 | * Use the access and other non-mapping-related flag bits from |
762 | * Use the access and other non-mapping-related flag bits from |
932 | * the memory placement flags to the current flags |
763 | * the memory placement flags to the current flags |
933 | */ |
764 | */ |
934 | ttm_flag_masked(&cur_flags, placement->busy_placement[i], |
765 | ttm_flag_masked(&cur_flags, placement->busy_placement[i], |
935 | ~TTM_PL_MASK_MEMTYPE); |
766 | ~TTM_PL_MASK_MEMTYPE); |
936 | - | ||
937 | 767 | ||
938 | if (mem_type == TTM_PL_SYSTEM) { |
768 | if (mem_type == TTM_PL_SYSTEM) { |
939 | mem->mem_type = mem_type; |
769 | mem->mem_type = mem_type; |
940 | mem->placement = cur_flags; |
770 | mem->placement = cur_flags; |
941 | mem->mm_node = NULL; |
771 | mem->mm_node = NULL; |
942 | return 0; |
772 | return 0; |
943 | } |
773 | } |
944 | 774 | ||
945 | ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem, |
775 | ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem, |
946 | interruptible, no_wait_gpu); |
776 | interruptible, no_wait_gpu); |
947 | if (ret == 0 && mem->mm_node) { |
777 | if (ret == 0 && mem->mm_node) { |
948 | mem->placement = cur_flags; |
778 | mem->placement = cur_flags; |
949 | return 0; |
779 | return 0; |
950 | } |
780 | } |
951 | if (ret == -ERESTARTSYS) |
781 | if (ret == -ERESTARTSYS) |
952 | has_erestartsys = true; |
782 | has_erestartsys = true; |
953 | } |
783 | } |
954 | ret = (has_erestartsys) ? -ERESTARTSYS : -ENOMEM; |
784 | ret = (has_erestartsys) ? -ERESTARTSYS : -ENOMEM; |
955 | return ret; |
785 | return ret; |
956 | } |
786 | } |
957 | EXPORT_SYMBOL(ttm_bo_mem_space); |
787 | EXPORT_SYMBOL(ttm_bo_mem_space); |
958 | 788 | ||
959 | static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, |
789 | static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, |
960 | struct ttm_placement *placement, |
790 | struct ttm_placement *placement, |
961 | bool interruptible, |
791 | bool interruptible, |
962 | bool no_wait_gpu) |
792 | bool no_wait_gpu) |
963 | { |
793 | { |
964 | int ret = 0; |
794 | int ret = 0; |
965 | struct ttm_mem_reg mem; |
795 | struct ttm_mem_reg mem; |
966 | struct ttm_bo_device *bdev = bo->bdev; |
796 | struct ttm_bo_device *bdev = bo->bdev; |
967 | 797 | ||
968 | // BUG_ON(!ttm_bo_is_reserved(bo)); |
798 | lockdep_assert_held(&bo->resv->lock.base); |
969 | 799 | ||
970 | /* |
800 | /* |
971 | * FIXME: It's possible to pipeline buffer moves. |
801 | * FIXME: It's possible to pipeline buffer moves. |
972 | * Have the driver move function wait for idle when necessary, |
802 | * Have the driver move function wait for idle when necessary, |
973 | * instead of doing it here. |
803 | * instead of doing it here. |
974 | */ |
804 | */ |
975 | spin_lock(&bdev->fence_lock); |
805 | spin_lock(&bdev->fence_lock); |
976 | ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); |
806 | ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); |
977 | spin_unlock(&bdev->fence_lock); |
807 | spin_unlock(&bdev->fence_lock); |
978 | if (ret) |
808 | if (ret) |
979 | return ret; |
809 | return ret; |
980 | mem.num_pages = bo->num_pages; |
810 | mem.num_pages = bo->num_pages; |
981 | mem.size = mem.num_pages << PAGE_SHIFT; |
811 | mem.size = mem.num_pages << PAGE_SHIFT; |
982 | mem.page_alignment = bo->mem.page_alignment; |
812 | mem.page_alignment = bo->mem.page_alignment; |
983 | mem.bus.io_reserved_vm = false; |
813 | mem.bus.io_reserved_vm = false; |
984 | mem.bus.io_reserved_count = 0; |
814 | mem.bus.io_reserved_count = 0; |
985 | /* |
815 | /* |
986 | * Determine where to move the buffer. |
816 | * Determine where to move the buffer. |
987 | */ |
817 | */ |
988 | ret = ttm_bo_mem_space(bo, placement, &mem, |
818 | ret = ttm_bo_mem_space(bo, placement, &mem, |
989 | interruptible, no_wait_gpu); |
819 | interruptible, no_wait_gpu); |
990 | if (ret) |
820 | if (ret) |
991 | goto out_unlock; |
821 | goto out_unlock; |
992 | ret = ttm_bo_handle_move_mem(bo, &mem, false, |
822 | ret = ttm_bo_handle_move_mem(bo, &mem, false, |
993 | interruptible, no_wait_gpu); |
823 | interruptible, no_wait_gpu); |
994 | out_unlock: |
824 | out_unlock: |
995 | if (ret && mem.mm_node) |
825 | if (ret && mem.mm_node) |
996 | ttm_bo_mem_put(bo, &mem); |
826 | ttm_bo_mem_put(bo, &mem); |
997 | return ret; |
827 | return ret; |
998 | } |
828 | } |
999 | #endif |
- | |
1000 | 829 | ||
1001 | static bool ttm_bo_mem_compat(struct ttm_placement *placement, |
830 | static bool ttm_bo_mem_compat(struct ttm_placement *placement, |
1002 | struct ttm_mem_reg *mem, |
831 | struct ttm_mem_reg *mem, |
1003 | uint32_t *new_flags) |
832 | uint32_t *new_flags) |
1004 | { |
833 | { |
1005 | int i; |
834 | int i; |
1006 | 835 | ||
1007 | if (mem->mm_node && placement->lpfn != 0 && |
836 | if (mem->mm_node && placement->lpfn != 0 && |
1008 | (mem->start < placement->fpfn || |
837 | (mem->start < placement->fpfn || |
1009 | mem->start + mem->num_pages > placement->lpfn)) |
838 | mem->start + mem->num_pages > placement->lpfn)) |
1010 | return false; |
839 | return false; |
1011 | 840 | ||
1012 | for (i = 0; i < placement->num_placement; i++) { |
841 | for (i = 0; i < placement->num_placement; i++) { |
1013 | *new_flags = placement->placement[i]; |
842 | *new_flags = placement->placement[i]; |
1014 | if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && |
843 | if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && |
1015 | (*new_flags & mem->placement & TTM_PL_MASK_MEM)) |
844 | (*new_flags & mem->placement & TTM_PL_MASK_MEM)) |
1016 | return true; |
845 | return true; |
1017 | } |
846 | } |
1018 | 847 | ||
1019 | for (i = 0; i < placement->num_busy_placement; i++) { |
848 | for (i = 0; i < placement->num_busy_placement; i++) { |
1020 | *new_flags = placement->busy_placement[i]; |
849 | *new_flags = placement->busy_placement[i]; |
1021 | if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && |
850 | if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && |
1022 | (*new_flags & mem->placement & TTM_PL_MASK_MEM)) |
851 | (*new_flags & mem->placement & TTM_PL_MASK_MEM)) |
1023 | return true; |
852 | return true; |
1024 | } |
853 | } |
1025 | 854 | ||
1026 | return false; |
855 | return false; |
1027 | } |
856 | } |
1028 | 857 | ||
1029 | int ttm_bo_validate(struct ttm_buffer_object *bo, |
858 | int ttm_bo_validate(struct ttm_buffer_object *bo, |
1030 | struct ttm_placement *placement, |
859 | struct ttm_placement *placement, |
1031 | bool interruptible, |
860 | bool interruptible, |
1032 | bool no_wait_gpu) |
861 | bool no_wait_gpu) |
1033 | { |
862 | { |
1034 | int ret; |
863 | int ret; |
1035 | uint32_t new_flags; |
864 | uint32_t new_flags; |
1036 | 865 | ||
1037 | // BUG_ON(!ttm_bo_is_reserved(bo)); |
866 | lockdep_assert_held(&bo->resv->lock.base); |
1038 | /* Check that range is valid */ |
867 | /* Check that range is valid */ |
1039 | if (placement->lpfn || placement->fpfn) |
868 | if (placement->lpfn || placement->fpfn) |
1040 | if (placement->fpfn > placement->lpfn || |
869 | if (placement->fpfn > placement->lpfn || |
1041 | (placement->lpfn - placement->fpfn) < bo->num_pages) |
870 | (placement->lpfn - placement->fpfn) < bo->num_pages) |
1042 | return -EINVAL; |
871 | return -EINVAL; |
1043 | /* |
872 | /* |
1044 | * Check whether we need to move buffer. |
873 | * Check whether we need to move buffer. |
1045 | */ |
874 | */ |
1046 | if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) { |
875 | if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) { |
1047 | // ret = ttm_bo_move_buffer(bo, placement, interruptible, |
876 | ret = ttm_bo_move_buffer(bo, placement, interruptible, |
1048 | // no_wait_gpu); |
877 | no_wait_gpu); |
1049 | if (ret) |
878 | if (ret) |
1050 | return ret; |
879 | return ret; |
1051 | } else { |
880 | } else { |
1052 | /* |
881 | /* |
1053 | * Use the access and other non-mapping-related flag bits from |
882 | * Use the access and other non-mapping-related flag bits from |
1054 | * the compatible memory placement flags to the active flags |
883 | * the compatible memory placement flags to the active flags |
1055 | */ |
884 | */ |
1056 | ttm_flag_masked(&bo->mem.placement, new_flags, |
885 | ttm_flag_masked(&bo->mem.placement, new_flags, |
1057 | ~TTM_PL_MASK_MEMTYPE); |
886 | ~TTM_PL_MASK_MEMTYPE); |
1058 | } |
887 | } |
1059 | /* |
888 | /* |
1060 | * We might need to add a TTM. |
889 | * We might need to add a TTM. |
1061 | */ |
890 | */ |
1062 | if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { |
891 | if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { |
1063 | ret = ttm_bo_add_ttm(bo, true); |
892 | ret = ttm_bo_add_ttm(bo, true); |
1064 | if (ret) |
893 | if (ret) |
1065 | return ret; |
894 | return ret; |
1066 | } |
895 | } |
1067 | return 0; |
896 | return 0; |
1068 | } |
897 | } |
1069 | EXPORT_SYMBOL(ttm_bo_validate); |
898 | EXPORT_SYMBOL(ttm_bo_validate); |
1070 | 899 | ||
1071 | int ttm_bo_check_placement(struct ttm_buffer_object *bo, |
900 | int ttm_bo_check_placement(struct ttm_buffer_object *bo, |
1072 | struct ttm_placement *placement) |
901 | struct ttm_placement *placement) |
1073 | { |
902 | { |
1074 | BUG_ON((placement->fpfn || placement->lpfn) && |
903 | BUG_ON((placement->fpfn || placement->lpfn) && |
1075 | (bo->mem.num_pages > (placement->lpfn - placement->fpfn))); |
904 | (bo->mem.num_pages > (placement->lpfn - placement->fpfn))); |
1076 | 905 | ||
1077 | return 0; |
906 | return 0; |
1078 | } |
907 | } |
1079 | 908 | ||
1080 | int ttm_bo_init(struct ttm_bo_device *bdev, |
909 | int ttm_bo_init(struct ttm_bo_device *bdev, |
1081 | struct ttm_buffer_object *bo, |
910 | struct ttm_buffer_object *bo, |
1082 | unsigned long size, |
911 | unsigned long size, |
1083 | enum ttm_bo_type type, |
912 | enum ttm_bo_type type, |
1084 | struct ttm_placement *placement, |
913 | struct ttm_placement *placement, |
1085 | uint32_t page_alignment, |
914 | uint32_t page_alignment, |
1086 | bool interruptible, |
915 | bool interruptible, |
1087 | struct file *persistent_swap_storage, |
916 | struct file *persistent_swap_storage, |
1088 | size_t acc_size, |
917 | size_t acc_size, |
1089 | struct sg_table *sg, |
918 | struct sg_table *sg, |
1090 | void (*destroy) (struct ttm_buffer_object *)) |
919 | void (*destroy) (struct ttm_buffer_object *)) |
1091 | { |
920 | { |
1092 | int ret = 0; |
921 | int ret = 0; |
1093 | unsigned long num_pages; |
922 | unsigned long num_pages; |
1094 | struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; |
- | |
1095 | bool locked; |
923 | bool locked; |
1096 | - | ||
1097 | // ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false); |
- | |
1098 | if (ret) { |
- | |
1099 | pr_err("Out of kernel memory\n"); |
- | |
1100 | if (destroy) |
- | |
1101 | (*destroy)(bo); |
- | |
1102 | else |
- | |
1103 | kfree(bo); |
- | |
1104 | return -ENOMEM; |
- | |
1105 | } |
- | |
1106 | 924 | ||
1107 | num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; |
925 | num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; |
1108 | if (num_pages == 0) { |
926 | if (num_pages == 0) { |
1109 | pr_err("Illegal buffer object size\n"); |
927 | pr_err("Illegal buffer object size\n"); |
1110 | if (destroy) |
928 | if (destroy) |
1111 | (*destroy)(bo); |
929 | (*destroy)(bo); |
1112 | else |
930 | else |
1113 | kfree(bo); |
931 | kfree(bo); |
1114 | // ttm_mem_global_free(mem_glob, acc_size); |
- | |
1115 | return -EINVAL; |
932 | return -EINVAL; |
1116 | } |
933 | } |
1117 | bo->destroy = destroy; |
934 | bo->destroy = destroy; |
1118 | 935 | ||
1119 | kref_init(&bo->kref); |
936 | kref_init(&bo->kref); |
1120 | kref_init(&bo->list_kref); |
937 | kref_init(&bo->list_kref); |
1121 | atomic_set(&bo->cpu_writers, 0); |
938 | atomic_set(&bo->cpu_writers, 0); |
1122 | INIT_LIST_HEAD(&bo->lru); |
939 | INIT_LIST_HEAD(&bo->lru); |
1123 | INIT_LIST_HEAD(&bo->ddestroy); |
940 | INIT_LIST_HEAD(&bo->ddestroy); |
1124 | INIT_LIST_HEAD(&bo->swap); |
941 | INIT_LIST_HEAD(&bo->swap); |
1125 | INIT_LIST_HEAD(&bo->io_reserve_lru); |
942 | INIT_LIST_HEAD(&bo->io_reserve_lru); |
1126 | mutex_init(&bo->wu_mutex); |
943 | mutex_init(&bo->wu_mutex); |
1127 | bo->bdev = bdev; |
944 | bo->bdev = bdev; |
1128 | bo->glob = bdev->glob; |
945 | bo->glob = bdev->glob; |
1129 | bo->type = type; |
946 | bo->type = type; |
1130 | bo->num_pages = num_pages; |
947 | bo->num_pages = num_pages; |
1131 | bo->mem.size = num_pages << PAGE_SHIFT; |
948 | bo->mem.size = num_pages << PAGE_SHIFT; |
1132 | bo->mem.mem_type = TTM_PL_SYSTEM; |
949 | bo->mem.mem_type = TTM_PL_SYSTEM; |
1133 | bo->mem.num_pages = bo->num_pages; |
950 | bo->mem.num_pages = bo->num_pages; |
1134 | bo->mem.mm_node = NULL; |
951 | bo->mem.mm_node = NULL; |
1135 | bo->mem.page_alignment = page_alignment; |
952 | bo->mem.page_alignment = page_alignment; |
1136 | bo->mem.bus.io_reserved_vm = false; |
953 | bo->mem.bus.io_reserved_vm = false; |
1137 | bo->mem.bus.io_reserved_count = 0; |
954 | bo->mem.bus.io_reserved_count = 0; |
1138 | bo->priv_flags = 0; |
955 | bo->priv_flags = 0; |
1139 | bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); |
956 | bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); |
1140 | bo->persistent_swap_storage = persistent_swap_storage; |
957 | bo->persistent_swap_storage = persistent_swap_storage; |
1141 | bo->acc_size = acc_size; |
958 | bo->acc_size = acc_size; |
1142 | bo->sg = sg; |
959 | bo->sg = sg; |
1143 | bo->resv = &bo->ttm_resv; |
960 | bo->resv = &bo->ttm_resv; |
1144 | // reservation_object_init(bo->resv); |
961 | reservation_object_init(bo->resv); |
1145 | atomic_inc(&bo->glob->bo_count); |
962 | atomic_inc(&bo->glob->bo_count); |
1146 | drm_vma_node_reset(&bo->vma_node); |
963 | drm_vma_node_reset(&bo->vma_node); |
1147 | 964 | ||
1148 | ret = ttm_bo_check_placement(bo, placement); |
965 | ret = ttm_bo_check_placement(bo, placement); |
1149 | 966 | ||
1150 | /* |
967 | /* |
1151 | * For ttm_bo_type_device buffers, allocate |
968 | * For ttm_bo_type_device buffers, allocate |
1152 | * address space from the device. |
969 | * address space from the device. |
1153 | */ |
970 | */ |
1154 | // if (likely(!ret) && |
971 | if (likely(!ret) && |
1155 | // (bo->type == ttm_bo_type_device || |
972 | (bo->type == ttm_bo_type_device || |
1156 | // bo->type == ttm_bo_type_sg)) |
973 | bo->type == ttm_bo_type_sg)) |
- | 974 | ret = drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node, |
|
1157 | // ret = ttm_bo_setup_vm(bo); |
975 | bo->mem.num_pages); |
1158 | 976 | ||
1159 | // if (likely(!ret)) |
977 | locked = ww_mutex_trylock(&bo->resv->lock); |
1160 | // ret = ttm_bo_validate(bo, placement, interruptible, false); |
978 | WARN_ON(!locked); |
- | 979 | ||
- | 980 | if (likely(!ret)) |
|
- | 981 | ret = ttm_bo_validate(bo, placement, interruptible, false); |
|
1161 | 982 | ||
1162 | // ttm_bo_unreserve(bo); |
983 | ttm_bo_unreserve(bo); |
1163 | 984 | ||
1164 | // if (unlikely(ret)) |
985 | if (unlikely(ret)) |
1165 | // ttm_bo_unref(&bo); |
986 | ttm_bo_unref(&bo); |
1166 | 987 | ||
1167 | return ret; |
988 | return ret; |
1168 | } |
989 | } |
1169 | EXPORT_SYMBOL(ttm_bo_init); |
990 | EXPORT_SYMBOL(ttm_bo_init); |
1170 | 991 | ||
1171 | size_t ttm_bo_acc_size(struct ttm_bo_device *bdev, |
992 | size_t ttm_bo_acc_size(struct ttm_bo_device *bdev, |
1172 | unsigned long bo_size, |
993 | unsigned long bo_size, |
1173 | unsigned struct_size) |
994 | unsigned struct_size) |
1174 | { |
995 | { |
1175 | unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; |
996 | unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; |
1176 | size_t size = 0; |
997 | size_t size = 0; |
1177 | 998 | ||
1178 | size += ttm_round_pot(struct_size); |
999 | size += ttm_round_pot(struct_size); |
1179 | size += PAGE_ALIGN(npages * sizeof(void *)); |
1000 | size += PAGE_ALIGN(npages * sizeof(void *)); |
1180 | size += ttm_round_pot(sizeof(struct ttm_tt)); |
1001 | size += ttm_round_pot(sizeof(struct ttm_tt)); |
1181 | return size; |
1002 | return size; |
1182 | } |
1003 | } |
1183 | EXPORT_SYMBOL(ttm_bo_acc_size); |
1004 | EXPORT_SYMBOL(ttm_bo_acc_size); |
1184 | 1005 | ||
1185 | size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, |
1006 | size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, |
1186 | unsigned long bo_size, |
1007 | unsigned long bo_size, |
1187 | unsigned struct_size) |
1008 | unsigned struct_size) |
1188 | { |
1009 | { |
1189 | unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; |
1010 | unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; |
1190 | size_t size = 0; |
1011 | size_t size = 0; |
1191 | 1012 | ||
1192 | size += ttm_round_pot(struct_size); |
1013 | size += ttm_round_pot(struct_size); |
1193 | size += PAGE_ALIGN(npages * sizeof(void *)); |
1014 | size += PAGE_ALIGN(npages * sizeof(void *)); |
1194 | size += PAGE_ALIGN(npages * sizeof(dma_addr_t)); |
1015 | size += PAGE_ALIGN(npages * sizeof(dma_addr_t)); |
1195 | size += ttm_round_pot(sizeof(struct ttm_dma_tt)); |
1016 | size += ttm_round_pot(sizeof(struct ttm_dma_tt)); |
1196 | return size; |
1017 | return size; |
1197 | } |
1018 | } |
1198 | EXPORT_SYMBOL(ttm_bo_dma_acc_size); |
1019 | EXPORT_SYMBOL(ttm_bo_dma_acc_size); |
1199 | - | ||
1200 | int ttm_bo_create(struct ttm_bo_device *bdev, |
- | |
1201 | unsigned long size, |
- | |
1202 | enum ttm_bo_type type, |
- | |
1203 | struct ttm_placement *placement, |
- | |
1204 | uint32_t page_alignment, |
- | |
1205 | bool interruptible, |
- | |
1206 | struct file *persistent_swap_storage, |
- | |
1207 | struct ttm_buffer_object **p_bo) |
- | |
1208 | { |
- | |
1209 | struct ttm_buffer_object *bo; |
- | |
1210 | size_t acc_size; |
- | |
1211 | int ret; |
- | |
1212 | - | ||
1213 | bo = kzalloc(sizeof(*bo), GFP_KERNEL); |
- | |
1214 | if (unlikely(bo == NULL)) |
- | |
1215 | return -ENOMEM; |
- | |
1216 | - | ||
1217 | acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); |
- | |
1218 | ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, |
- | |
1219 | interruptible, persistent_swap_storage, acc_size, |
- | |
1220 | NULL, NULL); |
- | |
1221 | if (likely(ret == 0)) |
- | |
1222 | *p_bo = bo; |
- | |
1223 | - | ||
1224 | return ret; |
- | |
1225 | } |
- | |
1226 | EXPORT_SYMBOL(ttm_bo_create); |
- | |
1227 | - | ||
1228 | - | ||
1229 | - | ||
1230 | - | ||
1231 | - | ||
1232 | - | ||
1233 | - | ||
1234 | - | ||
1235 | - | ||
1236 | - | ||
1237 | - | ||
1238 | - | ||
1239 | - | ||
1240 | - | ||
1241 | - | ||
1242 | - | ||
1243 | - | ||
1244 | - | ||
1245 | - | ||
1246 | 1020 | ||
1247 | int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, |
1021 | int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, |
1248 | unsigned long p_size) |
1022 | unsigned long p_size) |
1249 | { |
1023 | { |
1250 | int ret = -EINVAL; |
1024 | int ret = -EINVAL; |
1251 | struct ttm_mem_type_manager *man; |
1025 | struct ttm_mem_type_manager *man; |
1252 | - | ||
1253 | ENTER(); |
- | |
1254 | 1026 | ||
1255 | BUG_ON(type >= TTM_NUM_MEM_TYPES); |
1027 | BUG_ON(type >= TTM_NUM_MEM_TYPES); |
1256 | man = &bdev->man[type]; |
1028 | man = &bdev->man[type]; |
1257 | BUG_ON(man->has_type); |
1029 | BUG_ON(man->has_type); |
1258 | man->io_reserve_fastpath = true; |
1030 | man->io_reserve_fastpath = true; |
1259 | man->use_io_reserve_lru = false; |
1031 | man->use_io_reserve_lru = false; |
1260 | mutex_init(&man->io_reserve_mutex); |
1032 | mutex_init(&man->io_reserve_mutex); |
1261 | INIT_LIST_HEAD(&man->io_reserve_lru); |
1033 | INIT_LIST_HEAD(&man->io_reserve_lru); |
1262 | 1034 | ||
1263 | ret = bdev->driver->init_mem_type(bdev, type, man); |
1035 | ret = bdev->driver->init_mem_type(bdev, type, man); |
1264 | if (ret) |
1036 | if (ret) |
1265 | return ret; |
1037 | return ret; |
1266 | man->bdev = bdev; |
1038 | man->bdev = bdev; |
1267 | 1039 | ||
1268 | ret = 0; |
1040 | ret = 0; |
1269 | if (type != TTM_PL_SYSTEM) { |
1041 | if (type != TTM_PL_SYSTEM) { |
1270 | ret = (*man->func->init)(man, p_size); |
1042 | ret = (*man->func->init)(man, p_size); |
1271 | if (ret) |
1043 | if (ret) |
1272 | return ret; |
1044 | return ret; |
1273 | } |
1045 | } |
1274 | man->has_type = true; |
1046 | man->has_type = true; |
1275 | man->use_type = true; |
1047 | man->use_type = true; |
1276 | man->size = p_size; |
1048 | man->size = p_size; |
1277 | 1049 | ||
1278 | INIT_LIST_HEAD(&man->lru); |
1050 | INIT_LIST_HEAD(&man->lru); |
1279 | - | ||
1280 | LEAVE(); |
- | |
1281 | 1051 | ||
1282 | return 0; |
1052 | return 0; |
1283 | } |
- | |
1284 | - | ||
- | 1053 | } |
|
1285 | 1054 | EXPORT_SYMBOL(ttm_bo_init_mm); |
|
1286 | void ttm_bo_global_release(struct drm_global_reference *ref) |
1055 | void ttm_bo_global_release(struct drm_global_reference *ref) |
1287 | { |
1056 | { |
1288 | struct ttm_bo_global *glob = ref->object; |
1057 | struct ttm_bo_global *glob = ref->object; |
1289 | 1058 | ||
1290 | } |
1059 | } |
1291 | EXPORT_SYMBOL(ttm_bo_global_release); |
1060 | EXPORT_SYMBOL(ttm_bo_global_release); |
1292 | 1061 | ||
1293 | int ttm_bo_global_init(struct drm_global_reference *ref) |
1062 | int ttm_bo_global_init(struct drm_global_reference *ref) |
1294 | { |
1063 | { |
1295 | struct ttm_bo_global_ref *bo_ref = |
1064 | struct ttm_bo_global_ref *bo_ref = |
1296 | container_of(ref, struct ttm_bo_global_ref, ref); |
1065 | container_of(ref, struct ttm_bo_global_ref, ref); |
1297 | struct ttm_bo_global *glob = ref->object; |
1066 | struct ttm_bo_global *glob = ref->object; |
1298 | int ret; |
1067 | int ret; |
1299 | - | ||
1300 | ENTER(); |
- | |
1301 | 1068 | ||
1302 | mutex_init(&glob->device_list_mutex); |
1069 | mutex_init(&glob->device_list_mutex); |
1303 | spin_lock_init(&glob->lru_lock); |
1070 | spin_lock_init(&glob->lru_lock); |
1304 | glob->mem_glob = bo_ref->mem_glob; |
1071 | glob->mem_glob = bo_ref->mem_glob; |
1305 | glob->dummy_read_page = AllocPage(); |
1072 | glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32); |
1306 | 1073 | ||
1307 | if (unlikely(glob->dummy_read_page == NULL)) { |
1074 | if (unlikely(glob->dummy_read_page == NULL)) { |
1308 | ret = -ENOMEM; |
1075 | ret = -ENOMEM; |
1309 | goto out_no_drp; |
1076 | goto out_no_drp; |
1310 | } |
1077 | } |
1311 | 1078 | ||
1312 | INIT_LIST_HEAD(&glob->swap_lru); |
1079 | INIT_LIST_HEAD(&glob->swap_lru); |
1313 | INIT_LIST_HEAD(&glob->device_list); |
1080 | INIT_LIST_HEAD(&glob->device_list); |
1314 | 1081 | ||
1315 | atomic_set(&glob->bo_count, 0); |
1082 | atomic_set(&glob->bo_count, 0); |
1316 | - | ||
1317 | LEAVE(); |
- | |
1318 | 1083 | ||
1319 | return 0; |
1084 | return 0; |
1320 | 1085 | ||
1321 | out_no_drp: |
1086 | out_no_drp: |
1322 | kfree(glob); |
1087 | kfree(glob); |
1323 | return ret; |
1088 | return ret; |
1324 | } |
1089 | } |
1325 | EXPORT_SYMBOL(ttm_bo_global_init); |
1090 | EXPORT_SYMBOL(ttm_bo_global_init); |
1326 | - | ||
1327 | 1091 | ||
1328 | int ttm_bo_device_init(struct ttm_bo_device *bdev, |
1092 | int ttm_bo_device_init(struct ttm_bo_device *bdev, |
1329 | struct ttm_bo_global *glob, |
1093 | struct ttm_bo_global *glob, |
1330 | struct ttm_bo_driver *driver, |
1094 | struct ttm_bo_driver *driver, |
- | 1095 | struct address_space *mapping, |
|
1331 | uint64_t file_page_offset, |
1096 | uint64_t file_page_offset, |
1332 | bool need_dma32) |
1097 | bool need_dma32) |
1333 | { |
1098 | { |
1334 | int ret = -EINVAL; |
1099 | int ret = -EINVAL; |
1335 | - | ||
1336 | ENTER(); |
- | |
1337 | 1100 | ||
1338 | bdev->driver = driver; |
1101 | bdev->driver = driver; |
1339 | 1102 | ||
1340 | memset(bdev->man, 0, sizeof(bdev->man)); |
1103 | memset(bdev->man, 0, sizeof(bdev->man)); |
1341 | 1104 | ||
1342 | /* |
1105 | /* |
1343 | * Initialize the system memory buffer type. |
1106 | * Initialize the system memory buffer type. |
1344 | * Other types need to be driver / IOCTL initialized. |
1107 | * Other types need to be driver / IOCTL initialized. |
1345 | */ |
1108 | */ |
1346 | ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0); |
1109 | ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0); |
1347 | if (unlikely(ret != 0)) |
1110 | if (unlikely(ret != 0)) |
1348 | goto out_no_sys; |
1111 | goto out_no_sys; |
1349 | 1112 | ||
1350 | drm_vma_offset_manager_init(&bdev->vma_manager, file_page_offset, |
1113 | drm_vma_offset_manager_init(&bdev->vma_manager, file_page_offset, |
1351 | 0x10000000); |
1114 | 0x10000000); |
1352 | // INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); |
1115 | INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); |
1353 | INIT_LIST_HEAD(&bdev->ddestroy); |
1116 | INIT_LIST_HEAD(&bdev->ddestroy); |
1354 | bdev->dev_mapping = NULL; |
1117 | bdev->dev_mapping = mapping; |
1355 | bdev->glob = glob; |
1118 | bdev->glob = glob; |
1356 | bdev->need_dma32 = need_dma32; |
1119 | bdev->need_dma32 = need_dma32; |
1357 | bdev->val_seq = 0; |
1120 | bdev->val_seq = 0; |
1358 | spin_lock_init(&bdev->fence_lock); |
1121 | spin_lock_init(&bdev->fence_lock); |
1359 | mutex_lock(&glob->device_list_mutex); |
1122 | mutex_lock(&glob->device_list_mutex); |
1360 | list_add_tail(&bdev->device_list, &glob->device_list); |
1123 | list_add_tail(&bdev->device_list, &glob->device_list); |
1361 | mutex_unlock(&glob->device_list_mutex); |
1124 | mutex_unlock(&glob->device_list_mutex); |
1362 | - | ||
1363 | LEAVE(); |
- | |
1364 | 1125 | ||
1365 | return 0; |
1126 | return 0; |
1366 | out_no_sys: |
1127 | out_no_sys: |
1367 | return ret; |
1128 | return ret; |
1368 | } |
1129 | } |
1369 | EXPORT_SYMBOL(ttm_bo_device_init); |
1130 | EXPORT_SYMBOL(ttm_bo_device_init); |
1370 | 1131 | ||
1371 | /* |
1132 | /* |
1372 | * buffer object vm functions. |
1133 | * buffer object vm functions. |
1373 | */ |
1134 | */ |
1374 | 1135 | ||
1375 | bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) |
1136 | bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) |
1376 | { |
1137 | { |
1377 | struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; |
1138 | struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; |
1378 | 1139 | ||
1379 | if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) { |
1140 | if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) { |
1380 | if (mem->mem_type == TTM_PL_SYSTEM) |
1141 | if (mem->mem_type == TTM_PL_SYSTEM) |
1381 | return false; |
1142 | return false; |
1382 | 1143 | ||
1383 | if (man->flags & TTM_MEMTYPE_FLAG_CMA) |
1144 | if (man->flags & TTM_MEMTYPE_FLAG_CMA) |
1384 | return false; |
1145 | return false; |
1385 | 1146 | ||
1386 | if (mem->placement & TTM_PL_FLAG_CACHED) |
1147 | if (mem->placement & TTM_PL_FLAG_CACHED) |
1387 | return false; |
1148 | return false; |
1388 | } |
1149 | } |
1389 | return true; |
1150 | return true; |
1390 | } |
1151 | } |
- | 1152 | ||
- | 1153 | void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) |
|
- | 1154 | { |
|
- | 1155 | struct ttm_bo_device *bdev = bo->bdev; |
|
- | 1156 | ||
- | 1157 | drm_vma_node_unmap(&bo->vma_node, bdev->dev_mapping); |
|
- | 1158 | ttm_mem_io_free_vm(bo); |
|
- | 1159 | } |
|
- | 1160 | ||
- | 1161 | void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) |
|
- | 1162 | { |
|
- | 1163 | struct ttm_bo_device *bdev = bo->bdev; |
|
- | 1164 | struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; |
|
- | 1165 | ||
- | 1166 | ttm_mem_io_lock(man, false); |
|
- | 1167 | ttm_bo_unmap_virtual_locked(bo); |
|
- | 1168 | ttm_mem_io_unlock(man); |
|
- | 1169 | } |
|
- | 1170 | ||
- | 1171 | ||
- | 1172 | EXPORT_SYMBOL(ttm_bo_unmap_virtual); |
|
- | 1173 | ||
1391 | 1174 | ||
1392 | int ttm_bo_wait(struct ttm_buffer_object *bo, |
1175 | int ttm_bo_wait(struct ttm_buffer_object *bo, |
1393 | bool lazy, bool interruptible, bool no_wait) |
1176 | bool lazy, bool interruptible, bool no_wait) |
1394 | { |
1177 | { |
1395 | struct ttm_bo_driver *driver = bo->bdev->driver; |
1178 | struct ttm_bo_driver *driver = bo->bdev->driver; |
1396 | struct ttm_bo_device *bdev = bo->bdev; |
1179 | struct ttm_bo_device *bdev = bo->bdev; |
1397 | void *sync_obj; |
1180 | void *sync_obj; |
1398 | int ret = 0; |
1181 | int ret = 0; |
1399 | 1182 | ||
1400 | if (likely(bo->sync_obj == NULL)) |
1183 | if (likely(bo->sync_obj == NULL)) |
1401 | return 0; |
1184 | return 0; |
1402 | - | ||
1403 | return 0; |
- | |
1404 | } |
1185 | |
1405 | EXPORT_SYMBOL(ttm_bo_wait); |
1186 | while (bo->sync_obj) { |
1406 | - | ||
1407 | int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait) |
1187 | |
- | 1188 | if (driver->sync_obj_signaled(bo->sync_obj)) { |
|
- | 1189 | void *tmp_obj = bo->sync_obj; |
|
- | 1190 | bo->sync_obj = NULL; |
|
- | 1191 | clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); |
|
- | 1192 | spin_unlock(&bdev->fence_lock); |
|
1408 | { |
1193 | driver->sync_obj_unref(&tmp_obj); |
- | 1194 | spin_lock(&bdev->fence_lock); |
|
1409 | struct ttm_bo_device *bdev = bo->bdev; |
1195 | continue; |
1410 | int ret = 0; |
- | |
1411 | 1196 | } |
|
- | 1197 | ||
- | 1198 | if (no_wait) |
|
- | 1199 | return -EBUSY; |
|
- | 1200 | ||
- | 1201 | sync_obj = driver->sync_obj_ref(bo->sync_obj); |
|
- | 1202 | spin_unlock(&bdev->fence_lock); |
|
- | 1203 | ret = driver->sync_obj_wait(sync_obj, |
|
1412 | /* |
1204 | lazy, interruptible); |
1413 | * Using ttm_bo_reserve makes sure the lru lists are updated. |
1205 | if (unlikely(ret != 0)) { |
- | 1206 | driver->sync_obj_unref(&sync_obj); |
|
- | 1207 | spin_lock(&bdev->fence_lock); |
|
- | 1208 | return ret; |
|
- | 1209 | } |
|
- | 1210 | spin_lock(&bdev->fence_lock); |
|
- | 1211 | if (likely(bo->sync_obj == sync_obj)) { |
|
- | 1212 | void *tmp_obj = bo->sync_obj; |
|
1414 | */ |
1213 | bo->sync_obj = NULL; |
- | 1214 | clear_bit(TTM_BO_PRIV_FLAG_MOVING, |
|
- | 1215 | &bo->priv_flags); |
|
1415 | 1216 | spin_unlock(&bdev->fence_lock); |
|
- | 1217 | driver->sync_obj_unref(&sync_obj); |
|
1416 | return ret; |
1218 | driver->sync_obj_unref(&tmp_obj); |
- | 1219 | spin_lock(&bdev->fence_lock); |
|
- | 1220 | } else { |
|
1417 | } |
1221 | spin_unlock(&bdev->fence_lock); |
1418 | EXPORT_SYMBOL(ttm_bo_synccpu_write_grab); |
1222 | driver->sync_obj_unref(&sync_obj); |
1419 | 1223 | spin_lock(&bdev->fence_lock); |
|
1420 | void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo) |
1224 | } |
- | 1225 | } |
|
1421 | { |
1226 | return 0; |
1422 | atomic_dec(&bo->cpu_writers); |
1227 | } |
1423 | } |
1228 | EXPORT_SYMBOL(ttm_bo_wait);><>>>>>><>>>>>><>><>><>><>><>=> |
1424 | EXPORT_SYMBOL(ttm_bo_synccpu_write_release);><>>>>>><>>>>>>><>><>><>><>> |
- |