Rev 5078 | Rev 6661 | Go to most recent revision | Show entire file | Regard whitespace | Details | Blame | Last modification | View Log | RSS feed
Rev 5078 | Rev 5271 | ||
---|---|---|---|
Line 164... | Line 164... | ||
164 | } |
164 | } |
Line 165... | Line 165... | ||
165 | 165 | ||
166 | static void radeon_evict_flags(struct ttm_buffer_object *bo, |
166 | static void radeon_evict_flags(struct ttm_buffer_object *bo, |
167 | struct ttm_placement *placement) |
167 | struct ttm_placement *placement) |
- | 168 | { |
|
- | 169 | static struct ttm_place placements = { |
|
- | 170 | .fpfn = 0, |
|
- | 171 | .lpfn = 0, |
|
- | 172 | .flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM |
|
- | 173 | }; |
|
168 | { |
174 | |
169 | struct radeon_bo *rbo; |
- | |
Line 170... | Line 175... | ||
170 | static u32 placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM; |
175 | struct radeon_bo *rbo; |
171 | - | ||
172 | if (!radeon_ttm_bo_is_radeon_bo(bo)) { |
- | |
173 | placement->fpfn = 0; |
176 | |
174 | placement->lpfn = 0; |
177 | if (!radeon_ttm_bo_is_radeon_bo(bo)) { |
175 | placement->placement = &placements; |
178 | placement->placement = &placements; |
176 | placement->busy_placement = &placements; |
179 | placement->busy_placement = &placements; |
177 | placement->num_placement = 1; |
180 | placement->num_placement = 1; |
178 | placement->num_busy_placement = 1; |
181 | placement->num_busy_placement = 1; |
179 | return; |
182 | return; |
180 | } |
183 | } |
181 | rbo = container_of(bo, struct radeon_bo, tbo); |
184 | rbo = container_of(bo, struct radeon_bo, tbo); |
182 | switch (bo->mem.mem_type) { |
185 | switch (bo->mem.mem_type) { |
183 | case TTM_PL_VRAM: |
186 | case TTM_PL_VRAM: |
- | 187 | if (rbo->rdev->ring[radeon_copy_ring_index(rbo->rdev)].ready == false) |
|
- | 188 | radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU); |
|
- | 189 | else if (rbo->rdev->mc.visible_vram_size < rbo->rdev->mc.real_vram_size && |
|
- | 190 | bo->mem.start < (rbo->rdev->mc.visible_vram_size >> PAGE_SHIFT)) { |
|
- | 191 | unsigned fpfn = rbo->rdev->mc.visible_vram_size >> PAGE_SHIFT; |
|
- | 192 | int i; |
|
- | 193 | ||
- | 194 | /* Try evicting to the CPU inaccessible part of VRAM |
|
- | 195 | * first, but only set GTT as busy placement, so this |
|
- | 196 | * BO will be evicted to GTT rather than causing other |
|
- | 197 | * BOs to be evicted from VRAM |
|
- | 198 | */ |
|
- | 199 | radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_VRAM | |
|
- | 200 | RADEON_GEM_DOMAIN_GTT); |
|
- | 201 | rbo->placement.num_busy_placement = 0; |
|
- | 202 | for (i = 0; i < rbo->placement.num_placement; i++) { |
|
- | 203 | if (rbo->placements[i].flags & TTM_PL_FLAG_VRAM) { |
|
- | 204 | if (rbo->placements[0].fpfn < fpfn) |
|
- | 205 | rbo->placements[0].fpfn = fpfn; |
|
- | 206 | } else { |
|
- | 207 | rbo->placement.busy_placement = |
|
- | 208 | &rbo->placements[i]; |
|
- | 209 | rbo->placement.num_busy_placement = 1; |
|
184 | if (rbo->rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready == false) |
210 | } |
185 | radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU); |
211 | } |
186 | else |
212 | } else |
187 | radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_GTT); |
213 | radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_GTT); |
188 | break; |
214 | break; |
189 | case TTM_PL_TT: |
215 | case TTM_PL_TT: |
Line 214... | Line 240... | ||
214 | struct ttm_mem_reg *old_mem) |
240 | struct ttm_mem_reg *old_mem) |
215 | { |
241 | { |
216 | struct radeon_device *rdev; |
242 | struct radeon_device *rdev; |
217 | uint64_t old_start, new_start; |
243 | uint64_t old_start, new_start; |
218 | struct radeon_fence *fence; |
244 | struct radeon_fence *fence; |
- | 245 | unsigned num_pages; |
|
219 | int r, ridx; |
246 | int r, ridx; |
Line 220... | Line 247... | ||
220 | 247 | ||
221 | rdev = radeon_get_rdev(bo->bdev); |
248 | rdev = radeon_get_rdev(bo->bdev); |
222 | ridx = radeon_copy_ring_index(rdev); |
249 | ridx = radeon_copy_ring_index(rdev); |
Line 250... | Line 277... | ||
250 | return -EINVAL; |
277 | return -EINVAL; |
251 | } |
278 | } |
Line 252... | Line 279... | ||
252 | 279 | ||
Line 253... | Line -... | ||
253 | BUILD_BUG_ON((PAGE_SIZE % RADEON_GPU_PAGE_SIZE) != 0); |
- | |
254 | 280 | BUILD_BUG_ON((PAGE_SIZE % RADEON_GPU_PAGE_SIZE) != 0); |
|
255 | /* sync other rings */ |
281 | |
256 | fence = bo->sync_obj; |
- | |
257 | r = radeon_copy(rdev, old_start, new_start, |
282 | num_pages = new_mem->num_pages * (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); |
258 | new_mem->num_pages * (PAGE_SIZE / RADEON_GPU_PAGE_SIZE), /* GPU pages */ |
283 | fence = radeon_copy(rdev, old_start, new_start, num_pages, bo->resv); |
- | 284 | if (IS_ERR(fence)) |
|
259 | &fence); |
285 | return PTR_ERR(fence); |
260 | /* FIXME: handle copy error */ |
286 | |
261 | r = ttm_bo_move_accel_cleanup(bo, (void *)fence, |
287 | r = ttm_bo_move_accel_cleanup(bo, &fence->base, |
262 | evict, no_wait_gpu, new_mem); |
288 | evict, no_wait_gpu, new_mem); |
263 | radeon_fence_unref(&fence); |
289 | radeon_fence_unref(&fence); |
Line 270... | Line 296... | ||
270 | struct ttm_mem_reg *new_mem) |
296 | struct ttm_mem_reg *new_mem) |
271 | { |
297 | { |
272 | struct radeon_device *rdev; |
298 | struct radeon_device *rdev; |
273 | struct ttm_mem_reg *old_mem = &bo->mem; |
299 | struct ttm_mem_reg *old_mem = &bo->mem; |
274 | struct ttm_mem_reg tmp_mem; |
300 | struct ttm_mem_reg tmp_mem; |
275 | u32 placements; |
301 | struct ttm_place placements; |
276 | struct ttm_placement placement; |
302 | struct ttm_placement placement; |
277 | int r; |
303 | int r; |
Line 278... | Line 304... | ||
278 | 304 | ||
279 | rdev = radeon_get_rdev(bo->bdev); |
305 | rdev = radeon_get_rdev(bo->bdev); |
280 | tmp_mem = *new_mem; |
306 | tmp_mem = *new_mem; |
281 | tmp_mem.mm_node = NULL; |
- | |
282 | placement.fpfn = 0; |
- | |
283 | placement.lpfn = 0; |
307 | tmp_mem.mm_node = NULL; |
284 | placement.num_placement = 1; |
308 | placement.num_placement = 1; |
285 | placement.placement = &placements; |
309 | placement.placement = &placements; |
286 | placement.num_busy_placement = 1; |
310 | placement.num_busy_placement = 1; |
- | 311 | placement.busy_placement = &placements; |
|
- | 312 | placements.fpfn = 0; |
|
287 | placement.busy_placement = &placements; |
313 | placements.lpfn = 0; |
288 | placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; |
314 | placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; |
289 | r = ttm_bo_mem_space(bo, &placement, &tmp_mem, |
315 | r = ttm_bo_mem_space(bo, &placement, &tmp_mem, |
290 | interruptible, no_wait_gpu); |
316 | interruptible, no_wait_gpu); |
291 | if (unlikely(r)) { |
317 | if (unlikely(r)) { |
292 | return r; |
318 | return r; |
Line 318... | Line 344... | ||
318 | { |
344 | { |
319 | struct radeon_device *rdev; |
345 | struct radeon_device *rdev; |
320 | struct ttm_mem_reg *old_mem = &bo->mem; |
346 | struct ttm_mem_reg *old_mem = &bo->mem; |
321 | struct ttm_mem_reg tmp_mem; |
347 | struct ttm_mem_reg tmp_mem; |
322 | struct ttm_placement placement; |
348 | struct ttm_placement placement; |
323 | u32 placements; |
349 | struct ttm_place placements; |
324 | int r; |
350 | int r; |
Line 325... | Line 351... | ||
325 | 351 | ||
326 | rdev = radeon_get_rdev(bo->bdev); |
352 | rdev = radeon_get_rdev(bo->bdev); |
327 | tmp_mem = *new_mem; |
353 | tmp_mem = *new_mem; |
328 | tmp_mem.mm_node = NULL; |
- | |
329 | placement.fpfn = 0; |
- | |
330 | placement.lpfn = 0; |
354 | tmp_mem.mm_node = NULL; |
331 | placement.num_placement = 1; |
355 | placement.num_placement = 1; |
332 | placement.placement = &placements; |
356 | placement.placement = &placements; |
333 | placement.num_busy_placement = 1; |
357 | placement.num_busy_placement = 1; |
- | 358 | placement.busy_placement = &placements; |
|
- | 359 | placements.fpfn = 0; |
|
334 | placement.busy_placement = &placements; |
360 | placements.lpfn = 0; |
335 | placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; |
361 | placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; |
336 | r = ttm_bo_mem_space(bo, &placement, &tmp_mem, |
362 | r = ttm_bo_mem_space(bo, &placement, &tmp_mem, |
337 | interruptible, no_wait_gpu); |
363 | interruptible, no_wait_gpu); |
338 | if (unlikely(r)) { |
364 | if (unlikely(r)) { |
339 | return r; |
365 | return r; |
Line 469... | Line 495... | ||
469 | 495 | ||
470 | static void radeon_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) |
496 | static void radeon_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) |
471 | { |
497 | { |
Line 472... | Line -... | ||
472 | } |
- | |
473 | - | ||
474 | static int radeon_sync_obj_wait(void *sync_obj, bool lazy, bool interruptible) |
- | |
475 | { |
- | |
476 | return radeon_fence_wait((struct radeon_fence *)sync_obj, interruptible); |
- | |
477 | } |
- | |
478 | - | ||
479 | static int radeon_sync_obj_flush(void *sync_obj) |
- | |
480 | { |
- | |
481 | return 0; |
- | |
482 | } |
- | |
483 | - | ||
484 | static void radeon_sync_obj_unref(void **sync_obj) |
- | |
485 | { |
- | |
486 | radeon_fence_unref((struct radeon_fence **)sync_obj); |
- | |
487 | } |
- | |
488 | - | ||
489 | static void *radeon_sync_obj_ref(void *sync_obj) |
- | |
490 | { |
- | |
491 | return radeon_fence_ref((struct radeon_fence *)sync_obj); |
- | |
492 | } |
- | |
493 | - | ||
494 | static bool radeon_sync_obj_signaled(void *sync_obj) |
- | |
495 | { |
- | |
496 | return radeon_fence_signaled((struct radeon_fence *)sync_obj); |
- | |
497 | } |
498 | } |
498 | 499 | ||
499 | /* |
500 | /* |
500 | * TTM backend functions. |
501 | * TTM backend functions. |
501 | */ |
502 | */ |
502 | struct radeon_ttm_tt { |
503 | struct radeon_ttm_tt { |
503 | struct ttm_dma_tt ttm; |
504 | struct ttm_dma_tt ttm; |
- | 505 | struct radeon_device *rdev; |
|
- | 506 | u64 offset; |
|
- | 507 | ||
- | 508 | uint64_t userptr; |
|
504 | struct radeon_device *rdev; |
509 | struct mm_struct *usermm; |
Line 505... | Line 510... | ||
505 | u64 offset; |
510 | uint32_t userflags; |
506 | }; |
511 | }; |
507 | 512 | ||
Line 578... | Line 583... | ||
578 | return NULL; |
583 | return NULL; |
579 | } |
584 | } |
580 | return >t->ttm.ttm; |
585 | return >t->ttm.ttm; |
581 | } |
586 | } |
Line -... | Line 587... | ||
- | 587 | ||
- | 588 | static struct radeon_ttm_tt *radeon_ttm_tt_to_gtt(struct ttm_tt *ttm) |
|
- | 589 | { |
|
- | 590 | if (!ttm || ttm->func != &radeon_backend_func) |
|
- | 591 | return NULL; |
|
- | 592 | return (struct radeon_ttm_tt *)ttm; |
|
- | 593 | } |
|
582 | 594 | ||
583 | static int radeon_ttm_tt_populate(struct ttm_tt *ttm) |
595 | static int radeon_ttm_tt_populate(struct ttm_tt *ttm) |
- | 596 | { |
|
584 | { |
597 | struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(ttm); |
585 | struct radeon_device *rdev; |
- | |
586 | struct radeon_ttm_tt *gtt = (void *)ttm; |
598 | struct radeon_device *rdev; |
587 | unsigned i; |
599 | unsigned i; |
588 | int r; |
600 | int r; |
Line 589... | Line 601... | ||
589 | bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); |
601 | bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); |
Line 626... | Line 638... | ||
626 | } |
638 | } |
Line 627... | Line 639... | ||
627 | 639 | ||
628 | static void radeon_ttm_tt_unpopulate(struct ttm_tt *ttm) |
640 | static void radeon_ttm_tt_unpopulate(struct ttm_tt *ttm) |
629 | { |
641 | { |
630 | struct radeon_device *rdev; |
642 | struct radeon_device *rdev; |
631 | struct radeon_ttm_tt *gtt = (void *)ttm; |
643 | struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(ttm); |
632 | unsigned i; |
644 | unsigned i; |
Line 633... | Line 645... | ||
633 | bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); |
645 | bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); |
634 | 646 | ||
Line 661... | Line 673... | ||
661 | .invalidate_caches = &radeon_invalidate_caches, |
673 | .invalidate_caches = &radeon_invalidate_caches, |
662 | .init_mem_type = &radeon_init_mem_type, |
674 | .init_mem_type = &radeon_init_mem_type, |
663 | .evict_flags = &radeon_evict_flags, |
675 | .evict_flags = &radeon_evict_flags, |
664 | .move = &radeon_bo_move, |
676 | .move = &radeon_bo_move, |
665 | .verify_access = &radeon_verify_access, |
677 | .verify_access = &radeon_verify_access, |
666 | .sync_obj_signaled = &radeon_sync_obj_signaled, |
- | |
667 | .sync_obj_wait = &radeon_sync_obj_wait, |
- | |
668 | .sync_obj_flush = &radeon_sync_obj_flush, |
- | |
669 | .sync_obj_unref = &radeon_sync_obj_unref, |
- | |
670 | .sync_obj_ref = &radeon_sync_obj_ref, |
- | |
671 | .move_notify = &radeon_bo_move_notify, |
678 | .move_notify = &radeon_bo_move_notify, |
672 | // .fault_reserve_notify = &radeon_bo_fault_reserve_notify, |
679 | // .fault_reserve_notify = &radeon_bo_fault_reserve_notify, |
673 | .io_mem_reserve = &radeon_ttm_io_mem_reserve, |
680 | .io_mem_reserve = &radeon_ttm_io_mem_reserve, |
674 | .io_mem_free = &radeon_ttm_io_mem_free, |
681 | .io_mem_free = &radeon_ttm_io_mem_free, |
675 | }; |
682 | }; |
Line 702... | Line 709... | ||
702 | } |
709 | } |
703 | /* Change the size here instead of the init above so only lpfn is affected */ |
710 | /* Change the size here instead of the init above so only lpfn is affected */ |
704 | radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); |
711 | radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); |
Line 705... | Line 712... | ||
705 | 712 | ||
706 | r = radeon_bo_create(rdev, 16*1024*1024, PAGE_SIZE, true, |
713 | r = radeon_bo_create(rdev, 16 * 1024 * 1024, PAGE_SIZE, true, |
707 | RADEON_GEM_DOMAIN_VRAM, 0, |
714 | RADEON_GEM_DOMAIN_VRAM, 0, NULL, |
708 | NULL, &rdev->stollen_vga_memory); |
715 | NULL, &rdev->stollen_vga_memory); |
709 | if (r) { |
716 | if (r) { |
710 | return r; |
717 | return r; |
711 | } |
718 | } |