Rev 5179 | Rev 6104 | Go to most recent revision | Show entire file | Regard whitespace | Details | Blame | Last modification | View Log | RSS feed
Rev 5179 | Rev 5271 | ||
---|---|---|---|
Line 132... | Line 132... | ||
132 | */ |
132 | */ |
133 | void cik_sdma_ring_ib_execute(struct radeon_device *rdev, |
133 | void cik_sdma_ring_ib_execute(struct radeon_device *rdev, |
134 | struct radeon_ib *ib) |
134 | struct radeon_ib *ib) |
135 | { |
135 | { |
136 | struct radeon_ring *ring = &rdev->ring[ib->ring]; |
136 | struct radeon_ring *ring = &rdev->ring[ib->ring]; |
137 | u32 extra_bits = (ib->vm ? ib->vm->id : 0) & 0xf; |
137 | u32 extra_bits = (ib->vm ? ib->vm->ids[ib->ring].id : 0) & 0xf; |
Line 138... | Line 138... | ||
138 | 138 | ||
139 | if (rdev->wb.enabled) { |
139 | if (rdev->wb.enabled) { |
140 | u32 next_rptr = ring->wptr + 5; |
140 | u32 next_rptr = ring->wptr + 5; |
141 | while ((next_rptr & 7) != 4) |
141 | while ((next_rptr & 7) != 4) |
Line 528... | Line 528... | ||
528 | * |
528 | * |
529 | * @rdev: radeon_device pointer |
529 | * @rdev: radeon_device pointer |
530 | * @src_offset: src GPU address |
530 | * @src_offset: src GPU address |
531 | * @dst_offset: dst GPU address |
531 | * @dst_offset: dst GPU address |
532 | * @num_gpu_pages: number of GPU pages to xfer |
532 | * @num_gpu_pages: number of GPU pages to xfer |
533 | * @fence: radeon fence object |
533 | * @resv: reservation object to sync to |
534 | * |
534 | * |
535 | * Copy GPU paging using the DMA engine (CIK). |
535 | * Copy GPU paging using the DMA engine (CIK). |
536 | * Used by the radeon ttm implementation to move pages if |
536 | * Used by the radeon ttm implementation to move pages if |
537 | * registered as the asic copy callback. |
537 | * registered as the asic copy callback. |
538 | */ |
538 | */ |
539 | int cik_copy_dma(struct radeon_device *rdev, |
539 | struct radeon_fence *cik_copy_dma(struct radeon_device *rdev, |
540 | uint64_t src_offset, uint64_t dst_offset, |
540 | uint64_t src_offset, uint64_t dst_offset, |
541 | unsigned num_gpu_pages, |
541 | unsigned num_gpu_pages, |
542 | struct radeon_fence **fence) |
542 | struct reservation_object *resv) |
543 | { |
543 | { |
544 | struct radeon_semaphore *sem = NULL; |
544 | struct radeon_fence *fence; |
- | 545 | struct radeon_sync sync; |
|
545 | int ring_index = rdev->asic->copy.dma_ring_index; |
546 | int ring_index = rdev->asic->copy.dma_ring_index; |
546 | struct radeon_ring *ring = &rdev->ring[ring_index]; |
547 | struct radeon_ring *ring = &rdev->ring[ring_index]; |
547 | u32 size_in_bytes, cur_size_in_bytes; |
548 | u32 size_in_bytes, cur_size_in_bytes; |
548 | int i, num_loops; |
549 | int i, num_loops; |
549 | int r = 0; |
550 | int r = 0; |
Line 550... | Line 551... | ||
550 | 551 | ||
551 | r = radeon_semaphore_create(rdev, &sem); |
- | |
552 | if (r) { |
- | |
553 | DRM_ERROR("radeon: moving bo (%d).\n", r); |
- | |
554 | return r; |
- | |
Line 555... | Line 552... | ||
555 | } |
552 | radeon_sync_create(&sync); |
556 | 553 | ||
557 | size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT); |
554 | size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT); |
558 | num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff); |
555 | num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff); |
559 | r = radeon_ring_lock(rdev, ring, num_loops * 7 + 14); |
556 | r = radeon_ring_lock(rdev, ring, num_loops * 7 + 14); |
560 | if (r) { |
557 | if (r) { |
561 | DRM_ERROR("radeon: moving bo (%d).\n", r); |
558 | DRM_ERROR("radeon: moving bo (%d).\n", r); |
562 | radeon_semaphore_free(rdev, &sem, NULL); |
559 | radeon_sync_free(rdev, &sync, NULL); |
Line 563... | Line 560... | ||
563 | return r; |
560 | return ERR_PTR(r); |
564 | } |
561 | } |
Line 565... | Line 562... | ||
565 | 562 | ||
566 | radeon_semaphore_sync_to(sem, *fence); |
563 | radeon_sync_resv(rdev, &sync, resv, false); |
567 | radeon_semaphore_sync_rings(rdev, sem, ring->idx); |
564 | radeon_sync_rings(rdev, &sync, ring->idx); |
568 | 565 | ||
Line 580... | Line 577... | ||
580 | radeon_ring_write(ring, upper_32_bits(dst_offset)); |
577 | radeon_ring_write(ring, upper_32_bits(dst_offset)); |
581 | src_offset += cur_size_in_bytes; |
578 | src_offset += cur_size_in_bytes; |
582 | dst_offset += cur_size_in_bytes; |
579 | dst_offset += cur_size_in_bytes; |
583 | } |
580 | } |
Line 584... | Line 581... | ||
584 | 581 | ||
585 | r = radeon_fence_emit(rdev, fence, ring->idx); |
582 | r = radeon_fence_emit(rdev, &fence, ring->idx); |
586 | if (r) { |
583 | if (r) { |
587 | radeon_ring_unlock_undo(rdev, ring); |
584 | radeon_ring_unlock_undo(rdev, ring); |
588 | radeon_semaphore_free(rdev, &sem, NULL); |
585 | radeon_sync_free(rdev, &sync, NULL); |
589 | return r; |
586 | return ERR_PTR(r); |
Line 590... | Line 587... | ||
590 | } |
587 | } |
591 | 588 | ||
Line 592... | Line 589... | ||
592 | radeon_ring_unlock_commit(rdev, ring, false); |
589 | radeon_ring_unlock_commit(rdev, ring, false); |
593 | radeon_semaphore_free(rdev, &sem, *fence); |
590 | radeon_sync_free(rdev, &sync, fence); |
Line 594... | Line 591... | ||
594 | 591 | ||
595 | return r; |
592 | return fence; |
596 | } |
593 | } |
Line 664... | Line 661... | ||
664 | */ |
661 | */ |
665 | int cik_sdma_ib_test(struct radeon_device *rdev, struct radeon_ring *ring) |
662 | int cik_sdma_ib_test(struct radeon_device *rdev, struct radeon_ring *ring) |
666 | { |
663 | { |
667 | struct radeon_ib ib; |
664 | struct radeon_ib ib; |
668 | unsigned i; |
665 | unsigned i; |
- | 666 | unsigned index; |
|
669 | int r; |
667 | int r; |
670 | void __iomem *ptr = (void *)rdev->vram_scratch.ptr; |
- | |
671 | u32 tmp = 0; |
668 | u32 tmp = 0; |
- | 669 | u64 gpu_addr; |
|
Line 672... | Line 670... | ||
672 | 670 | ||
673 | if (!ptr) { |
671 | if (ring->idx == R600_RING_TYPE_DMA_INDEX) |
- | 672 | index = R600_WB_DMA_RING_TEST_OFFSET; |
|
674 | DRM_ERROR("invalid vram scratch pointer\n"); |
673 | else |
675 | return -EINVAL; |
674 | index = CAYMAN_WB_DMA1_RING_TEST_OFFSET; |
- | 675 | ||
Line 676... | Line 676... | ||
676 | } |
676 | gpu_addr = rdev->wb.gpu_addr + index; |
677 | 677 | ||
Line 678... | Line 678... | ||
678 | tmp = 0xCAFEDEAD; |
678 | tmp = 0xCAFEDEAD; |
679 | writel(tmp, ptr); |
679 | rdev->wb.wb[index/4] = cpu_to_le32(tmp); |
680 | 680 | ||
681 | r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256); |
681 | r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256); |
682 | if (r) { |
682 | if (r) { |
Line 683... | Line 683... | ||
683 | DRM_ERROR("radeon: failed to get ib (%d).\n", r); |
683 | DRM_ERROR("radeon: failed to get ib (%d).\n", r); |
684 | return r; |
684 | return r; |
685 | } |
685 | } |
686 | 686 | ||
687 | ib.ptr[0] = SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0); |
687 | ib.ptr[0] = SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0); |
688 | ib.ptr[1] = rdev->vram_scratch.gpu_addr & 0xfffffffc; |
688 | ib.ptr[1] = lower_32_bits(gpu_addr); |
Line 689... | Line 689... | ||
689 | ib.ptr[2] = upper_32_bits(rdev->vram_scratch.gpu_addr); |
689 | ib.ptr[2] = upper_32_bits(gpu_addr); |
Line 701... | Line 701... | ||
701 | if (r) { |
701 | if (r) { |
702 | DRM_ERROR("radeon: fence wait failed (%d).\n", r); |
702 | DRM_ERROR("radeon: fence wait failed (%d).\n", r); |
703 | return r; |
703 | return r; |
704 | } |
704 | } |
705 | for (i = 0; i < rdev->usec_timeout; i++) { |
705 | for (i = 0; i < rdev->usec_timeout; i++) { |
706 | tmp = readl(ptr); |
706 | tmp = le32_to_cpu(rdev->wb.wb[index/4]); |
707 | if (tmp == 0xDEADBEEF) |
707 | if (tmp == 0xDEADBEEF) |
708 | break; |
708 | break; |
709 | DRM_UDELAY(1); |
709 | DRM_UDELAY(1); |
710 | } |
710 | } |
711 | if (i < rdev->usec_timeout) { |
711 | if (i < rdev->usec_timeout) { |
Line 898... | Line 898... | ||
898 | * @rdev: radeon_device pointer |
898 | * @rdev: radeon_device pointer |
899 | * |
899 | * |
900 | * Update the page table base and flush the VM TLB |
900 | * Update the page table base and flush the VM TLB |
901 | * using sDMA (CIK). |
901 | * using sDMA (CIK). |
902 | */ |
902 | */ |
903 | void cik_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) |
903 | void cik_dma_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, |
- | 904 | unsigned vm_id, uint64_t pd_addr) |
|
904 | { |
905 | { |
905 | struct radeon_ring *ring = &rdev->ring[ridx]; |
- | |
906 | - | ||
907 | if (vm == NULL) |
- | |
908 | return; |
- | |
909 | - | ||
910 | radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); |
906 | radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); |
911 | if (vm->id < 8) { |
907 | if (vm_id < 8) { |
912 | radeon_ring_write(ring, (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2); |
908 | radeon_ring_write(ring, (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2); |
913 | } else { |
909 | } else { |
914 | radeon_ring_write(ring, (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2); |
910 | radeon_ring_write(ring, (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2); |
915 | } |
911 | } |
916 | radeon_ring_write(ring, vm->pd_gpu_addr >> 12); |
912 | radeon_ring_write(ring, pd_addr >> 12); |
Line 917... | Line 913... | ||
917 | 913 | ||
918 | /* update SH_MEM_* regs */ |
914 | /* update SH_MEM_* regs */ |
919 | radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); |
915 | radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); |
920 | radeon_ring_write(ring, SRBM_GFX_CNTL >> 2); |
916 | radeon_ring_write(ring, SRBM_GFX_CNTL >> 2); |
Line 921... | Line 917... | ||
921 | radeon_ring_write(ring, VMID(vm->id)); |
917 | radeon_ring_write(ring, VMID(vm_id)); |
922 | 918 | ||
923 | radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); |
919 | radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); |
Line 939... | Line 935... | ||
939 | radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); |
935 | radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); |
940 | radeon_ring_write(ring, SRBM_GFX_CNTL >> 2); |
936 | radeon_ring_write(ring, SRBM_GFX_CNTL >> 2); |
941 | radeon_ring_write(ring, VMID(0)); |
937 | radeon_ring_write(ring, VMID(0)); |
Line 942... | Line 938... | ||
942 | 938 | ||
943 | /* flush HDP */ |
939 | /* flush HDP */ |
Line 944... | Line 940... | ||
944 | cik_sdma_hdp_flush_ring_emit(rdev, ridx); |
940 | cik_sdma_hdp_flush_ring_emit(rdev, ring->idx); |
945 | 941 | ||
946 | /* flush TLB */ |
942 | /* flush TLB */ |
947 | radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); |
943 | radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); |
948 | radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); |
944 | radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); |