Rev 3764 | Rev 5271 | Go to most recent revision | Show entire file | Regard whitespace | Details | Blame | Last modification | View Log | RSS feed
Rev 3764 | Rev 5078 | ||
---|---|---|---|
Line 42... | Line 42... | ||
42 | #include "radeon.h" |
42 | #include "radeon.h" |
Line 43... | Line 43... | ||
43 | 43 | ||
Line 44... | Line 44... | ||
44 | #define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT) |
44 | #define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT) |
- | 45 | ||
Line 45... | Line 46... | ||
45 | 46 | static int radeon_ttm_debugfs_init(struct radeon_device *rdev); |
|
46 | static int radeon_ttm_debugfs_init(struct radeon_device *rdev); |
47 | static void radeon_ttm_debugfs_fini(struct radeon_device *rdev); |
47 | 48 | ||
48 | static struct radeon_device *radeon_get_rdev(struct ttm_bo_device *bdev) |
49 | static struct radeon_device *radeon_get_rdev(struct ttm_bo_device *bdev) |
Line 72... | Line 73... | ||
72 | static int radeon_ttm_global_init(struct radeon_device *rdev) |
73 | static int radeon_ttm_global_init(struct radeon_device *rdev) |
73 | { |
74 | { |
74 | struct drm_global_reference *global_ref; |
75 | struct drm_global_reference *global_ref; |
75 | int r; |
76 | int r; |
Line 76... | Line -... | ||
76 | - | ||
77 | ENTER(); |
- | |
78 | 77 | ||
79 | rdev->mman.mem_global_referenced = false; |
78 | rdev->mman.mem_global_referenced = false; |
80 | global_ref = &rdev->mman.mem_global_ref; |
79 | global_ref = &rdev->mman.mem_global_ref; |
81 | global_ref->global_type = DRM_GLOBAL_TTM_MEM; |
80 | global_ref->global_type = DRM_GLOBAL_TTM_MEM; |
82 | global_ref->size = sizeof(struct ttm_mem_global); |
81 | global_ref->size = sizeof(struct ttm_mem_global); |
Line 102... | Line 101... | ||
102 | drm_global_item_unref(&rdev->mman.mem_global_ref); |
101 | drm_global_item_unref(&rdev->mman.mem_global_ref); |
103 | return r; |
102 | return r; |
104 | } |
103 | } |
Line 105... | Line 104... | ||
105 | 104 | ||
106 | rdev->mman.mem_global_referenced = true; |
- | |
107 | - | ||
108 | LEAVE(); |
- | |
109 | 105 | rdev->mman.mem_global_referenced = true; |
|
110 | return 0; |
106 | return 0; |
Line 111... | Line -... | ||
111 | } |
- | |
112 | 107 | } |
|
113 | 108 | ||
114 | 109 | ||
115 | static int radeon_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags) |
110 | static int radeon_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags) |
Line 116... | Line 111... | ||
116 | { |
111 | { |
117 | return 0; |
112 | return 0; |
118 | } |
113 | } |
119 | 114 | ||
Line 120... | Line -... | ||
120 | static int radeon_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, |
- | |
121 | struct ttm_mem_type_manager *man) |
- | |
122 | { |
115 | static int radeon_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, |
Line 123... | Line 116... | ||
123 | struct radeon_device *rdev; |
116 | struct ttm_mem_type_manager *man) |
124 | 117 | { |
|
125 | ENTER(); |
118 | struct radeon_device *rdev; |
Line 139... | Line 132... | ||
139 | man->available_caching = TTM_PL_MASK_CACHING; |
132 | man->available_caching = TTM_PL_MASK_CACHING; |
140 | man->default_caching = TTM_PL_FLAG_CACHED; |
133 | man->default_caching = TTM_PL_FLAG_CACHED; |
141 | man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA; |
134 | man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA; |
142 | #if __OS_HAS_AGP |
135 | #if __OS_HAS_AGP |
143 | if (rdev->flags & RADEON_IS_AGP) { |
136 | if (rdev->flags & RADEON_IS_AGP) { |
144 | if (!(drm_core_has_AGP(rdev->ddev) && rdev->ddev->agp)) { |
137 | if (!rdev->ddev->agp) { |
145 | DRM_ERROR("AGP is not enabled for memory type %u\n", |
138 | DRM_ERROR("AGP is not enabled for memory type %u\n", |
146 | (unsigned)type); |
139 | (unsigned)type); |
147 | return -EINVAL; |
140 | return -EINVAL; |
148 | } |
141 | } |
149 | if (!rdev->ddev->agp->cant_use_aperture) |
142 | if (!rdev->ddev->agp->cant_use_aperture) |
Line 165... | Line 158... | ||
165 | break; |
158 | break; |
166 | default: |
159 | default: |
167 | DRM_ERROR("Unsupported memory type %u\n", (unsigned)type); |
160 | DRM_ERROR("Unsupported memory type %u\n", (unsigned)type); |
168 | return -EINVAL; |
161 | return -EINVAL; |
169 | } |
162 | } |
170 | - | ||
171 | LEAVE(); |
- | |
172 | - | ||
173 | return 0; |
163 | return 0; |
174 | } |
164 | } |
Line 175... | Line 165... | ||
175 | 165 | ||
176 | static void radeon_evict_flags(struct ttm_buffer_object *bo, |
166 | static void radeon_evict_flags(struct ttm_buffer_object *bo, |
Line 216... | Line 206... | ||
216 | BUG_ON(old_mem->mm_node != NULL); |
206 | BUG_ON(old_mem->mm_node != NULL); |
217 | *old_mem = *new_mem; |
207 | *old_mem = *new_mem; |
218 | new_mem->mm_node = NULL; |
208 | new_mem->mm_node = NULL; |
219 | } |
209 | } |
Line -... | Line 210... | ||
- | 210 | ||
- | 211 | static int radeon_move_blit(struct ttm_buffer_object *bo, |
|
- | 212 | bool evict, bool no_wait_gpu, |
|
- | 213 | struct ttm_mem_reg *new_mem, |
|
- | 214 | struct ttm_mem_reg *old_mem) |
|
- | 215 | { |
|
- | 216 | struct radeon_device *rdev; |
|
- | 217 | uint64_t old_start, new_start; |
|
- | 218 | struct radeon_fence *fence; |
|
- | 219 | int r, ridx; |
|
- | 220 | ||
- | 221 | rdev = radeon_get_rdev(bo->bdev); |
|
- | 222 | ridx = radeon_copy_ring_index(rdev); |
|
- | 223 | old_start = old_mem->start << PAGE_SHIFT; |
|
- | 224 | new_start = new_mem->start << PAGE_SHIFT; |
|
- | 225 | ||
- | 226 | switch (old_mem->mem_type) { |
|
- | 227 | case TTM_PL_VRAM: |
|
- | 228 | old_start += rdev->mc.vram_start; |
|
- | 229 | break; |
|
- | 230 | case TTM_PL_TT: |
|
- | 231 | old_start += rdev->mc.gtt_start; |
|
- | 232 | break; |
|
- | 233 | default: |
|
- | 234 | DRM_ERROR("Unknown placement %d\n", old_mem->mem_type); |
|
- | 235 | return -EINVAL; |
|
- | 236 | } |
|
- | 237 | switch (new_mem->mem_type) { |
|
- | 238 | case TTM_PL_VRAM: |
|
- | 239 | new_start += rdev->mc.vram_start; |
|
- | 240 | break; |
|
- | 241 | case TTM_PL_TT: |
|
- | 242 | new_start += rdev->mc.gtt_start; |
|
- | 243 | break; |
|
- | 244 | default: |
|
- | 245 | DRM_ERROR("Unknown placement %d\n", old_mem->mem_type); |
|
- | 246 | return -EINVAL; |
|
- | 247 | } |
|
- | 248 | if (!rdev->ring[ridx].ready) { |
|
- | 249 | DRM_ERROR("Trying to move memory with ring turned off.\n"); |
|
- | 250 | return -EINVAL; |
|
- | 251 | } |
|
- | 252 | ||
- | 253 | BUILD_BUG_ON((PAGE_SIZE % RADEON_GPU_PAGE_SIZE) != 0); |
|
- | 254 | ||
- | 255 | /* sync other rings */ |
|
- | 256 | fence = bo->sync_obj; |
|
- | 257 | r = radeon_copy(rdev, old_start, new_start, |
|
- | 258 | new_mem->num_pages * (PAGE_SIZE / RADEON_GPU_PAGE_SIZE), /* GPU pages */ |
|
- | 259 | &fence); |
|
- | 260 | /* FIXME: handle copy error */ |
|
- | 261 | r = ttm_bo_move_accel_cleanup(bo, (void *)fence, |
|
- | 262 | evict, no_wait_gpu, new_mem); |
|
- | 263 | radeon_fence_unref(&fence); |
|
- | 264 | return r; |
|
- | 265 | } |
|
- | 266 | ||
- | 267 | static int radeon_move_vram_ram(struct ttm_buffer_object *bo, |
|
- | 268 | bool evict, bool interruptible, |
|
- | 269 | bool no_wait_gpu, |
|
- | 270 | struct ttm_mem_reg *new_mem) |
|
- | 271 | { |
|
- | 272 | struct radeon_device *rdev; |
|
- | 273 | struct ttm_mem_reg *old_mem = &bo->mem; |
|
- | 274 | struct ttm_mem_reg tmp_mem; |
|
- | 275 | u32 placements; |
|
- | 276 | struct ttm_placement placement; |
|
- | 277 | int r; |
|
- | 278 | ||
- | 279 | rdev = radeon_get_rdev(bo->bdev); |
|
- | 280 | tmp_mem = *new_mem; |
|
- | 281 | tmp_mem.mm_node = NULL; |
|
- | 282 | placement.fpfn = 0; |
|
- | 283 | placement.lpfn = 0; |
|
- | 284 | placement.num_placement = 1; |
|
- | 285 | placement.placement = &placements; |
|
- | 286 | placement.num_busy_placement = 1; |
|
- | 287 | placement.busy_placement = &placements; |
|
- | 288 | placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; |
|
- | 289 | r = ttm_bo_mem_space(bo, &placement, &tmp_mem, |
|
- | 290 | interruptible, no_wait_gpu); |
|
- | 291 | if (unlikely(r)) { |
|
- | 292 | return r; |
|
- | 293 | } |
|
- | 294 | ||
- | 295 | r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement); |
|
- | 296 | if (unlikely(r)) { |
|
- | 297 | goto out_cleanup; |
|
- | 298 | } |
|
- | 299 | ||
- | 300 | r = ttm_tt_bind(bo->ttm, &tmp_mem); |
|
- | 301 | if (unlikely(r)) { |
|
- | 302 | goto out_cleanup; |
|
- | 303 | } |
|
- | 304 | r = radeon_move_blit(bo, true, no_wait_gpu, &tmp_mem, old_mem); |
|
- | 305 | if (unlikely(r)) { |
|
- | 306 | goto out_cleanup; |
|
- | 307 | } |
|
- | 308 | r = ttm_bo_move_ttm(bo, true, no_wait_gpu, new_mem); |
|
- | 309 | out_cleanup: |
|
- | 310 | ttm_bo_mem_put(bo, &tmp_mem); |
|
- | 311 | return r; |
|
- | 312 | } |
|
- | 313 | ||
- | 314 | static int radeon_move_ram_vram(struct ttm_buffer_object *bo, |
|
- | 315 | bool evict, bool interruptible, |
|
- | 316 | bool no_wait_gpu, |
|
- | 317 | struct ttm_mem_reg *new_mem) |
|
- | 318 | { |
|
- | 319 | struct radeon_device *rdev; |
|
- | 320 | struct ttm_mem_reg *old_mem = &bo->mem; |
|
- | 321 | struct ttm_mem_reg tmp_mem; |
|
- | 322 | struct ttm_placement placement; |
|
- | 323 | u32 placements; |
|
- | 324 | int r; |
|
- | 325 | ||
- | 326 | rdev = radeon_get_rdev(bo->bdev); |
|
- | 327 | tmp_mem = *new_mem; |
|
- | 328 | tmp_mem.mm_node = NULL; |
|
- | 329 | placement.fpfn = 0; |
|
- | 330 | placement.lpfn = 0; |
|
- | 331 | placement.num_placement = 1; |
|
- | 332 | placement.placement = &placements; |
|
- | 333 | placement.num_busy_placement = 1; |
|
- | 334 | placement.busy_placement = &placements; |
|
- | 335 | placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; |
|
- | 336 | r = ttm_bo_mem_space(bo, &placement, &tmp_mem, |
|
- | 337 | interruptible, no_wait_gpu); |
|
- | 338 | if (unlikely(r)) { |
|
- | 339 | return r; |
|
- | 340 | } |
|
- | 341 | r = ttm_bo_move_ttm(bo, true, no_wait_gpu, &tmp_mem); |
|
- | 342 | if (unlikely(r)) { |
|
- | 343 | goto out_cleanup; |
|
- | 344 | } |
|
- | 345 | r = radeon_move_blit(bo, true, no_wait_gpu, new_mem, old_mem); |
|
- | 346 | if (unlikely(r)) { |
|
- | 347 | goto out_cleanup; |
|
- | 348 | } |
|
- | 349 | out_cleanup: |
|
- | 350 | ttm_bo_mem_put(bo, &tmp_mem); |
|
- | 351 | return r; |
|
- | 352 | } |
|
- | 353 | ||
- | 354 | static int radeon_bo_move(struct ttm_buffer_object *bo, |
|
- | 355 | bool evict, bool interruptible, |
|
- | 356 | bool no_wait_gpu, |
|
- | 357 | struct ttm_mem_reg *new_mem) |
|
- | 358 | { |
|
- | 359 | struct radeon_device *rdev; |
|
- | 360 | struct ttm_mem_reg *old_mem = &bo->mem; |
|
- | 361 | int r; |
|
- | 362 | ||
- | 363 | rdev = radeon_get_rdev(bo->bdev); |
|
- | 364 | if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { |
|
- | 365 | radeon_move_null(bo, new_mem); |
|
- | 366 | return 0; |
|
- | 367 | } |
|
- | 368 | if ((old_mem->mem_type == TTM_PL_TT && |
|
- | 369 | new_mem->mem_type == TTM_PL_SYSTEM) || |
|
- | 370 | (old_mem->mem_type == TTM_PL_SYSTEM && |
|
- | 371 | new_mem->mem_type == TTM_PL_TT)) { |
|
- | 372 | /* bind is enough */ |
|
- | 373 | radeon_move_null(bo, new_mem); |
|
- | 374 | return 0; |
|
- | 375 | } |
|
- | 376 | if (!rdev->ring[radeon_copy_ring_index(rdev)].ready || |
|
- | 377 | rdev->asic->copy.copy == NULL) { |
|
- | 378 | /* use memcpy */ |
|
- | 379 | goto memcpy; |
|
- | 380 | } |
|
- | 381 | ||
- | 382 | if (old_mem->mem_type == TTM_PL_VRAM && |
|
- | 383 | new_mem->mem_type == TTM_PL_SYSTEM) { |
|
- | 384 | r = radeon_move_vram_ram(bo, evict, interruptible, |
|
- | 385 | no_wait_gpu, new_mem); |
|
- | 386 | } else if (old_mem->mem_type == TTM_PL_SYSTEM && |
|
- | 387 | new_mem->mem_type == TTM_PL_VRAM) { |
|
- | 388 | r = radeon_move_ram_vram(bo, evict, interruptible, |
|
- | 389 | no_wait_gpu, new_mem); |
|
- | 390 | } else { |
|
- | 391 | r = radeon_move_blit(bo, evict, no_wait_gpu, new_mem, old_mem); |
|
- | 392 | } |
|
- | 393 | ||
- | 394 | if (r) { |
|
- | 395 | memcpy: |
|
- | 396 | r = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem); |
|
- | 397 | if (r) { |
|
- | 398 | return r; |
|
- | 399 | } |
|
- | 400 | } |
|
- | 401 | ||
- | 402 | /* update statistics */ |
|
- | 403 | // atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &rdev->num_bytes_moved); |
|
- | 404 | return 0; |
|
- | 405 | } |
|
- | 406 | ||
- | 407 | static int radeon_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) |
|
- | 408 | { |
|
- | 409 | struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; |
|
- | 410 | struct radeon_device *rdev = radeon_get_rdev(bdev); |
|
- | 411 | ||
- | 412 | mem->bus.addr = NULL; |
|
- | 413 | mem->bus.offset = 0; |
|
- | 414 | mem->bus.size = mem->num_pages << PAGE_SHIFT; |
|
- | 415 | mem->bus.base = 0; |
|
- | 416 | mem->bus.is_iomem = false; |
|
- | 417 | if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE)) |
|
- | 418 | return -EINVAL; |
|
- | 419 | switch (mem->mem_type) { |
|
- | 420 | case TTM_PL_SYSTEM: |
|
- | 421 | /* system memory */ |
|
- | 422 | return 0; |
|
- | 423 | case TTM_PL_TT: |
|
- | 424 | #if __OS_HAS_AGP |
|
- | 425 | if (rdev->flags & RADEON_IS_AGP) { |
|
- | 426 | /* RADEON_IS_AGP is set only if AGP is active */ |
|
- | 427 | mem->bus.offset = mem->start << PAGE_SHIFT; |
|
- | 428 | mem->bus.base = rdev->mc.agp_base; |
|
- | 429 | mem->bus.is_iomem = !rdev->ddev->agp->cant_use_aperture; |
|
- | 430 | } |
|
- | 431 | #endif |
|
- | 432 | break; |
|
- | 433 | case TTM_PL_VRAM: |
|
- | 434 | mem->bus.offset = mem->start << PAGE_SHIFT; |
|
- | 435 | /* check if it's visible */ |
|
- | 436 | if ((mem->bus.offset + mem->bus.size) > rdev->mc.visible_vram_size) |
|
- | 437 | return -EINVAL; |
|
- | 438 | mem->bus.base = rdev->mc.aper_base; |
|
- | 439 | mem->bus.is_iomem = true; |
|
- | 440 | #ifdef __alpha__ |
|
- | 441 | /* |
|
- | 442 | * Alpha: use bus.addr to hold the ioremap() return, |
|
- | 443 | * so we can modify bus.base below. |
|
- | 444 | */ |
|
- | 445 | if (mem->placement & TTM_PL_FLAG_WC) |
|
- | 446 | mem->bus.addr = |
|
- | 447 | ioremap_wc(mem->bus.base + mem->bus.offset, |
|
- | 448 | mem->bus.size); |
|
- | 449 | else |
|
- | 450 | mem->bus.addr = |
|
- | 451 | ioremap_nocache(mem->bus.base + mem->bus.offset, |
|
- | 452 | mem->bus.size); |
|
- | 453 | ||
- | 454 | /* |
|
- | 455 | * Alpha: Use just the bus offset plus |
|
- | 456 | * the hose/domain memory base for bus.base. |
|
- | 457 | * It then can be used to build PTEs for VRAM |
|
- | 458 | * access, as done in ttm_bo_vm_fault(). |
|
- | 459 | */ |
|
- | 460 | mem->bus.base = (mem->bus.base & 0x0ffffffffUL) + |
|
- | 461 | rdev->ddev->hose->dense_mem_base; |
|
- | 462 | #endif |
|
- | 463 | break; |
|
- | 464 | default: |
|
- | 465 | return -EINVAL; |
|
- | 466 | } |
|
- | 467 | return 0; |
|
- | 468 | } |
|
220 | 469 | ||
221 | static void radeon_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) |
470 | static void radeon_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) |
222 | { |
471 | { |
Line 223... | Line 472... | ||
223 | } |
472 | } |
Line 258... | Line 507... | ||
258 | 507 | ||
259 | static int radeon_ttm_backend_bind(struct ttm_tt *ttm, |
508 | static int radeon_ttm_backend_bind(struct ttm_tt *ttm, |
260 | struct ttm_mem_reg *bo_mem) |
509 | struct ttm_mem_reg *bo_mem) |
261 | { |
510 | { |
- | 511 | struct radeon_ttm_tt *gtt = (void*)ttm; |
|
- | 512 | uint32_t flags = RADEON_GART_PAGE_VALID | RADEON_GART_PAGE_READ | |
|
262 | struct radeon_ttm_tt *gtt = (void*)ttm; |
513 | RADEON_GART_PAGE_WRITE; |
Line 263... | Line 514... | ||
263 | int r; |
514 | int r; |
264 | 515 | ||
265 | gtt->offset = (unsigned long)(bo_mem->start << PAGE_SHIFT); |
516 | gtt->offset = (unsigned long)(bo_mem->start << PAGE_SHIFT); |
266 | if (!ttm->num_pages) { |
517 | if (!ttm->num_pages) { |
267 | WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n", |
518 | WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n", |
- | 519 | ttm->num_pages, bo_mem, ttm); |
|
- | 520 | } |
|
268 | ttm->num_pages, bo_mem, ttm); |
521 | if (ttm->caching_state == tt_cached) |
269 | } |
522 | flags |= RADEON_GART_PAGE_SNOOP; |
270 | r = radeon_gart_bind(gtt->rdev, gtt->offset, |
523 | r = radeon_gart_bind(gtt->rdev, gtt->offset, ttm->num_pages, |
271 | ttm->num_pages, ttm->pages, gtt->ttm.dma_address); |
524 | ttm->pages, gtt->ttm.dma_address, flags); |
272 | if (r) { |
525 | if (r) { |
273 | DRM_ERROR("failed to bind %lu pages at 0x%08X\n", |
526 | DRM_ERROR("failed to bind %lu pages at 0x%08X\n", |
274 | ttm->num_pages, (unsigned)gtt->offset); |
527 | ttm->num_pages, (unsigned)gtt->offset); |
Line 287... | Line 540... | ||
287 | 540 | ||
288 | static void radeon_ttm_backend_destroy(struct ttm_tt *ttm) |
541 | static void radeon_ttm_backend_destroy(struct ttm_tt *ttm) |
289 | { |
542 | { |
Line 290... | Line 543... | ||
290 | struct radeon_ttm_tt *gtt = (void *)ttm; |
543 | struct radeon_ttm_tt *gtt = (void *)ttm; |
291 | 544 | ||
292 | ttm_dma_tt_fini(>t->ttm); |
545 | // ttm_dma_tt_fini(>t->ttm); |
Line 293... | Line 546... | ||
293 | kfree(gtt); |
546 | kfree(gtt); |
294 | } |
547 | } |
Line 325... | Line 578... | ||
325 | return NULL; |
578 | return NULL; |
326 | } |
579 | } |
327 | return >t->ttm.ttm; |
580 | return >t->ttm.ttm; |
328 | } |
581 | } |
Line -... | Line 582... | ||
- | 582 | ||
- | 583 | static int radeon_ttm_tt_populate(struct ttm_tt *ttm) |
|
- | 584 | { |
|
- | 585 | struct radeon_device *rdev; |
|
- | 586 | struct radeon_ttm_tt *gtt = (void *)ttm; |
|
- | 587 | unsigned i; |
|
- | 588 | int r; |
|
- | 589 | bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); |
|
- | 590 | ||
- | 591 | if (ttm->state != tt_unpopulated) |
|
- | 592 | return 0; |
|
- | 593 | ||
- | 594 | if (slave && ttm->sg) { |
|
- | 595 | drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages, |
|
- | 596 | gtt->ttm.dma_address, ttm->num_pages); |
|
- | 597 | ttm->state = tt_unbound; |
|
- | 598 | return 0; |
|
- | 599 | } |
|
- | 600 | ||
- | 601 | rdev = radeon_get_rdev(ttm->bdev); |
|
- | 602 | #if __OS_HAS_AGP |
|
- | 603 | if (rdev->flags & RADEON_IS_AGP) { |
|
- | 604 | return ttm_agp_tt_populate(ttm); |
|
- | 605 | } |
|
- | 606 | #endif |
|
- | 607 | ||
- | 608 | #ifdef CONFIG_SWIOTLB |
|
- | 609 | if (swiotlb_nr_tbl()) { |
|
- | 610 | return ttm_dma_populate(>t->ttm, rdev->dev); |
|
- | 611 | } |
|
- | 612 | #endif |
|
- | 613 | ||
- | 614 | r = ttm_pool_populate(ttm); |
|
- | 615 | if (r) { |
|
- | 616 | return r; |
|
- | 617 | } |
|
- | 618 | ||
- | 619 | for (i = 0; i < ttm->num_pages; i++) { |
|
- | 620 | gtt->ttm.dma_address[i] = pci_map_page(rdev->pdev, ttm->pages[i], |
|
- | 621 | 0, PAGE_SIZE, |
|
- | 622 | PCI_DMA_BIDIRECTIONAL); |
|
- | 623 | ||
- | 624 | } |
|
- | 625 | return 0; |
|
- | 626 | } |
|
- | 627 | ||
- | 628 | static void radeon_ttm_tt_unpopulate(struct ttm_tt *ttm) |
|
- | 629 | { |
|
- | 630 | struct radeon_device *rdev; |
|
- | 631 | struct radeon_ttm_tt *gtt = (void *)ttm; |
|
- | 632 | unsigned i; |
|
- | 633 | bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); |
|
- | 634 | ||
- | 635 | if (slave) |
|
- | 636 | return; |
|
- | 637 | ||
- | 638 | rdev = radeon_get_rdev(ttm->bdev); |
|
- | 639 | #if __OS_HAS_AGP |
|
- | 640 | if (rdev->flags & RADEON_IS_AGP) { |
|
- | 641 | ttm_agp_tt_unpopulate(ttm); |
|
- | 642 | return; |
|
- | 643 | } |
|
- | 644 | #endif |
|
- | 645 | ||
- | 646 | #ifdef CONFIG_SWIOTLB |
|
- | 647 | if (swiotlb_nr_tbl()) { |
|
- | 648 | ttm_dma_unpopulate(>t->ttm, rdev->dev); |
|
- | 649 | return; |
|
- | 650 | } |
|
- | 651 | #endif |
|
- | 652 | ||
- | 653 | ||
- | 654 | ttm_pool_unpopulate(ttm); |
|
- | 655 | } |
|
329 | 656 | ||
330 | static struct ttm_bo_driver radeon_bo_driver = { |
657 | static struct ttm_bo_driver radeon_bo_driver = { |
331 | .ttm_tt_create = &radeon_ttm_tt_create, |
658 | .ttm_tt_create = &radeon_ttm_tt_create, |
332 | // .ttm_tt_populate = &radeon_ttm_tt_populate, |
659 | .ttm_tt_populate = &radeon_ttm_tt_populate, |
333 | // .ttm_tt_unpopulate = &radeon_ttm_tt_unpopulate, |
660 | .ttm_tt_unpopulate = &radeon_ttm_tt_unpopulate, |
334 | // .invalidate_caches = &radeon_invalidate_caches, |
661 | .invalidate_caches = &radeon_invalidate_caches, |
335 | .init_mem_type = &radeon_init_mem_type, |
662 | .init_mem_type = &radeon_init_mem_type, |
336 | // .evict_flags = &radeon_evict_flags, |
663 | .evict_flags = &radeon_evict_flags, |
337 | // .move = &radeon_bo_move, |
664 | .move = &radeon_bo_move, |
338 | // .verify_access = &radeon_verify_access, |
665 | .verify_access = &radeon_verify_access, |
339 | // .sync_obj_signaled = &radeon_sync_obj_signaled, |
666 | .sync_obj_signaled = &radeon_sync_obj_signaled, |
340 | // .sync_obj_wait = &radeon_sync_obj_wait, |
667 | .sync_obj_wait = &radeon_sync_obj_wait, |
341 | // .sync_obj_flush = &radeon_sync_obj_flush, |
668 | .sync_obj_flush = &radeon_sync_obj_flush, |
342 | // .sync_obj_unref = &radeon_sync_obj_unref, |
669 | .sync_obj_unref = &radeon_sync_obj_unref, |
343 | // .sync_obj_ref = &radeon_sync_obj_ref, |
670 | .sync_obj_ref = &radeon_sync_obj_ref, |
344 | // .move_notify = &radeon_bo_move_notify, |
671 | .move_notify = &radeon_bo_move_notify, |
345 | // .fault_reserve_notify = &radeon_bo_fault_reserve_notify, |
672 | // .fault_reserve_notify = &radeon_bo_fault_reserve_notify, |
346 | // .io_mem_reserve = &radeon_ttm_io_mem_reserve, |
673 | .io_mem_reserve = &radeon_ttm_io_mem_reserve, |
347 | // .io_mem_free = &radeon_ttm_io_mem_free, |
674 | .io_mem_free = &radeon_ttm_io_mem_free, |
Line 348... | Line 675... | ||
348 | }; |
675 | }; |
349 | 676 | ||
350 | int radeon_ttm_init(struct radeon_device *rdev) |
677 | int radeon_ttm_init(struct radeon_device *rdev) |
Line 351... | Line -... | ||
351 | { |
- | |
352 | int r; |
- | |
353 | 678 | { |
|
354 | ENTER(); |
679 | int r; |
355 | 680 | ||
356 | r = radeon_ttm_global_init(rdev); |
681 | r = radeon_ttm_global_init(rdev); |
357 | if (r) { |
682 | if (r) { |
358 | return r; |
683 | return r; |
359 | } |
684 | } |
- | 685 | /* No others user of address space so set it to 0 */ |
|
- | 686 | r = ttm_bo_device_init(&rdev->mman.bdev, |
|
360 | /* No others user of address space so set it to 0 */ |
687 | rdev->mman.bo_global_ref.ref.object, |
361 | r = ttm_bo_device_init(&rdev->mman.bdev, |
688 | &radeon_bo_driver, |
362 | rdev->mman.bo_global_ref.ref.object, |
689 | NULL, |
363 | &radeon_bo_driver, DRM_FILE_PAGE_OFFSET, |
690 | DRM_FILE_PAGE_OFFSET, |
364 | rdev->need_dma32); |
691 | rdev->need_dma32); |
365 | if (r) { |
692 | if (r) { |
Line 371... | Line 698... | ||
371 | rdev->mc.real_vram_size >> PAGE_SHIFT); |
698 | rdev->mc.real_vram_size >> PAGE_SHIFT); |
372 | if (r) { |
699 | if (r) { |
373 | DRM_ERROR("Failed initializing VRAM heap.\n"); |
700 | DRM_ERROR("Failed initializing VRAM heap.\n"); |
374 | return r; |
701 | return r; |
375 | } |
702 | } |
- | 703 | /* Change the size here instead of the init above so only lpfn is affected */ |
|
- | 704 | radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); |
|
Line 376... | Line 705... | ||
376 | 705 | ||
377 | // r = radeon_bo_create(rdev, 256 * 1024, PAGE_SIZE, true, |
706 | r = radeon_bo_create(rdev, 16*1024*1024, PAGE_SIZE, true, |
378 | // RADEON_GEM_DOMAIN_VRAM, |
707 | RADEON_GEM_DOMAIN_VRAM, 0, |
379 | // NULL, &rdev->stollen_vga_memory); |
708 | NULL, &rdev->stollen_vga_memory); |
380 | // if (r) { |
709 | if (r) { |
381 | // return r; |
710 | return r; |
382 | // } |
711 | } |
383 | // r = radeon_bo_reserve(rdev->stollen_vga_memory, false); |
712 | r = radeon_bo_reserve(rdev->stollen_vga_memory, false); |
384 | // if (r) |
713 | if (r) |
385 | // return r; |
714 | return r; |
386 | // r = radeon_bo_pin(rdev->stollen_vga_memory, RADEON_GEM_DOMAIN_VRAM, NULL); |
715 | r = radeon_bo_pin(rdev->stollen_vga_memory, RADEON_GEM_DOMAIN_VRAM, NULL); |
387 | // radeon_bo_unreserve(rdev->stollen_vga_memory); |
716 | radeon_bo_unreserve(rdev->stollen_vga_memory); |
388 | // if (r) { |
717 | if (r) { |
389 | // radeon_bo_unref(&rdev->stollen_vga_memory); |
718 | radeon_bo_unref(&rdev->stollen_vga_memory); |
390 | // return r; |
- | |
391 | // } |
719 | return r; |
392 | 720 | } |
|
393 | DRM_INFO("radeon: %uM of VRAM memory ready\n", |
721 | DRM_INFO("radeon: %uM of VRAM memory ready\n", |
394 | (unsigned)rdev->mc.real_vram_size / (1024 * 1024)); |
722 | (unsigned) (rdev->mc.real_vram_size / (1024 * 1024))); |
395 | r = ttm_bo_init_mm(&rdev->mman.bdev, TTM_PL_TT, |
723 | r = ttm_bo_init_mm(&rdev->mman.bdev, TTM_PL_TT, |
396 | rdev->mc.gtt_size >> PAGE_SHIFT); |
724 | rdev->mc.gtt_size >> PAGE_SHIFT); |
397 | if (r) { |
725 | if (r) { |
398 | DRM_ERROR("Failed initializing GTT heap.\n"); |
726 | DRM_ERROR("Failed initializing GTT heap.\n"); |
399 | return r; |
727 | return r; |
400 | } |
728 | } |
401 | DRM_INFO("radeon: %uM of GTT memory ready.\n", |
729 | DRM_INFO("radeon: %uM of GTT memory ready.\n", |
402 | (unsigned)(rdev->mc.gtt_size / (1024 * 1024))); |
- | |
403 | rdev->mman.bdev.dev_mapping = rdev->ddev->dev_mapping; |
- | |
404 | - | ||
Line 405... | Line 730... | ||
405 | LEAVE(); |
730 | (unsigned)(rdev->mc.gtt_size / (1024 * 1024))); |
406 | 731 | ||
Line 472... | Line 797... | ||
472 | #endif6>6>6>6>><> |
797 | #endif |
Line -... | Line 798... | ||
- | 798 | ||
- | 799 | ||
- | 800 | ||
- | 801 | ||
- | 802 | int drm_prime_sg_to_page_addr_arrays(struct sg_table *sgt, struct page **pages, |
|
- | 803 | dma_addr_t *addrs, int max_pages) |
|
- | 804 | { |
|
- | 805 | unsigned count; |
|
- | 806 | struct scatterlist *sg; |
|
- | 807 | struct page *page; |
|
- | 808 | u32 len; |
|
- | 809 | int pg_index; |
|
- | 810 | dma_addr_t addr; |
|
- | 811 | ||
- | 812 | pg_index = 0; |
|
- | 813 | for_each_sg(sgt->sgl, sg, sgt->nents, count) { |
|
- | 814 | len = sg->length; |
|
- | 815 | page = sg_page(sg); |
|
- | 816 | addr = sg_dma_address(sg); |
|
- | 817 | ||
- | 818 | while (len > 0) { |
|
- | 819 | if (WARN_ON(pg_index >= max_pages)) |
|
- | 820 | return -1; |
|
- | 821 | pages[pg_index] = page; |
|
- | 822 | if (addrs) |
|
- | 823 | addrs[pg_index] = addr; |
|
- | 824 | ||
- | 825 | page++; |
|
- | 826 | addr += PAGE_SIZE; |
|
- | 827 | len -= PAGE_SIZE; |
|
- | 828 | pg_index++; |