From 70f54ff98d5c40772d60fc20861ff4a3884df437 Mon Sep 17 00:00:00 2001 From: Roger He Date: Thu, 21 Dec 2017 17:42:53 +0800 Subject: [PATCH 3736/4131] drm/ttm: enable swapout for reserved BOs during allocation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit if the bo shares same reservation object then not lock it again at swapout time to make it possible to swap out. v2: refine the commmit message Reviewed-by: Thomas Hellström Reviewed-by: Christian König Reviewed-by: Chuming Zhou Signed-off-by: Roger He Signed-off-by: Alex Deucher --- drivers/gpu/drm/ttm/ttm_bo.c | 142 ++++++++++++++++++++------------------- drivers/gpu/drm/ttm/ttm_memory.c | 45 ++++++------- 2 files changed, 95 insertions(+), 92 deletions(-) diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index d3ea59c..6c1ef7a 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -1737,95 +1737,101 @@ EXPORT_SYMBOL(ttm_bo_synccpu_write_release); * buffer object on the bo_global::swap_lru list. */ -static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) +int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx) { - struct ttm_bo_global *glob = - container_of(shrink, struct ttm_bo_global, shrink); - struct ttm_buffer_object *bo; - int ret = -EBUSY; - unsigned i; + struct ttm_buffer_object *bo; + int ret = -EBUSY; + bool locked; + unsigned i; + + spin_lock(&glob->lru_lock); + for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { + list_for_each_entry(bo, &glob->swap_lru[i], swap) { + if (ttm_bo_evict_swapout_allowable(bo, ctx, &locked)) { + ret = 0; + break; + } + } + if (!ret) + break; + } - spin_lock(&glob->lru_lock); - for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { - list_for_each_entry(bo, &glob->swap_lru[i], swap) { - ret = __ttm_bo_reserve(bo, false, true, NULL); - if (!ret) - break; - } - if (!ret) - break; - } + if (ret) { + spin_unlock(&glob->lru_lock); + return ret; + } - if (ret) { - spin_unlock(&glob->lru_lock); - return ret; - } + kref_get(&bo->list_kref); - kref_get(&bo->list_kref); + if (!list_empty(&bo->ddestroy)) { + ret = ttm_bo_cleanup_refs(bo, false, false, locked); + kref_put(&bo->list_kref, ttm_bo_release_list); + return ret; + } - if (!list_empty(&bo->ddestroy)) { - ret = ttm_bo_cleanup_refs_and_unlock(bo, false, false); - kref_put(&bo->list_kref, ttm_bo_release_list); - return ret; - } + ttm_bo_del_from_lru(bo); + spin_unlock(&glob->lru_lock); - ttm_bo_del_from_lru(bo); - spin_unlock(&glob->lru_lock); + /** + * Move to system cached + */ - /** - * Move to system cached - */ + if (bo->mem.mem_type != TTM_PL_SYSTEM || + bo->ttm->caching_state != tt_cached) { + struct ttm_operation_ctx ctx = { false, false }; + struct ttm_mem_reg evict_mem; - if (bo->mem.mem_type != TTM_PL_SYSTEM || - bo->ttm->caching_state != tt_cached) { - struct ttm_operation_ctx ctx = { false, false }; - struct ttm_mem_reg evict_mem; + evict_mem = bo->mem; + evict_mem.mm_node = NULL; + evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED; + evict_mem.mem_type = TTM_PL_SYSTEM; - evict_mem = bo->mem; - evict_mem.mm_node = NULL; - evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED; - evict_mem.mem_type = TTM_PL_SYSTEM; - - ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx); - if (unlikely(ret != 0)) - goto out; - } + ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx); + if (unlikely(ret != 0)) + goto out; + } - /** - * Make sure BO is idle. - */ + /** + * Make sure BO is idle. + */ - ret = ttm_bo_wait(bo, false, false); - if (unlikely(ret != 0)) - goto out; + ret = ttm_bo_wait(bo, false, false); + if (unlikely(ret != 0)) + goto out; - ttm_bo_unmap_virtual(bo); + ttm_bo_unmap_virtual(bo); - /** - * Swap out. Buffer will be swapped in again as soon as - * anyone tries to access a ttm page. - */ + /** + * Swap out. Buffer will be swapped in again as soon as + * anyone tries to access a ttm page. + */ - if (bo->bdev->driver->swap_notify) - bo->bdev->driver->swap_notify(bo); + if (bo->bdev->driver->swap_notify) + bo->bdev->driver->swap_notify(bo); - ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage); + ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage); out: - /** - * - * Unreserve without putting on LRU to avoid swapping out an - * already swapped buffer. - */ - - __ttm_bo_unreserve(bo); - kref_put(&bo->list_kref, ttm_bo_release_list); - return ret; + /** + * + * Unreserve without putting on LRU to avoid swapping out an + * already swapped buffer. + */ + if (locked) + kcl_reservation_object_unlock(bo->resv); + kref_put(&bo->list_kref, ttm_bo_release_list); + return ret; } +EXPORT_SYMBOL(ttm_bo_swapout); void ttm_bo_swapout_all(struct ttm_bo_device *bdev) { - while (ttm_bo_swapout(&bdev->glob->shrink) == 0) + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false + }; + + while (ttm_bo_swapout(bdev->glob, &ctx) == 0) ; } EXPORT_SYMBOL(ttm_bo_swapout_all); diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c index c1cbefe..dc25904 100644 --- a/drivers/gpu/drm/ttm/ttm_memory.c +++ b/drivers/gpu/drm/ttm/ttm_memory.c @@ -209,37 +209,34 @@ static bool ttm_zones_above_swap_target(struct ttm_mem_global *glob, * Note that this function is reentrant: * many threads may try to swap out at any given time. */ - static void ttm_shrink(struct ttm_mem_global *glob, bool from_wq, - uint64_t extra) + uint64_t extra, struct ttm_operation_ctx *ctx) { - int ret; - struct ttm_mem_shrink *shrink; + int ret; - spin_lock(&glob->lock); - if (glob->shrink == NULL) - goto out; - - while (ttm_zones_above_swap_target(glob, from_wq, extra)) { - shrink = glob->shrink; - spin_unlock(&glob->lock); - ret = shrink->do_shrink(shrink); - spin_lock(&glob->lock); - if (unlikely(ret != 0)) - goto out; - } -out: - spin_unlock(&glob->lock); -} + spin_lock(&glob->lock); + while (ttm_zones_above_swap_target(glob, from_wq, extra)) { + spin_unlock(&glob->lock); + ret = ttm_bo_swapout(glob->bo_glob, ctx); + spin_lock(&glob->lock); + if (unlikely(ret != 0)) + break; + } + spin_unlock(&glob->lock); +} static void ttm_shrink_work(struct work_struct *work) { - struct ttm_mem_global *glob = - container_of(work, struct ttm_mem_global, work); - - ttm_shrink(glob, true, 0ULL); + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false + }; + struct ttm_mem_global *glob = + container_of(work, struct ttm_mem_global, work); + + ttm_shrink(glob, true, 0ULL, &ctx); } static int ttm_mem_init_kernel_zone(struct ttm_mem_global *glob, @@ -526,7 +523,7 @@ static int ttm_mem_global_alloc_zone(struct ttm_mem_global *glob, return -ENOMEM; if (unlikely(count-- == 0)) return -ENOMEM; - ttm_shrink(glob, false, memory + (memory >> 2) + 16); + ttm_shrink(glob, false, memory + (memory >> 2) + 16, ctx); } return 0; -- 2.7.4