From 17f5622d02c6a20b55238692f556aa4caf5b63a9 Mon Sep 17 00:00:00 2001 From: Kalyan Alle Date: Mon, 23 Jul 2018 14:28:55 +0530 Subject: [PATCH 3739/4131] drm/ttm: ttm code modifications to fix the build/compilation issues Signed-off-by: Kalyan Alle --- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 7 +- drivers/gpu/drm/ast/ast_ttm.c | 1 - drivers/gpu/drm/cirrus/cirrus_ttm.c | 1 - drivers/gpu/drm/mgag200/mgag200_ttm.c | 1 - drivers/gpu/drm/nouveau/nouveau_bo.c | 10 +- drivers/gpu/drm/nouveau/nouveau_gem.c | 2 +- drivers/gpu/drm/qxl/qxl_release.c | 4 +- drivers/gpu/drm/qxl/qxl_ttm.c | 4 +- drivers/gpu/drm/radeon/radeon_ttm.c | 8 +- drivers/gpu/drm/ttm/Makefile | 19 +- drivers/gpu/drm/ttm/ttm_bo.c | 294 +++++++---------- drivers/gpu/drm/ttm/ttm_bo_util.c | 52 ++- drivers/gpu/drm/ttm/ttm_bo_vm.c | 61 ++-- drivers/gpu/drm/ttm/ttm_execbuf_util.c | 28 +- drivers/gpu/drm/ttm/ttm_memory.c | 143 +++++++-- drivers/gpu/drm/ttm/ttm_module.c | 4 +- drivers/gpu/drm/ttm/ttm_page_alloc.c | 531 ++++++++++++++++++++++--------- drivers/gpu/drm/ttm/ttm_page_alloc_dma.c | 359 ++++++++++++++------- drivers/gpu/drm/ttm/ttm_tt.c | 68 +++- drivers/gpu/drm/virtio/virtgpu_ttm.c | 1 - drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c | 1 - include/drm/ttm/ttm_bo_driver.h | 87 ++--- include/drm/ttm/ttm_memory.h | 69 +--- include/drm/ttm/ttm_page_alloc.h | 7 +- include/linux/reservation.h | 22 ++ 25 files changed, 1100 insertions(+), 684 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 548239c..2c4eed2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -537,7 +537,7 @@ static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, bool evict, if (unlikely(r)) { goto out_cleanup; } - r = ttm_bo_move_ttm(bo, ctx->interruptible, ctx->no_wait_gpu, new_mem); + r = ttm_bo_move_ttm(bo, ctx, new_mem); out_cleanup: ttm_bo_mem_put(bo, &tmp_mem); return r; @@ -568,7 +568,7 @@ static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, bool evict, if (unlikely(r)) { return r; } - r = ttm_bo_move_ttm(bo, ctx->interruptible, ctx->no_wait_gpu, &tmp_mem); + r = ttm_bo_move_ttm(bo, ctx, &tmp_mem); if (unlikely(r)) { goto out_cleanup; } @@ -630,8 +630,7 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict, if (r) { memcpy: - r = ttm_bo_move_memcpy(bo, ctx->interruptible, - ctx->no_wait_gpu, new_mem); + r = ttm_bo_move_memcpy(bo, ctx, new_mem); if (r) { return r; } diff --git a/drivers/gpu/drm/ast/ast_ttm.c b/drivers/gpu/drm/ast/ast_ttm.c index 10c36c9..846e4d8 100644 --- a/drivers/gpu/drm/ast/ast_ttm.c +++ b/drivers/gpu/drm/ast/ast_ttm.c @@ -237,7 +237,6 @@ struct ttm_bo_driver ast_bo_driver = { .verify_access = ast_bo_verify_access, .io_mem_reserve = &ast_ttm_io_mem_reserve, .io_mem_free = &ast_ttm_io_mem_free, - .io_mem_pfn = ttm_bo_default_io_mem_pfn, }; int ast_mm_init(struct ast_private *ast) diff --git a/drivers/gpu/drm/cirrus/cirrus_ttm.c b/drivers/gpu/drm/cirrus/cirrus_ttm.c index b40e294..4a9c232 100644 --- a/drivers/gpu/drm/cirrus/cirrus_ttm.c +++ b/drivers/gpu/drm/cirrus/cirrus_ttm.c @@ -237,7 +237,6 @@ struct ttm_bo_driver cirrus_bo_driver = { .verify_access = cirrus_bo_verify_access, .io_mem_reserve = &cirrus_ttm_io_mem_reserve, .io_mem_free = &cirrus_ttm_io_mem_free, - .io_mem_pfn = ttm_bo_default_io_mem_pfn, }; int cirrus_mm_init(struct cirrus_device *cirrus) diff --git a/drivers/gpu/drm/mgag200/mgag200_ttm.c b/drivers/gpu/drm/mgag200/mgag200_ttm.c index 85e39c3..882e902 100644 --- a/drivers/gpu/drm/mgag200/mgag200_ttm.c +++ b/drivers/gpu/drm/mgag200/mgag200_ttm.c @@ -237,7 +237,6 @@ struct ttm_bo_driver mgag200_bo_driver = { .verify_access = mgag200_bo_verify_access, .io_mem_reserve = &mgag200_ttm_io_mem_reserve, .io_mem_free = &mgag200_ttm_io_mem_free, - .io_mem_pfn = ttm_bo_default_io_mem_pfn, }; int mgag200_mm_init(struct mga_device *mdev) diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index d25ccd4..47827a4 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -1154,7 +1154,7 @@ nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr, if (ret) goto out; - ret = ttm_bo_move_ttm(bo, intr, no_wait_gpu, new_reg); + ret = ttm_bo_move_ttm(bo, &ctx, new_reg); out: ttm_bo_mem_put(bo, &tmp_reg); return ret; @@ -1183,7 +1183,7 @@ nouveau_bo_move_flips(struct ttm_buffer_object *bo, bool evict, bool intr, if (ret) return ret; - ret = ttm_bo_move_ttm(bo, intr, no_wait_gpu, &tmp_reg); + ret = ttm_bo_move_ttm(bo, &ctx, &tmp_reg); if (ret) goto out; @@ -1307,8 +1307,7 @@ nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, /* Fallback to software copy. */ ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu); if (ret == 0) - ret = ttm_bo_move_memcpy(bo, ctx->interruptible, - ctx->no_wait_gpu, new_reg); + ret = ttm_bo_move_memcpy(bo, ctx, new_reg); out: if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { @@ -1446,7 +1445,7 @@ nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo) } static int -nouveau_ttm_tt_populate(struct ttm_tt *ttm) +nouveau_ttm_tt_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) { struct ttm_dma_tt *ttm_dma = (void *)ttm; struct nouveau_drm *drm; @@ -1579,7 +1578,6 @@ struct ttm_bo_driver nouveau_bo_driver = { .fault_reserve_notify = &nouveau_ttm_fault_reserve_notify, .io_mem_reserve = &nouveau_ttm_io_mem_reserve, .io_mem_free = &nouveau_ttm_io_mem_free, - .io_mem_pfn = ttm_bo_default_io_mem_pfn, }; struct nvkm_vma * diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c index 60ffb70..75ec8b1 100644 --- a/drivers/gpu/drm/nouveau/nouveau_gem.c +++ b/drivers/gpu/drm/nouveau/nouveau_gem.c @@ -349,7 +349,7 @@ validate_fini_no_ticket(struct validate_op *op, struct nouveau_fence *fence, list_del(&nvbo->entry); nvbo->reserved_by = NULL; - ttm_bo_unreserve_ticket(&nvbo->bo, &op->ticket); + ttm_bo_unreserve(&nvbo->bo); drm_gem_object_unreference_unlocked(&nvbo->gem); } } diff --git a/drivers/gpu/drm/qxl/qxl_release.c b/drivers/gpu/drm/qxl/qxl_release.c index 1f90d0e..c010d9c 100644 --- a/drivers/gpu/drm/qxl/qxl_release.c +++ b/drivers/gpu/drm/qxl/qxl_release.c @@ -459,7 +459,7 @@ void qxl_release_fence_buffer_objects(struct qxl_release *release) trace_dma_fence_emit(&release->base); driver = bdev->driver; - glob = bo->glob; + glob = bdev->glob; spin_lock(&glob->lru_lock); @@ -469,7 +469,7 @@ void qxl_release_fence_buffer_objects(struct qxl_release *release) reservation_object_add_shared_fence(bo->resv, &release->base); ttm_bo_add_to_lru(bo); - __ttm_bo_unreserve(bo); + reservation_object_unlock(bo->resv); } spin_unlock(&glob->lru_lock); ww_acquire_fini(&release->ticket); diff --git a/drivers/gpu/drm/qxl/qxl_ttm.c b/drivers/gpu/drm/qxl/qxl_ttm.c index a283352..0944f7c 100644 --- a/drivers/gpu/drm/qxl/qxl_ttm.c +++ b/drivers/gpu/drm/qxl/qxl_ttm.c @@ -359,8 +359,7 @@ static int qxl_bo_move(struct ttm_buffer_object *bo, bool evict, qxl_move_null(bo, new_mem); return 0; } - return ttm_bo_move_memcpy(bo, ctx->interruptible, ctx->no_wait_gpu, - new_mem); + return ttm_bo_move_memcpy(bo, ctx, new_mem); } static void qxl_bo_move_notify(struct ttm_buffer_object *bo, @@ -391,7 +390,6 @@ static struct ttm_bo_driver qxl_bo_driver = { .verify_access = &qxl_verify_access, .io_mem_reserve = &qxl_ttm_io_mem_reserve, .io_mem_free = &qxl_ttm_io_mem_free, - .io_mem_pfn = ttm_bo_default_io_mem_pfn, .move_notify = &qxl_bo_move_notify, }; diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c index e651589..801249d 100644 --- a/drivers/gpu/drm/radeon/radeon_ttm.c +++ b/drivers/gpu/drm/radeon/radeon_ttm.c @@ -347,7 +347,7 @@ static int radeon_move_vram_ram(struct ttm_buffer_object *bo, if (unlikely(r)) { goto out_cleanup; } - r = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, new_mem); + r = ttm_bo_move_ttm(bo, &ctx, new_mem); out_cleanup: ttm_bo_mem_put(bo, &tmp_mem); return r; @@ -380,7 +380,7 @@ static int radeon_move_ram_vram(struct ttm_buffer_object *bo, if (unlikely(r)) { return r; } - r = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, &tmp_mem); + r = ttm_bo_move_ttm(bo, &ctx, &tmp_mem); if (unlikely(r)) { goto out_cleanup; } @@ -445,8 +445,7 @@ static int radeon_bo_move(struct ttm_buffer_object *bo, bool evict, if (r) { memcpy: - r = ttm_bo_move_memcpy(bo, ctx->interruptible, - ctx->no_wait_gpu, new_mem); + r = ttm_bo_move_memcpy(bo, ctx, new_mem); if (r) { return r; } @@ -845,7 +844,6 @@ static struct ttm_bo_driver radeon_bo_driver = { .fault_reserve_notify = &radeon_bo_fault_reserve_notify, .io_mem_reserve = &radeon_ttm_io_mem_reserve, .io_mem_free = &radeon_ttm_io_mem_free, - .io_mem_pfn = ttm_bo_default_io_mem_pfn, }; int radeon_ttm_init(struct radeon_device *rdev) diff --git a/drivers/gpu/drm/ttm/Makefile b/drivers/gpu/drm/ttm/Makefile index 7c57dac..e2c3ed2 100644 --- a/drivers/gpu/drm/ttm/Makefile +++ b/drivers/gpu/drm/ttm/Makefile @@ -2,11 +2,16 @@ # # Makefile for the drm device driver. This driver provides support for the -ccflags-y := -I$(src)/. -ttm-y := ttm_memory.o ttm_tt.o ttm_bo.o \ - ttm_bo_util.o ttm_bo_vm.o ttm_module.o \ - ttm_object.o ttm_lock.o ttm_execbuf_util.o ttm_page_alloc.o \ - ttm_bo_manager.o ttm_page_alloc_dma.o ttm_debug.o ttm_tracepoints.o -ttm-$(CONFIG_AGP) += ttm_agp_backend.o +ifneq (,$(BUILD_AS_DKMS)) + TTM_NAME = amdttm +else + TTM_NAME = ttm +endif -obj-$(CONFIG_DRM_TTM) += ttm.o +$(TTM_NAME)-y := ttm_memory.o ttm_tt.o ttm_bo.o \ + ttm_bo_util.o ttm_bo_vm.o ttm_module.o \ + ttm_object.o ttm_lock.o ttm_execbuf_util.o ttm_page_alloc.o \ + ttm_bo_manager.o ttm_page_alloc_dma.o +$(TTM_NAME)-$(CONFIG_AGP) += ttm_agp_backend.o + +obj-$(CONFIG_DRM_TTM) += $(TTM_NAME).o diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index cf3aef0..7f0f16b 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -28,6 +28,7 @@ * Authors: Thomas Hellstrom */ +#undef pr_fmt #define pr_fmt(fmt) "[TTM] " fmt #include @@ -42,11 +43,6 @@ #include #include -#define TTM_ASSERT_LOCKED(param) -#define TTM_DEBUG(fmt, arg...) -#define TTM_BO_HASH_ORDER 13 - -static int ttm_bo_swapout(struct ttm_mem_shrink *shrink); static void ttm_bo_global_kobj_release(struct kobject *kobj); static struct attribute ttm_bo_count = { @@ -54,6 +50,12 @@ static struct attribute ttm_bo_count = { .mode = S_IRUGO }; +/* default destructor */ +static void ttm_bo_default_destroy(struct ttm_buffer_object *bo) +{ + kfree(bo); +} + static inline int ttm_mem_type_from_place(const struct ttm_place *place, uint32_t *mem_type) { @@ -148,15 +150,11 @@ static void ttm_bo_release_list(struct kref *list_kref) BUG_ON(!list_empty(&bo->lru)); BUG_ON(!list_empty(&bo->ddestroy)); ttm_tt_destroy(bo->ttm); - atomic_dec(&bo->glob->bo_count); + atomic_dec(&bo->bdev->glob->bo_count); dma_fence_put(bo->moving); reservation_object_fini(&bo->ttm_resv); mutex_destroy(&bo->wu_mutex); - if (bo->destroy) - bo->destroy(bo); - else { - kfree(bo); - } + bo->destroy(bo); ttm_mem_global_free(bdev->glob->mem_glob, acc_size); } @@ -165,7 +163,7 @@ void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_type_manager *man; - lockdep_assert_held(&bo->resv->lock.base); + reservation_object_assert_held(bo->resv); if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { @@ -175,10 +173,10 @@ void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) list_add_tail(&bo->lru, &man->lru[bo->priority]); kref_get(&bo->list_kref); - if (bo->ttm && !(bo->ttm->page_flags & - (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED))) { + if (bo->ttm && !(bo->ttm->page_flags & + (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED))) { list_add_tail(&bo->swap, - &bo->glob->swap_lru[bo->priority]); + &bdev->glob->swap_lru[bo->priority]); kref_get(&bo->list_kref); } } @@ -209,63 +207,23 @@ void ttm_bo_del_from_lru(struct ttm_buffer_object *bo) void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo) { - spin_lock(&bo->glob->lru_lock); + struct ttm_bo_global *glob = bo->bdev->glob; + + spin_lock(&glob->lru_lock); ttm_bo_del_from_lru(bo); - spin_unlock(&bo->glob->lru_lock); + spin_unlock(&glob->lru_lock); } EXPORT_SYMBOL(ttm_bo_del_sub_from_lru); void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo) { - lockdep_assert_held(&bo->resv->lock.base); + reservation_object_assert_held(bo->resv); ttm_bo_del_from_lru(bo); ttm_bo_add_to_lru(bo); } EXPORT_SYMBOL(ttm_bo_move_to_lru_tail); -/* - * Call bo->mutex locked. - */ -static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) -{ - struct ttm_bo_device *bdev = bo->bdev; - int ret = 0; - uint32_t page_flags = 0; - - TTM_ASSERT_LOCKED(&bo->mutex); - bo->ttm = NULL; - - if (bdev->need_dma32) - page_flags |= TTM_PAGE_FLAG_DMA32; - - switch (bo->type) { - case ttm_bo_type_device: - if (zero_alloc) - page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC; - case ttm_bo_type_kernel: - bo->ttm = bdev->driver->ttm_tt_create(bo, page_flags); - if (unlikely(bo->ttm == NULL)) - ret = -ENOMEM; - break; - case ttm_bo_type_sg: - bo->ttm = bdev->driver->ttm_tt_create(bo, - page_flags | TTM_PAGE_FLAG_SG); - if (unlikely(bo->ttm == NULL)) { - ret = -ENOMEM; - break; - } - bo->ttm->sg = bo->sg; - break; - default: - pr_err("Illegal buffer object type\n"); - ret = -EINVAL; - break; - } - - return ret; -} - static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem, bool evict, struct ttm_operation_ctx *ctx) @@ -293,7 +251,7 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) { if (bo->ttm == NULL) { bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED); - ret = ttm_bo_add_ttm(bo, zero); + ret = ttm_tt_create(bo, zero); if (ret) goto out_err; } @@ -322,14 +280,12 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) - ret = ttm_bo_move_ttm(bo, ctx->interruptible, - ctx->no_wait_gpu, mem); + ret = ttm_bo_move_ttm(bo, ctx, mem); else if (bdev->driver->move) ret = bdev->driver->move(bo, evict, ctx, mem); else - ret = ttm_bo_move_memcpy(bo, ctx->interruptible, - ctx->no_wait_gpu, mem); - + ret = ttm_bo_move_memcpy(bo, ctx, mem); + if (ret) { if (bdev->driver->move_notify) { struct ttm_mem_reg tmp_mem = *mem; @@ -353,13 +309,12 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, bo->evicted = false; } - if (bo->mem.mm_node) { + if (bo->mem.mm_node) bo->offset = (bo->mem.start << PAGE_SHIFT) + bdev->man[bo->mem.mem_type].gpu_offset; - bo->cur_placement = bo->mem.placement; - } else + else bo->offset = 0; - + ctx->bytes_moved += bo->num_pages << PAGE_SHIFT; return 0; @@ -389,8 +344,6 @@ static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) ttm_tt_destroy(bo->ttm); bo->ttm = NULL; ttm_bo_mem_put(bo, &bo->mem); - - ww_mutex_unlock (&bo->resv->lock); } static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo) @@ -432,7 +385,7 @@ static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) { struct ttm_bo_device *bdev = bo->bdev; - struct ttm_bo_global *glob = bo->glob; + struct ttm_bo_global *glob = bdev->glob; int ret; ret = ttm_bo_individualize_resv(bo); @@ -447,7 +400,7 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) } spin_lock(&glob->lru_lock); - ret = __ttm_bo_reserve(bo, false, true, NULL); + ret = reservation_object_trylock(bo->resv) ? 0 : -EBUSY; if (!ret) { if (reservation_object_test_signaled_rcu(&bo->ttm_resv, true)) { ttm_bo_del_from_lru(bo); @@ -456,6 +409,7 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) reservation_object_unlock(&bo->ttm_resv); ttm_bo_cleanup_memtype_use(bo); + reservation_object_unlock(bo->resv); return; } @@ -471,8 +425,8 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) ttm_bo_add_to_lru(bo); } - __ttm_bo_unreserve(bo); - } + reservation_object_unlock(bo->resv); + } if (bo->resv != &bo->ttm_resv) reservation_object_unlock(&bo->ttm_resv); @@ -497,11 +451,11 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. */ -static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, - bool interruptible, - bool no_wait_gpu) +static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, + bool interruptible, bool no_wait_gpu, + bool unlock_resv) { - struct ttm_bo_global *glob = bo->glob; + struct ttm_bo_global *glob = bo->bdev->glob; struct reservation_object *resv; int ret; @@ -517,8 +471,10 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, if (ret && !no_wait_gpu) { long lret; - ww_mutex_unlock(&bo->resv->lock); - spin_unlock(&glob->lru_lock); + + if (unlock_resv) + reservation_object_unlock(bo->resv); + spin_unlock(&glob->lru_lock); lret = reservation_object_wait_timeout_rcu(resv, true, interruptible, @@ -530,7 +486,7 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, return -EBUSY; spin_lock(&glob->lru_lock); - ret = __ttm_bo_reserve(bo, false, true, NULL); + if (unlock_resv && !reservation_object_trylock(bo->resv)) { /* * We raced, and lost, someone else holds the reservation now, @@ -540,14 +496,15 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, * delayed destruction would succeed, so just return success * here. */ - if (ret) { - spin_unlock(&glob->lru_lock); + spin_unlock(&glob->lru_lock); return 0; } + ret = 0; } if (ret || unlikely(list_empty(&bo->ddestroy))) { - __ttm_bo_unreserve(bo); + if (unlock_resv) + reservation_object_unlock(bo->resv); spin_unlock(&glob->lru_lock); return ret; } @@ -559,6 +516,9 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, spin_unlock(&glob->lru_lock); ttm_bo_cleanup_memtype_use(bo); + if (unlock_resv) + reservation_object_unlock(bo->resv); + return 0; } @@ -566,60 +526,44 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, * Traverse the delayed list, and call ttm_bo_cleanup_refs on all * encountered buffers. */ - -static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) +static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) { struct ttm_bo_global *glob = bdev->glob; - struct ttm_buffer_object *entry = NULL; - int ret = 0; + struct list_head removed; + bool empty; - spin_lock(&glob->lru_lock); - if (list_empty(&bdev->ddestroy)) - goto out_unlock; + INIT_LIST_HEAD(&removed); - entry = list_first_entry(&bdev->ddestroy, - struct ttm_buffer_object, ddestroy); - kref_get(&entry->list_kref); - - for (;;) { - struct ttm_buffer_object *nentry = NULL; + spin_lock(&glob->lru_lock); + while (!list_empty(&bdev->ddestroy)) { + struct ttm_buffer_object *bo; - if (entry->ddestroy.next != &bdev->ddestroy) { - nentry = list_first_entry(&entry->ddestroy, - struct ttm_buffer_object, ddestroy); - kref_get(&nentry->list_kref); - } + bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object, + ddestroy); + kref_get(&bo->list_kref); + list_move_tail(&bo->ddestroy, &removed); - ret = __ttm_bo_reserve(entry, false, true, NULL); - if (remove_all && ret) { + if (remove_all || bo->resv != &bo->ttm_resv) { spin_unlock(&glob->lru_lock); - ret = __ttm_bo_reserve(entry, false, false, NULL); + reservation_object_lock(bo->resv, NULL); + spin_lock(&glob->lru_lock); - } + ttm_bo_cleanup_refs(bo, false, !remove_all, true); - if (!ret) - ret = ttm_bo_cleanup_refs_and_unlock(entry, false, - !remove_all); - else + } else if (reservation_object_trylock(bo->resv)) { + ttm_bo_cleanup_refs(bo, false, !remove_all, true); + } else { spin_unlock(&glob->lru_lock); + } - kref_put(&entry->list_kref, ttm_bo_release_list); - entry = nentry; - - if (ret || !entry) - goto out; - + kref_put(&bo->list_kref, ttm_bo_release_list); spin_lock(&glob->lru_lock); - if (list_empty(&entry->ddestroy)) - break; } - -out_unlock: + list_splice_tail(&removed, &bdev->ddestroy); + empty = list_empty(&bdev->ddestroy); spin_unlock(&glob->lru_lock); -out: - if (entry) - kref_put(&entry->list_kref, ttm_bo_release_list); - return ret; + + return empty; } static void ttm_bo_delayed_workqueue(struct work_struct *work) @@ -627,10 +571,9 @@ static void ttm_bo_delayed_workqueue(struct work_struct *work) struct ttm_bo_device *bdev = container_of(work, struct ttm_bo_device, wq.work); - if (ttm_bo_delayed_delete(bdev, false)) { + if (!ttm_bo_delayed_delete(bdev, false)) schedule_delayed_work(&bdev->wq, ((HZ / 100) < 1) ? 1 : HZ / 100); - } } static void ttm_bo_release(struct kref *kref) @@ -679,16 +622,25 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, struct ttm_placement placement; int ret = 0; - lockdep_assert_held(&bo->resv->lock.base); + reservation_object_assert_held(bo->resv); + + placement.num_placement = 0; + placement.num_busy_placement = 0; + bdev->driver->evict_flags(bo, &placement); + + if (!placement.num_placement && !placement.num_busy_placement) { + ret = ttm_bo_pipeline_gutting(bo); + if (ret) + return ret; + + return ttm_tt_create(bo, false); + } evict_mem = bo->mem; evict_mem.mm_node = NULL; evict_mem.bus.io_reserved_vm = false; evict_mem.bus.io_reserved_count = 0; - placement.num_placement = 0; - placement.num_busy_placement = 0; - bdev->driver->evict_flags(bo, &placement); ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx); if (ret) { if (ret != -ERESTARTSYS) { @@ -743,7 +695,8 @@ static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, *locked = false; if (bo->resv == ctx->resv) { reservation_object_assert_held(bo->resv); - if (ctx->allow_reserved_eviction || !list_empty(&bo->ddestroy)) + if (ctx->flags & TTM_OPT_FLAG_ALLOW_RES_EVICT + || !list_empty(&bo->ddestroy)) ret = true; } else { *locked = reservation_object_trylock(bo->resv); @@ -876,7 +829,7 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, return ret; if (mem->mm_node) break; - ret = ttm_mem_evict_first(bdev, bo->resv, mem_type, place, ctx); + ret = ttm_mem_evict_first(bdev, mem_type, place, ctx); if (unlikely(ret != 0)) return ret; } while (1); @@ -1052,14 +1005,13 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, EXPORT_SYMBOL(ttm_bo_mem_space); static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, - struct ttm_placement *placement, - struct ttm_placement *placement, - struct ttm_operation_ctx *ctx) + struct ttm_placement *placement, + struct ttm_operation_ctx *ctx) { int ret = 0; struct ttm_mem_reg mem; - - lockdep_assert_held(&bo->resv->lock.base); + + reservation_object_assert_held(bo->resv); mem.num_pages = bo->num_pages; mem.size = mem.num_pages << PAGE_SHIFT; @@ -1129,7 +1081,7 @@ int ttm_bo_validate(struct ttm_buffer_object *bo, int ret; uint32_t new_flags; - lockdep_assert_held(&bo->resv->lock.base); + reservation_object_assert_held(bo->resv); /* * Check whether we need to move buffer. */ @@ -1149,7 +1101,7 @@ int ttm_bo_validate(struct ttm_buffer_object *bo, * We might need to add a TTM. */ if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { - ret = ttm_bo_add_ttm(bo, true); + ret = ttm_tt_create(bo, true); if (ret) return ret; } @@ -1194,7 +1146,7 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, ttm_mem_global_free(mem_glob, acc_size); return -EINVAL; } - bo->destroy = destroy; + bo->destroy = destroy ? destroy : ttm_bo_default_destroy; kref_init(&bo->kref); kref_init(&bo->list_kref); @@ -1205,7 +1157,6 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, INIT_LIST_HEAD(&bo->io_reserve_lru); mutex_init(&bo->wu_mutex); bo->bdev = bdev; - bo->glob = bdev->glob; bo->type = type; bo->num_pages = num_pages; bo->mem.size = num_pages << PAGE_SHIFT; @@ -1219,14 +1170,15 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); bo->acc_size = acc_size; bo->sg = sg; + bo->ssg_can_map = false; if (resv) { bo->resv = resv; - lockdep_assert_held(&bo->resv->lock.base); + reservation_object_assert_held(bo->resv); } else { bo->resv = &bo->ttm_resv; } reservation_object_init(&bo->ttm_resv); - atomic_inc(&bo->glob->bo_count); + atomic_inc(&bo->bdev->glob->bo_count); drm_vma_node_reset(&bo->vma_node); bo->priority = 0; @@ -1243,15 +1195,12 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, * since otherwise lockdep will be angered in radeon. */ if (!resv) { - locked = ww_mutex_trylock(&bo->resv->lock); + locked = reservation_object_trylock(bo->resv); WARN_ON(!locked); } - if (likely(!ret)) { - struct ttm_operation_ctx ctx = { interruptible, false }; - - ret = ttm_bo_validate(bo, placement, &ctx); - } + if (likely(!ret)) + ret = ttm_bo_validate(bo, placement, ctx); if (unlikely(ret)) { if (!resv) @@ -1262,9 +1211,9 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, } if (resv && !(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { - spin_lock(&bo->glob->lru_lock); + spin_lock(&bdev->glob->lru_lock); ttm_bo_add_to_lru(bo); - spin_unlock(&bo->glob->lru_lock); + spin_unlock(&bdev->glob->lru_lock); } return ret; @@ -1357,7 +1306,11 @@ EXPORT_SYMBOL(ttm_bo_create); static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, unsigned mem_type) { - struct ttm_operation_ctx ctx = { false, false }; + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false, + .flags = TTM_OPT_FLAG_FORCE_ALLOC + }; struct ttm_mem_type_manager *man = &bdev->man[mem_type]; struct ttm_bo_global *glob = bdev->glob; struct dma_fence *fence; @@ -1372,8 +1325,7 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { while (!list_empty(&man->lru[i])) { spin_unlock(&glob->lru_lock); - ret = ttm_mem_evict_first(bdev, NULL, mem_type, - NULL, &ctx); + ret = ttm_mem_evict_first(bdev, mem_type, NULL, &ctx); if (ret) return ret; spin_lock(&glob->lru_lock); @@ -1494,7 +1446,6 @@ static void ttm_bo_global_kobj_release(struct kobject *kobj) struct ttm_bo_global *glob = container_of(kobj, struct ttm_bo_global, kobj); - ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink); __free_page(glob->dummy_read_page); kfree(glob); } @@ -1519,6 +1470,7 @@ int ttm_bo_global_init(struct drm_global_reference *ref) mutex_init(&glob->device_list_mutex); spin_lock_init(&glob->lru_lock); glob->mem_glob = bo_ref->mem_glob; + glob->mem_glob->bo_glob = glob; glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32); if (unlikely(glob->dummy_read_page == NULL)) { @@ -1529,14 +1481,6 @@ int ttm_bo_global_init(struct drm_global_reference *ref) for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) INIT_LIST_HEAD(&glob->swap_lru[i]); INIT_LIST_HEAD(&glob->device_list); - - ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout); - ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink); - if (unlikely(ret != 0)) { - pr_err("Could not register buffer object swapout\n"); - goto out_no_shrink; - } - atomic_set(&glob->bo_count, 0); ret = kobject_init_and_add( @@ -1544,8 +1488,6 @@ int ttm_bo_global_init(struct drm_global_reference *ref) if (unlikely(ret != 0)) kobject_put(&glob->kobj); return ret; -out_no_shrink: - __free_page(glob->dummy_read_page); out_no_drp: kfree(glob); return ret; @@ -1579,16 +1521,13 @@ int ttm_bo_device_release(struct ttm_bo_device *bdev) cancel_delayed_work_sync(&bdev->wq); - while (ttm_bo_delayed_delete(bdev, true)) - ; + if (ttm_bo_delayed_delete(bdev, true)) + pr_debug("Delayed destroy list was clean\n"); spin_lock(&glob->lru_lock); - if (list_empty(&bdev->ddestroy)) - TTM_DEBUG("Delayed destroy list was clean\n"); - for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) if (list_empty(&bdev->man[0].lru[0])) - TTM_DEBUG("Swap list %d was clean\n", i); + pr_debug("Swap list %d was clean\n", i); spin_unlock(&glob->lru_lock); drm_vma_offset_manager_destroy(&bdev->vma_manager); @@ -1731,7 +1670,6 @@ EXPORT_SYMBOL(ttm_bo_synccpu_write_release); * A buffer object shrink method that tries to swap out the first * buffer object on the bo_global::swap_lru list. */ - int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx) { struct ttm_buffer_object *bo; @@ -1813,7 +1751,7 @@ int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx) * already swapped buffer. */ if (locked) - kcl_reservation_object_unlock(bo->resv); + reservation_object_unlock(bo->resv); kref_put(&bo->list_kref, ttm_bo_release_list); return ret; } @@ -1853,10 +1791,12 @@ int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo) return -ERESTARTSYS; if (!ww_mutex_is_locked(&bo->resv->lock)) goto out_unlock; - ret = __ttm_bo_reserve(bo, true, false, NULL); + ret = reservation_object_lock_interruptible(bo->resv, NULL); + if (ret == -EINTR) + ret = -ERESTARTSYS; if (unlikely(ret != 0)) goto out_unlock; - __ttm_bo_unreserve(bo); + reservation_object_unlock(bo->resv); out_unlock: mutex_unlock(&bo->wu_mutex); diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index 4ef7a59..de9161f 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -45,7 +45,7 @@ void ttm_bo_free_old_node(struct ttm_buffer_object *bo) } int ttm_bo_move_ttm(struct ttm_buffer_object *bo, - bool interruptible, bool no_wait_gpu, + struct ttm_operation_ctx *ctx, struct ttm_mem_reg *new_mem) { struct ttm_tt *ttm = bo->ttm; @@ -53,7 +53,7 @@ int ttm_bo_move_ttm(struct ttm_buffer_object *bo, int ret; if (old_mem->mem_type != TTM_PL_SYSTEM) { - ret = ttm_bo_wait(bo, interruptible, no_wait_gpu); + ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu); if (unlikely(ret != 0)) { if (ret != -ERESTARTSYS) @@ -329,7 +329,7 @@ static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, } int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, - bool interruptible, bool no_wait_gpu, + struct ttm_operation_ctx *ctx, struct ttm_mem_reg *new_mem) { struct ttm_bo_device *bdev = bo->bdev; @@ -345,7 +345,7 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, unsigned long add = 0; int dir; - ret = ttm_bo_wait(bo, interruptible, no_wait_gpu); + ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu); if (ret) return ret; @@ -375,8 +375,8 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, /* * TTM might be null for moves within the same region. */ - if (ttm && ttm->state == tt_unpopulated) { - ret = ttm->bdev->driver->ttm_tt_populate(ttm, ctx); + if (ttm) { + ret = ttm_tt_populate(ttm, ctx); if (ret) goto out1; } @@ -402,8 +402,9 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, PAGE_KERNEL); ret = ttm_copy_io_ttm_page(ttm, old_iomap, page, prot); - } else + } else { ret = ttm_copy_io_page(new_iomap, old_iomap, page); + } if (ret) goto out1; } @@ -469,7 +470,7 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, * TODO: Explicit member copy would probably be better here. */ - atomic_inc(&bo->glob->bo_count); + atomic_inc(&bo->bdev->glob->bo_count); INIT_LIST_HEAD(&fbo->ddestroy); INIT_LIST_HEAD(&fbo->lru); INIT_LIST_HEAD(&fbo->swap); @@ -485,7 +486,7 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, fbo->acc_size = 0; fbo->resv = &fbo->ttm_resv; reservation_object_init(fbo->resv); - ret = ww_mutex_trylock(&fbo->resv->lock); + ret = reservation_object_trylock(fbo->resv); WARN_ON(!ret); *new_obj = fbo; @@ -556,11 +557,9 @@ static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, BUG_ON(!ttm); - if (ttm->state == tt_unpopulated) { - ret = ttm->bdev->driver->ttm_tt_populate(ttm, &ctx); - if (ret) - return ret; - } + ret = ttm_tt_populate(ttm, &ctx); + if (ret) + return ret; if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) { /* @@ -593,7 +592,6 @@ int ttm_bo_kmap(struct ttm_buffer_object *bo, unsigned long offset, size; int ret; - BUG_ON(!list_empty(&bo->swap)); map->virtual = NULL; map->bo = bo; if (num_pages > bo->num_pages) @@ -803,3 +801,27 @@ int ttm_bo_pipeline_move(struct ttm_buffer_object *bo, return 0; } EXPORT_SYMBOL(ttm_bo_pipeline_move); + +int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo) +{ + struct ttm_buffer_object *ghost; + int ret; + + ret = ttm_buffer_object_transfer(bo, &ghost); + if (ret) + return ret; + + ret = reservation_object_copy_fences(ghost->resv, bo->resv); + /* Last resort, wait for the BO to be idle when we are OOM */ + if (ret) + ttm_bo_wait(bo, false, false); + + memset(&bo->mem, 0, sizeof(bo->mem)); + bo->mem.mem_type = TTM_PL_SYSTEM; + bo->ttm = NULL; + + ttm_bo_unreserve(ghost); + ttm_bo_unref(&ghost); + + return 0; +} diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index 77e6577..a18131e 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -92,6 +92,17 @@ static int ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo, return ret; } +static unsigned long ttm_bo_io_mem_pfn(struct ttm_buffer_object *bo, + unsigned long page_offset) +{ + struct ttm_bo_device *bdev = bo->bdev; + + if (bdev->driver->io_mem_pfn) + return bdev->driver->io_mem_pfn(bo, page_offset); + + return ((bo->mem.bus.base + bo->mem.bus.offset) >> PAGE_SHIFT) + + page_offset; +} static int ttm_bo_vm_fault(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; @@ -125,7 +136,7 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) { if (!(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { ttm_bo_reference(bo); - up_read(&vmf->vma->vm_mm->mmap_sem); + up_read(&vma->vm_mm->mmap_sem); (void) ttm_bo_wait_unreserved(bo); ttm_bo_unref(&bo); } @@ -146,7 +157,7 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) * (if at all) by redirecting mmap to the exporter. */ if (bo->ttm && (bo->ttm->page_flags & TTM_PAGE_FLAG_SG)) { - retval = VM_FAULT_SIGBUS; + ret = VM_FAULT_SIGBUS; goto out_unlock; } @@ -157,10 +168,10 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) break; case -EBUSY: case -ERESTARTSYS: - retval = VM_FAULT_NOPAGE; + ret = VM_FAULT_NOPAGE; goto out_unlock; default: - retval = VM_FAULT_SIGBUS; + ret = VM_FAULT_SIGBUS; goto out_unlock; } } @@ -171,12 +182,10 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) */ ret = ttm_bo_vm_fault_idle(bo, vmf); if (unlikely(ret != 0)) { - retval = ret; - - if (retval == VM_FAULT_RETRY && + if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { /* The BO has already been unreserved. */ - return retval; + return ret; } goto out_unlock; @@ -184,12 +193,12 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) ret = ttm_mem_io_lock(man, true); if (unlikely(ret != 0)) { - retval = VM_FAULT_NOPAGE; + ret = VM_FAULT_NOPAGE; goto out_unlock; } ret = ttm_mem_io_reserve_vm(bo); if (unlikely(ret != 0)) { - retval = VM_FAULT_SIGBUS; + ret = VM_FAULT_SIGBUS; goto out_io_unlock; } @@ -199,7 +208,7 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) drm_vma_node_start(&bo->vma_node); if (unlikely(page_offset >= bo->num_pages)) { - retval = VM_FAULT_SIGBUS; + ret = VM_FAULT_SIGBUS; goto out_io_unlock; } @@ -217,7 +226,9 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) } else { struct ttm_operation_ctx ctx = { .interruptible = false, - .no_wait_gpu = false + .no_wait_gpu = false, + .flags = TTM_OPT_FLAG_FORCE_ALLOC + }; ttm = bo->ttm; @@ -225,8 +236,8 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) cvma.vm_page_prot); /* Allocate all page at once, most common usage */ - if (ttm->bdev->driver->ttm_tt_populate(ttm, &ctx)) { - retval = VM_FAULT_OOM; + if (ttm_tt_populate(ttm, &ctx)) { + ret = VM_FAULT_OOM; goto out_io_unlock; } } @@ -239,16 +250,15 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) if (bo->mem.bus.is_iomem) { /* Iomem should not be marked encrypted */ cvma.vm_page_prot = pgprot_decrypted(cvma.vm_page_prot); - pfn = bdev->driver->io_mem_pfn(bo, page_offset); + pfn = ttm_bo_io_mem_pfn(bo, page_offset); } else { page = ttm->pages[page_offset]; if (unlikely(!page && i == 0)) { - retval = VM_FAULT_OOM; + ret = VM_FAULT_OOM; goto out_io_unlock; } else if (unlikely(!page)) { break; } - page->mapping = vma->vm_file->f_mapping; page->index = drm_vma_node_start(&bo->vma_node) + page_offset; pfn = page_to_pfn(page); @@ -256,7 +266,7 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) if (vma->vm_flags & VM_MIXEDMAP) ret = vm_insert_mixed(&cvma, address, - __pfn_to_pfn_t(pfn, PFN_DEV)); + __pfn_to_pfn_t(pfn, PFN_DEV | (bo->ssg_can_map ? PFN_MAP : 0))); else ret = vm_insert_pfn(&cvma, address, pfn); @@ -268,7 +278,7 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) if (unlikely((ret == -EBUSY) || (ret != 0 && i > 0))) break; else if (unlikely(ret != 0)) { - retval = + ret = (ret == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS; goto out_io_unlock; } @@ -277,11 +287,12 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) if (unlikely(++page_offset >= page_last)) break; } + ret = VM_FAULT_NOPAGE; out_io_unlock: ttm_mem_io_unlock(man); out_unlock: ttm_bo_unreserve(bo); - return retval; + return ret; } static void ttm_bo_vm_open(struct vm_area_struct *vma) @@ -410,14 +421,6 @@ static struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_bo_device *bdev, return bo; } -unsigned long ttm_bo_default_io_mem_pfn(struct ttm_buffer_object *bo, - unsigned long page_offset) -{ - return ((bo->mem.bus.base + bo->mem.bus.offset) >> PAGE_SHIFT) - + page_offset; -} -EXPORT_SYMBOL(ttm_bo_default_io_mem_pfn); - int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma, struct ttm_bo_device *bdev) { @@ -455,7 +458,7 @@ int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma, * VM_MIXEDMAP on all mappings. See freedesktop.org bug #75719 */ vma->vm_flags |= VM_MIXEDMAP; - vma->vm_flags |= VM_IO | VM_DONTEXPAND | VM_DONTDUMP; + vma->vm_flags |= (bo->ssg_can_map ? 0 : VM_IO) | VM_DONTEXPAND | VM_DONTDUMP; return 0; out_unref: ttm_bo_unref(&bo); diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c index 5e1bcab..3dca206e 100644 --- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c +++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c @@ -38,7 +38,7 @@ static void ttm_eu_backoff_reservation_reverse(struct list_head *list, list_for_each_entry_continue_reverse(entry, list, head) { struct ttm_buffer_object *bo = entry->bo; - __ttm_bo_unreserve(bo); + reservation_object_unlock(bo->resv); } } @@ -62,14 +62,14 @@ void ttm_eu_backoff_reservation(struct ww_acquire_ctx *ticket, return; entry = list_first_entry(list, struct ttm_validate_buffer, head); - glob = entry->bo->glob; + glob = entry->bo->bdev->glob; spin_lock(&glob->lru_lock); list_for_each_entry(entry, list, head) { struct ttm_buffer_object *bo = entry->bo; ttm_bo_add_to_lru(bo); - __ttm_bo_unreserve(bo); + reservation_object_unlock(bo->resv); } spin_unlock(&glob->lru_lock); @@ -102,7 +102,7 @@ int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket, return 0; entry = list_first_entry(list, struct ttm_validate_buffer, head); - glob = entry->bo->glob; + glob = entry->bo->bdev->glob; if (ticket) ww_acquire_init(ticket, &reservation_ww_class); @@ -112,7 +112,7 @@ int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket, ret = __ttm_bo_reserve(bo, intr, (ticket == NULL), ticket); if (!ret && unlikely(atomic_read(&bo->cpu_writers) > 0)) { - __ttm_bo_unreserve(bo); + reservation_object_unlock(bo->resv); ret = -EBUSY; @@ -139,12 +139,14 @@ int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket, */ ttm_eu_backoff_reservation_reverse(list, entry); - if (ret == -EDEADLK && intr) { - ret = ww_mutex_lock_slow_interruptible(&bo->resv->lock, - ticket); - } else if (ret == -EDEADLK) { - ww_mutex_lock_slow(&bo->resv->lock, ticket); - ret = 0; + if (ret == -EDEADLK) { + if (intr) { + ret = ww_mutex_lock_slow_interruptible(&bo->resv->lock, + ticket); + } else { + ww_mutex_lock_slow(&bo->resv->lock, ticket); + ret = 0; + } } if (!ret && entry->shared) @@ -192,7 +194,7 @@ void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket, bo = list_first_entry(list, struct ttm_validate_buffer, head)->bo; bdev = bo->bdev; driver = bdev->driver; - glob = bo->glob; + glob = bo->bdev->glob; spin_lock(&glob->lru_lock); @@ -203,7 +205,7 @@ void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket, else reservation_object_add_excl_fence(bo->resv, fence); ttm_bo_add_to_lru(bo); - __ttm_bo_unreserve(bo); + reservation_object_unlock(bo->resv); } spin_unlock(&glob->lru_lock); if (ticket) diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c index dc25904..5bd82a9 100644 --- a/drivers/gpu/drm/ttm/ttm_memory.c +++ b/drivers/gpu/drm/ttm/ttm_memory.c @@ -36,6 +36,7 @@ #include #include #include +#include #define TTM_MEMORY_ALLOC_RETRIES 4 @@ -166,6 +167,54 @@ static struct kobj_type ttm_mem_zone_kobj_type = { .default_attrs = ttm_mem_zone_attrs, }; +static struct attribute ttm_mem_global_lower_mem_limit = { + .name = "lower_mem_limit", + .mode = S_IRUGO | S_IWUSR +}; + +static ssize_t ttm_mem_global_show(struct kobject *kobj, + struct attribute *attr, + char *buffer) +{ + struct ttm_mem_global *glob = + container_of(kobj, struct ttm_mem_global, kobj); + uint64_t val = 0; + + spin_lock(&glob->lock); + val = glob->lower_mem_limit; + spin_unlock(&glob->lock); + /* convert from number of pages to KB */ + val <<= (PAGE_SHIFT - 10); + return snprintf(buffer, PAGE_SIZE, "%llu\n", + (unsigned long long) val); +} + +static ssize_t ttm_mem_global_store(struct kobject *kobj, + struct attribute *attr, + const char *buffer, + size_t size) +{ + int chars; + uint64_t val64; + unsigned long val; + struct ttm_mem_global *glob = + container_of(kobj, struct ttm_mem_global, kobj); + + chars = sscanf(buffer, "%lu", &val); + if (chars == 0) + return size; + + val64 = val; + /* convert from KB to number of pages */ + val64 >>= (PAGE_SHIFT - 10); + + spin_lock(&glob->lock); + glob->lower_mem_limit = val64; + spin_unlock(&glob->lock); + + return size; +} + static void ttm_mem_global_kobj_release(struct kobject *kobj) { struct ttm_mem_global *glob = @@ -174,8 +223,20 @@ static void ttm_mem_global_kobj_release(struct kobject *kobj) kfree(glob); } +static struct attribute *ttm_mem_global_attrs[] = { + &ttm_mem_global_lower_mem_limit, + NULL +}; + +static const struct sysfs_ops ttm_mem_global_ops = { + .show = &ttm_mem_global_show, + .store = &ttm_mem_global_store, +}; + static struct kobj_type ttm_mem_glob_kobj_type = { .release = &ttm_mem_global_kobj_release, + .sysfs_ops = &ttm_mem_global_ops, + .default_attrs = ttm_mem_global_attrs, }; static bool ttm_zones_above_swap_target(struct ttm_mem_global *glob, @@ -209,34 +270,35 @@ static bool ttm_zones_above_swap_target(struct ttm_mem_global *glob, * Note that this function is reentrant: * many threads may try to swap out at any given time. */ + static void ttm_shrink(struct ttm_mem_global *glob, bool from_wq, - uint64_t extra, struct ttm_operation_ctx *ctx) + uint64_t extra, struct ttm_operation_ctx *ctx) { - int ret; + int ret; - spin_lock(&glob->lock); + spin_lock(&glob->lock); - while (ttm_zones_above_swap_target(glob, from_wq, extra)) { - spin_unlock(&glob->lock); - ret = ttm_bo_swapout(glob->bo_glob, ctx); - spin_lock(&glob->lock); - if (unlikely(ret != 0)) - break; - } + while (ttm_zones_above_swap_target(glob, from_wq, extra)) { + spin_unlock(&glob->lock); + ret = ttm_bo_swapout(glob->bo_glob, ctx); + spin_lock(&glob->lock); + if (unlikely(ret != 0)) + break; + } - spin_unlock(&glob->lock); + spin_unlock(&glob->lock); } static void ttm_shrink_work(struct work_struct *work) { - struct ttm_operation_ctx ctx = { - .interruptible = false, - .no_wait_gpu = false - }; - struct ttm_mem_global *glob = - container_of(work, struct ttm_mem_global, work); - - ttm_shrink(glob, true, 0ULL, &ctx); + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false + }; + struct ttm_mem_global *glob = + container_of(work, struct ttm_mem_global, work); + + ttm_shrink(glob, true, 0ULL, &ctx); } static int ttm_mem_init_kernel_zone(struct ttm_mem_global *glob, @@ -374,6 +436,9 @@ int ttm_mem_global_init(struct ttm_mem_global *glob) si_meminfo(&si); + /* set it as 0 by default to keep original behavior of OOM */ + glob->lower_mem_limit = 0; + ret = ttm_mem_init_kernel_zone(glob, &si); if (unlikely(ret != 0)) goto out_no_zone; @@ -468,6 +533,36 @@ void ttm_mem_global_free(struct ttm_mem_global *glob, } EXPORT_SYMBOL(ttm_mem_global_free); +/* + * check if the available mem is under lower memory limit + * + * a. if no swap disk at all or free swap space is under swap_mem_limit + * but available system mem is bigger than sys_mem_limit, allow TTM + * allocation; + * + * b. if the available system mem is less than sys_mem_limit but free + * swap disk is bigger than swap_mem_limit, allow TTM allocation. + */ +bool +ttm_check_under_lowerlimit(struct ttm_mem_global *glob, + uint64_t num_pages, + struct ttm_operation_ctx *ctx) +{ + int64_t available; + + if (ctx->flags & TTM_OPT_FLAG_FORCE_ALLOC) + return false; + + available = get_nr_swap_pages() + si_mem_available(); + + available -= num_pages; + if (available < glob->lower_mem_limit) + return true; + + return false; +} +EXPORT_SYMBOL(ttm_check_under_lowerlimit); + static int ttm_mem_global_reserve(struct ttm_mem_global *glob, struct ttm_mem_zone *single_zone, uint64_t amount, bool reserve) @@ -542,10 +637,9 @@ int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory, EXPORT_SYMBOL(ttm_mem_global_alloc); int ttm_mem_global_alloc_page(struct ttm_mem_global *glob, - struct page *page, uint64_t size, - struct ttm_operation_ctx *ctx) + struct page *page, uint64_t size, + struct ttm_operation_ctx *ctx) { - struct ttm_mem_zone *zone = NULL; /** @@ -563,7 +657,8 @@ int ttm_mem_global_alloc_page(struct ttm_mem_global *glob, return ttm_mem_global_alloc_zone(glob, zone, size, ctx); } -void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page) +void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page, + uint64_t size) { struct ttm_mem_zone *zone = NULL; @@ -574,7 +669,7 @@ void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page) if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL) zone = glob->zone_kernel; #endif - ttm_mem_global_free_zone(glob, zone, PAGE_SIZE); + ttm_mem_global_free_zone(glob, zone, size); } diff --git a/drivers/gpu/drm/ttm/ttm_module.c b/drivers/gpu/drm/ttm/ttm_module.c index 66fc639..7370e34 100644 --- a/drivers/gpu/drm/ttm/ttm_module.c +++ b/drivers/gpu/drm/ttm/ttm_module.c @@ -38,7 +38,7 @@ static DECLARE_WAIT_QUEUE_HEAD(exit_q); static atomic_t device_released; static struct device_type ttm_drm_class_type = { - .name = "ttm", + .name = "amdttm", /** * Add pm ops here. */ @@ -66,7 +66,7 @@ static int __init ttm_init(void) { int ret; - ret = dev_set_name(&ttm_drm_class_device, "ttm"); + ret = dev_set_name(&ttm_drm_class_device, "amdttm"); if (unlikely(ret != 0)) return ret; diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c index 55c9a90..9f9f181 100644 --- a/drivers/gpu/drm/ttm/ttm_page_alloc.c +++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c @@ -81,6 +81,7 @@ struct ttm_page_pool { char *name; unsigned long nfrees; unsigned long nrefills; + unsigned int order; }; /** @@ -95,7 +96,7 @@ struct ttm_pool_opts { unsigned small; }; -#define NUM_POOLS 4 +#define NUM_POOLS 6 /** * struct ttm_pool_manager - Holds memory pools for fst allocation @@ -122,6 +123,8 @@ struct ttm_pool_manager { struct ttm_page_pool uc_pool; struct ttm_page_pool wc_pool_dma32; struct ttm_page_pool uc_pool_dma32; + struct ttm_page_pool wc_pool_huge; + struct ttm_page_pool uc_pool_huge; } ; }; }; @@ -220,6 +223,17 @@ static struct kobj_type ttm_pool_kobj_type = { static struct ttm_pool_manager *_manager; #ifndef CONFIG_X86 +static int set_pages_wb(struct page *page, int numpages) +{ +#if IS_ENABLED(CONFIG_AGP) + int i; + + for (i = 0; i < numpages; i++) + unmap_page_from_agp(page++); +#endif + return 0; +} + static int set_pages_array_wb(struct page **pages, int addrinarray) { #if IS_ENABLED(CONFIG_AGP) @@ -256,8 +270,8 @@ static int set_pages_array_uc(struct page **pages, int addrinarray) /** * Select the right pool or requested caching state and ttm flags. */ -static struct ttm_page_pool *ttm_get_pool(int flags, - enum ttm_caching_state cstate) +static struct ttm_page_pool *ttm_get_pool(int flags, bool huge, + enum ttm_caching_state cstate) { int pool_index; @@ -269,20 +283,36 @@ static struct ttm_page_pool *ttm_get_pool(int flags, else pool_index = 0x1; - if (flags & TTM_PAGE_FLAG_DMA32) + if (flags & TTM_PAGE_FLAG_DMA32) { + if (huge) + return NULL; pool_index |= 0x2; + } else if (huge) { + pool_index |= 0x4; + } + return &_manager->pools[pool_index]; } /* set memory back to wb and free the pages. */ -static void ttm_pages_put(struct page *pages[], unsigned npages) +static void ttm_pages_put(struct page *pages[], unsigned npages, + unsigned int order) { - unsigned i; - if (set_pages_array_wb(pages, npages)) - pr_err("Failed to set %d pages to wb!\n", npages); - for (i = 0; i < npages; ++i) - __free_page(pages[i]); + unsigned int i, pages_nr = (1 << order); + + if (order == 0) { + if (set_pages_array_wb(pages, npages)) + pr_err("Failed to set %d pages to wb!\n", npages); + } + + for (i = 0; i < npages; ++i) { + if (order > 0) { + if (set_pages_wb(pages[i], pages_nr)) + pr_err("Failed to set %d pages to wb!\n", pages_nr); + } + __free_pages(pages[i], order); + } } static void ttm_pool_update_free_locked(struct ttm_page_pool *pool, @@ -321,7 +351,7 @@ static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free, pages_to_free = kmalloc(npages_to_free * sizeof(struct page *), GFP_KERNEL); if (!pages_to_free) { - pr_err("Failed to allocate memory for pool free operation\n"); + pr_debug("Failed to allocate memory for pool free operation\n"); return 0; } @@ -345,7 +375,7 @@ static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free, */ spin_unlock_irqrestore(&pool->lock, irq_flags); - ttm_pages_put(pages_to_free, freed_pages); + ttm_pages_put(pages_to_free, freed_pages, pool->order); if (likely(nr_free != FREE_ALL_PAGES)) nr_free -= freed_pages; @@ -380,7 +410,7 @@ static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free, spin_unlock_irqrestore(&pool->lock, irq_flags); if (freed_pages) - ttm_pages_put(pages_to_free, freed_pages); + ttm_pages_put(pages_to_free, freed_pages, pool->order); out: if (pages_to_free != static_buf) kfree(pages_to_free); @@ -404,6 +434,7 @@ ttm_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) struct ttm_page_pool *pool; int shrink_pages = sc->nr_to_scan; unsigned long freed = 0; + unsigned int nr_free_pool; if (!mutex_trylock(&lock)) return SHRINK_STOP; @@ -411,12 +442,20 @@ ttm_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) /* select start pool in round robin fashion */ for (i = 0; i < NUM_POOLS; ++i) { unsigned nr_free = shrink_pages; + unsigned page_nr; + if (shrink_pages == 0) break; + pool = &_manager->pools[(i + pool_offset)%NUM_POOLS]; + page_nr = (1 << pool->order); /* OK to use static buffer since global mutex is held. */ - shrink_pages = ttm_page_pool_free(pool, nr_free, true); - freed += nr_free - shrink_pages; + nr_free_pool = roundup(nr_free, page_nr) >> pool->order; + shrink_pages = ttm_page_pool_free(pool, nr_free_pool, true); + freed += (nr_free_pool - shrink_pages) << pool->order; + if (freed >= sc->nr_to_scan) + break; + shrink_pages <<= pool->order; } mutex_unlock(&lock); return freed; @@ -428,19 +467,22 @@ ttm_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc) { unsigned i; unsigned long count = 0; + struct ttm_page_pool *pool; - for (i = 0; i < NUM_POOLS; ++i) - count += _manager->pools[i].npages; + for (i = 0; i < NUM_POOLS; ++i) { + pool = &_manager->pools[i]; + count += (pool->npages << pool->order); + } return count; } -static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager) +static int ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager) { manager->mm_shrink.count_objects = ttm_pool_shrink_count; manager->mm_shrink.scan_objects = ttm_pool_shrink_scan; manager->mm_shrink.seeks = 1; - register_shrinker(&manager->mm_shrink); + return register_shrinker(&manager->mm_shrink); } static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager) @@ -494,28 +536,29 @@ static void ttm_handle_caching_state_failure(struct list_head *pages, * pages returned in pages array. */ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, - int ttm_flags, enum ttm_caching_state cstate, unsigned count) + int ttm_flags, enum ttm_caching_state cstate, + unsigned count, unsigned order) { struct page **caching_array; struct page *p; int r = 0; - unsigned i, cpages; - unsigned max_cpages = min(count, - (unsigned)(PAGE_SIZE/sizeof(struct page *))); + unsigned i, j, cpages; + unsigned npages = 1 << order; + unsigned max_cpages = min(count << order, (unsigned)NUM_PAGES_TO_ALLOC); /* allocate array for page caching change */ caching_array = kmalloc(max_cpages*sizeof(struct page *), GFP_KERNEL); if (!caching_array) { - pr_err("Unable to allocate table for new pages\n"); + pr_debug("Unable to allocate table for new pages\n"); return -ENOMEM; } for (i = 0, cpages = 0; i < count; ++i) { - p = alloc_page(gfp_flags); + p = alloc_pages(gfp_flags, order); if (!p) { - pr_err("Unable to get page %u\n", i); + pr_debug("Unable to get page %u\n", i); /* store already allocated pages in the pool after * setting the caching state */ @@ -531,14 +574,18 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, goto out; } + list_add(&p->lru, pages); + #ifdef CONFIG_HIGHMEM /* gfp flags of highmem page should never be dma32 so we * we should be fine in such case */ - if (!PageHighMem(p)) + if (PageHighMem(p)) + continue; + #endif - { - caching_array[cpages++] = p; + for (j = 0; j < npages; ++j) { + caching_array[cpages++] = p++; if (cpages == max_cpages) { r = ttm_set_pages_caching(caching_array, @@ -552,8 +599,6 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, cpages = 0; } } - - list_add(&p->lru, pages); } if (cpages) { @@ -573,9 +618,9 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, * Fill the given pool if there aren't enough pages and the requested number of * pages is small. */ -static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, - int ttm_flags, enum ttm_caching_state cstate, unsigned count, - unsigned long *irq_flags) +static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, int ttm_flags, + enum ttm_caching_state cstate, + unsigned count, unsigned long *irq_flags) { struct page *p; int r; @@ -605,7 +650,7 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, INIT_LIST_HEAD(&new_pages); r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags, - cstate, alloc_size); + cstate, alloc_size, 0); spin_lock_irqsave(&pool->lock, *irq_flags); if (!r) { @@ -613,7 +658,7 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, ++pool->nrefills; pool->npages += alloc_size; } else { - pr_err("Failed to fill pool (%p)\n", pool); + pr_debug("Failed to fill pool (%p)\n", pool); /* If we have any pages left put them to the pool. */ list_for_each_entry(p, &new_pages, lru) { ++cpages; @@ -627,22 +672,25 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, } /** - * Cut 'count' number of pages from the pool and put them on the return list. + * Allocate pages from the pool and put them on the return list. * - * @return count of pages still required to fulfill the request. + * @return zero for success or negative error code. */ -static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool, - struct list_head *pages, - int ttm_flags, - enum ttm_caching_state cstate, - unsigned count) +static int ttm_page_pool_get_pages(struct ttm_page_pool *pool, + struct list_head *pages, + int ttm_flags, + enum ttm_caching_state cstate, + unsigned count, unsigned order) { unsigned long irq_flags; struct list_head *p; unsigned i; + int r = 0; spin_lock_irqsave(&pool->lock, irq_flags); - ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, &irq_flags); + if (!order) + ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, + &irq_flags); if (count >= pool->npages) { /* take all pages from the pool */ @@ -672,32 +720,131 @@ static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool, count = 0; out: spin_unlock_irqrestore(&pool->lock, irq_flags); - return count; + + /* clear the pages coming from the pool if requested */ + if (ttm_flags & TTM_PAGE_FLAG_ZERO_ALLOC) { + struct page *page; + + list_for_each_entry(page, pages, lru) { + if (PageHighMem(page)) + clear_highpage(page); + else + clear_page(page_address(page)); + } + } + + /* If pool didn't have enough pages allocate new one. */ + if (count) { + gfp_t gfp_flags = pool->gfp_flags; + + /* set zero flag for page allocation if required */ + if (ttm_flags & TTM_PAGE_FLAG_ZERO_ALLOC) + gfp_flags |= __GFP_ZERO; + + if (ttm_flags & TTM_PAGE_FLAG_NO_RETRY) + gfp_flags |= __GFP_RETRY_MAYFAIL; + + /* ttm_alloc_new_pages doesn't reference pool so we can run + * multiple requests in parallel. + **/ + r = ttm_alloc_new_pages(pages, gfp_flags, ttm_flags, cstate, + count, order); + } + + return r; } /* Put all pages in pages list to correct pool to wait for reuse */ static void ttm_put_pages(struct page **pages, unsigned npages, int flags, enum ttm_caching_state cstate) { + struct ttm_page_pool *pool = ttm_get_pool(flags, false, cstate); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + struct ttm_page_pool *huge = ttm_get_pool(flags, true, cstate); +#endif unsigned long irq_flags; - struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); unsigned i; if (pool == NULL) { /* No pool for this memory type so free the pages */ - for (i = 0; i < npages; i++) { - if (pages[i]) { - if (page_count(pages[i]) != 1) - pr_err("Erroneous page count. Leaking pages.\n"); - __free_page(pages[i]); - pages[i] = NULL; + i = 0; + while (i < npages) { +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + struct page *p = pages[i]; +#endif + unsigned order = 0, j; + + if (!pages[i]) { + ++i; + continue; + } + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + if (!(flags & TTM_PAGE_FLAG_DMA32)) { + for (j = 0; j < HPAGE_PMD_NR; ++j) + if (p++ != pages[i + j]) + break; + + if (j == HPAGE_PMD_NR) + order = HPAGE_PMD_ORDER; + } +#endif + + if (page_count(pages[i]) != 1) + pr_err("Erroneous page count. Leaking pages.\n"); + __free_pages(pages[i], order); + + j = 1 << order; + while (j) { + pages[i++] = NULL; + --j; } } return; } + i = 0; +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + if (huge) { + unsigned max_size, n2free; + + spin_lock_irqsave(&huge->lock, irq_flags); + while (i < npages) { + struct page *p = pages[i]; + unsigned j; + + if (!p) + break; + + for (j = 0; j < HPAGE_PMD_NR; ++j) + if (p++ != pages[i + j]) + break; + + if (j != HPAGE_PMD_NR) + break; + + list_add_tail(&pages[i]->lru, &huge->list); + + for (j = 0; j < HPAGE_PMD_NR; ++j) + pages[i++] = NULL; + huge->npages++; + } + + /* Check that we don't go over the pool limit */ + max_size = _manager->options.max_size; + max_size /= HPAGE_PMD_NR; + if (huge->npages > max_size) + n2free = huge->npages - max_size; + else + n2free = 0; + spin_unlock_irqrestore(&huge->lock, irq_flags); + if (n2free) + ttm_page_pool_free(huge, n2free, false); + } +#endif + spin_lock_irqsave(&pool->lock, irq_flags); - for (i = 0; i < npages; i++) { + while (i < npages) { if (pages[i]) { if (page_count(pages[i]) != 1) pr_err("Erroneous page count. Leaking pages.\n"); @@ -705,6 +852,7 @@ static void ttm_put_pages(struct page **pages, unsigned npages, int flags, pages[i] = NULL; pool->npages++; } + ++i; } /* Check that we don't go over the pool limit */ npages = 0; @@ -727,82 +875,120 @@ static void ttm_put_pages(struct page **pages, unsigned npages, int flags, static int ttm_get_pages(struct page **pages, unsigned npages, int flags, enum ttm_caching_state cstate) { - struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); + struct ttm_page_pool *pool = ttm_get_pool(flags, false, cstate); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + struct ttm_page_pool *huge = ttm_get_pool(flags, true, cstate); +#endif struct list_head plist; struct page *p = NULL; - gfp_t gfp_flags = GFP_USER; - unsigned count; + unsigned count, first; int r; - /* set zero flag for page allocation if required */ - if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) - gfp_flags |= __GFP_ZERO; - /* No pool for cached pages */ if (pool == NULL) { + gfp_t gfp_flags = GFP_USER; + unsigned i; +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + unsigned j; +#endif + + /* set zero flag for page allocation if required */ + if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) + gfp_flags |= __GFP_ZERO; + + if (flags & TTM_PAGE_FLAG_NO_RETRY) + gfp_flags |= __GFP_RETRY_MAYFAIL; + if (flags & TTM_PAGE_FLAG_DMA32) gfp_flags |= GFP_DMA32; else gfp_flags |= GFP_HIGHUSER; - for (r = 0; r < npages; ++r) { + i = 0; +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + if (!(gfp_flags & GFP_DMA32)) { + while (npages >= HPAGE_PMD_NR) { + gfp_t huge_flags = gfp_flags; + + huge_flags |= GFP_TRANSHUGE; + huge_flags &= ~__GFP_MOVABLE; + huge_flags &= ~__GFP_COMP; + p = alloc_pages(huge_flags, HPAGE_PMD_ORDER); + if (!p) + break; + + for (j = 0; j < HPAGE_PMD_NR; ++j) + pages[i++] = p++; + + npages -= HPAGE_PMD_NR; + } + } +#endif + + first = i; + while (npages) { p = alloc_page(gfp_flags); if (!p) { - - pr_err("Unable to allocate page\n"); + pr_debug("Unable to allocate page\n"); return -ENOMEM; } - pages[r] = p; + /* Swap the pages if we detect consecutive order */ + if (i > first && pages[i - 1] == p - 1) + swap(p, pages[i - 1]); + + pages[i++] = p; + --npages; } return 0; } - /* combine zero flag to pool flags */ - gfp_flags |= pool->gfp_flags; - - /* First we take pages from the pool */ - INIT_LIST_HEAD(&plist); - npages = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages); count = 0; - list_for_each_entry(p, &plist, lru) { - pages[count++] = p; - } - /* clear the pages coming from the pool if requested */ - if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) { +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + if (huge && npages >= HPAGE_PMD_NR) { + INIT_LIST_HEAD(&plist); + ttm_page_pool_get_pages(huge, &plist, flags, cstate, + npages / HPAGE_PMD_NR, + HPAGE_PMD_ORDER); + list_for_each_entry(p, &plist, lru) { - if (PageHighMem(p)) - clear_highpage(p); - else - clear_page(page_address(p)); + unsigned j; + + for (j = 0; j < HPAGE_PMD_NR; ++j) + pages[count++] = &p[j]; } } +#endif - /* If pool didn't have enough pages allocate new one. */ - if (npages > 0) { - /* ttm_alloc_new_pages doesn't reference pool so we can run - * multiple requests in parallel. - **/ - INIT_LIST_HEAD(&plist); - r = ttm_alloc_new_pages(&plist, gfp_flags, flags, cstate, npages); - list_for_each_entry(p, &plist, lru) { - pages[count++] = p; - } - if (r) { - /* If there is any pages in the list put them back to - * the pool. */ - pr_err("Failed to allocate extra pages for large request\n"); - ttm_put_pages(pages, count, flags, cstate); - return r; - } + INIT_LIST_HEAD(&plist); + r = ttm_page_pool_get_pages(pool, &plist, flags, cstate, + npages - count, 0); + + first = count; + list_for_each_entry(p, &plist, lru) { + struct page *tmp = p; + + /* Swap the pages if we detect consecutive order */ + if (count > first && pages[count - 1] == tmp - 1) + swap(tmp, pages[count - 1]); + pages[count++] = tmp; + } + + if (r) { + /* If there is any pages in the list put them back to + * the pool. + */ + pr_debug("Failed to allocate extra pages for large request\n"); + ttm_put_pages(pages, count, flags, cstate); + return r; } return 0; } static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, gfp_t flags, - char *name) + char *name, unsigned int order) { spin_lock_init(&pool->lock); pool->fill_lock = false; @@ -810,11 +996,17 @@ static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, gfp_t flags, pool->npages = pool->nfrees = 0; pool->gfp_flags = flags; pool->name = name; + pool->order = order; } int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) { int ret; +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + unsigned order = HPAGE_PMD_ORDER; +#else + unsigned order = 0; +#endif WARN_ON(_manager); @@ -824,31 +1016,41 @@ int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) if (!_manager) return -ENOMEM; - ttm_page_pool_init_locked(&_manager->wc_pool, GFP_HIGHUSER, "wc"); + ttm_page_pool_init_locked(&_manager->wc_pool, GFP_HIGHUSER, "wc", 0); - ttm_page_pool_init_locked(&_manager->uc_pool, GFP_HIGHUSER, "uc"); + ttm_page_pool_init_locked(&_manager->uc_pool, GFP_HIGHUSER, "uc", 0); ttm_page_pool_init_locked(&_manager->wc_pool_dma32, - GFP_USER | GFP_DMA32, "wc dma"); - + GFP_USER | GFP_DMA32, "wc dma", 0); ttm_page_pool_init_locked(&_manager->uc_pool_dma32, - GFP_USER | GFP_DMA32, "uc dma"); + GFP_USER | GFP_DMA32, "uc dma", 0); + ttm_page_pool_init_locked(&_manager->wc_pool_huge, + GFP_TRANSHUGE & ~(__GFP_MOVABLE | __GFP_COMP), + "wc huge", order); + + ttm_page_pool_init_locked(&_manager->uc_pool_huge, + GFP_TRANSHUGE & ~(__GFP_MOVABLE | __GFP_COMP) + , "uc huge", order); + _manager->options.max_size = max_pages; _manager->options.small = SMALL_ALLOCATION; _manager->options.alloc_size = NUM_PAGES_TO_ALLOC; ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type, &glob->kobj, "pool"); - if (unlikely(ret != 0)) { - kobject_put(&_manager->kobj); - _manager = NULL; - return ret; - } - - ttm_pool_mm_shrink_init(_manager); + if (unlikely(ret != 0)) + goto error; + ret = ttm_pool_mm_shrink_init(_manager); + if (unlikely(ret != 0)) + goto error; return 0; + +error: + kobject_put(&_manager->kobj); + _manager = NULL; + return ret; } void ttm_page_alloc_fini(void) @@ -866,28 +1068,52 @@ void ttm_page_alloc_fini(void) _manager = NULL; } +static void +ttm_pool_unpopulate_helper(struct ttm_tt *ttm, unsigned mem_count_update) +{ + struct ttm_mem_global *mem_glob = ttm->bdev->glob->mem_glob; + unsigned i; + + if (mem_count_update == 0) + goto put_pages; + + for (i = 0; i < mem_count_update; ++i) { + if (!ttm->pages[i]) + continue; + + ttm_mem_global_free_page(mem_glob, ttm->pages[i], PAGE_SIZE); + } + +put_pages: + ttm_put_pages(ttm->pages, ttm->num_pages, ttm->page_flags, + ttm->caching_state); + ttm->state = tt_unpopulated; +} + int ttm_pool_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) { - struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; + struct ttm_mem_global *mem_glob = ttm->bdev->glob->mem_glob; unsigned i; int ret; if (ttm->state != tt_unpopulated) return 0; - for (i = 0; i < ttm->num_pages; ++i) { - ret = ttm_get_pages(&ttm->pages[i], 1, - ttm->page_flags, - ttm->caching_state); - if (ret != 0) { - ttm_pool_unpopulate(ttm); - return -ENOMEM; - } + if (ttm_check_under_lowerlimit(mem_glob, ttm->num_pages, ctx)) + return -ENOMEM; + + ret = ttm_get_pages(ttm->pages, ttm->num_pages, ttm->page_flags, + ttm->caching_state); + if (unlikely(ret != 0)) { + ttm_pool_unpopulate_helper(ttm, 0); + return ret; + } + for (i = 0; i < ttm->num_pages; ++i) { ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], - PAGE_SIZE, ctx); + PAGE_SIZE, ctx); if (unlikely(ret != 0)) { - ttm_pool_unpopulate(ttm); + ttm_pool_unpopulate_helper(ttm, i); return -ENOMEM; } } @@ -907,34 +1133,33 @@ EXPORT_SYMBOL(ttm_pool_populate); void ttm_pool_unpopulate(struct ttm_tt *ttm) { - unsigned i; - - for (i = 0; i < ttm->num_pages; ++i) { - if (ttm->pages[i]) { - ttm_mem_global_free_page(ttm->glob->mem_glob, - ttm->pages[i]); - ttm_put_pages(&ttm->pages[i], 1, - ttm->page_flags, - ttm->caching_state); - } - } - ttm->state = tt_unpopulated; + ttm_pool_unpopulate_helper(ttm, ttm->num_pages); } EXPORT_SYMBOL(ttm_pool_unpopulate); int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt, struct ttm_operation_ctx *ctx) { - unsigned i; + unsigned i, j; int r; r = ttm_pool_populate(&tt->ttm, ctx); if (r) return r; - for (i = 0; i < tt->ttm.num_pages; i++) { + for (i = 0; i < tt->ttm.num_pages; ++i) { + struct page *p = tt->ttm.pages[i]; + size_t num_pages = 1; + + for (j = i + 1; j < tt->ttm.num_pages; ++j) { + if (++p != tt->ttm.pages[j]) + break; + + ++num_pages; + } + tt->dma_address[i] = dma_map_page(dev, tt->ttm.pages[i], - 0, PAGE_SIZE, + 0, num_pages * PAGE_SIZE, DMA_BIDIRECTIONAL); if (dma_mapping_error(dev, tt->dma_address[i])) { while (i--) { @@ -945,6 +1170,11 @@ int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt, ttm_pool_unpopulate(&tt->ttm); return -EFAULT; } + + for (j = 1; j < num_pages; ++j) { + tt->dma_address[i + 1] = tt->dma_address[i] + PAGE_SIZE; + ++i; + } } return 0; } @@ -952,13 +1182,28 @@ EXPORT_SYMBOL(ttm_populate_and_map_pages); void ttm_unmap_and_unpopulate_pages(struct device *dev, struct ttm_dma_tt *tt) { - unsigned i; - - for (i = 0; i < tt->ttm.num_pages; i++) { - if (tt->dma_address[i]) { - dma_unmap_page(dev, tt->dma_address[i], - PAGE_SIZE, DMA_BIDIRECTIONAL); + unsigned i, j; + + for (i = 0; i < tt->ttm.num_pages;) { + struct page *p = tt->ttm.pages[i]; + size_t num_pages = 1; + + if (!tt->dma_address[i] || !tt->ttm.pages[i]) { + ++i; + continue; + } + + for (j = i + 1; j < tt->ttm.num_pages; ++j) { + if (++p != tt->ttm.pages[j]) + break; + + ++num_pages; } + + dma_unmap_page(dev, tt->dma_address[i], num_pages * PAGE_SIZE, + DMA_BIDIRECTIONAL); + + i += num_pages; } ttm_pool_unpopulate(&tt->ttm); } @@ -973,12 +1218,12 @@ int ttm_page_alloc_debugfs(struct seq_file *m, void *data) seq_printf(m, "No pool allocator running.\n"); return 0; } - seq_printf(m, "%6s %12s %13s %8s\n", + seq_printf(m, "%7s %12s %13s %8s\n", h[0], h[1], h[2], h[3]); for (i = 0; i < NUM_POOLS; ++i) { p = &_manager->pools[i]; - seq_printf(m, "%6s %12ld %13ld %8d\n", + seq_printf(m, "%7s %12ld %13ld %8d\n", p->name, p->nrefills, p->nfrees, p->npages); } diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c index 47ba7be..52fccc5 100644 --- a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c +++ b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c @@ -60,37 +60,32 @@ #define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(struct page *)) #define SMALL_ALLOCATION 4 #define FREE_ALL_PAGES (~0U) -/* times are in msecs */ -#define IS_UNDEFINED (0) -#define IS_WC (1<<1) -#define IS_UC (1<<2) -#define IS_CACHED (1<<3) -#define IS_DMA32 (1<<4) +#define VADDR_FLAG_HUGE_POOL 1UL +#define VADDR_FLAG_UPDATED_COUNT 2UL enum pool_type { - POOL_IS_UNDEFINED, - POOL_IS_WC = IS_WC, - POOL_IS_UC = IS_UC, - POOL_IS_CACHED = IS_CACHED, - POOL_IS_WC_DMA32 = IS_WC | IS_DMA32, - POOL_IS_UC_DMA32 = IS_UC | IS_DMA32, - POOL_IS_CACHED_DMA32 = IS_CACHED | IS_DMA32, + IS_UNDEFINED = 0, + IS_WC = 1 << 1, + IS_UC = 1 << 2, + IS_CACHED = 1 << 3, + IS_DMA32 = 1 << 4, + IS_HUGE = 1 << 5 }; /* - * The pool structure. There are usually six pools: + * The pool structure. There are up to nine pools: * - generic (not restricted to DMA32): * - write combined, uncached, cached. * - dma32 (up to 2^32 - so up 4GB): * - write combined, uncached, cached. + * - huge (not restricted to DMA32): + * - write combined, uncached, cached. * for each 'struct device'. The 'cached' is for pages that are actively used. * The other ones can be shrunk by the shrinker API if neccessary. * @pools: The 'struct device->dma_pools' link. * @type: Type of the pool - * @lock: Protects the inuse_list and free_list from concurrnet access. Must be + * @lock: Protects the free_list from concurrnet access. Must be * used with irqsave/irqrestore variants because pool allocator maybe called * from delayed work. - * @inuse_list: Pool of pages that are in use. The order is very important and - * it is in the order that the TTM pages that are put back are in. * @free_list: Pool of pages that are free to be used. No order requirements. * @dev: The device that is associated with these pools. * @size: Size used during DMA allocation. @@ -107,7 +102,6 @@ struct dma_pool { struct list_head pools; /* The 'struct device->dma_pools link */ enum pool_type type; spinlock_t lock; - struct list_head inuse_list; struct list_head free_list; struct device *dev; unsigned size; @@ -124,13 +118,14 @@ struct dma_pool { * The accounting page keeping track of the allocated page along with * the DMA address. * @page_list: The link to the 'page_list' in 'struct dma_pool'. - * @vaddr: The virtual address of the page + * @vaddr: The virtual address of the page and a flag if the page belongs to a + * huge pool * @dma: The bus address of the page. If the page is not allocated * via the DMA API, it will be -1. */ struct dma_page { struct list_head page_list; - void *vaddr; + unsigned long vaddr; struct page *p; dma_addr_t dma; }; @@ -221,11 +216,11 @@ static ssize_t ttm_pool_store(struct kobject *kobj, struct attribute *attr, /* Convert kb to number of pages */ val = val / (PAGE_SIZE >> 10); - if (attr == &ttm_page_pool_max) + if (attr == &ttm_page_pool_max) { m->options.max_size = val; - else if (attr == &ttm_page_pool_small) + } else if (attr == &ttm_page_pool_small) { m->options.small = val; - else if (attr == &ttm_page_pool_alloc_size) { + } else if (attr == &ttm_page_pool_alloc_size) { if (val > NUM_PAGES_TO_ALLOC*8) { pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n", NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7), @@ -329,7 +324,8 @@ static int ttm_set_pages_caching(struct dma_pool *pool, static void __ttm_dma_free_page(struct dma_pool *pool, struct dma_page *d_page) { dma_addr_t dma = d_page->dma; - dma_free_coherent(pool->dev, pool->size, d_page->vaddr, dma); + d_page->vaddr &= ~VADDR_FLAG_HUGE_POOL; + dma_free_coherent(pool->dev, pool->size, (void *)d_page->vaddr, dma); kfree(d_page); d_page = NULL; @@ -337,19 +333,26 @@ static void __ttm_dma_free_page(struct dma_pool *pool, struct dma_page *d_page) static struct dma_page *__ttm_dma_alloc_page(struct dma_pool *pool) { struct dma_page *d_page; + unsigned long attrs = 0; + void *vaddr; d_page = kmalloc(sizeof(struct dma_page), GFP_KERNEL); if (!d_page) return NULL; - d_page->vaddr = dma_alloc_coherent(pool->dev, pool->size, - &d_page->dma, - pool->gfp_flags); - if (d_page->vaddr) { - if (is_vmalloc_addr(d_page->vaddr)) - d_page->p = vmalloc_to_page(d_page->vaddr); + if (pool->type & IS_HUGE) + attrs = DMA_ATTR_NO_WARN; + + vaddr = dma_alloc_attrs(pool->dev, pool->size, &d_page->dma, + pool->gfp_flags, attrs); + if (vaddr) { + if (is_vmalloc_addr(vaddr)) + d_page->p = vmalloc_to_page(vaddr); else - d_page->p = virt_to_page(d_page->vaddr); + d_page->p = virt_to_page(vaddr); + d_page->vaddr = (unsigned long)vaddr; + if (pool->type & IS_HUGE) + d_page->vaddr |= VADDR_FLAG_HUGE_POOL; } else { kfree(d_page); d_page = NULL; @@ -381,11 +384,38 @@ static void ttm_pool_update_free_locked(struct dma_pool *pool, } /* set memory back to wb and free the pages. */ +static void ttm_dma_page_put(struct dma_pool *pool, struct dma_page *d_page) +{ + struct page *page = d_page->p; + unsigned i, num_pages; + + /* Don't set WB on WB page pool. */ + if (!(pool->type & IS_CACHED)) { + num_pages = pool->size / PAGE_SIZE; + for (i = 0; i < num_pages; ++i, ++page) { + if (set_pages_array_wb(&page, 1)) { + pr_err("%s: Failed to set %d pages to wb!\n", + pool->dev_name, 1); + } + } + } + + list_del(&d_page->page_list); + __ttm_dma_free_page(pool, d_page); +} + static void ttm_dma_pages_put(struct dma_pool *pool, struct list_head *d_pages, struct page *pages[], unsigned npages) { struct dma_page *d_page, *tmp; + if (pool->type & IS_HUGE) { + list_for_each_entry_safe(d_page, tmp, d_pages, page_list) + ttm_dma_page_put(pool, d_page); + + return; + } + /* Don't set WB on WB page pool. */ if (npages && !(pool->type & IS_CACHED) && set_pages_array_wb(pages, npages)) @@ -398,17 +428,6 @@ static void ttm_dma_pages_put(struct dma_pool *pool, struct list_head *d_pages, } } -static void ttm_dma_page_put(struct dma_pool *pool, struct dma_page *d_page) -{ - /* Don't set WB on WB page pool. */ - if (!(pool->type & IS_CACHED) && set_pages_array_wb(&d_page->p, 1)) - pr_err("%s: Failed to set %d pages to wb!\n", - pool->dev_name, 1); - - list_del(&d_page->page_list); - __ttm_dma_free_page(pool, d_page); -} - /* * Free pages from pool. * @@ -446,7 +465,7 @@ static unsigned ttm_dma_page_pool_free(struct dma_pool *pool, unsigned nr_free, GFP_KERNEL); if (!pages_to_free) { - pr_err("%s: Failed to allocate memory for pool free operation\n", + pr_debug("%s: Failed to allocate memory for pool free operation\n", pool->dev_name); return 0; } @@ -577,8 +596,8 @@ static int ttm_dma_pool_match(struct device *dev, void *res, void *match_data) static struct dma_pool *ttm_dma_pool_init(struct device *dev, gfp_t flags, enum pool_type type) { - char *n[] = {"wc", "uc", "cached", " dma32", "unknown",}; - enum pool_type t[] = {IS_WC, IS_UC, IS_CACHED, IS_DMA32, IS_UNDEFINED}; + const char *n[] = {"wc", "uc", "cached", " dma32", "huge"}; + enum pool_type t[] = {IS_WC, IS_UC, IS_CACHED, IS_DMA32, IS_HUGE}; struct device_pools *sec_pool = NULL; struct dma_pool *pool = NULL, **ptr; unsigned i; @@ -609,18 +628,24 @@ static struct dma_pool *ttm_dma_pool_init(struct device *dev, gfp_t flags, sec_pool->pool = pool; INIT_LIST_HEAD(&pool->free_list); - INIT_LIST_HEAD(&pool->inuse_list); INIT_LIST_HEAD(&pool->pools); spin_lock_init(&pool->lock); pool->dev = dev; pool->npages_free = pool->npages_in_use = 0; pool->nfrees = 0; pool->gfp_flags = flags; - pool->size = PAGE_SIZE; + if (type & IS_HUGE) +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + pool->size = HPAGE_PMD_SIZE; +#else + BUG(); +#endif + else + pool->size = PAGE_SIZE; pool->type = type; pool->nrefills = 0; p = pool->name; - for (i = 0; i < 5; i++) { + for (i = 0; i < ARRAY_SIZE(t); i++) { if (type & t[i]) { p += snprintf(p, sizeof(pool->name) - (p - pool->name), "%s", n[i]); @@ -653,10 +678,10 @@ static struct dma_pool *ttm_dma_pool_init(struct device *dev, gfp_t flags, static struct dma_pool *ttm_dma_find_pool(struct device *dev, enum pool_type type) { - struct dma_pool *pool, *tmp, *found = NULL; + struct dma_pool *pool, *tmp; if (type == IS_UNDEFINED) - return found; + return NULL; /* NB: We iterate on the 'struct dev' which has no spinlock, but * it does have a kref which we have taken. The kref is taken during @@ -669,13 +694,10 @@ static struct dma_pool *ttm_dma_find_pool(struct device *dev, * thing is at that point of time there are no pages associated with the * driver so this function will not be called. */ - list_for_each_entry_safe(pool, tmp, &dev->dma_pools, pools) { - if (pool->type != type) - continue; - found = pool; - break; - } - return found; + list_for_each_entry_safe(pool, tmp, &dev->dma_pools, pools) + if (pool->type == type) + return pool; + return NULL; } /* @@ -724,7 +746,7 @@ static int ttm_dma_pool_alloc_new_pages(struct dma_pool *pool, struct dma_page *dma_p; struct page *p; int r = 0; - unsigned i, cpages; + unsigned i, j, npages, cpages; unsigned max_cpages = min(count, (unsigned)(PAGE_SIZE/sizeof(struct page *))); @@ -732,21 +754,20 @@ static int ttm_dma_pool_alloc_new_pages(struct dma_pool *pool, caching_array = kmalloc(max_cpages*sizeof(struct page *), GFP_KERNEL); if (!caching_array) { - pr_err("%s: Unable to allocate table for new pages\n", + pr_debug("%s: Unable to allocate table for new pages\n", pool->dev_name); return -ENOMEM; } - if (count > 1) { + if (count > 1) pr_debug("%s: (%s:%d) Getting %d pages\n", pool->dev_name, pool->name, current->pid, count); - } for (i = 0, cpages = 0; i < count; ++i) { dma_p = __ttm_dma_alloc_page(pool); if (!dma_p) { - pr_err("%s: Unable to get page %u\n", - pool->dev_name, i); + pr_debug("%s: Unable to get page %u\n", + pool->dev_name, i); /* store already allocated pages in the pool after * setting the caching state */ @@ -762,28 +783,32 @@ static int ttm_dma_pool_alloc_new_pages(struct dma_pool *pool, goto out; } p = dma_p->p; + list_add(&dma_p->page_list, d_pages); + #ifdef CONFIG_HIGHMEM /* gfp flags of highmem page should never be dma32 so we * we should be fine in such case */ - if (!PageHighMem(p)) + if (PageHighMem(p)) + continue; #endif - { - caching_array[cpages++] = p; + + npages = pool->size / PAGE_SIZE; + for (j = 0; j < npages; ++j) { + caching_array[cpages++] = p + j; if (cpages == max_cpages) { /* Note: Cannot hold the spinlock */ r = ttm_set_pages_caching(pool, caching_array, - cpages); + cpages); if (r) { ttm_dma_handle_caching_state_failure( - pool, d_pages, caching_array, - cpages); + pool, d_pages, caching_array, + cpages); goto out; } cpages = 0; } } - list_add(&dma_p->page_list, d_pages); } if (cpages) { @@ -828,8 +853,8 @@ static int ttm_dma_page_pool_fill_locked(struct dma_pool *pool, struct dma_page *d_page; unsigned cpages = 0; - pr_err("%s: Failed to fill %s pool (r:%d)!\n", - pool->dev_name, pool->name, r); + pr_debug("%s: Failed to fill %s pool (r:%d)!\n", + pool->dev_name, pool->name, r); list_for_each_entry(d_page, &d_pages, page_list) { cpages++; @@ -843,18 +868,18 @@ static int ttm_dma_page_pool_fill_locked(struct dma_pool *pool, } /* - * @return count of pages still required to fulfill the request. * The populate list is actually a stack (not that is matters as TTM * allocates one page at a time. + * return dma_page pointer if success, otherwise NULL. */ -static int ttm_dma_pool_get_pages(struct dma_pool *pool, +static struct dma_page *ttm_dma_pool_get_pages(struct dma_pool *pool, struct ttm_dma_tt *ttm_dma, unsigned index) { - struct dma_page *d_page; + struct dma_page *d_page = NULL; struct ttm_tt *ttm = &ttm_dma->ttm; unsigned long irq_flags; - int count, r = -ENOMEM; + int count; spin_lock_irqsave(&pool->lock, irq_flags); count = ttm_dma_page_pool_fill_locked(pool, &irq_flags); @@ -863,12 +888,35 @@ static int ttm_dma_pool_get_pages(struct dma_pool *pool, ttm->pages[index] = d_page->p; ttm_dma->dma_address[index] = d_page->dma; list_move_tail(&d_page->page_list, &ttm_dma->pages_list); - r = 0; pool->npages_in_use += 1; pool->npages_free -= 1; } spin_unlock_irqrestore(&pool->lock, irq_flags); - return r; + return d_page; +} + +static gfp_t ttm_dma_pool_gfp_flags(struct ttm_dma_tt *ttm_dma, bool huge) +{ + struct ttm_tt *ttm = &ttm_dma->ttm; + gfp_t gfp_flags; + + if (ttm->page_flags & TTM_PAGE_FLAG_DMA32) + gfp_flags = GFP_USER | GFP_DMA32; + else + gfp_flags = GFP_HIGHUSER; + if (ttm->page_flags & TTM_PAGE_FLAG_ZERO_ALLOC) + gfp_flags |= __GFP_ZERO; + + if (huge) { + gfp_flags |= GFP_TRANSHUGE; + gfp_flags &= ~__GFP_MOVABLE; + gfp_flags &= ~__GFP_COMP; + } + + if (ttm->page_flags & TTM_PAGE_FLAG_NO_RETRY) + gfp_flags |= __GFP_RETRY_MAYFAIL; + + return gfp_flags; } /* @@ -879,37 +927,78 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev, struct ttm_operation_ctx *ctx) { struct ttm_tt *ttm = &ttm_dma->ttm; - struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; + struct ttm_mem_global *mem_glob = ttm->bdev->glob->mem_glob; unsigned long num_pages = ttm->num_pages; struct dma_pool *pool; + struct dma_page *d_page; enum pool_type type; unsigned i; - gfp_t gfp_flags; int ret; if (ttm->state != tt_unpopulated) return 0; + if (ttm_check_under_lowerlimit(mem_glob, num_pages, ctx)) + return -ENOMEM; + + INIT_LIST_HEAD(&ttm_dma->pages_list); + i = 0; + type = ttm_to_type(ttm->page_flags, ttm->caching_state); + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE if (ttm->page_flags & TTM_PAGE_FLAG_DMA32) - gfp_flags = GFP_USER | GFP_DMA32; - else - gfp_flags = GFP_HIGHUSER; - if (ttm->page_flags & TTM_PAGE_FLAG_ZERO_ALLOC) - gfp_flags |= __GFP_ZERO; + goto skip_huge; + + pool = ttm_dma_find_pool(dev, type | IS_HUGE); + if (!pool) { + gfp_t gfp_flags = ttm_dma_pool_gfp_flags(ttm_dma, true); + + pool = ttm_dma_pool_init(dev, gfp_flags, type | IS_HUGE); + if (IS_ERR_OR_NULL(pool)) + goto skip_huge; + } + + while (num_pages >= HPAGE_PMD_NR) { + unsigned j; + + d_page = ttm_dma_pool_get_pages(pool, ttm_dma, i); + if (!d_page) + break; + + ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], + pool->size, ctx); + if (unlikely(ret != 0)) { + ttm_dma_unpopulate(ttm_dma, dev); + return -ENOMEM; + } + + d_page->vaddr |= VADDR_FLAG_UPDATED_COUNT; + for (j = i + 1; j < (i + HPAGE_PMD_NR); ++j) { + ttm->pages[j] = ttm->pages[j - 1] + 1; + ttm_dma->dma_address[j] = ttm_dma->dma_address[j - 1] + + PAGE_SIZE; + } + + i += HPAGE_PMD_NR; + num_pages -= HPAGE_PMD_NR; + } + +skip_huge: +#endif pool = ttm_dma_find_pool(dev, type); if (!pool) { + gfp_t gfp_flags = ttm_dma_pool_gfp_flags(ttm_dma, false); + pool = ttm_dma_pool_init(dev, gfp_flags, type); - if (IS_ERR_OR_NULL(pool)) { + if (IS_ERR_OR_NULL(pool)) return -ENOMEM; - } } - INIT_LIST_HEAD(&ttm_dma->pages_list); - for (i = 0; i < ttm->num_pages; ++i) { - ret = ttm_dma_pool_get_pages(pool, ttm_dma, i); - if (ret != 0) { + while (num_pages) { + d_page = ttm_dma_pool_get_pages(pool, ttm_dma, i); + if (!d_page) { ttm_dma_unpopulate(ttm_dma, dev); return -ENOMEM; } @@ -920,6 +1009,10 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev, ttm_dma_unpopulate(ttm_dma, dev); return -ENOMEM; } + + d_page->vaddr |= VADDR_FLAG_UPDATED_COUNT; + ++i; + --num_pages; } if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) { @@ -939,14 +1032,41 @@ EXPORT_SYMBOL_GPL(ttm_dma_populate); void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev) { struct ttm_tt *ttm = &ttm_dma->ttm; + struct ttm_mem_global *mem_glob = ttm->bdev->glob->mem_glob; struct dma_pool *pool; struct dma_page *d_page, *next; enum pool_type type; bool is_cached = false; - unsigned count = 0, i, npages = 0; + unsigned count, i, npages = 0; unsigned long irq_flags; type = ttm_to_type(ttm->page_flags, ttm->caching_state); + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + pool = ttm_dma_find_pool(dev, type | IS_HUGE); + if (pool) { + count = 0; + list_for_each_entry_safe(d_page, next, &ttm_dma->pages_list, + page_list) { + if (!(d_page->vaddr & VADDR_FLAG_HUGE_POOL)) + continue; + + count++; + if (d_page->vaddr & VADDR_FLAG_UPDATED_COUNT) { + ttm_mem_global_free_page(mem_glob, d_page->p, + pool->size); + d_page->vaddr &= ~VADDR_FLAG_UPDATED_COUNT; + } + ttm_dma_page_put(pool, d_page); + } + + spin_lock_irqsave(&pool->lock, irq_flags); + pool->npages_in_use -= count; + pool->nfrees += count; + spin_unlock_irqrestore(&pool->lock, irq_flags); + } +#endif + pool = ttm_dma_find_pool(dev, type); if (!pool) return; @@ -955,9 +1075,20 @@ void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev) ttm_to_type(ttm->page_flags, tt_cached)) == pool); /* make sure pages array match list and count number of pages */ - list_for_each_entry(d_page, &ttm_dma->pages_list, page_list) { + count = 0; + list_for_each_entry_safe(d_page, next, &ttm_dma->pages_list, + page_list) { ttm->pages[count] = d_page->p; count++; + + if (d_page->vaddr & VADDR_FLAG_UPDATED_COUNT) { + ttm_mem_global_free_page(mem_glob, d_page->p, + pool->size); + d_page->vaddr &= ~VADDR_FLAG_UPDATED_COUNT; + } + + if (is_cached) + ttm_dma_page_put(pool, d_page); } spin_lock_irqsave(&pool->lock, irq_flags); @@ -977,19 +1108,6 @@ void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev) } spin_unlock_irqrestore(&pool->lock, irq_flags); - if (is_cached) { - list_for_each_entry_safe(d_page, next, &ttm_dma->pages_list, page_list) { - ttm_mem_global_free_page(ttm->glob->mem_glob, - d_page->p); - ttm_dma_page_put(pool, d_page); - } - } else { - for (i = 0; i < count; i++) { - ttm_mem_global_free_page(ttm->glob->mem_glob, - ttm->pages[i]); - } - } - INIT_LIST_HEAD(&ttm_dma->pages_list); for (i = 0; i < ttm->num_pages; i++) { ttm->pages[i] = NULL; @@ -1067,12 +1185,12 @@ ttm_dma_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc) return count; } -static void ttm_dma_pool_mm_shrink_init(struct ttm_pool_manager *manager) +static int ttm_dma_pool_mm_shrink_init(struct ttm_pool_manager *manager) { manager->mm_shrink.count_objects = ttm_dma_pool_shrink_count; manager->mm_shrink.scan_objects = &ttm_dma_pool_shrink_scan; manager->mm_shrink.seeks = 1; - register_shrinker(&manager->mm_shrink); + return register_shrinker(&manager->mm_shrink); } static void ttm_dma_pool_mm_shrink_fini(struct ttm_pool_manager *manager) @@ -1082,7 +1200,7 @@ static void ttm_dma_pool_mm_shrink_fini(struct ttm_pool_manager *manager) int ttm_dma_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) { - int ret = -ENOMEM; + int ret; WARN_ON(_manager); @@ -1090,7 +1208,7 @@ int ttm_dma_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) _manager = kzalloc(sizeof(*_manager), GFP_KERNEL); if (!_manager) - goto err; + return -ENOMEM; mutex_init(&_manager->lock); INIT_LIST_HEAD(&_manager->pools); @@ -1102,13 +1220,17 @@ int ttm_dma_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) /* This takes care of auto-freeing the _manager */ ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type, &glob->kobj, "dma_pool"); - if (unlikely(ret != 0)) { - kobject_put(&_manager->kobj); - goto err; - } - ttm_dma_pool_mm_shrink_init(_manager); + if (unlikely(ret != 0)) + goto error; + + ret = ttm_dma_pool_mm_shrink_init(_manager); + if (unlikely(ret != 0)) + goto error; return 0; -err: + +error: + kobject_put(&_manager->kobj); + _manager = NULL; return ret; } @@ -1134,15 +1256,12 @@ int ttm_dma_page_alloc_debugfs(struct seq_file *m, void *data) { struct device_pools *p; struct dma_pool *pool = NULL; - char *h[] = {"pool", "refills", "pages freed", "inuse", "available", - "name", "virt", "busaddr"}; if (!_manager) { seq_printf(m, "No pool allocator running.\n"); return 0; } - seq_printf(m, "%13s %12s %13s %8s %8s %8s\n", - h[0], h[1], h[2], h[3], h[4], h[5]); + seq_printf(m, " pool refills pages freed inuse available name\n"); mutex_lock(&_manager->lock); list_for_each_entry(p, &_manager->pools, pools) { struct device *dev = p->dev; diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c index 522a80d..a9acda4 100644 --- a/drivers/gpu/drm/ttm/ttm_tt.c +++ b/drivers/gpu/drm/ttm/ttm_tt.c @@ -83,16 +83,13 @@ int ttm_tt_create(struct ttm_buffer_object *bo, bool zero_alloc) if (unlikely(bo->ttm == NULL)) return -ENOMEM; - if (bo->type == ttm_bo_type_sg) - bo->ttm->sg = bo->sg; - - return 0; + return 0; } /** * Allocates storage for pointers to the pages that back the ttm. */ -static void ttm_tt_alloc_page_directory(struct ttm_tt *ttm) +static int ttm_tt_alloc_page_directory(struct ttm_tt *ttm) { ttm->pages = kvmalloc_array(ttm->num_pages, sizeof(void*), GFP_KERNEL | __GFP_ZERO); @@ -105,6 +102,7 @@ static int ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt *ttm) sizeof(*ttm->dma_address), GFP_KERNEL | __GFP_ZERO); ttm->dma_address = (void *) (ttm->ttm.pages + ttm->ttm.num_pages); + return 0; } static int ttm_sg_tt_alloc_page_directory(struct ttm_dma_tt *ttm) @@ -246,6 +244,7 @@ void ttm_tt_init_fields(struct ttm_tt *ttm, struct ttm_buffer_object *bo, ttm->page_flags = page_flags; ttm->state = tt_unpopulated; ttm->swap_storage = NULL; + ttm->sg = bo->sg; } int ttm_tt_init(struct ttm_tt *ttm, struct ttm_buffer_object *bo, @@ -253,8 +252,7 @@ int ttm_tt_init(struct ttm_tt *ttm, struct ttm_buffer_object *bo, { ttm_tt_init_fields(ttm, bo, page_flags); - ttm_tt_alloc_page_directory(ttm); - if (!ttm->pages) { + if (ttm_tt_alloc_page_directory(ttm)) { ttm_tt_destroy(ttm); pr_err("Failed allocating page table\n"); return -ENOMEM; @@ -278,8 +276,7 @@ int ttm_dma_tt_init(struct ttm_dma_tt *ttm_dma, struct ttm_buffer_object *bo, ttm_tt_init_fields(ttm, bo, page_flags); INIT_LIST_HEAD(&ttm_dma->pages_list); - ttm_dma_tt_alloc_page_directory(ttm_dma); - if (!ttm->pages) { + if (ttm_dma_tt_alloc_page_directory(ttm_dma)) { ttm_tt_destroy(ttm); pr_err("Failed allocating page table\n"); return -ENOMEM; @@ -314,7 +311,10 @@ void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma) { struct ttm_tt *ttm = &ttm_dma->ttm; + if (ttm->pages) kvfree(ttm->pages); + else + kvfree(ttm_dma->dma_address); ttm->pages = NULL; ttm_dma->dma_address = NULL; } @@ -342,7 +342,7 @@ int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem, if (ttm->state == tt_bound) return 0; - ret = ttm->bdev->driver->ttm_tt_populate(ttm, ctx); + ret = ttm_tt_populate(ttm, ctx); if (ret) return ret; @@ -371,7 +371,11 @@ int ttm_tt_swapin(struct ttm_tt *ttm) swap_space = swap_storage->f_mapping; for (i = 0; i < ttm->num_pages; ++i) { - from_page = shmem_read_mapping_page(swap_space, i); + gfp_t gfp_mask = mapping_gfp_mask(swap_space); + + gfp_mask |= (ttm->page_flags & TTM_PAGE_FLAG_NO_RETRY ? __GFP_RETRY_MAYFAIL : 0); + from_page = shmem_read_mapping_page_gfp(swap_space, i, gfp_mask); + if (IS_ERR(from_page)) { ret = PTR_ERR(from_page); goto out_err; @@ -414,16 +418,22 @@ int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistent_swap_storage) pr_err("Failed allocating swap storage\n"); return PTR_ERR(swap_storage); } - } else + } else { swap_storage = persistent_swap_storage; + } swap_space = swap_storage->f_mapping; for (i = 0; i < ttm->num_pages; ++i) { + gfp_t gfp_mask = mapping_gfp_mask(swap_space); + + gfp_mask |= (ttm->page_flags & TTM_PAGE_FLAG_NO_RETRY ? __GFP_RETRY_MAYFAIL : 0); + from_page = ttm->pages[i]; if (unlikely(from_page == NULL)) continue; - to_page = shmem_read_mapping_page(swap_space, i); + + to_page = shmem_read_mapping_page_gfp(swap_space, i, gfp_mask); if (IS_ERR(to_page)) { ret = PTR_ERR(to_page); goto out_err; @@ -448,6 +458,33 @@ int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistent_swap_storage) return ret; } +static void ttm_tt_add_mapping(struct ttm_tt *ttm) +{ + pgoff_t i; + + if (ttm->page_flags & TTM_PAGE_FLAG_SG) + return; + + for (i = 0; i < ttm->num_pages; ++i) + ttm->pages[i]->mapping = ttm->bdev->dev_mapping; +} + +int ttm_tt_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) +{ + int ret; + + if (ttm->state != tt_unpopulated) + return 0; + + if (ttm->bdev->driver->ttm_tt_populate) + ret = ttm->bdev->driver->ttm_tt_populate(ttm, ctx); + else + ret = ttm_pool_populate(ttm, ctx); + if (!ret) + ttm_tt_add_mapping(ttm); + return ret; +} + static void ttm_tt_clear_mapping(struct ttm_tt *ttm) { pgoff_t i; @@ -468,5 +505,8 @@ void ttm_tt_unpopulate(struct ttm_tt *ttm) return; ttm_tt_clear_mapping(ttm); - ttm->bdev->driver->ttm_tt_unpopulate(ttm); + if (ttm->bdev->driver->ttm_tt_unpopulate) + ttm->bdev->driver->ttm_tt_unpopulate(ttm); + else + ttm_pool_unpopulate(ttm); } diff --git a/drivers/gpu/drm/virtio/virtgpu_ttm.c b/drivers/gpu/drm/virtio/virtgpu_ttm.c index a9f2fb0..3848903 100644 --- a/drivers/gpu/drm/virtio/virtgpu_ttm.c +++ b/drivers/gpu/drm/virtio/virtgpu_ttm.c @@ -428,7 +428,6 @@ static struct ttm_bo_driver virtio_gpu_bo_driver = { .verify_access = &virtio_gpu_verify_access, .io_mem_reserve = &virtio_gpu_ttm_io_mem_reserve, .io_mem_free = &virtio_gpu_ttm_io_mem_free, - .io_mem_pfn = ttm_bo_default_io_mem_pfn, .move_notify = &virtio_gpu_bo_move_notify, .swap_notify = &virtio_gpu_bo_swap_notify, }; diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c index bab10e3..7177eec 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c @@ -860,5 +860,4 @@ struct ttm_bo_driver vmw_bo_driver = { .fault_reserve_notify = &vmw_ttm_fault_reserve_notify, .io_mem_reserve = &vmw_ttm_io_mem_reserve, .io_mem_free = &vmw_ttm_io_mem_free, - .io_mem_pfn = ttm_bo_default_io_mem_pfn, }; diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h index 2f71269..1822c4e 100644 --- a/include/drm/ttm/ttm_bo_driver.h +++ b/include/drm/ttm/ttm_bo_driver.h @@ -609,6 +609,17 @@ ttm_flag_masked(uint32_t *old, uint32_t new, uint32_t mask) } /** + * ttm_tt_create + * + * @bo: pointer to a struct ttm_buffer_object + * @zero_alloc: true if allocated pages needs to be zeroed + * + * Make sure we have a TTM structure allocated for the given BO. + * No pages are actually allocated. + */ +int ttm_tt_create(struct ttm_buffer_object *bo, bool zero_alloc); + +/** * ttm_tt_init * * @ttm: The struct ttm_tt. @@ -647,8 +658,8 @@ extern void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma); * * Bind the pages of @ttm to an aperture location identified by @bo_mem */ -extern int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem); - +extern int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem, + struct ttm_operation_ctx *ctx); /** * ttm_ttm_destroy: * @@ -694,6 +705,15 @@ extern int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistent_swap_storage); /** + * ttm_tt_populate - allocate pages for a ttm + * + * @ttm: Pointer to the ttm_tt structure + * + * Calls the driver method to allocate pages for a ttm + */ +extern int ttm_tt_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx); + +/** * ttm_tt_unpopulate - free pages from a ttm * * @ttm: Pointer to the ttm_tt structure @@ -823,24 +843,24 @@ static inline int __ttm_bo_reserve(struct ttm_buffer_object *bo, bool interruptible, bool no_wait, struct ww_acquire_ctx *ticket) { - int ret = 0; - - if (no_wait) { - bool success; - if (WARN_ON(ticket)) - return -EBUSY; - - success = ww_mutex_trylock(&bo->resv->lock); - return success ? 0 : -EBUSY; - } - - if (interruptible) - ret = ww_mutex_lock_interruptible(&bo->resv->lock, ticket); - else - ret = ww_mutex_lock(&bo->resv->lock, ticket); - if (ret == -EINTR) - return -ERESTARTSYS; - return ret; + int ret = 0; + + if (no_wait) { + bool success; + if (WARN_ON(ticket)) + return -EBUSY; + + success = reservation_object_trylock(bo->resv); + return success ? 0 : -EBUSY; + } + + if (interruptible) + ret = reservation_object_lock_interruptible(bo->resv, ticket); + else + ret = reservation_object_lock(bo->resv, ticket); + if (ret == -EINTR) + return -ERESTARTSYS; + return ret; } /** @@ -935,18 +955,6 @@ static inline int ttm_bo_reserve_slowpath(struct ttm_buffer_object *bo, } /** - * __ttm_bo_unreserve - * @bo: A pointer to a struct ttm_buffer_object. - * - * Unreserve a previous reservation of @bo where the buffer object is - * already on lru lists. - */ -static inline void __ttm_bo_unreserve(struct ttm_buffer_object *bo) -{ - ww_mutex_unlock(&bo->resv->lock); -} - -/** * ttm_bo_unreserve * * @bo: A pointer to a struct ttm_buffer_object. @@ -955,13 +963,12 @@ static inline void __ttm_bo_unreserve(struct ttm_buffer_object *bo) */ static inline void ttm_bo_unreserve(struct ttm_buffer_object *bo) { - if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { - spin_lock(&bo->bdev->glob->lru_lock); - ttm_bo_add_to_lru(bo); - spin_unlock(&bo->glob->lru_lock); - spin_unlock(&bo->bdev->glob->lru_lock); - } - __ttm_bo_unreserve(bo); + if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { + spin_lock(&bo->bdev->glob->lru_lock); + ttm_bo_add_to_lru(bo); + spin_unlock(&bo->bdev->glob->lru_lock); + } + reservation_object_unlock(bo->resv); } /** @@ -1113,7 +1120,7 @@ extern const struct ttm_mem_type_manager_func ttm_bo_manager_func; struct ttm_tt *ttm_agp_tt_create(struct ttm_buffer_object *bo, struct agp_bridge_data *bridge, uint32_t page_flags); -int ttm_agp_tt_populate(struct ttm_tt *ttm); +int ttm_agp_tt_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx); void ttm_agp_tt_unpopulate(struct ttm_tt *ttm); #endif diff --git a/include/drm/ttm/ttm_memory.h b/include/drm/ttm/ttm_memory.h index c8885bc9..bc8b22e 100644 --- a/include/drm/ttm/ttm_memory.h +++ b/include/drm/ttm/ttm_memory.h @@ -38,20 +38,6 @@ #include "ttm_bo_api.h" /** - * struct ttm_mem_shrink - callback to shrink TTM memory usage. - * - * @do_shrink: The callback function. - * - * Arguments to the do_shrink functions are intended to be passed using - * inheritance. That is, the argument class derives from struct ttm_mem_shrink, - * and can be accessed using container_of(). - */ - -struct ttm_mem_shrink { - int (*do_shrink) (struct ttm_mem_shrink *); -}; - -/** * struct ttm_mem_global - Global memory accounting structure. * * @shrink: A single callback to shrink TTM memory usage. Extend this @@ -94,59 +80,6 @@ struct ttm_mem_global { #endif }; -/** - * ttm_mem_init_shrink - initialize a struct ttm_mem_shrink object - * - * @shrink: The object to initialize. - * @func: The callback function. - */ - -static inline void ttm_mem_init_shrink(struct ttm_mem_shrink *shrink, - int (*func) (struct ttm_mem_shrink *)) -{ - shrink->do_shrink = func; -} - -/** - * ttm_mem_register_shrink - register a struct ttm_mem_shrink object. - * - * @glob: The struct ttm_mem_global object to register with. - * @shrink: An initialized struct ttm_mem_shrink object to register. - * - * Returns: - * -EBUSY: There's already a callback registered. (May change). - */ - -static inline int ttm_mem_register_shrink(struct ttm_mem_global *glob, - struct ttm_mem_shrink *shrink) -{ - spin_lock(&glob->lock); - if (glob->shrink != NULL) { - spin_unlock(&glob->lock); - return -EBUSY; - } - glob->shrink = shrink; - spin_unlock(&glob->lock); - return 0; -} - -/** - * ttm_mem_unregister_shrink - unregister a struct ttm_mem_shrink object. - * - * @glob: The struct ttm_mem_global object to unregister from. - * @shrink: A previously registert struct ttm_mem_shrink object. - * - */ - -static inline void ttm_mem_unregister_shrink(struct ttm_mem_global *glob, - struct ttm_mem_shrink *shrink) -{ - spin_lock(&glob->lock); - BUG_ON(glob->shrink != shrink); - glob->shrink = NULL; - spin_unlock(&glob->lock); -} - extern int ttm_mem_global_init(struct ttm_mem_global *glob); extern void ttm_mem_global_release(struct ttm_mem_global *glob); extern int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory, @@ -157,7 +90,7 @@ extern int ttm_mem_global_alloc_page(struct ttm_mem_global *glob, struct page *page, uint64_t size, struct ttm_operation_ctx *ctx); extern void ttm_mem_global_free_page(struct ttm_mem_global *glob, - struct page *page); + struct page *page, uint64_t size); extern size_t ttm_round_pot(size_t size); extern uint64_t ttm_get_kernel_zone_memory_size(struct ttm_mem_global *glob); extern bool ttm_check_under_lowerlimit(struct ttm_mem_global *glob, diff --git a/include/drm/ttm/ttm_page_alloc.h b/include/drm/ttm/ttm_page_alloc.h index 80aa53c..606a3ab 100644 --- a/include/drm/ttm/ttm_page_alloc.h +++ b/include/drm/ttm/ttm_page_alloc.h @@ -58,7 +58,7 @@ extern int ttm_pool_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx); */ extern void ttm_pool_unpopulate(struct ttm_tt *ttm); -extern +int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt, +extern int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt, struct ttm_operation_ctx *ctx); /** @@ -92,11 +92,6 @@ extern int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev, extern void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev); /** - * Populates and DMA maps pages to fullfil a ttm_dma_populate() request - */ -int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt); - -/** * Unpopulates and DMA unmaps pages as part of a * ttm_dma_unpopulate() request */ void ttm_unmap_and_unpopulate_pages(struct device *dev, struct ttm_dma_tt *tt); diff --git a/include/linux/reservation.h b/include/linux/reservation.h index 21fc84d..eb444ff 100644 --- a/include/linux/reservation.h +++ b/include/linux/reservation.h @@ -167,6 +167,28 @@ reservation_object_lock(struct reservation_object *obj, } /** + * reservation_object_lock_interruptible - lock the reservation object + * @obj: the reservation object + * @ctx: the locking context + * + * Locks the reservation object interruptible for exclusive access and + * modification. Note, that the lock is only against other writers, readers + * will run concurrently with a writer under RCU. The seqlock is used to + * notify readers if they overlap with a writer. + * + * As the reservation object may be locked by multiple parties in an + * undefined order, a #ww_acquire_ctx is passed to unwind if a cycle + * is detected. See ww_mutex_lock() and ww_acquire_init(). A reservation + * object may be locked by itself by passing NULL as @ctx. + */ +static inline int +reservation_object_lock_interruptible(struct reservation_object *obj, + struct ww_acquire_ctx *ctx) +{ + return ww_mutex_lock_interruptible(&obj->lock, ctx); +} + +/** * reservation_object_trylock - trylock the reservation object * @obj: the reservation object * -- 2.7.4