diff options
Diffstat (limited to 'meta-amd-bsp/recipes-kernel/linux/linux-yocto-4.14.71/3739-drm-ttm-ttm-code-modifications-to-fix-the.patch')
| -rw-r--r-- | meta-amd-bsp/recipes-kernel/linux/linux-yocto-4.14.71/3739-drm-ttm-ttm-code-modifications-to-fix-the.patch | 3640 |
1 files changed, 3640 insertions, 0 deletions
diff --git a/meta-amd-bsp/recipes-kernel/linux/linux-yocto-4.14.71/3739-drm-ttm-ttm-code-modifications-to-fix-the.patch b/meta-amd-bsp/recipes-kernel/linux/linux-yocto-4.14.71/3739-drm-ttm-ttm-code-modifications-to-fix-the.patch new file mode 100644 index 00000000..c3606598 --- /dev/null +++ b/meta-amd-bsp/recipes-kernel/linux/linux-yocto-4.14.71/3739-drm-ttm-ttm-code-modifications-to-fix-the.patch @@ -0,0 +1,3640 @@ +From 17f5622d02c6a20b55238692f556aa4caf5b63a9 Mon Sep 17 00:00:00 2001 +From: Kalyan Alle <kalyan.alle@amd.com> +Date: Mon, 23 Jul 2018 14:28:55 +0530 +Subject: [PATCH 3739/4131] drm/ttm: ttm code modifications to fix the + +build/compilation issues + +Signed-off-by: Kalyan Alle <kalyan.alle@amd.com> +--- + drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 7 +- + drivers/gpu/drm/ast/ast_ttm.c | 1 - + drivers/gpu/drm/cirrus/cirrus_ttm.c | 1 - + drivers/gpu/drm/mgag200/mgag200_ttm.c | 1 - + drivers/gpu/drm/nouveau/nouveau_bo.c | 10 +- + drivers/gpu/drm/nouveau/nouveau_gem.c | 2 +- + drivers/gpu/drm/qxl/qxl_release.c | 4 +- + drivers/gpu/drm/qxl/qxl_ttm.c | 4 +- + drivers/gpu/drm/radeon/radeon_ttm.c | 8 +- + drivers/gpu/drm/ttm/Makefile | 19 +- + drivers/gpu/drm/ttm/ttm_bo.c | 294 +++++++---------- + drivers/gpu/drm/ttm/ttm_bo_util.c | 52 ++- + drivers/gpu/drm/ttm/ttm_bo_vm.c | 61 ++-- + drivers/gpu/drm/ttm/ttm_execbuf_util.c | 28 +- + drivers/gpu/drm/ttm/ttm_memory.c | 143 +++++++-- + drivers/gpu/drm/ttm/ttm_module.c | 4 +- + drivers/gpu/drm/ttm/ttm_page_alloc.c | 531 ++++++++++++++++++++++--------- + drivers/gpu/drm/ttm/ttm_page_alloc_dma.c | 359 ++++++++++++++------- + drivers/gpu/drm/ttm/ttm_tt.c | 68 +++- + drivers/gpu/drm/virtio/virtgpu_ttm.c | 1 - + drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c | 1 - + include/drm/ttm/ttm_bo_driver.h | 87 ++--- + include/drm/ttm/ttm_memory.h | 69 +--- + include/drm/ttm/ttm_page_alloc.h | 7 +- + include/linux/reservation.h | 22 ++ + 25 files changed, 1100 insertions(+), 684 deletions(-) + +diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +index 548239c..2c4eed2 100644 +--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c ++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +@@ -537,7 +537,7 @@ static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, bool evict, + if (unlikely(r)) { + goto out_cleanup; + } +- r = ttm_bo_move_ttm(bo, ctx->interruptible, ctx->no_wait_gpu, new_mem); ++ r = ttm_bo_move_ttm(bo, ctx, new_mem); + out_cleanup: + ttm_bo_mem_put(bo, &tmp_mem); + return r; +@@ -568,7 +568,7 @@ static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, bool evict, + if (unlikely(r)) { + return r; + } +- r = ttm_bo_move_ttm(bo, ctx->interruptible, ctx->no_wait_gpu, &tmp_mem); ++ r = ttm_bo_move_ttm(bo, ctx, &tmp_mem); + if (unlikely(r)) { + goto out_cleanup; + } +@@ -630,8 +630,7 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict, + + if (r) { + memcpy: +- r = ttm_bo_move_memcpy(bo, ctx->interruptible, +- ctx->no_wait_gpu, new_mem); ++ r = ttm_bo_move_memcpy(bo, ctx, new_mem); + if (r) { + return r; + } +diff --git a/drivers/gpu/drm/ast/ast_ttm.c b/drivers/gpu/drm/ast/ast_ttm.c +index 10c36c9..846e4d8 100644 +--- a/drivers/gpu/drm/ast/ast_ttm.c ++++ b/drivers/gpu/drm/ast/ast_ttm.c +@@ -237,7 +237,6 @@ struct ttm_bo_driver ast_bo_driver = { + .verify_access = ast_bo_verify_access, + .io_mem_reserve = &ast_ttm_io_mem_reserve, + .io_mem_free = &ast_ttm_io_mem_free, +- .io_mem_pfn = ttm_bo_default_io_mem_pfn, + }; + + int ast_mm_init(struct ast_private *ast) +diff --git a/drivers/gpu/drm/cirrus/cirrus_ttm.c b/drivers/gpu/drm/cirrus/cirrus_ttm.c +index b40e294..4a9c232 100644 +--- a/drivers/gpu/drm/cirrus/cirrus_ttm.c ++++ b/drivers/gpu/drm/cirrus/cirrus_ttm.c +@@ -237,7 +237,6 @@ struct ttm_bo_driver cirrus_bo_driver = { + .verify_access = cirrus_bo_verify_access, + .io_mem_reserve = &cirrus_ttm_io_mem_reserve, + .io_mem_free = &cirrus_ttm_io_mem_free, +- .io_mem_pfn = ttm_bo_default_io_mem_pfn, + }; + + int cirrus_mm_init(struct cirrus_device *cirrus) +diff --git a/drivers/gpu/drm/mgag200/mgag200_ttm.c b/drivers/gpu/drm/mgag200/mgag200_ttm.c +index 85e39c3..882e902 100644 +--- a/drivers/gpu/drm/mgag200/mgag200_ttm.c ++++ b/drivers/gpu/drm/mgag200/mgag200_ttm.c +@@ -237,7 +237,6 @@ struct ttm_bo_driver mgag200_bo_driver = { + .verify_access = mgag200_bo_verify_access, + .io_mem_reserve = &mgag200_ttm_io_mem_reserve, + .io_mem_free = &mgag200_ttm_io_mem_free, +- .io_mem_pfn = ttm_bo_default_io_mem_pfn, + }; + + int mgag200_mm_init(struct mga_device *mdev) +diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c +index d25ccd4..47827a4 100644 +--- a/drivers/gpu/drm/nouveau/nouveau_bo.c ++++ b/drivers/gpu/drm/nouveau/nouveau_bo.c +@@ -1154,7 +1154,7 @@ nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr, + if (ret) + goto out; + +- ret = ttm_bo_move_ttm(bo, intr, no_wait_gpu, new_reg); ++ ret = ttm_bo_move_ttm(bo, &ctx, new_reg); + out: + ttm_bo_mem_put(bo, &tmp_reg); + return ret; +@@ -1183,7 +1183,7 @@ nouveau_bo_move_flips(struct ttm_buffer_object *bo, bool evict, bool intr, + if (ret) + return ret; + +- ret = ttm_bo_move_ttm(bo, intr, no_wait_gpu, &tmp_reg); ++ ret = ttm_bo_move_ttm(bo, &ctx, &tmp_reg); + if (ret) + goto out; + +@@ -1307,8 +1307,7 @@ nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, + /* Fallback to software copy. */ + ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu); + if (ret == 0) +- ret = ttm_bo_move_memcpy(bo, ctx->interruptible, +- ctx->no_wait_gpu, new_reg); ++ ret = ttm_bo_move_memcpy(bo, ctx, new_reg); + + out: + if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { +@@ -1446,7 +1445,7 @@ nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo) + } + + static int +-nouveau_ttm_tt_populate(struct ttm_tt *ttm) ++nouveau_ttm_tt_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) + { + struct ttm_dma_tt *ttm_dma = (void *)ttm; + struct nouveau_drm *drm; +@@ -1579,7 +1578,6 @@ struct ttm_bo_driver nouveau_bo_driver = { + .fault_reserve_notify = &nouveau_ttm_fault_reserve_notify, + .io_mem_reserve = &nouveau_ttm_io_mem_reserve, + .io_mem_free = &nouveau_ttm_io_mem_free, +- .io_mem_pfn = ttm_bo_default_io_mem_pfn, + }; + + struct nvkm_vma * +diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c +index 60ffb70..75ec8b1 100644 +--- a/drivers/gpu/drm/nouveau/nouveau_gem.c ++++ b/drivers/gpu/drm/nouveau/nouveau_gem.c +@@ -349,7 +349,7 @@ validate_fini_no_ticket(struct validate_op *op, struct nouveau_fence *fence, + + list_del(&nvbo->entry); + nvbo->reserved_by = NULL; +- ttm_bo_unreserve_ticket(&nvbo->bo, &op->ticket); ++ ttm_bo_unreserve(&nvbo->bo); + drm_gem_object_unreference_unlocked(&nvbo->gem); + } + } +diff --git a/drivers/gpu/drm/qxl/qxl_release.c b/drivers/gpu/drm/qxl/qxl_release.c +index 1f90d0e..c010d9c 100644 +--- a/drivers/gpu/drm/qxl/qxl_release.c ++++ b/drivers/gpu/drm/qxl/qxl_release.c +@@ -459,7 +459,7 @@ void qxl_release_fence_buffer_objects(struct qxl_release *release) + trace_dma_fence_emit(&release->base); + + driver = bdev->driver; +- glob = bo->glob; ++ glob = bdev->glob; + + spin_lock(&glob->lru_lock); + +@@ -469,7 +469,7 @@ void qxl_release_fence_buffer_objects(struct qxl_release *release) + + reservation_object_add_shared_fence(bo->resv, &release->base); + ttm_bo_add_to_lru(bo); +- __ttm_bo_unreserve(bo); ++ reservation_object_unlock(bo->resv); + } + spin_unlock(&glob->lru_lock); + ww_acquire_fini(&release->ticket); +diff --git a/drivers/gpu/drm/qxl/qxl_ttm.c b/drivers/gpu/drm/qxl/qxl_ttm.c +index a283352..0944f7c 100644 +--- a/drivers/gpu/drm/qxl/qxl_ttm.c ++++ b/drivers/gpu/drm/qxl/qxl_ttm.c +@@ -359,8 +359,7 @@ static int qxl_bo_move(struct ttm_buffer_object *bo, bool evict, + qxl_move_null(bo, new_mem); + return 0; + } +- return ttm_bo_move_memcpy(bo, ctx->interruptible, ctx->no_wait_gpu, +- new_mem); ++ return ttm_bo_move_memcpy(bo, ctx, new_mem); + } + + static void qxl_bo_move_notify(struct ttm_buffer_object *bo, +@@ -391,7 +390,6 @@ static struct ttm_bo_driver qxl_bo_driver = { + .verify_access = &qxl_verify_access, + .io_mem_reserve = &qxl_ttm_io_mem_reserve, + .io_mem_free = &qxl_ttm_io_mem_free, +- .io_mem_pfn = ttm_bo_default_io_mem_pfn, + .move_notify = &qxl_bo_move_notify, + }; + +diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c +index e651589..801249d 100644 +--- a/drivers/gpu/drm/radeon/radeon_ttm.c ++++ b/drivers/gpu/drm/radeon/radeon_ttm.c +@@ -347,7 +347,7 @@ static int radeon_move_vram_ram(struct ttm_buffer_object *bo, + if (unlikely(r)) { + goto out_cleanup; + } +- r = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, new_mem); ++ r = ttm_bo_move_ttm(bo, &ctx, new_mem); + out_cleanup: + ttm_bo_mem_put(bo, &tmp_mem); + return r; +@@ -380,7 +380,7 @@ static int radeon_move_ram_vram(struct ttm_buffer_object *bo, + if (unlikely(r)) { + return r; + } +- r = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, &tmp_mem); ++ r = ttm_bo_move_ttm(bo, &ctx, &tmp_mem); + if (unlikely(r)) { + goto out_cleanup; + } +@@ -445,8 +445,7 @@ static int radeon_bo_move(struct ttm_buffer_object *bo, bool evict, + + if (r) { + memcpy: +- r = ttm_bo_move_memcpy(bo, ctx->interruptible, +- ctx->no_wait_gpu, new_mem); ++ r = ttm_bo_move_memcpy(bo, ctx, new_mem); + if (r) { + return r; + } +@@ -845,7 +844,6 @@ static struct ttm_bo_driver radeon_bo_driver = { + .fault_reserve_notify = &radeon_bo_fault_reserve_notify, + .io_mem_reserve = &radeon_ttm_io_mem_reserve, + .io_mem_free = &radeon_ttm_io_mem_free, +- .io_mem_pfn = ttm_bo_default_io_mem_pfn, + }; + + int radeon_ttm_init(struct radeon_device *rdev) +diff --git a/drivers/gpu/drm/ttm/Makefile b/drivers/gpu/drm/ttm/Makefile +index 7c57dac..e2c3ed2 100644 +--- a/drivers/gpu/drm/ttm/Makefile ++++ b/drivers/gpu/drm/ttm/Makefile +@@ -2,11 +2,16 @@ + # + # Makefile for the drm device driver. This driver provides support for the + +-ccflags-y := -I$(src)/. +-ttm-y := ttm_memory.o ttm_tt.o ttm_bo.o \ +- ttm_bo_util.o ttm_bo_vm.o ttm_module.o \ +- ttm_object.o ttm_lock.o ttm_execbuf_util.o ttm_page_alloc.o \ +- ttm_bo_manager.o ttm_page_alloc_dma.o ttm_debug.o ttm_tracepoints.o +-ttm-$(CONFIG_AGP) += ttm_agp_backend.o ++ifneq (,$(BUILD_AS_DKMS)) ++ TTM_NAME = amdttm ++else ++ TTM_NAME = ttm ++endif + +-obj-$(CONFIG_DRM_TTM) += ttm.o ++$(TTM_NAME)-y := ttm_memory.o ttm_tt.o ttm_bo.o \ ++ ttm_bo_util.o ttm_bo_vm.o ttm_module.o \ ++ ttm_object.o ttm_lock.o ttm_execbuf_util.o ttm_page_alloc.o \ ++ ttm_bo_manager.o ttm_page_alloc_dma.o ++$(TTM_NAME)-$(CONFIG_AGP) += ttm_agp_backend.o ++ ++obj-$(CONFIG_DRM_TTM) += $(TTM_NAME).o +diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c +index cf3aef0..7f0f16b 100644 +--- a/drivers/gpu/drm/ttm/ttm_bo.c ++++ b/drivers/gpu/drm/ttm/ttm_bo.c +@@ -28,6 +28,7 @@ + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + */ + ++#undef pr_fmt + #define pr_fmt(fmt) "[TTM] " fmt + + #include <drm/ttm/ttm_module.h> +@@ -42,11 +43,6 @@ + #include <linux/atomic.h> + #include <linux/reservation.h> + +-#define TTM_ASSERT_LOCKED(param) +-#define TTM_DEBUG(fmt, arg...) +-#define TTM_BO_HASH_ORDER 13 +- +-static int ttm_bo_swapout(struct ttm_mem_shrink *shrink); + static void ttm_bo_global_kobj_release(struct kobject *kobj); + + static struct attribute ttm_bo_count = { +@@ -54,6 +50,12 @@ static struct attribute ttm_bo_count = { + .mode = S_IRUGO + }; + ++/* default destructor */ ++static void ttm_bo_default_destroy(struct ttm_buffer_object *bo) ++{ ++ kfree(bo); ++} ++ + static inline int ttm_mem_type_from_place(const struct ttm_place *place, + uint32_t *mem_type) + { +@@ -148,15 +150,11 @@ static void ttm_bo_release_list(struct kref *list_kref) + BUG_ON(!list_empty(&bo->lru)); + BUG_ON(!list_empty(&bo->ddestroy)); + ttm_tt_destroy(bo->ttm); +- atomic_dec(&bo->glob->bo_count); ++ atomic_dec(&bo->bdev->glob->bo_count); + dma_fence_put(bo->moving); + reservation_object_fini(&bo->ttm_resv); + mutex_destroy(&bo->wu_mutex); +- if (bo->destroy) +- bo->destroy(bo); +- else { +- kfree(bo); +- } ++ bo->destroy(bo); + ttm_mem_global_free(bdev->glob->mem_glob, acc_size); + } + +@@ -165,7 +163,7 @@ void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_mem_type_manager *man; + +- lockdep_assert_held(&bo->resv->lock.base); ++ reservation_object_assert_held(bo->resv); + + if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { + +@@ -175,10 +173,10 @@ void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) + list_add_tail(&bo->lru, &man->lru[bo->priority]); + kref_get(&bo->list_kref); + +- if (bo->ttm && !(bo->ttm->page_flags & +- (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED))) { ++ if (bo->ttm && !(bo->ttm->page_flags & ++ (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED))) { + list_add_tail(&bo->swap, +- &bo->glob->swap_lru[bo->priority]); ++ &bdev->glob->swap_lru[bo->priority]); + kref_get(&bo->list_kref); + } + } +@@ -209,63 +207,23 @@ void ttm_bo_del_from_lru(struct ttm_buffer_object *bo) + + void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo) + { +- spin_lock(&bo->glob->lru_lock); ++ struct ttm_bo_global *glob = bo->bdev->glob; ++ ++ spin_lock(&glob->lru_lock); + ttm_bo_del_from_lru(bo); +- spin_unlock(&bo->glob->lru_lock); ++ spin_unlock(&glob->lru_lock); + } + EXPORT_SYMBOL(ttm_bo_del_sub_from_lru); + + void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo) + { +- lockdep_assert_held(&bo->resv->lock.base); ++ reservation_object_assert_held(bo->resv); + + ttm_bo_del_from_lru(bo); + ttm_bo_add_to_lru(bo); + } + EXPORT_SYMBOL(ttm_bo_move_to_lru_tail); + +-/* +- * Call bo->mutex locked. +- */ +-static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) +-{ +- struct ttm_bo_device *bdev = bo->bdev; +- int ret = 0; +- uint32_t page_flags = 0; +- +- TTM_ASSERT_LOCKED(&bo->mutex); +- bo->ttm = NULL; +- +- if (bdev->need_dma32) +- page_flags |= TTM_PAGE_FLAG_DMA32; +- +- switch (bo->type) { +- case ttm_bo_type_device: +- if (zero_alloc) +- page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC; +- case ttm_bo_type_kernel: +- bo->ttm = bdev->driver->ttm_tt_create(bo, page_flags); +- if (unlikely(bo->ttm == NULL)) +- ret = -ENOMEM; +- break; +- case ttm_bo_type_sg: +- bo->ttm = bdev->driver->ttm_tt_create(bo, +- page_flags | TTM_PAGE_FLAG_SG); +- if (unlikely(bo->ttm == NULL)) { +- ret = -ENOMEM; +- break; +- } +- bo->ttm->sg = bo->sg; +- break; +- default: +- pr_err("Illegal buffer object type\n"); +- ret = -EINVAL; +- break; +- } +- +- return ret; +-} +- + static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, + struct ttm_mem_reg *mem, bool evict, + struct ttm_operation_ctx *ctx) +@@ -293,7 +251,7 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, + if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) { + if (bo->ttm == NULL) { + bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED); +- ret = ttm_bo_add_ttm(bo, zero); ++ ret = ttm_tt_create(bo, zero); + if (ret) + goto out_err; + } +@@ -322,14 +280,12 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, + + if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && + !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) +- ret = ttm_bo_move_ttm(bo, ctx->interruptible, +- ctx->no_wait_gpu, mem); ++ ret = ttm_bo_move_ttm(bo, ctx, mem); + else if (bdev->driver->move) + ret = bdev->driver->move(bo, evict, ctx, mem); + else +- ret = ttm_bo_move_memcpy(bo, ctx->interruptible, +- ctx->no_wait_gpu, mem); +- ++ ret = ttm_bo_move_memcpy(bo, ctx, mem); ++ + if (ret) { + if (bdev->driver->move_notify) { + struct ttm_mem_reg tmp_mem = *mem; +@@ -353,13 +309,12 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, + bo->evicted = false; + } + +- if (bo->mem.mm_node) { ++ if (bo->mem.mm_node) + bo->offset = (bo->mem.start << PAGE_SHIFT) + + bdev->man[bo->mem.mem_type].gpu_offset; +- bo->cur_placement = bo->mem.placement; +- } else ++ else + bo->offset = 0; +- ++ + ctx->bytes_moved += bo->num_pages << PAGE_SHIFT; + return 0; + +@@ -389,8 +344,6 @@ static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) + ttm_tt_destroy(bo->ttm); + bo->ttm = NULL; + ttm_bo_mem_put(bo, &bo->mem); +- +- ww_mutex_unlock (&bo->resv->lock); + } + + static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo) +@@ -432,7 +385,7 @@ static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) + static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) + { + struct ttm_bo_device *bdev = bo->bdev; +- struct ttm_bo_global *glob = bo->glob; ++ struct ttm_bo_global *glob = bdev->glob; + int ret; + + ret = ttm_bo_individualize_resv(bo); +@@ -447,7 +400,7 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) + } + + spin_lock(&glob->lru_lock); +- ret = __ttm_bo_reserve(bo, false, true, NULL); ++ ret = reservation_object_trylock(bo->resv) ? 0 : -EBUSY; + if (!ret) { + if (reservation_object_test_signaled_rcu(&bo->ttm_resv, true)) { + ttm_bo_del_from_lru(bo); +@@ -456,6 +409,7 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) + reservation_object_unlock(&bo->ttm_resv); + + ttm_bo_cleanup_memtype_use(bo); ++ reservation_object_unlock(bo->resv); + return; + } + +@@ -471,8 +425,8 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) + ttm_bo_add_to_lru(bo); + } + +- __ttm_bo_unreserve(bo); +- } ++ reservation_object_unlock(bo->resv); ++ } + if (bo->resv != &bo->ttm_resv) + reservation_object_unlock(&bo->ttm_resv); + +@@ -497,11 +451,11 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) + * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. + */ + +-static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, +- bool interruptible, +- bool no_wait_gpu) ++static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, ++ bool interruptible, bool no_wait_gpu, ++ bool unlock_resv) + { +- struct ttm_bo_global *glob = bo->glob; ++ struct ttm_bo_global *glob = bo->bdev->glob; + struct reservation_object *resv; + int ret; + +@@ -517,8 +471,10 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, + + if (ret && !no_wait_gpu) { + long lret; +- ww_mutex_unlock(&bo->resv->lock); +- spin_unlock(&glob->lru_lock); ++ ++ if (unlock_resv) ++ reservation_object_unlock(bo->resv); ++ spin_unlock(&glob->lru_lock); + + lret = reservation_object_wait_timeout_rcu(resv, true, + interruptible, +@@ -530,7 +486,7 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, + return -EBUSY; + + spin_lock(&glob->lru_lock); +- ret = __ttm_bo_reserve(bo, false, true, NULL); ++ if (unlock_resv && !reservation_object_trylock(bo->resv)) { + + /* + * We raced, and lost, someone else holds the reservation now, +@@ -540,14 +496,15 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, + * delayed destruction would succeed, so just return success + * here. + */ +- if (ret) { +- spin_unlock(&glob->lru_lock); ++ spin_unlock(&glob->lru_lock); + return 0; + } ++ ret = 0; + } + + if (ret || unlikely(list_empty(&bo->ddestroy))) { +- __ttm_bo_unreserve(bo); ++ if (unlock_resv) ++ reservation_object_unlock(bo->resv); + spin_unlock(&glob->lru_lock); + return ret; + } +@@ -559,6 +516,9 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, + spin_unlock(&glob->lru_lock); + ttm_bo_cleanup_memtype_use(bo); + ++ if (unlock_resv) ++ reservation_object_unlock(bo->resv); ++ + return 0; + } + +@@ -566,60 +526,44 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, + * Traverse the delayed list, and call ttm_bo_cleanup_refs on all + * encountered buffers. + */ +- +-static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) ++static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) + { + struct ttm_bo_global *glob = bdev->glob; +- struct ttm_buffer_object *entry = NULL; +- int ret = 0; ++ struct list_head removed; ++ bool empty; + +- spin_lock(&glob->lru_lock); +- if (list_empty(&bdev->ddestroy)) +- goto out_unlock; ++ INIT_LIST_HEAD(&removed); + +- entry = list_first_entry(&bdev->ddestroy, +- struct ttm_buffer_object, ddestroy); +- kref_get(&entry->list_kref); +- +- for (;;) { +- struct ttm_buffer_object *nentry = NULL; ++ spin_lock(&glob->lru_lock); ++ while (!list_empty(&bdev->ddestroy)) { ++ struct ttm_buffer_object *bo; + +- if (entry->ddestroy.next != &bdev->ddestroy) { +- nentry = list_first_entry(&entry->ddestroy, +- struct ttm_buffer_object, ddestroy); +- kref_get(&nentry->list_kref); +- } ++ bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object, ++ ddestroy); ++ kref_get(&bo->list_kref); ++ list_move_tail(&bo->ddestroy, &removed); + +- ret = __ttm_bo_reserve(entry, false, true, NULL); +- if (remove_all && ret) { ++ if (remove_all || bo->resv != &bo->ttm_resv) { + spin_unlock(&glob->lru_lock); +- ret = __ttm_bo_reserve(entry, false, false, NULL); ++ reservation_object_lock(bo->resv, NULL); ++ + spin_lock(&glob->lru_lock); +- } ++ ttm_bo_cleanup_refs(bo, false, !remove_all, true); + +- if (!ret) +- ret = ttm_bo_cleanup_refs_and_unlock(entry, false, +- !remove_all); +- else ++ } else if (reservation_object_trylock(bo->resv)) { ++ ttm_bo_cleanup_refs(bo, false, !remove_all, true); ++ } else { + spin_unlock(&glob->lru_lock); ++ } + +- kref_put(&entry->list_kref, ttm_bo_release_list); +- entry = nentry; +- +- if (ret || !entry) +- goto out; +- ++ kref_put(&bo->list_kref, ttm_bo_release_list); + spin_lock(&glob->lru_lock); +- if (list_empty(&entry->ddestroy)) +- break; + } +- +-out_unlock: ++ list_splice_tail(&removed, &bdev->ddestroy); ++ empty = list_empty(&bdev->ddestroy); + spin_unlock(&glob->lru_lock); +-out: +- if (entry) +- kref_put(&entry->list_kref, ttm_bo_release_list); +- return ret; ++ ++ return empty; + } + + static void ttm_bo_delayed_workqueue(struct work_struct *work) +@@ -627,10 +571,9 @@ static void ttm_bo_delayed_workqueue(struct work_struct *work) + struct ttm_bo_device *bdev = + container_of(work, struct ttm_bo_device, wq.work); + +- if (ttm_bo_delayed_delete(bdev, false)) { ++ if (!ttm_bo_delayed_delete(bdev, false)) + schedule_delayed_work(&bdev->wq, + ((HZ / 100) < 1) ? 1 : HZ / 100); +- } + } + + static void ttm_bo_release(struct kref *kref) +@@ -679,16 +622,25 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, + struct ttm_placement placement; + int ret = 0; + +- lockdep_assert_held(&bo->resv->lock.base); ++ reservation_object_assert_held(bo->resv); ++ ++ placement.num_placement = 0; ++ placement.num_busy_placement = 0; ++ bdev->driver->evict_flags(bo, &placement); ++ ++ if (!placement.num_placement && !placement.num_busy_placement) { ++ ret = ttm_bo_pipeline_gutting(bo); ++ if (ret) ++ return ret; ++ ++ return ttm_tt_create(bo, false); ++ } + + evict_mem = bo->mem; + evict_mem.mm_node = NULL; + evict_mem.bus.io_reserved_vm = false; + evict_mem.bus.io_reserved_count = 0; + +- placement.num_placement = 0; +- placement.num_busy_placement = 0; +- bdev->driver->evict_flags(bo, &placement); + ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx); + if (ret) { + if (ret != -ERESTARTSYS) { +@@ -743,7 +695,8 @@ static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, + *locked = false; + if (bo->resv == ctx->resv) { + reservation_object_assert_held(bo->resv); +- if (ctx->allow_reserved_eviction || !list_empty(&bo->ddestroy)) ++ if (ctx->flags & TTM_OPT_FLAG_ALLOW_RES_EVICT ++ || !list_empty(&bo->ddestroy)) + ret = true; + } else { + *locked = reservation_object_trylock(bo->resv); +@@ -876,7 +829,7 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, + return ret; + if (mem->mm_node) + break; +- ret = ttm_mem_evict_first(bdev, bo->resv, mem_type, place, ctx); ++ ret = ttm_mem_evict_first(bdev, mem_type, place, ctx); + if (unlikely(ret != 0)) + return ret; + } while (1); +@@ -1052,14 +1005,13 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, + EXPORT_SYMBOL(ttm_bo_mem_space); + + static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, +- struct ttm_placement *placement, +- struct ttm_placement *placement, +- struct ttm_operation_ctx *ctx) ++ struct ttm_placement *placement, ++ struct ttm_operation_ctx *ctx) + { + int ret = 0; + struct ttm_mem_reg mem; +- +- lockdep_assert_held(&bo->resv->lock.base); ++ ++ reservation_object_assert_held(bo->resv); + + mem.num_pages = bo->num_pages; + mem.size = mem.num_pages << PAGE_SHIFT; +@@ -1129,7 +1081,7 @@ int ttm_bo_validate(struct ttm_buffer_object *bo, + int ret; + uint32_t new_flags; + +- lockdep_assert_held(&bo->resv->lock.base); ++ reservation_object_assert_held(bo->resv); + /* + * Check whether we need to move buffer. + */ +@@ -1149,7 +1101,7 @@ int ttm_bo_validate(struct ttm_buffer_object *bo, + * We might need to add a TTM. + */ + if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { +- ret = ttm_bo_add_ttm(bo, true); ++ ret = ttm_tt_create(bo, true); + if (ret) + return ret; + } +@@ -1194,7 +1146,7 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, + ttm_mem_global_free(mem_glob, acc_size); + return -EINVAL; + } +- bo->destroy = destroy; ++ bo->destroy = destroy ? destroy : ttm_bo_default_destroy; + + kref_init(&bo->kref); + kref_init(&bo->list_kref); +@@ -1205,7 +1157,6 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, + INIT_LIST_HEAD(&bo->io_reserve_lru); + mutex_init(&bo->wu_mutex); + bo->bdev = bdev; +- bo->glob = bdev->glob; + bo->type = type; + bo->num_pages = num_pages; + bo->mem.size = num_pages << PAGE_SHIFT; +@@ -1219,14 +1170,15 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, + bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); + bo->acc_size = acc_size; + bo->sg = sg; ++ bo->ssg_can_map = false; + if (resv) { + bo->resv = resv; +- lockdep_assert_held(&bo->resv->lock.base); ++ reservation_object_assert_held(bo->resv); + } else { + bo->resv = &bo->ttm_resv; + } + reservation_object_init(&bo->ttm_resv); +- atomic_inc(&bo->glob->bo_count); ++ atomic_inc(&bo->bdev->glob->bo_count); + drm_vma_node_reset(&bo->vma_node); + bo->priority = 0; + +@@ -1243,15 +1195,12 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, + * since otherwise lockdep will be angered in radeon. + */ + if (!resv) { +- locked = ww_mutex_trylock(&bo->resv->lock); ++ locked = reservation_object_trylock(bo->resv); + WARN_ON(!locked); + } + +- if (likely(!ret)) { +- struct ttm_operation_ctx ctx = { interruptible, false }; +- +- ret = ttm_bo_validate(bo, placement, &ctx); +- } ++ if (likely(!ret)) ++ ret = ttm_bo_validate(bo, placement, ctx); + + if (unlikely(ret)) { + if (!resv) +@@ -1262,9 +1211,9 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, + } + + if (resv && !(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { +- spin_lock(&bo->glob->lru_lock); ++ spin_lock(&bdev->glob->lru_lock); + ttm_bo_add_to_lru(bo); +- spin_unlock(&bo->glob->lru_lock); ++ spin_unlock(&bdev->glob->lru_lock); + } + + return ret; +@@ -1357,7 +1306,11 @@ EXPORT_SYMBOL(ttm_bo_create); + static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, + unsigned mem_type) + { +- struct ttm_operation_ctx ctx = { false, false }; ++ struct ttm_operation_ctx ctx = { ++ .interruptible = false, ++ .no_wait_gpu = false, ++ .flags = TTM_OPT_FLAG_FORCE_ALLOC ++ }; + struct ttm_mem_type_manager *man = &bdev->man[mem_type]; + struct ttm_bo_global *glob = bdev->glob; + struct dma_fence *fence; +@@ -1372,8 +1325,7 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, + for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { + while (!list_empty(&man->lru[i])) { + spin_unlock(&glob->lru_lock); +- ret = ttm_mem_evict_first(bdev, NULL, mem_type, +- NULL, &ctx); ++ ret = ttm_mem_evict_first(bdev, mem_type, NULL, &ctx); + if (ret) + return ret; + spin_lock(&glob->lru_lock); +@@ -1494,7 +1446,6 @@ static void ttm_bo_global_kobj_release(struct kobject *kobj) + struct ttm_bo_global *glob = + container_of(kobj, struct ttm_bo_global, kobj); + +- ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink); + __free_page(glob->dummy_read_page); + kfree(glob); + } +@@ -1519,6 +1470,7 @@ int ttm_bo_global_init(struct drm_global_reference *ref) + mutex_init(&glob->device_list_mutex); + spin_lock_init(&glob->lru_lock); + glob->mem_glob = bo_ref->mem_glob; ++ glob->mem_glob->bo_glob = glob; + glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32); + + if (unlikely(glob->dummy_read_page == NULL)) { +@@ -1529,14 +1481,6 @@ int ttm_bo_global_init(struct drm_global_reference *ref) + for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) + INIT_LIST_HEAD(&glob->swap_lru[i]); + INIT_LIST_HEAD(&glob->device_list); +- +- ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout); +- ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink); +- if (unlikely(ret != 0)) { +- pr_err("Could not register buffer object swapout\n"); +- goto out_no_shrink; +- } +- + atomic_set(&glob->bo_count, 0); + + ret = kobject_init_and_add( +@@ -1544,8 +1488,6 @@ int ttm_bo_global_init(struct drm_global_reference *ref) + if (unlikely(ret != 0)) + kobject_put(&glob->kobj); + return ret; +-out_no_shrink: +- __free_page(glob->dummy_read_page); + out_no_drp: + kfree(glob); + return ret; +@@ -1579,16 +1521,13 @@ int ttm_bo_device_release(struct ttm_bo_device *bdev) + + cancel_delayed_work_sync(&bdev->wq); + +- while (ttm_bo_delayed_delete(bdev, true)) +- ; ++ if (ttm_bo_delayed_delete(bdev, true)) ++ pr_debug("Delayed destroy list was clean\n"); + + spin_lock(&glob->lru_lock); +- if (list_empty(&bdev->ddestroy)) +- TTM_DEBUG("Delayed destroy list was clean\n"); +- + for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) + if (list_empty(&bdev->man[0].lru[0])) +- TTM_DEBUG("Swap list %d was clean\n", i); ++ pr_debug("Swap list %d was clean\n", i); + spin_unlock(&glob->lru_lock); + + drm_vma_offset_manager_destroy(&bdev->vma_manager); +@@ -1731,7 +1670,6 @@ EXPORT_SYMBOL(ttm_bo_synccpu_write_release); + * A buffer object shrink method that tries to swap out the first + * buffer object on the bo_global::swap_lru list. + */ +- + int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx) + { + struct ttm_buffer_object *bo; +@@ -1813,7 +1751,7 @@ int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx) + * already swapped buffer. + */ + if (locked) +- kcl_reservation_object_unlock(bo->resv); ++ reservation_object_unlock(bo->resv); + kref_put(&bo->list_kref, ttm_bo_release_list); + return ret; + } +@@ -1853,10 +1791,12 @@ int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo) + return -ERESTARTSYS; + if (!ww_mutex_is_locked(&bo->resv->lock)) + goto out_unlock; +- ret = __ttm_bo_reserve(bo, true, false, NULL); ++ ret = reservation_object_lock_interruptible(bo->resv, NULL); ++ if (ret == -EINTR) ++ ret = -ERESTARTSYS; + if (unlikely(ret != 0)) + goto out_unlock; +- __ttm_bo_unreserve(bo); ++ reservation_object_unlock(bo->resv); + + out_unlock: + mutex_unlock(&bo->wu_mutex); +diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c +index 4ef7a59..de9161f 100644 +--- a/drivers/gpu/drm/ttm/ttm_bo_util.c ++++ b/drivers/gpu/drm/ttm/ttm_bo_util.c +@@ -45,7 +45,7 @@ void ttm_bo_free_old_node(struct ttm_buffer_object *bo) + } + + int ttm_bo_move_ttm(struct ttm_buffer_object *bo, +- bool interruptible, bool no_wait_gpu, ++ struct ttm_operation_ctx *ctx, + struct ttm_mem_reg *new_mem) + { + struct ttm_tt *ttm = bo->ttm; +@@ -53,7 +53,7 @@ int ttm_bo_move_ttm(struct ttm_buffer_object *bo, + int ret; + + if (old_mem->mem_type != TTM_PL_SYSTEM) { +- ret = ttm_bo_wait(bo, interruptible, no_wait_gpu); ++ ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu); + + if (unlikely(ret != 0)) { + if (ret != -ERESTARTSYS) +@@ -329,7 +329,7 @@ static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, + } + + int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, +- bool interruptible, bool no_wait_gpu, ++ struct ttm_operation_ctx *ctx, + struct ttm_mem_reg *new_mem) + { + struct ttm_bo_device *bdev = bo->bdev; +@@ -345,7 +345,7 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, + unsigned long add = 0; + int dir; + +- ret = ttm_bo_wait(bo, interruptible, no_wait_gpu); ++ ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu); + if (ret) + return ret; + +@@ -375,8 +375,8 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, + /* + * TTM might be null for moves within the same region. + */ +- if (ttm && ttm->state == tt_unpopulated) { +- ret = ttm->bdev->driver->ttm_tt_populate(ttm, ctx); ++ if (ttm) { ++ ret = ttm_tt_populate(ttm, ctx); + if (ret) + goto out1; + } +@@ -402,8 +402,9 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, + PAGE_KERNEL); + ret = ttm_copy_io_ttm_page(ttm, old_iomap, page, + prot); +- } else ++ } else { + ret = ttm_copy_io_page(new_iomap, old_iomap, page); ++ } + if (ret) + goto out1; + } +@@ -469,7 +470,7 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, + * TODO: Explicit member copy would probably be better here. + */ + +- atomic_inc(&bo->glob->bo_count); ++ atomic_inc(&bo->bdev->glob->bo_count); + INIT_LIST_HEAD(&fbo->ddestroy); + INIT_LIST_HEAD(&fbo->lru); + INIT_LIST_HEAD(&fbo->swap); +@@ -485,7 +486,7 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, + fbo->acc_size = 0; + fbo->resv = &fbo->ttm_resv; + reservation_object_init(fbo->resv); +- ret = ww_mutex_trylock(&fbo->resv->lock); ++ ret = reservation_object_trylock(fbo->resv); + WARN_ON(!ret); + + *new_obj = fbo; +@@ -556,11 +557,9 @@ static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, + + BUG_ON(!ttm); + +- if (ttm->state == tt_unpopulated) { +- ret = ttm->bdev->driver->ttm_tt_populate(ttm, &ctx); +- if (ret) +- return ret; +- } ++ ret = ttm_tt_populate(ttm, &ctx); ++ if (ret) ++ return ret; + + if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) { + /* +@@ -593,7 +592,6 @@ int ttm_bo_kmap(struct ttm_buffer_object *bo, + unsigned long offset, size; + int ret; + +- BUG_ON(!list_empty(&bo->swap)); + map->virtual = NULL; + map->bo = bo; + if (num_pages > bo->num_pages) +@@ -803,3 +801,27 @@ int ttm_bo_pipeline_move(struct ttm_buffer_object *bo, + return 0; + } + EXPORT_SYMBOL(ttm_bo_pipeline_move); ++ ++int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo) ++{ ++ struct ttm_buffer_object *ghost; ++ int ret; ++ ++ ret = ttm_buffer_object_transfer(bo, &ghost); ++ if (ret) ++ return ret; ++ ++ ret = reservation_object_copy_fences(ghost->resv, bo->resv); ++ /* Last resort, wait for the BO to be idle when we are OOM */ ++ if (ret) ++ ttm_bo_wait(bo, false, false); ++ ++ memset(&bo->mem, 0, sizeof(bo->mem)); ++ bo->mem.mem_type = TTM_PL_SYSTEM; ++ bo->ttm = NULL; ++ ++ ttm_bo_unreserve(ghost); ++ ttm_bo_unref(&ghost); ++ ++ return 0; ++} +diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c +index 77e6577..a18131e 100644 +--- a/drivers/gpu/drm/ttm/ttm_bo_vm.c ++++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c +@@ -92,6 +92,17 @@ static int ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo, + return ret; + } + ++static unsigned long ttm_bo_io_mem_pfn(struct ttm_buffer_object *bo, ++ unsigned long page_offset) ++{ ++ struct ttm_bo_device *bdev = bo->bdev; ++ ++ if (bdev->driver->io_mem_pfn) ++ return bdev->driver->io_mem_pfn(bo, page_offset); ++ ++ return ((bo->mem.bus.base + bo->mem.bus.offset) >> PAGE_SHIFT) ++ + page_offset; ++} + static int ttm_bo_vm_fault(struct vm_fault *vmf) + { + struct vm_area_struct *vma = vmf->vma; +@@ -125,7 +136,7 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) + if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) { + if (!(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { + ttm_bo_reference(bo); +- up_read(&vmf->vma->vm_mm->mmap_sem); ++ up_read(&vma->vm_mm->mmap_sem); + (void) ttm_bo_wait_unreserved(bo); + ttm_bo_unref(&bo); + } +@@ -146,7 +157,7 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) + * (if at all) by redirecting mmap to the exporter. + */ + if (bo->ttm && (bo->ttm->page_flags & TTM_PAGE_FLAG_SG)) { +- retval = VM_FAULT_SIGBUS; ++ ret = VM_FAULT_SIGBUS; + goto out_unlock; + } + +@@ -157,10 +168,10 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) + break; + case -EBUSY: + case -ERESTARTSYS: +- retval = VM_FAULT_NOPAGE; ++ ret = VM_FAULT_NOPAGE; + goto out_unlock; + default: +- retval = VM_FAULT_SIGBUS; ++ ret = VM_FAULT_SIGBUS; + goto out_unlock; + } + } +@@ -171,12 +182,10 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) + */ + ret = ttm_bo_vm_fault_idle(bo, vmf); + if (unlikely(ret != 0)) { +- retval = ret; +- +- if (retval == VM_FAULT_RETRY && ++ if (ret == VM_FAULT_RETRY && + !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { + /* The BO has already been unreserved. */ +- return retval; ++ return ret; + } + + goto out_unlock; +@@ -184,12 +193,12 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) + + ret = ttm_mem_io_lock(man, true); + if (unlikely(ret != 0)) { +- retval = VM_FAULT_NOPAGE; ++ ret = VM_FAULT_NOPAGE; + goto out_unlock; + } + ret = ttm_mem_io_reserve_vm(bo); + if (unlikely(ret != 0)) { +- retval = VM_FAULT_SIGBUS; ++ ret = VM_FAULT_SIGBUS; + goto out_io_unlock; + } + +@@ -199,7 +208,7 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) + drm_vma_node_start(&bo->vma_node); + + if (unlikely(page_offset >= bo->num_pages)) { +- retval = VM_FAULT_SIGBUS; ++ ret = VM_FAULT_SIGBUS; + goto out_io_unlock; + } + +@@ -217,7 +226,9 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) + } else { + struct ttm_operation_ctx ctx = { + .interruptible = false, +- .no_wait_gpu = false ++ .no_wait_gpu = false, ++ .flags = TTM_OPT_FLAG_FORCE_ALLOC ++ + }; + + ttm = bo->ttm; +@@ -225,8 +236,8 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) + cvma.vm_page_prot); + + /* Allocate all page at once, most common usage */ +- if (ttm->bdev->driver->ttm_tt_populate(ttm, &ctx)) { +- retval = VM_FAULT_OOM; ++ if (ttm_tt_populate(ttm, &ctx)) { ++ ret = VM_FAULT_OOM; + goto out_io_unlock; + } + } +@@ -239,16 +250,15 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) + if (bo->mem.bus.is_iomem) { + /* Iomem should not be marked encrypted */ + cvma.vm_page_prot = pgprot_decrypted(cvma.vm_page_prot); +- pfn = bdev->driver->io_mem_pfn(bo, page_offset); ++ pfn = ttm_bo_io_mem_pfn(bo, page_offset); + } else { + page = ttm->pages[page_offset]; + if (unlikely(!page && i == 0)) { +- retval = VM_FAULT_OOM; ++ ret = VM_FAULT_OOM; + goto out_io_unlock; + } else if (unlikely(!page)) { + break; + } +- page->mapping = vma->vm_file->f_mapping; + page->index = drm_vma_node_start(&bo->vma_node) + + page_offset; + pfn = page_to_pfn(page); +@@ -256,7 +266,7 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) + + if (vma->vm_flags & VM_MIXEDMAP) + ret = vm_insert_mixed(&cvma, address, +- __pfn_to_pfn_t(pfn, PFN_DEV)); ++ __pfn_to_pfn_t(pfn, PFN_DEV | (bo->ssg_can_map ? PFN_MAP : 0))); + else + ret = vm_insert_pfn(&cvma, address, pfn); + +@@ -268,7 +278,7 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) + if (unlikely((ret == -EBUSY) || (ret != 0 && i > 0))) + break; + else if (unlikely(ret != 0)) { +- retval = ++ ret = + (ret == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + goto out_io_unlock; + } +@@ -277,11 +287,12 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) + if (unlikely(++page_offset >= page_last)) + break; + } ++ ret = VM_FAULT_NOPAGE; + out_io_unlock: + ttm_mem_io_unlock(man); + out_unlock: + ttm_bo_unreserve(bo); +- return retval; ++ return ret; + } + + static void ttm_bo_vm_open(struct vm_area_struct *vma) +@@ -410,14 +421,6 @@ static struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_bo_device *bdev, + return bo; + } + +-unsigned long ttm_bo_default_io_mem_pfn(struct ttm_buffer_object *bo, +- unsigned long page_offset) +-{ +- return ((bo->mem.bus.base + bo->mem.bus.offset) >> PAGE_SHIFT) +- + page_offset; +-} +-EXPORT_SYMBOL(ttm_bo_default_io_mem_pfn); +- + int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma, + struct ttm_bo_device *bdev) + { +@@ -455,7 +458,7 @@ int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma, + * VM_MIXEDMAP on all mappings. See freedesktop.org bug #75719 + */ + vma->vm_flags |= VM_MIXEDMAP; +- vma->vm_flags |= VM_IO | VM_DONTEXPAND | VM_DONTDUMP; ++ vma->vm_flags |= (bo->ssg_can_map ? 0 : VM_IO) | VM_DONTEXPAND | VM_DONTDUMP; + return 0; + out_unref: + ttm_bo_unref(&bo); +diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c +index 5e1bcab..3dca206e 100644 +--- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c ++++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c +@@ -38,7 +38,7 @@ static void ttm_eu_backoff_reservation_reverse(struct list_head *list, + list_for_each_entry_continue_reverse(entry, list, head) { + struct ttm_buffer_object *bo = entry->bo; + +- __ttm_bo_unreserve(bo); ++ reservation_object_unlock(bo->resv); + } + } + +@@ -62,14 +62,14 @@ void ttm_eu_backoff_reservation(struct ww_acquire_ctx *ticket, + return; + + entry = list_first_entry(list, struct ttm_validate_buffer, head); +- glob = entry->bo->glob; ++ glob = entry->bo->bdev->glob; + + spin_lock(&glob->lru_lock); + list_for_each_entry(entry, list, head) { + struct ttm_buffer_object *bo = entry->bo; + + ttm_bo_add_to_lru(bo); +- __ttm_bo_unreserve(bo); ++ reservation_object_unlock(bo->resv); + } + spin_unlock(&glob->lru_lock); + +@@ -102,7 +102,7 @@ int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket, + return 0; + + entry = list_first_entry(list, struct ttm_validate_buffer, head); +- glob = entry->bo->glob; ++ glob = entry->bo->bdev->glob; + + if (ticket) + ww_acquire_init(ticket, &reservation_ww_class); +@@ -112,7 +112,7 @@ int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket, + + ret = __ttm_bo_reserve(bo, intr, (ticket == NULL), ticket); + if (!ret && unlikely(atomic_read(&bo->cpu_writers) > 0)) { +- __ttm_bo_unreserve(bo); ++ reservation_object_unlock(bo->resv); + + ret = -EBUSY; + +@@ -139,12 +139,14 @@ int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket, + */ + ttm_eu_backoff_reservation_reverse(list, entry); + +- if (ret == -EDEADLK && intr) { +- ret = ww_mutex_lock_slow_interruptible(&bo->resv->lock, +- ticket); +- } else if (ret == -EDEADLK) { +- ww_mutex_lock_slow(&bo->resv->lock, ticket); +- ret = 0; ++ if (ret == -EDEADLK) { ++ if (intr) { ++ ret = ww_mutex_lock_slow_interruptible(&bo->resv->lock, ++ ticket); ++ } else { ++ ww_mutex_lock_slow(&bo->resv->lock, ticket); ++ ret = 0; ++ } + } + + if (!ret && entry->shared) +@@ -192,7 +194,7 @@ void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket, + bo = list_first_entry(list, struct ttm_validate_buffer, head)->bo; + bdev = bo->bdev; + driver = bdev->driver; +- glob = bo->glob; ++ glob = bo->bdev->glob; + + spin_lock(&glob->lru_lock); + +@@ -203,7 +205,7 @@ void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket, + else + reservation_object_add_excl_fence(bo->resv, fence); + ttm_bo_add_to_lru(bo); +- __ttm_bo_unreserve(bo); ++ reservation_object_unlock(bo->resv); + } + spin_unlock(&glob->lru_lock); + if (ticket) +diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c +index dc25904..5bd82a9 100644 +--- a/drivers/gpu/drm/ttm/ttm_memory.c ++++ b/drivers/gpu/drm/ttm/ttm_memory.c +@@ -36,6 +36,7 @@ + #include <linux/mm.h> + #include <linux/module.h> + #include <linux/slab.h> ++#include <linux/swap.h> + + #define TTM_MEMORY_ALLOC_RETRIES 4 + +@@ -166,6 +167,54 @@ static struct kobj_type ttm_mem_zone_kobj_type = { + .default_attrs = ttm_mem_zone_attrs, + }; + ++static struct attribute ttm_mem_global_lower_mem_limit = { ++ .name = "lower_mem_limit", ++ .mode = S_IRUGO | S_IWUSR ++}; ++ ++static ssize_t ttm_mem_global_show(struct kobject *kobj, ++ struct attribute *attr, ++ char *buffer) ++{ ++ struct ttm_mem_global *glob = ++ container_of(kobj, struct ttm_mem_global, kobj); ++ uint64_t val = 0; ++ ++ spin_lock(&glob->lock); ++ val = glob->lower_mem_limit; ++ spin_unlock(&glob->lock); ++ /* convert from number of pages to KB */ ++ val <<= (PAGE_SHIFT - 10); ++ return snprintf(buffer, PAGE_SIZE, "%llu\n", ++ (unsigned long long) val); ++} ++ ++static ssize_t ttm_mem_global_store(struct kobject *kobj, ++ struct attribute *attr, ++ const char *buffer, ++ size_t size) ++{ ++ int chars; ++ uint64_t val64; ++ unsigned long val; ++ struct ttm_mem_global *glob = ++ container_of(kobj, struct ttm_mem_global, kobj); ++ ++ chars = sscanf(buffer, "%lu", &val); ++ if (chars == 0) ++ return size; ++ ++ val64 = val; ++ /* convert from KB to number of pages */ ++ val64 >>= (PAGE_SHIFT - 10); ++ ++ spin_lock(&glob->lock); ++ glob->lower_mem_limit = val64; ++ spin_unlock(&glob->lock); ++ ++ return size; ++} ++ + static void ttm_mem_global_kobj_release(struct kobject *kobj) + { + struct ttm_mem_global *glob = +@@ -174,8 +223,20 @@ static void ttm_mem_global_kobj_release(struct kobject *kobj) + kfree(glob); + } + ++static struct attribute *ttm_mem_global_attrs[] = { ++ &ttm_mem_global_lower_mem_limit, ++ NULL ++}; ++ ++static const struct sysfs_ops ttm_mem_global_ops = { ++ .show = &ttm_mem_global_show, ++ .store = &ttm_mem_global_store, ++}; ++ + static struct kobj_type ttm_mem_glob_kobj_type = { + .release = &ttm_mem_global_kobj_release, ++ .sysfs_ops = &ttm_mem_global_ops, ++ .default_attrs = ttm_mem_global_attrs, + }; + + static bool ttm_zones_above_swap_target(struct ttm_mem_global *glob, +@@ -209,34 +270,35 @@ static bool ttm_zones_above_swap_target(struct ttm_mem_global *glob, + * Note that this function is reentrant: + * many threads may try to swap out at any given time. + */ ++ + static void ttm_shrink(struct ttm_mem_global *glob, bool from_wq, +- uint64_t extra, struct ttm_operation_ctx *ctx) ++ uint64_t extra, struct ttm_operation_ctx *ctx) + { +- int ret; ++ int ret; + +- spin_lock(&glob->lock); ++ spin_lock(&glob->lock); + +- while (ttm_zones_above_swap_target(glob, from_wq, extra)) { +- spin_unlock(&glob->lock); +- ret = ttm_bo_swapout(glob->bo_glob, ctx); +- spin_lock(&glob->lock); +- if (unlikely(ret != 0)) +- break; +- } ++ while (ttm_zones_above_swap_target(glob, from_wq, extra)) { ++ spin_unlock(&glob->lock); ++ ret = ttm_bo_swapout(glob->bo_glob, ctx); ++ spin_lock(&glob->lock); ++ if (unlikely(ret != 0)) ++ break; ++ } + +- spin_unlock(&glob->lock); ++ spin_unlock(&glob->lock); + } + + static void ttm_shrink_work(struct work_struct *work) + { +- struct ttm_operation_ctx ctx = { +- .interruptible = false, +- .no_wait_gpu = false +- }; +- struct ttm_mem_global *glob = +- container_of(work, struct ttm_mem_global, work); +- +- ttm_shrink(glob, true, 0ULL, &ctx); ++ struct ttm_operation_ctx ctx = { ++ .interruptible = false, ++ .no_wait_gpu = false ++ }; ++ struct ttm_mem_global *glob = ++ container_of(work, struct ttm_mem_global, work); ++ ++ ttm_shrink(glob, true, 0ULL, &ctx); + } + + static int ttm_mem_init_kernel_zone(struct ttm_mem_global *glob, +@@ -374,6 +436,9 @@ int ttm_mem_global_init(struct ttm_mem_global *glob) + + si_meminfo(&si); + ++ /* set it as 0 by default to keep original behavior of OOM */ ++ glob->lower_mem_limit = 0; ++ + ret = ttm_mem_init_kernel_zone(glob, &si); + if (unlikely(ret != 0)) + goto out_no_zone; +@@ -468,6 +533,36 @@ void ttm_mem_global_free(struct ttm_mem_global *glob, + } + EXPORT_SYMBOL(ttm_mem_global_free); + ++/* ++ * check if the available mem is under lower memory limit ++ * ++ * a. if no swap disk at all or free swap space is under swap_mem_limit ++ * but available system mem is bigger than sys_mem_limit, allow TTM ++ * allocation; ++ * ++ * b. if the available system mem is less than sys_mem_limit but free ++ * swap disk is bigger than swap_mem_limit, allow TTM allocation. ++ */ ++bool ++ttm_check_under_lowerlimit(struct ttm_mem_global *glob, ++ uint64_t num_pages, ++ struct ttm_operation_ctx *ctx) ++{ ++ int64_t available; ++ ++ if (ctx->flags & TTM_OPT_FLAG_FORCE_ALLOC) ++ return false; ++ ++ available = get_nr_swap_pages() + si_mem_available(); ++ ++ available -= num_pages; ++ if (available < glob->lower_mem_limit) ++ return true; ++ ++ return false; ++} ++EXPORT_SYMBOL(ttm_check_under_lowerlimit); ++ + static int ttm_mem_global_reserve(struct ttm_mem_global *glob, + struct ttm_mem_zone *single_zone, + uint64_t amount, bool reserve) +@@ -542,10 +637,9 @@ int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory, + EXPORT_SYMBOL(ttm_mem_global_alloc); + + int ttm_mem_global_alloc_page(struct ttm_mem_global *glob, +- struct page *page, uint64_t size, +- struct ttm_operation_ctx *ctx) ++ struct page *page, uint64_t size, ++ struct ttm_operation_ctx *ctx) + { +- + struct ttm_mem_zone *zone = NULL; + + /** +@@ -563,7 +657,8 @@ int ttm_mem_global_alloc_page(struct ttm_mem_global *glob, + return ttm_mem_global_alloc_zone(glob, zone, size, ctx); + } + +-void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page) ++void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page, ++ uint64_t size) + { + struct ttm_mem_zone *zone = NULL; + +@@ -574,7 +669,7 @@ void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page) + if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL) + zone = glob->zone_kernel; + #endif +- ttm_mem_global_free_zone(glob, zone, PAGE_SIZE); ++ ttm_mem_global_free_zone(glob, zone, size); + } + + +diff --git a/drivers/gpu/drm/ttm/ttm_module.c b/drivers/gpu/drm/ttm/ttm_module.c +index 66fc639..7370e34 100644 +--- a/drivers/gpu/drm/ttm/ttm_module.c ++++ b/drivers/gpu/drm/ttm/ttm_module.c +@@ -38,7 +38,7 @@ static DECLARE_WAIT_QUEUE_HEAD(exit_q); + static atomic_t device_released; + + static struct device_type ttm_drm_class_type = { +- .name = "ttm", ++ .name = "amdttm", + /** + * Add pm ops here. + */ +@@ -66,7 +66,7 @@ static int __init ttm_init(void) + { + int ret; + +- ret = dev_set_name(&ttm_drm_class_device, "ttm"); ++ ret = dev_set_name(&ttm_drm_class_device, "amdttm"); + if (unlikely(ret != 0)) + return ret; + +diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c +index 55c9a90..9f9f181 100644 +--- a/drivers/gpu/drm/ttm/ttm_page_alloc.c ++++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c +@@ -81,6 +81,7 @@ struct ttm_page_pool { + char *name; + unsigned long nfrees; + unsigned long nrefills; ++ unsigned int order; + }; + + /** +@@ -95,7 +96,7 @@ struct ttm_pool_opts { + unsigned small; + }; + +-#define NUM_POOLS 4 ++#define NUM_POOLS 6 + + /** + * struct ttm_pool_manager - Holds memory pools for fst allocation +@@ -122,6 +123,8 @@ struct ttm_pool_manager { + struct ttm_page_pool uc_pool; + struct ttm_page_pool wc_pool_dma32; + struct ttm_page_pool uc_pool_dma32; ++ struct ttm_page_pool wc_pool_huge; ++ struct ttm_page_pool uc_pool_huge; + } ; + }; + }; +@@ -220,6 +223,17 @@ static struct kobj_type ttm_pool_kobj_type = { + static struct ttm_pool_manager *_manager; + + #ifndef CONFIG_X86 ++static int set_pages_wb(struct page *page, int numpages) ++{ ++#if IS_ENABLED(CONFIG_AGP) ++ int i; ++ ++ for (i = 0; i < numpages; i++) ++ unmap_page_from_agp(page++); ++#endif ++ return 0; ++} ++ + static int set_pages_array_wb(struct page **pages, int addrinarray) + { + #if IS_ENABLED(CONFIG_AGP) +@@ -256,8 +270,8 @@ static int set_pages_array_uc(struct page **pages, int addrinarray) + + /** + * Select the right pool or requested caching state and ttm flags. */ +-static struct ttm_page_pool *ttm_get_pool(int flags, +- enum ttm_caching_state cstate) ++static struct ttm_page_pool *ttm_get_pool(int flags, bool huge, ++ enum ttm_caching_state cstate) + { + int pool_index; + +@@ -269,20 +283,36 @@ static struct ttm_page_pool *ttm_get_pool(int flags, + else + pool_index = 0x1; + +- if (flags & TTM_PAGE_FLAG_DMA32) ++ if (flags & TTM_PAGE_FLAG_DMA32) { ++ if (huge) ++ return NULL; + pool_index |= 0x2; + ++ } else if (huge) { ++ pool_index |= 0x4; ++ } ++ + return &_manager->pools[pool_index]; + } + + /* set memory back to wb and free the pages. */ +-static void ttm_pages_put(struct page *pages[], unsigned npages) ++static void ttm_pages_put(struct page *pages[], unsigned npages, ++ unsigned int order) + { +- unsigned i; +- if (set_pages_array_wb(pages, npages)) +- pr_err("Failed to set %d pages to wb!\n", npages); +- for (i = 0; i < npages; ++i) +- __free_page(pages[i]); ++ unsigned int i, pages_nr = (1 << order); ++ ++ if (order == 0) { ++ if (set_pages_array_wb(pages, npages)) ++ pr_err("Failed to set %d pages to wb!\n", npages); ++ } ++ ++ for (i = 0; i < npages; ++i) { ++ if (order > 0) { ++ if (set_pages_wb(pages[i], pages_nr)) ++ pr_err("Failed to set %d pages to wb!\n", pages_nr); ++ } ++ __free_pages(pages[i], order); ++ } + } + + static void ttm_pool_update_free_locked(struct ttm_page_pool *pool, +@@ -321,7 +351,7 @@ static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free, + pages_to_free = kmalloc(npages_to_free * sizeof(struct page *), + GFP_KERNEL); + if (!pages_to_free) { +- pr_err("Failed to allocate memory for pool free operation\n"); ++ pr_debug("Failed to allocate memory for pool free operation\n"); + return 0; + } + +@@ -345,7 +375,7 @@ static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free, + */ + spin_unlock_irqrestore(&pool->lock, irq_flags); + +- ttm_pages_put(pages_to_free, freed_pages); ++ ttm_pages_put(pages_to_free, freed_pages, pool->order); + if (likely(nr_free != FREE_ALL_PAGES)) + nr_free -= freed_pages; + +@@ -380,7 +410,7 @@ static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free, + spin_unlock_irqrestore(&pool->lock, irq_flags); + + if (freed_pages) +- ttm_pages_put(pages_to_free, freed_pages); ++ ttm_pages_put(pages_to_free, freed_pages, pool->order); + out: + if (pages_to_free != static_buf) + kfree(pages_to_free); +@@ -404,6 +434,7 @@ ttm_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) + struct ttm_page_pool *pool; + int shrink_pages = sc->nr_to_scan; + unsigned long freed = 0; ++ unsigned int nr_free_pool; + + if (!mutex_trylock(&lock)) + return SHRINK_STOP; +@@ -411,12 +442,20 @@ ttm_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) + /* select start pool in round robin fashion */ + for (i = 0; i < NUM_POOLS; ++i) { + unsigned nr_free = shrink_pages; ++ unsigned page_nr; ++ + if (shrink_pages == 0) + break; ++ + pool = &_manager->pools[(i + pool_offset)%NUM_POOLS]; ++ page_nr = (1 << pool->order); + /* OK to use static buffer since global mutex is held. */ +- shrink_pages = ttm_page_pool_free(pool, nr_free, true); +- freed += nr_free - shrink_pages; ++ nr_free_pool = roundup(nr_free, page_nr) >> pool->order; ++ shrink_pages = ttm_page_pool_free(pool, nr_free_pool, true); ++ freed += (nr_free_pool - shrink_pages) << pool->order; ++ if (freed >= sc->nr_to_scan) ++ break; ++ shrink_pages <<= pool->order; + } + mutex_unlock(&lock); + return freed; +@@ -428,19 +467,22 @@ ttm_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc) + { + unsigned i; + unsigned long count = 0; ++ struct ttm_page_pool *pool; + +- for (i = 0; i < NUM_POOLS; ++i) +- count += _manager->pools[i].npages; ++ for (i = 0; i < NUM_POOLS; ++i) { ++ pool = &_manager->pools[i]; ++ count += (pool->npages << pool->order); ++ } + + return count; + } + +-static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager) ++static int ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager) + { + manager->mm_shrink.count_objects = ttm_pool_shrink_count; + manager->mm_shrink.scan_objects = ttm_pool_shrink_scan; + manager->mm_shrink.seeks = 1; +- register_shrinker(&manager->mm_shrink); ++ return register_shrinker(&manager->mm_shrink); + } + + static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager) +@@ -494,28 +536,29 @@ static void ttm_handle_caching_state_failure(struct list_head *pages, + * pages returned in pages array. + */ + static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, +- int ttm_flags, enum ttm_caching_state cstate, unsigned count) ++ int ttm_flags, enum ttm_caching_state cstate, ++ unsigned count, unsigned order) + { + struct page **caching_array; + struct page *p; + int r = 0; +- unsigned i, cpages; +- unsigned max_cpages = min(count, +- (unsigned)(PAGE_SIZE/sizeof(struct page *))); ++ unsigned i, j, cpages; ++ unsigned npages = 1 << order; ++ unsigned max_cpages = min(count << order, (unsigned)NUM_PAGES_TO_ALLOC); + + /* allocate array for page caching change */ + caching_array = kmalloc(max_cpages*sizeof(struct page *), GFP_KERNEL); + + if (!caching_array) { +- pr_err("Unable to allocate table for new pages\n"); ++ pr_debug("Unable to allocate table for new pages\n"); + return -ENOMEM; + } + + for (i = 0, cpages = 0; i < count; ++i) { +- p = alloc_page(gfp_flags); ++ p = alloc_pages(gfp_flags, order); + + if (!p) { +- pr_err("Unable to get page %u\n", i); ++ pr_debug("Unable to get page %u\n", i); + + /* store already allocated pages in the pool after + * setting the caching state */ +@@ -531,14 +574,18 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, + goto out; + } + ++ list_add(&p->lru, pages); ++ + #ifdef CONFIG_HIGHMEM + /* gfp flags of highmem page should never be dma32 so we + * we should be fine in such case + */ +- if (!PageHighMem(p)) ++ if (PageHighMem(p)) ++ continue; ++ + #endif +- { +- caching_array[cpages++] = p; ++ for (j = 0; j < npages; ++j) { ++ caching_array[cpages++] = p++; + if (cpages == max_cpages) { + + r = ttm_set_pages_caching(caching_array, +@@ -552,8 +599,6 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, + cpages = 0; + } + } +- +- list_add(&p->lru, pages); + } + + if (cpages) { +@@ -573,9 +618,9 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, + * Fill the given pool if there aren't enough pages and the requested number of + * pages is small. + */ +-static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, +- int ttm_flags, enum ttm_caching_state cstate, unsigned count, +- unsigned long *irq_flags) ++static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, int ttm_flags, ++ enum ttm_caching_state cstate, ++ unsigned count, unsigned long *irq_flags) + { + struct page *p; + int r; +@@ -605,7 +650,7 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, + + INIT_LIST_HEAD(&new_pages); + r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags, +- cstate, alloc_size); ++ cstate, alloc_size, 0); + spin_lock_irqsave(&pool->lock, *irq_flags); + + if (!r) { +@@ -613,7 +658,7 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, + ++pool->nrefills; + pool->npages += alloc_size; + } else { +- pr_err("Failed to fill pool (%p)\n", pool); ++ pr_debug("Failed to fill pool (%p)\n", pool); + /* If we have any pages left put them to the pool. */ + list_for_each_entry(p, &new_pages, lru) { + ++cpages; +@@ -627,22 +672,25 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, + } + + /** +- * Cut 'count' number of pages from the pool and put them on the return list. ++ * Allocate pages from the pool and put them on the return list. + * +- * @return count of pages still required to fulfill the request. ++ * @return zero for success or negative error code. + */ +-static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool, +- struct list_head *pages, +- int ttm_flags, +- enum ttm_caching_state cstate, +- unsigned count) ++static int ttm_page_pool_get_pages(struct ttm_page_pool *pool, ++ struct list_head *pages, ++ int ttm_flags, ++ enum ttm_caching_state cstate, ++ unsigned count, unsigned order) + { + unsigned long irq_flags; + struct list_head *p; + unsigned i; ++ int r = 0; + + spin_lock_irqsave(&pool->lock, irq_flags); +- ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, &irq_flags); ++ if (!order) ++ ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, ++ &irq_flags); + + if (count >= pool->npages) { + /* take all pages from the pool */ +@@ -672,32 +720,131 @@ static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool, + count = 0; + out: + spin_unlock_irqrestore(&pool->lock, irq_flags); +- return count; ++ ++ /* clear the pages coming from the pool if requested */ ++ if (ttm_flags & TTM_PAGE_FLAG_ZERO_ALLOC) { ++ struct page *page; ++ ++ list_for_each_entry(page, pages, lru) { ++ if (PageHighMem(page)) ++ clear_highpage(page); ++ else ++ clear_page(page_address(page)); ++ } ++ } ++ ++ /* If pool didn't have enough pages allocate new one. */ ++ if (count) { ++ gfp_t gfp_flags = pool->gfp_flags; ++ ++ /* set zero flag for page allocation if required */ ++ if (ttm_flags & TTM_PAGE_FLAG_ZERO_ALLOC) ++ gfp_flags |= __GFP_ZERO; ++ ++ if (ttm_flags & TTM_PAGE_FLAG_NO_RETRY) ++ gfp_flags |= __GFP_RETRY_MAYFAIL; ++ ++ /* ttm_alloc_new_pages doesn't reference pool so we can run ++ * multiple requests in parallel. ++ **/ ++ r = ttm_alloc_new_pages(pages, gfp_flags, ttm_flags, cstate, ++ count, order); ++ } ++ ++ return r; + } + + /* Put all pages in pages list to correct pool to wait for reuse */ + static void ttm_put_pages(struct page **pages, unsigned npages, int flags, + enum ttm_caching_state cstate) + { ++ struct ttm_page_pool *pool = ttm_get_pool(flags, false, cstate); ++#ifdef CONFIG_TRANSPARENT_HUGEPAGE ++ struct ttm_page_pool *huge = ttm_get_pool(flags, true, cstate); ++#endif + unsigned long irq_flags; +- struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); + unsigned i; + + if (pool == NULL) { + /* No pool for this memory type so free the pages */ +- for (i = 0; i < npages; i++) { +- if (pages[i]) { +- if (page_count(pages[i]) != 1) +- pr_err("Erroneous page count. Leaking pages.\n"); +- __free_page(pages[i]); +- pages[i] = NULL; ++ i = 0; ++ while (i < npages) { ++#ifdef CONFIG_TRANSPARENT_HUGEPAGE ++ struct page *p = pages[i]; ++#endif ++ unsigned order = 0, j; ++ ++ if (!pages[i]) { ++ ++i; ++ continue; ++ } ++ ++#ifdef CONFIG_TRANSPARENT_HUGEPAGE ++ if (!(flags & TTM_PAGE_FLAG_DMA32)) { ++ for (j = 0; j < HPAGE_PMD_NR; ++j) ++ if (p++ != pages[i + j]) ++ break; ++ ++ if (j == HPAGE_PMD_NR) ++ order = HPAGE_PMD_ORDER; ++ } ++#endif ++ ++ if (page_count(pages[i]) != 1) ++ pr_err("Erroneous page count. Leaking pages.\n"); ++ __free_pages(pages[i], order); ++ ++ j = 1 << order; ++ while (j) { ++ pages[i++] = NULL; ++ --j; + } + } + return; + } + ++ i = 0; ++#ifdef CONFIG_TRANSPARENT_HUGEPAGE ++ if (huge) { ++ unsigned max_size, n2free; ++ ++ spin_lock_irqsave(&huge->lock, irq_flags); ++ while (i < npages) { ++ struct page *p = pages[i]; ++ unsigned j; ++ ++ if (!p) ++ break; ++ ++ for (j = 0; j < HPAGE_PMD_NR; ++j) ++ if (p++ != pages[i + j]) ++ break; ++ ++ if (j != HPAGE_PMD_NR) ++ break; ++ ++ list_add_tail(&pages[i]->lru, &huge->list); ++ ++ for (j = 0; j < HPAGE_PMD_NR; ++j) ++ pages[i++] = NULL; ++ huge->npages++; ++ } ++ ++ /* Check that we don't go over the pool limit */ ++ max_size = _manager->options.max_size; ++ max_size /= HPAGE_PMD_NR; ++ if (huge->npages > max_size) ++ n2free = huge->npages - max_size; ++ else ++ n2free = 0; ++ spin_unlock_irqrestore(&huge->lock, irq_flags); ++ if (n2free) ++ ttm_page_pool_free(huge, n2free, false); ++ } ++#endif ++ + spin_lock_irqsave(&pool->lock, irq_flags); +- for (i = 0; i < npages; i++) { ++ while (i < npages) { + if (pages[i]) { + if (page_count(pages[i]) != 1) + pr_err("Erroneous page count. Leaking pages.\n"); +@@ -705,6 +852,7 @@ static void ttm_put_pages(struct page **pages, unsigned npages, int flags, + pages[i] = NULL; + pool->npages++; + } ++ ++i; + } + /* Check that we don't go over the pool limit */ + npages = 0; +@@ -727,82 +875,120 @@ static void ttm_put_pages(struct page **pages, unsigned npages, int flags, + static int ttm_get_pages(struct page **pages, unsigned npages, int flags, + enum ttm_caching_state cstate) + { +- struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); ++ struct ttm_page_pool *pool = ttm_get_pool(flags, false, cstate); ++#ifdef CONFIG_TRANSPARENT_HUGEPAGE ++ struct ttm_page_pool *huge = ttm_get_pool(flags, true, cstate); ++#endif + struct list_head plist; + struct page *p = NULL; +- gfp_t gfp_flags = GFP_USER; +- unsigned count; ++ unsigned count, first; + int r; + +- /* set zero flag for page allocation if required */ +- if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) +- gfp_flags |= __GFP_ZERO; +- + /* No pool for cached pages */ + if (pool == NULL) { ++ gfp_t gfp_flags = GFP_USER; ++ unsigned i; ++#ifdef CONFIG_TRANSPARENT_HUGEPAGE ++ unsigned j; ++#endif ++ ++ /* set zero flag for page allocation if required */ ++ if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) ++ gfp_flags |= __GFP_ZERO; ++ ++ if (flags & TTM_PAGE_FLAG_NO_RETRY) ++ gfp_flags |= __GFP_RETRY_MAYFAIL; ++ + if (flags & TTM_PAGE_FLAG_DMA32) + gfp_flags |= GFP_DMA32; + else + gfp_flags |= GFP_HIGHUSER; + +- for (r = 0; r < npages; ++r) { ++ i = 0; ++#ifdef CONFIG_TRANSPARENT_HUGEPAGE ++ if (!(gfp_flags & GFP_DMA32)) { ++ while (npages >= HPAGE_PMD_NR) { ++ gfp_t huge_flags = gfp_flags; ++ ++ huge_flags |= GFP_TRANSHUGE; ++ huge_flags &= ~__GFP_MOVABLE; ++ huge_flags &= ~__GFP_COMP; ++ p = alloc_pages(huge_flags, HPAGE_PMD_ORDER); ++ if (!p) ++ break; ++ ++ for (j = 0; j < HPAGE_PMD_NR; ++j) ++ pages[i++] = p++; ++ ++ npages -= HPAGE_PMD_NR; ++ } ++ } ++#endif ++ ++ first = i; ++ while (npages) { + p = alloc_page(gfp_flags); + if (!p) { +- +- pr_err("Unable to allocate page\n"); ++ pr_debug("Unable to allocate page\n"); + return -ENOMEM; + } + +- pages[r] = p; ++ /* Swap the pages if we detect consecutive order */ ++ if (i > first && pages[i - 1] == p - 1) ++ swap(p, pages[i - 1]); ++ ++ pages[i++] = p; ++ --npages; + } + return 0; + } + +- /* combine zero flag to pool flags */ +- gfp_flags |= pool->gfp_flags; +- +- /* First we take pages from the pool */ +- INIT_LIST_HEAD(&plist); +- npages = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages); + count = 0; +- list_for_each_entry(p, &plist, lru) { +- pages[count++] = p; +- } + +- /* clear the pages coming from the pool if requested */ +- if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) { ++#ifdef CONFIG_TRANSPARENT_HUGEPAGE ++ if (huge && npages >= HPAGE_PMD_NR) { ++ INIT_LIST_HEAD(&plist); ++ ttm_page_pool_get_pages(huge, &plist, flags, cstate, ++ npages / HPAGE_PMD_NR, ++ HPAGE_PMD_ORDER); ++ + list_for_each_entry(p, &plist, lru) { +- if (PageHighMem(p)) +- clear_highpage(p); +- else +- clear_page(page_address(p)); ++ unsigned j; ++ ++ for (j = 0; j < HPAGE_PMD_NR; ++j) ++ pages[count++] = &p[j]; + } + } ++#endif + +- /* If pool didn't have enough pages allocate new one. */ +- if (npages > 0) { +- /* ttm_alloc_new_pages doesn't reference pool so we can run +- * multiple requests in parallel. +- **/ +- INIT_LIST_HEAD(&plist); +- r = ttm_alloc_new_pages(&plist, gfp_flags, flags, cstate, npages); +- list_for_each_entry(p, &plist, lru) { +- pages[count++] = p; +- } +- if (r) { +- /* If there is any pages in the list put them back to +- * the pool. */ +- pr_err("Failed to allocate extra pages for large request\n"); +- ttm_put_pages(pages, count, flags, cstate); +- return r; +- } ++ INIT_LIST_HEAD(&plist); ++ r = ttm_page_pool_get_pages(pool, &plist, flags, cstate, ++ npages - count, 0); ++ ++ first = count; ++ list_for_each_entry(p, &plist, lru) { ++ struct page *tmp = p; ++ ++ /* Swap the pages if we detect consecutive order */ ++ if (count > first && pages[count - 1] == tmp - 1) ++ swap(tmp, pages[count - 1]); ++ pages[count++] = tmp; ++ } ++ ++ if (r) { ++ /* If there is any pages in the list put them back to ++ * the pool. ++ */ ++ pr_debug("Failed to allocate extra pages for large request\n"); ++ ttm_put_pages(pages, count, flags, cstate); ++ return r; + } + + return 0; + } + + static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, gfp_t flags, +- char *name) ++ char *name, unsigned int order) + { + spin_lock_init(&pool->lock); + pool->fill_lock = false; +@@ -810,11 +996,17 @@ static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, gfp_t flags, + pool->npages = pool->nfrees = 0; + pool->gfp_flags = flags; + pool->name = name; ++ pool->order = order; + } + + int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) + { + int ret; ++#ifdef CONFIG_TRANSPARENT_HUGEPAGE ++ unsigned order = HPAGE_PMD_ORDER; ++#else ++ unsigned order = 0; ++#endif + + WARN_ON(_manager); + +@@ -824,31 +1016,41 @@ int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) + if (!_manager) + return -ENOMEM; + +- ttm_page_pool_init_locked(&_manager->wc_pool, GFP_HIGHUSER, "wc"); ++ ttm_page_pool_init_locked(&_manager->wc_pool, GFP_HIGHUSER, "wc", 0); + +- ttm_page_pool_init_locked(&_manager->uc_pool, GFP_HIGHUSER, "uc"); ++ ttm_page_pool_init_locked(&_manager->uc_pool, GFP_HIGHUSER, "uc", 0); + + ttm_page_pool_init_locked(&_manager->wc_pool_dma32, +- GFP_USER | GFP_DMA32, "wc dma"); +- ++ GFP_USER | GFP_DMA32, "wc dma", 0); + ttm_page_pool_init_locked(&_manager->uc_pool_dma32, +- GFP_USER | GFP_DMA32, "uc dma"); ++ GFP_USER | GFP_DMA32, "uc dma", 0); + ++ ttm_page_pool_init_locked(&_manager->wc_pool_huge, ++ GFP_TRANSHUGE & ~(__GFP_MOVABLE | __GFP_COMP), ++ "wc huge", order); ++ ++ ttm_page_pool_init_locked(&_manager->uc_pool_huge, ++ GFP_TRANSHUGE & ~(__GFP_MOVABLE | __GFP_COMP) ++ , "uc huge", order); ++ + _manager->options.max_size = max_pages; + _manager->options.small = SMALL_ALLOCATION; + _manager->options.alloc_size = NUM_PAGES_TO_ALLOC; + + ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type, + &glob->kobj, "pool"); +- if (unlikely(ret != 0)) { +- kobject_put(&_manager->kobj); +- _manager = NULL; +- return ret; +- } +- +- ttm_pool_mm_shrink_init(_manager); ++ if (unlikely(ret != 0)) ++ goto error; + ++ ret = ttm_pool_mm_shrink_init(_manager); ++ if (unlikely(ret != 0)) ++ goto error; + return 0; ++ ++error: ++ kobject_put(&_manager->kobj); ++ _manager = NULL; ++ return ret; + } + + void ttm_page_alloc_fini(void) +@@ -866,28 +1068,52 @@ void ttm_page_alloc_fini(void) + _manager = NULL; + } + ++static void ++ttm_pool_unpopulate_helper(struct ttm_tt *ttm, unsigned mem_count_update) ++{ ++ struct ttm_mem_global *mem_glob = ttm->bdev->glob->mem_glob; ++ unsigned i; ++ ++ if (mem_count_update == 0) ++ goto put_pages; ++ ++ for (i = 0; i < mem_count_update; ++i) { ++ if (!ttm->pages[i]) ++ continue; ++ ++ ttm_mem_global_free_page(mem_glob, ttm->pages[i], PAGE_SIZE); ++ } ++ ++put_pages: ++ ttm_put_pages(ttm->pages, ttm->num_pages, ttm->page_flags, ++ ttm->caching_state); ++ ttm->state = tt_unpopulated; ++} ++ + int ttm_pool_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) + { +- struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; ++ struct ttm_mem_global *mem_glob = ttm->bdev->glob->mem_glob; + unsigned i; + int ret; + + if (ttm->state != tt_unpopulated) + return 0; + +- for (i = 0; i < ttm->num_pages; ++i) { +- ret = ttm_get_pages(&ttm->pages[i], 1, +- ttm->page_flags, +- ttm->caching_state); +- if (ret != 0) { +- ttm_pool_unpopulate(ttm); +- return -ENOMEM; +- } ++ if (ttm_check_under_lowerlimit(mem_glob, ttm->num_pages, ctx)) ++ return -ENOMEM; ++ ++ ret = ttm_get_pages(ttm->pages, ttm->num_pages, ttm->page_flags, ++ ttm->caching_state); ++ if (unlikely(ret != 0)) { ++ ttm_pool_unpopulate_helper(ttm, 0); ++ return ret; ++ } + ++ for (i = 0; i < ttm->num_pages; ++i) { + ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], +- PAGE_SIZE, ctx); ++ PAGE_SIZE, ctx); + if (unlikely(ret != 0)) { +- ttm_pool_unpopulate(ttm); ++ ttm_pool_unpopulate_helper(ttm, i); + return -ENOMEM; + } + } +@@ -907,34 +1133,33 @@ EXPORT_SYMBOL(ttm_pool_populate); + + void ttm_pool_unpopulate(struct ttm_tt *ttm) + { +- unsigned i; +- +- for (i = 0; i < ttm->num_pages; ++i) { +- if (ttm->pages[i]) { +- ttm_mem_global_free_page(ttm->glob->mem_glob, +- ttm->pages[i]); +- ttm_put_pages(&ttm->pages[i], 1, +- ttm->page_flags, +- ttm->caching_state); +- } +- } +- ttm->state = tt_unpopulated; ++ ttm_pool_unpopulate_helper(ttm, ttm->num_pages); + } + EXPORT_SYMBOL(ttm_pool_unpopulate); + + int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt, + struct ttm_operation_ctx *ctx) + { +- unsigned i; ++ unsigned i, j; + int r; + + r = ttm_pool_populate(&tt->ttm, ctx); + if (r) + return r; + +- for (i = 0; i < tt->ttm.num_pages; i++) { ++ for (i = 0; i < tt->ttm.num_pages; ++i) { ++ struct page *p = tt->ttm.pages[i]; ++ size_t num_pages = 1; ++ ++ for (j = i + 1; j < tt->ttm.num_pages; ++j) { ++ if (++p != tt->ttm.pages[j]) ++ break; ++ ++ ++num_pages; ++ } ++ + tt->dma_address[i] = dma_map_page(dev, tt->ttm.pages[i], +- 0, PAGE_SIZE, ++ 0, num_pages * PAGE_SIZE, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, tt->dma_address[i])) { + while (i--) { +@@ -945,6 +1170,11 @@ int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt, + ttm_pool_unpopulate(&tt->ttm); + return -EFAULT; + } ++ ++ for (j = 1; j < num_pages; ++j) { ++ tt->dma_address[i + 1] = tt->dma_address[i] + PAGE_SIZE; ++ ++i; ++ } + } + return 0; + } +@@ -952,13 +1182,28 @@ EXPORT_SYMBOL(ttm_populate_and_map_pages); + + void ttm_unmap_and_unpopulate_pages(struct device *dev, struct ttm_dma_tt *tt) + { +- unsigned i; +- +- for (i = 0; i < tt->ttm.num_pages; i++) { +- if (tt->dma_address[i]) { +- dma_unmap_page(dev, tt->dma_address[i], +- PAGE_SIZE, DMA_BIDIRECTIONAL); ++ unsigned i, j; ++ ++ for (i = 0; i < tt->ttm.num_pages;) { ++ struct page *p = tt->ttm.pages[i]; ++ size_t num_pages = 1; ++ ++ if (!tt->dma_address[i] || !tt->ttm.pages[i]) { ++ ++i; ++ continue; ++ } ++ ++ for (j = i + 1; j < tt->ttm.num_pages; ++j) { ++ if (++p != tt->ttm.pages[j]) ++ break; ++ ++ ++num_pages; + } ++ ++ dma_unmap_page(dev, tt->dma_address[i], num_pages * PAGE_SIZE, ++ DMA_BIDIRECTIONAL); ++ ++ i += num_pages; + } + ttm_pool_unpopulate(&tt->ttm); + } +@@ -973,12 +1218,12 @@ int ttm_page_alloc_debugfs(struct seq_file *m, void *data) + seq_printf(m, "No pool allocator running.\n"); + return 0; + } +- seq_printf(m, "%6s %12s %13s %8s\n", ++ seq_printf(m, "%7s %12s %13s %8s\n", + h[0], h[1], h[2], h[3]); + for (i = 0; i < NUM_POOLS; ++i) { + p = &_manager->pools[i]; + +- seq_printf(m, "%6s %12ld %13ld %8d\n", ++ seq_printf(m, "%7s %12ld %13ld %8d\n", + p->name, p->nrefills, + p->nfrees, p->npages); + } +diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c +index 47ba7be..52fccc5 100644 +--- a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c ++++ b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c +@@ -60,37 +60,32 @@ + #define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(struct page *)) + #define SMALL_ALLOCATION 4 + #define FREE_ALL_PAGES (~0U) +-/* times are in msecs */ +-#define IS_UNDEFINED (0) +-#define IS_WC (1<<1) +-#define IS_UC (1<<2) +-#define IS_CACHED (1<<3) +-#define IS_DMA32 (1<<4) ++#define VADDR_FLAG_HUGE_POOL 1UL ++#define VADDR_FLAG_UPDATED_COUNT 2UL + + enum pool_type { +- POOL_IS_UNDEFINED, +- POOL_IS_WC = IS_WC, +- POOL_IS_UC = IS_UC, +- POOL_IS_CACHED = IS_CACHED, +- POOL_IS_WC_DMA32 = IS_WC | IS_DMA32, +- POOL_IS_UC_DMA32 = IS_UC | IS_DMA32, +- POOL_IS_CACHED_DMA32 = IS_CACHED | IS_DMA32, ++ IS_UNDEFINED = 0, ++ IS_WC = 1 << 1, ++ IS_UC = 1 << 2, ++ IS_CACHED = 1 << 3, ++ IS_DMA32 = 1 << 4, ++ IS_HUGE = 1 << 5 + }; + /* +- * The pool structure. There are usually six pools: ++ * The pool structure. There are up to nine pools: + * - generic (not restricted to DMA32): + * - write combined, uncached, cached. + * - dma32 (up to 2^32 - so up 4GB): + * - write combined, uncached, cached. ++ * - huge (not restricted to DMA32): ++ * - write combined, uncached, cached. + * for each 'struct device'. The 'cached' is for pages that are actively used. + * The other ones can be shrunk by the shrinker API if neccessary. + * @pools: The 'struct device->dma_pools' link. + * @type: Type of the pool +- * @lock: Protects the inuse_list and free_list from concurrnet access. Must be ++ * @lock: Protects the free_list from concurrnet access. Must be + * used with irqsave/irqrestore variants because pool allocator maybe called + * from delayed work. +- * @inuse_list: Pool of pages that are in use. The order is very important and +- * it is in the order that the TTM pages that are put back are in. + * @free_list: Pool of pages that are free to be used. No order requirements. + * @dev: The device that is associated with these pools. + * @size: Size used during DMA allocation. +@@ -107,7 +102,6 @@ struct dma_pool { + struct list_head pools; /* The 'struct device->dma_pools link */ + enum pool_type type; + spinlock_t lock; +- struct list_head inuse_list; + struct list_head free_list; + struct device *dev; + unsigned size; +@@ -124,13 +118,14 @@ struct dma_pool { + * The accounting page keeping track of the allocated page along with + * the DMA address. + * @page_list: The link to the 'page_list' in 'struct dma_pool'. +- * @vaddr: The virtual address of the page ++ * @vaddr: The virtual address of the page and a flag if the page belongs to a ++ * huge pool + * @dma: The bus address of the page. If the page is not allocated + * via the DMA API, it will be -1. + */ + struct dma_page { + struct list_head page_list; +- void *vaddr; ++ unsigned long vaddr; + struct page *p; + dma_addr_t dma; + }; +@@ -221,11 +216,11 @@ static ssize_t ttm_pool_store(struct kobject *kobj, struct attribute *attr, + /* Convert kb to number of pages */ + val = val / (PAGE_SIZE >> 10); + +- if (attr == &ttm_page_pool_max) ++ if (attr == &ttm_page_pool_max) { + m->options.max_size = val; +- else if (attr == &ttm_page_pool_small) ++ } else if (attr == &ttm_page_pool_small) { + m->options.small = val; +- else if (attr == &ttm_page_pool_alloc_size) { ++ } else if (attr == &ttm_page_pool_alloc_size) { + if (val > NUM_PAGES_TO_ALLOC*8) { + pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n", + NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7), +@@ -329,7 +324,8 @@ static int ttm_set_pages_caching(struct dma_pool *pool, + static void __ttm_dma_free_page(struct dma_pool *pool, struct dma_page *d_page) + { + dma_addr_t dma = d_page->dma; +- dma_free_coherent(pool->dev, pool->size, d_page->vaddr, dma); ++ d_page->vaddr &= ~VADDR_FLAG_HUGE_POOL; ++ dma_free_coherent(pool->dev, pool->size, (void *)d_page->vaddr, dma); + + kfree(d_page); + d_page = NULL; +@@ -337,19 +333,26 @@ static void __ttm_dma_free_page(struct dma_pool *pool, struct dma_page *d_page) + static struct dma_page *__ttm_dma_alloc_page(struct dma_pool *pool) + { + struct dma_page *d_page; ++ unsigned long attrs = 0; ++ void *vaddr; + + d_page = kmalloc(sizeof(struct dma_page), GFP_KERNEL); + if (!d_page) + return NULL; + +- d_page->vaddr = dma_alloc_coherent(pool->dev, pool->size, +- &d_page->dma, +- pool->gfp_flags); +- if (d_page->vaddr) { +- if (is_vmalloc_addr(d_page->vaddr)) +- d_page->p = vmalloc_to_page(d_page->vaddr); ++ if (pool->type & IS_HUGE) ++ attrs = DMA_ATTR_NO_WARN; ++ ++ vaddr = dma_alloc_attrs(pool->dev, pool->size, &d_page->dma, ++ pool->gfp_flags, attrs); ++ if (vaddr) { ++ if (is_vmalloc_addr(vaddr)) ++ d_page->p = vmalloc_to_page(vaddr); + else +- d_page->p = virt_to_page(d_page->vaddr); ++ d_page->p = virt_to_page(vaddr); ++ d_page->vaddr = (unsigned long)vaddr; ++ if (pool->type & IS_HUGE) ++ d_page->vaddr |= VADDR_FLAG_HUGE_POOL; + } else { + kfree(d_page); + d_page = NULL; +@@ -381,11 +384,38 @@ static void ttm_pool_update_free_locked(struct dma_pool *pool, + } + + /* set memory back to wb and free the pages. */ ++static void ttm_dma_page_put(struct dma_pool *pool, struct dma_page *d_page) ++{ ++ struct page *page = d_page->p; ++ unsigned i, num_pages; ++ ++ /* Don't set WB on WB page pool. */ ++ if (!(pool->type & IS_CACHED)) { ++ num_pages = pool->size / PAGE_SIZE; ++ for (i = 0; i < num_pages; ++i, ++page) { ++ if (set_pages_array_wb(&page, 1)) { ++ pr_err("%s: Failed to set %d pages to wb!\n", ++ pool->dev_name, 1); ++ } ++ } ++ } ++ ++ list_del(&d_page->page_list); ++ __ttm_dma_free_page(pool, d_page); ++} ++ + static void ttm_dma_pages_put(struct dma_pool *pool, struct list_head *d_pages, + struct page *pages[], unsigned npages) + { + struct dma_page *d_page, *tmp; + ++ if (pool->type & IS_HUGE) { ++ list_for_each_entry_safe(d_page, tmp, d_pages, page_list) ++ ttm_dma_page_put(pool, d_page); ++ ++ return; ++ } ++ + /* Don't set WB on WB page pool. */ + if (npages && !(pool->type & IS_CACHED) && + set_pages_array_wb(pages, npages)) +@@ -398,17 +428,6 @@ static void ttm_dma_pages_put(struct dma_pool *pool, struct list_head *d_pages, + } + } + +-static void ttm_dma_page_put(struct dma_pool *pool, struct dma_page *d_page) +-{ +- /* Don't set WB on WB page pool. */ +- if (!(pool->type & IS_CACHED) && set_pages_array_wb(&d_page->p, 1)) +- pr_err("%s: Failed to set %d pages to wb!\n", +- pool->dev_name, 1); +- +- list_del(&d_page->page_list); +- __ttm_dma_free_page(pool, d_page); +-} +- + /* + * Free pages from pool. + * +@@ -446,7 +465,7 @@ static unsigned ttm_dma_page_pool_free(struct dma_pool *pool, unsigned nr_free, + GFP_KERNEL); + + if (!pages_to_free) { +- pr_err("%s: Failed to allocate memory for pool free operation\n", ++ pr_debug("%s: Failed to allocate memory for pool free operation\n", + pool->dev_name); + return 0; + } +@@ -577,8 +596,8 @@ static int ttm_dma_pool_match(struct device *dev, void *res, void *match_data) + static struct dma_pool *ttm_dma_pool_init(struct device *dev, gfp_t flags, + enum pool_type type) + { +- char *n[] = {"wc", "uc", "cached", " dma32", "unknown",}; +- enum pool_type t[] = {IS_WC, IS_UC, IS_CACHED, IS_DMA32, IS_UNDEFINED}; ++ const char *n[] = {"wc", "uc", "cached", " dma32", "huge"}; ++ enum pool_type t[] = {IS_WC, IS_UC, IS_CACHED, IS_DMA32, IS_HUGE}; + struct device_pools *sec_pool = NULL; + struct dma_pool *pool = NULL, **ptr; + unsigned i; +@@ -609,18 +628,24 @@ static struct dma_pool *ttm_dma_pool_init(struct device *dev, gfp_t flags, + sec_pool->pool = pool; + + INIT_LIST_HEAD(&pool->free_list); +- INIT_LIST_HEAD(&pool->inuse_list); + INIT_LIST_HEAD(&pool->pools); + spin_lock_init(&pool->lock); + pool->dev = dev; + pool->npages_free = pool->npages_in_use = 0; + pool->nfrees = 0; + pool->gfp_flags = flags; +- pool->size = PAGE_SIZE; ++ if (type & IS_HUGE) ++#ifdef CONFIG_TRANSPARENT_HUGEPAGE ++ pool->size = HPAGE_PMD_SIZE; ++#else ++ BUG(); ++#endif ++ else ++ pool->size = PAGE_SIZE; + pool->type = type; + pool->nrefills = 0; + p = pool->name; +- for (i = 0; i < 5; i++) { ++ for (i = 0; i < ARRAY_SIZE(t); i++) { + if (type & t[i]) { + p += snprintf(p, sizeof(pool->name) - (p - pool->name), + "%s", n[i]); +@@ -653,10 +678,10 @@ static struct dma_pool *ttm_dma_pool_init(struct device *dev, gfp_t flags, + static struct dma_pool *ttm_dma_find_pool(struct device *dev, + enum pool_type type) + { +- struct dma_pool *pool, *tmp, *found = NULL; ++ struct dma_pool *pool, *tmp; + + if (type == IS_UNDEFINED) +- return found; ++ return NULL; + + /* NB: We iterate on the 'struct dev' which has no spinlock, but + * it does have a kref which we have taken. The kref is taken during +@@ -669,13 +694,10 @@ static struct dma_pool *ttm_dma_find_pool(struct device *dev, + * thing is at that point of time there are no pages associated with the + * driver so this function will not be called. + */ +- list_for_each_entry_safe(pool, tmp, &dev->dma_pools, pools) { +- if (pool->type != type) +- continue; +- found = pool; +- break; +- } +- return found; ++ list_for_each_entry_safe(pool, tmp, &dev->dma_pools, pools) ++ if (pool->type == type) ++ return pool; ++ return NULL; + } + + /* +@@ -724,7 +746,7 @@ static int ttm_dma_pool_alloc_new_pages(struct dma_pool *pool, + struct dma_page *dma_p; + struct page *p; + int r = 0; +- unsigned i, cpages; ++ unsigned i, j, npages, cpages; + unsigned max_cpages = min(count, + (unsigned)(PAGE_SIZE/sizeof(struct page *))); + +@@ -732,21 +754,20 @@ static int ttm_dma_pool_alloc_new_pages(struct dma_pool *pool, + caching_array = kmalloc(max_cpages*sizeof(struct page *), GFP_KERNEL); + + if (!caching_array) { +- pr_err("%s: Unable to allocate table for new pages\n", ++ pr_debug("%s: Unable to allocate table for new pages\n", + pool->dev_name); + return -ENOMEM; + } + +- if (count > 1) { ++ if (count > 1) + pr_debug("%s: (%s:%d) Getting %d pages\n", + pool->dev_name, pool->name, current->pid, count); +- } + + for (i = 0, cpages = 0; i < count; ++i) { + dma_p = __ttm_dma_alloc_page(pool); + if (!dma_p) { +- pr_err("%s: Unable to get page %u\n", +- pool->dev_name, i); ++ pr_debug("%s: Unable to get page %u\n", ++ pool->dev_name, i); + + /* store already allocated pages in the pool after + * setting the caching state */ +@@ -762,28 +783,32 @@ static int ttm_dma_pool_alloc_new_pages(struct dma_pool *pool, + goto out; + } + p = dma_p->p; ++ list_add(&dma_p->page_list, d_pages); ++ + #ifdef CONFIG_HIGHMEM + /* gfp flags of highmem page should never be dma32 so we + * we should be fine in such case + */ +- if (!PageHighMem(p)) ++ if (PageHighMem(p)) ++ continue; + #endif +- { +- caching_array[cpages++] = p; ++ ++ npages = pool->size / PAGE_SIZE; ++ for (j = 0; j < npages; ++j) { ++ caching_array[cpages++] = p + j; + if (cpages == max_cpages) { + /* Note: Cannot hold the spinlock */ + r = ttm_set_pages_caching(pool, caching_array, +- cpages); ++ cpages); + if (r) { + ttm_dma_handle_caching_state_failure( +- pool, d_pages, caching_array, +- cpages); ++ pool, d_pages, caching_array, ++ cpages); + goto out; + } + cpages = 0; + } + } +- list_add(&dma_p->page_list, d_pages); + } + + if (cpages) { +@@ -828,8 +853,8 @@ static int ttm_dma_page_pool_fill_locked(struct dma_pool *pool, + struct dma_page *d_page; + unsigned cpages = 0; + +- pr_err("%s: Failed to fill %s pool (r:%d)!\n", +- pool->dev_name, pool->name, r); ++ pr_debug("%s: Failed to fill %s pool (r:%d)!\n", ++ pool->dev_name, pool->name, r); + + list_for_each_entry(d_page, &d_pages, page_list) { + cpages++; +@@ -843,18 +868,18 @@ static int ttm_dma_page_pool_fill_locked(struct dma_pool *pool, + } + + /* +- * @return count of pages still required to fulfill the request. + * The populate list is actually a stack (not that is matters as TTM + * allocates one page at a time. ++ * return dma_page pointer if success, otherwise NULL. + */ +-static int ttm_dma_pool_get_pages(struct dma_pool *pool, ++static struct dma_page *ttm_dma_pool_get_pages(struct dma_pool *pool, + struct ttm_dma_tt *ttm_dma, + unsigned index) + { +- struct dma_page *d_page; ++ struct dma_page *d_page = NULL; + struct ttm_tt *ttm = &ttm_dma->ttm; + unsigned long irq_flags; +- int count, r = -ENOMEM; ++ int count; + + spin_lock_irqsave(&pool->lock, irq_flags); + count = ttm_dma_page_pool_fill_locked(pool, &irq_flags); +@@ -863,12 +888,35 @@ static int ttm_dma_pool_get_pages(struct dma_pool *pool, + ttm->pages[index] = d_page->p; + ttm_dma->dma_address[index] = d_page->dma; + list_move_tail(&d_page->page_list, &ttm_dma->pages_list); +- r = 0; + pool->npages_in_use += 1; + pool->npages_free -= 1; + } + spin_unlock_irqrestore(&pool->lock, irq_flags); +- return r; ++ return d_page; ++} ++ ++static gfp_t ttm_dma_pool_gfp_flags(struct ttm_dma_tt *ttm_dma, bool huge) ++{ ++ struct ttm_tt *ttm = &ttm_dma->ttm; ++ gfp_t gfp_flags; ++ ++ if (ttm->page_flags & TTM_PAGE_FLAG_DMA32) ++ gfp_flags = GFP_USER | GFP_DMA32; ++ else ++ gfp_flags = GFP_HIGHUSER; ++ if (ttm->page_flags & TTM_PAGE_FLAG_ZERO_ALLOC) ++ gfp_flags |= __GFP_ZERO; ++ ++ if (huge) { ++ gfp_flags |= GFP_TRANSHUGE; ++ gfp_flags &= ~__GFP_MOVABLE; ++ gfp_flags &= ~__GFP_COMP; ++ } ++ ++ if (ttm->page_flags & TTM_PAGE_FLAG_NO_RETRY) ++ gfp_flags |= __GFP_RETRY_MAYFAIL; ++ ++ return gfp_flags; + } + + /* +@@ -879,37 +927,78 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev, + struct ttm_operation_ctx *ctx) + { + struct ttm_tt *ttm = &ttm_dma->ttm; +- struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; ++ struct ttm_mem_global *mem_glob = ttm->bdev->glob->mem_glob; + unsigned long num_pages = ttm->num_pages; + struct dma_pool *pool; ++ struct dma_page *d_page; + enum pool_type type; + unsigned i; +- gfp_t gfp_flags; + int ret; + + if (ttm->state != tt_unpopulated) + return 0; + ++ if (ttm_check_under_lowerlimit(mem_glob, num_pages, ctx)) ++ return -ENOMEM; ++ ++ INIT_LIST_HEAD(&ttm_dma->pages_list); ++ i = 0; ++ + type = ttm_to_type(ttm->page_flags, ttm->caching_state); ++ ++#ifdef CONFIG_TRANSPARENT_HUGEPAGE + if (ttm->page_flags & TTM_PAGE_FLAG_DMA32) +- gfp_flags = GFP_USER | GFP_DMA32; +- else +- gfp_flags = GFP_HIGHUSER; +- if (ttm->page_flags & TTM_PAGE_FLAG_ZERO_ALLOC) +- gfp_flags |= __GFP_ZERO; ++ goto skip_huge; ++ ++ pool = ttm_dma_find_pool(dev, type | IS_HUGE); ++ if (!pool) { ++ gfp_t gfp_flags = ttm_dma_pool_gfp_flags(ttm_dma, true); ++ ++ pool = ttm_dma_pool_init(dev, gfp_flags, type | IS_HUGE); ++ if (IS_ERR_OR_NULL(pool)) ++ goto skip_huge; ++ } ++ ++ while (num_pages >= HPAGE_PMD_NR) { ++ unsigned j; ++ ++ d_page = ttm_dma_pool_get_pages(pool, ttm_dma, i); ++ if (!d_page) ++ break; ++ ++ ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], ++ pool->size, ctx); ++ if (unlikely(ret != 0)) { ++ ttm_dma_unpopulate(ttm_dma, dev); ++ return -ENOMEM; ++ } ++ ++ d_page->vaddr |= VADDR_FLAG_UPDATED_COUNT; ++ for (j = i + 1; j < (i + HPAGE_PMD_NR); ++j) { ++ ttm->pages[j] = ttm->pages[j - 1] + 1; ++ ttm_dma->dma_address[j] = ttm_dma->dma_address[j - 1] + ++ PAGE_SIZE; ++ } ++ ++ i += HPAGE_PMD_NR; ++ num_pages -= HPAGE_PMD_NR; ++ } ++ ++skip_huge: ++#endif + + pool = ttm_dma_find_pool(dev, type); + if (!pool) { ++ gfp_t gfp_flags = ttm_dma_pool_gfp_flags(ttm_dma, false); ++ + pool = ttm_dma_pool_init(dev, gfp_flags, type); +- if (IS_ERR_OR_NULL(pool)) { ++ if (IS_ERR_OR_NULL(pool)) + return -ENOMEM; +- } + } + +- INIT_LIST_HEAD(&ttm_dma->pages_list); +- for (i = 0; i < ttm->num_pages; ++i) { +- ret = ttm_dma_pool_get_pages(pool, ttm_dma, i); +- if (ret != 0) { ++ while (num_pages) { ++ d_page = ttm_dma_pool_get_pages(pool, ttm_dma, i); ++ if (!d_page) { + ttm_dma_unpopulate(ttm_dma, dev); + return -ENOMEM; + } +@@ -920,6 +1009,10 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev, + ttm_dma_unpopulate(ttm_dma, dev); + return -ENOMEM; + } ++ ++ d_page->vaddr |= VADDR_FLAG_UPDATED_COUNT; ++ ++i; ++ --num_pages; + } + + if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) { +@@ -939,14 +1032,41 @@ EXPORT_SYMBOL_GPL(ttm_dma_populate); + void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev) + { + struct ttm_tt *ttm = &ttm_dma->ttm; ++ struct ttm_mem_global *mem_glob = ttm->bdev->glob->mem_glob; + struct dma_pool *pool; + struct dma_page *d_page, *next; + enum pool_type type; + bool is_cached = false; +- unsigned count = 0, i, npages = 0; ++ unsigned count, i, npages = 0; + unsigned long irq_flags; + + type = ttm_to_type(ttm->page_flags, ttm->caching_state); ++ ++#ifdef CONFIG_TRANSPARENT_HUGEPAGE ++ pool = ttm_dma_find_pool(dev, type | IS_HUGE); ++ if (pool) { ++ count = 0; ++ list_for_each_entry_safe(d_page, next, &ttm_dma->pages_list, ++ page_list) { ++ if (!(d_page->vaddr & VADDR_FLAG_HUGE_POOL)) ++ continue; ++ ++ count++; ++ if (d_page->vaddr & VADDR_FLAG_UPDATED_COUNT) { ++ ttm_mem_global_free_page(mem_glob, d_page->p, ++ pool->size); ++ d_page->vaddr &= ~VADDR_FLAG_UPDATED_COUNT; ++ } ++ ttm_dma_page_put(pool, d_page); ++ } ++ ++ spin_lock_irqsave(&pool->lock, irq_flags); ++ pool->npages_in_use -= count; ++ pool->nfrees += count; ++ spin_unlock_irqrestore(&pool->lock, irq_flags); ++ } ++#endif ++ + pool = ttm_dma_find_pool(dev, type); + if (!pool) + return; +@@ -955,9 +1075,20 @@ void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev) + ttm_to_type(ttm->page_flags, tt_cached)) == pool); + + /* make sure pages array match list and count number of pages */ +- list_for_each_entry(d_page, &ttm_dma->pages_list, page_list) { ++ count = 0; ++ list_for_each_entry_safe(d_page, next, &ttm_dma->pages_list, ++ page_list) { + ttm->pages[count] = d_page->p; + count++; ++ ++ if (d_page->vaddr & VADDR_FLAG_UPDATED_COUNT) { ++ ttm_mem_global_free_page(mem_glob, d_page->p, ++ pool->size); ++ d_page->vaddr &= ~VADDR_FLAG_UPDATED_COUNT; ++ } ++ ++ if (is_cached) ++ ttm_dma_page_put(pool, d_page); + } + + spin_lock_irqsave(&pool->lock, irq_flags); +@@ -977,19 +1108,6 @@ void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev) + } + spin_unlock_irqrestore(&pool->lock, irq_flags); + +- if (is_cached) { +- list_for_each_entry_safe(d_page, next, &ttm_dma->pages_list, page_list) { +- ttm_mem_global_free_page(ttm->glob->mem_glob, +- d_page->p); +- ttm_dma_page_put(pool, d_page); +- } +- } else { +- for (i = 0; i < count; i++) { +- ttm_mem_global_free_page(ttm->glob->mem_glob, +- ttm->pages[i]); +- } +- } +- + INIT_LIST_HEAD(&ttm_dma->pages_list); + for (i = 0; i < ttm->num_pages; i++) { + ttm->pages[i] = NULL; +@@ -1067,12 +1185,12 @@ ttm_dma_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc) + return count; + } + +-static void ttm_dma_pool_mm_shrink_init(struct ttm_pool_manager *manager) ++static int ttm_dma_pool_mm_shrink_init(struct ttm_pool_manager *manager) + { + manager->mm_shrink.count_objects = ttm_dma_pool_shrink_count; + manager->mm_shrink.scan_objects = &ttm_dma_pool_shrink_scan; + manager->mm_shrink.seeks = 1; +- register_shrinker(&manager->mm_shrink); ++ return register_shrinker(&manager->mm_shrink); + } + + static void ttm_dma_pool_mm_shrink_fini(struct ttm_pool_manager *manager) +@@ -1082,7 +1200,7 @@ static void ttm_dma_pool_mm_shrink_fini(struct ttm_pool_manager *manager) + + int ttm_dma_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) + { +- int ret = -ENOMEM; ++ int ret; + + WARN_ON(_manager); + +@@ -1090,7 +1208,7 @@ int ttm_dma_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) + + _manager = kzalloc(sizeof(*_manager), GFP_KERNEL); + if (!_manager) +- goto err; ++ return -ENOMEM; + + mutex_init(&_manager->lock); + INIT_LIST_HEAD(&_manager->pools); +@@ -1102,13 +1220,17 @@ int ttm_dma_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) + /* This takes care of auto-freeing the _manager */ + ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type, + &glob->kobj, "dma_pool"); +- if (unlikely(ret != 0)) { +- kobject_put(&_manager->kobj); +- goto err; +- } +- ttm_dma_pool_mm_shrink_init(_manager); ++ if (unlikely(ret != 0)) ++ goto error; ++ ++ ret = ttm_dma_pool_mm_shrink_init(_manager); ++ if (unlikely(ret != 0)) ++ goto error; + return 0; +-err: ++ ++error: ++ kobject_put(&_manager->kobj); ++ _manager = NULL; + return ret; + } + +@@ -1134,15 +1256,12 @@ int ttm_dma_page_alloc_debugfs(struct seq_file *m, void *data) + { + struct device_pools *p; + struct dma_pool *pool = NULL; +- char *h[] = {"pool", "refills", "pages freed", "inuse", "available", +- "name", "virt", "busaddr"}; + + if (!_manager) { + seq_printf(m, "No pool allocator running.\n"); + return 0; + } +- seq_printf(m, "%13s %12s %13s %8s %8s %8s\n", +- h[0], h[1], h[2], h[3], h[4], h[5]); ++ seq_printf(m, " pool refills pages freed inuse available name\n"); + mutex_lock(&_manager->lock); + list_for_each_entry(p, &_manager->pools, pools) { + struct device *dev = p->dev; +diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c +index 522a80d..a9acda4 100644 +--- a/drivers/gpu/drm/ttm/ttm_tt.c ++++ b/drivers/gpu/drm/ttm/ttm_tt.c +@@ -83,16 +83,13 @@ int ttm_tt_create(struct ttm_buffer_object *bo, bool zero_alloc) + if (unlikely(bo->ttm == NULL)) + return -ENOMEM; + +- if (bo->type == ttm_bo_type_sg) +- bo->ttm->sg = bo->sg; +- +- return 0; ++ return 0; + } + + /** + * Allocates storage for pointers to the pages that back the ttm. + */ +-static void ttm_tt_alloc_page_directory(struct ttm_tt *ttm) ++static int ttm_tt_alloc_page_directory(struct ttm_tt *ttm) + { + ttm->pages = kvmalloc_array(ttm->num_pages, sizeof(void*), + GFP_KERNEL | __GFP_ZERO); +@@ -105,6 +102,7 @@ static int ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt *ttm) + sizeof(*ttm->dma_address), + GFP_KERNEL | __GFP_ZERO); + ttm->dma_address = (void *) (ttm->ttm.pages + ttm->ttm.num_pages); ++ return 0; + } + + static int ttm_sg_tt_alloc_page_directory(struct ttm_dma_tt *ttm) +@@ -246,6 +244,7 @@ void ttm_tt_init_fields(struct ttm_tt *ttm, struct ttm_buffer_object *bo, + ttm->page_flags = page_flags; + ttm->state = tt_unpopulated; + ttm->swap_storage = NULL; ++ ttm->sg = bo->sg; + } + + int ttm_tt_init(struct ttm_tt *ttm, struct ttm_buffer_object *bo, +@@ -253,8 +252,7 @@ int ttm_tt_init(struct ttm_tt *ttm, struct ttm_buffer_object *bo, + { + ttm_tt_init_fields(ttm, bo, page_flags); + +- ttm_tt_alloc_page_directory(ttm); +- if (!ttm->pages) { ++ if (ttm_tt_alloc_page_directory(ttm)) { + ttm_tt_destroy(ttm); + pr_err("Failed allocating page table\n"); + return -ENOMEM; +@@ -278,8 +276,7 @@ int ttm_dma_tt_init(struct ttm_dma_tt *ttm_dma, struct ttm_buffer_object *bo, + ttm_tt_init_fields(ttm, bo, page_flags); + + INIT_LIST_HEAD(&ttm_dma->pages_list); +- ttm_dma_tt_alloc_page_directory(ttm_dma); +- if (!ttm->pages) { ++ if (ttm_dma_tt_alloc_page_directory(ttm_dma)) { + ttm_tt_destroy(ttm); + pr_err("Failed allocating page table\n"); + return -ENOMEM; +@@ -314,7 +311,10 @@ void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma) + { + struct ttm_tt *ttm = &ttm_dma->ttm; + ++ if (ttm->pages) + kvfree(ttm->pages); ++ else ++ kvfree(ttm_dma->dma_address); + ttm->pages = NULL; + ttm_dma->dma_address = NULL; + } +@@ -342,7 +342,7 @@ int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem, + if (ttm->state == tt_bound) + return 0; + +- ret = ttm->bdev->driver->ttm_tt_populate(ttm, ctx); ++ ret = ttm_tt_populate(ttm, ctx); + if (ret) + return ret; + +@@ -371,7 +371,11 @@ int ttm_tt_swapin(struct ttm_tt *ttm) + swap_space = swap_storage->f_mapping; + + for (i = 0; i < ttm->num_pages; ++i) { +- from_page = shmem_read_mapping_page(swap_space, i); ++ gfp_t gfp_mask = mapping_gfp_mask(swap_space); ++ ++ gfp_mask |= (ttm->page_flags & TTM_PAGE_FLAG_NO_RETRY ? __GFP_RETRY_MAYFAIL : 0); ++ from_page = shmem_read_mapping_page_gfp(swap_space, i, gfp_mask); ++ + if (IS_ERR(from_page)) { + ret = PTR_ERR(from_page); + goto out_err; +@@ -414,16 +418,22 @@ int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistent_swap_storage) + pr_err("Failed allocating swap storage\n"); + return PTR_ERR(swap_storage); + } +- } else ++ } else { + swap_storage = persistent_swap_storage; ++ } + + swap_space = swap_storage->f_mapping; + + for (i = 0; i < ttm->num_pages; ++i) { ++ gfp_t gfp_mask = mapping_gfp_mask(swap_space); ++ ++ gfp_mask |= (ttm->page_flags & TTM_PAGE_FLAG_NO_RETRY ? __GFP_RETRY_MAYFAIL : 0); ++ + from_page = ttm->pages[i]; + if (unlikely(from_page == NULL)) + continue; +- to_page = shmem_read_mapping_page(swap_space, i); ++ ++ to_page = shmem_read_mapping_page_gfp(swap_space, i, gfp_mask); + if (IS_ERR(to_page)) { + ret = PTR_ERR(to_page); + goto out_err; +@@ -448,6 +458,33 @@ int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistent_swap_storage) + return ret; + } + ++static void ttm_tt_add_mapping(struct ttm_tt *ttm) ++{ ++ pgoff_t i; ++ ++ if (ttm->page_flags & TTM_PAGE_FLAG_SG) ++ return; ++ ++ for (i = 0; i < ttm->num_pages; ++i) ++ ttm->pages[i]->mapping = ttm->bdev->dev_mapping; ++} ++ ++int ttm_tt_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) ++{ ++ int ret; ++ ++ if (ttm->state != tt_unpopulated) ++ return 0; ++ ++ if (ttm->bdev->driver->ttm_tt_populate) ++ ret = ttm->bdev->driver->ttm_tt_populate(ttm, ctx); ++ else ++ ret = ttm_pool_populate(ttm, ctx); ++ if (!ret) ++ ttm_tt_add_mapping(ttm); ++ return ret; ++} ++ + static void ttm_tt_clear_mapping(struct ttm_tt *ttm) + { + pgoff_t i; +@@ -468,5 +505,8 @@ void ttm_tt_unpopulate(struct ttm_tt *ttm) + return; + + ttm_tt_clear_mapping(ttm); +- ttm->bdev->driver->ttm_tt_unpopulate(ttm); ++ if (ttm->bdev->driver->ttm_tt_unpopulate) ++ ttm->bdev->driver->ttm_tt_unpopulate(ttm); ++ else ++ ttm_pool_unpopulate(ttm); + } +diff --git a/drivers/gpu/drm/virtio/virtgpu_ttm.c b/drivers/gpu/drm/virtio/virtgpu_ttm.c +index a9f2fb0..3848903 100644 +--- a/drivers/gpu/drm/virtio/virtgpu_ttm.c ++++ b/drivers/gpu/drm/virtio/virtgpu_ttm.c +@@ -428,7 +428,6 @@ static struct ttm_bo_driver virtio_gpu_bo_driver = { + .verify_access = &virtio_gpu_verify_access, + .io_mem_reserve = &virtio_gpu_ttm_io_mem_reserve, + .io_mem_free = &virtio_gpu_ttm_io_mem_free, +- .io_mem_pfn = ttm_bo_default_io_mem_pfn, + .move_notify = &virtio_gpu_bo_move_notify, + .swap_notify = &virtio_gpu_bo_swap_notify, + }; +diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c +index bab10e3..7177eec 100644 +--- a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c ++++ b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c +@@ -860,5 +860,4 @@ struct ttm_bo_driver vmw_bo_driver = { + .fault_reserve_notify = &vmw_ttm_fault_reserve_notify, + .io_mem_reserve = &vmw_ttm_io_mem_reserve, + .io_mem_free = &vmw_ttm_io_mem_free, +- .io_mem_pfn = ttm_bo_default_io_mem_pfn, + }; +diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h +index 2f71269..1822c4e 100644 +--- a/include/drm/ttm/ttm_bo_driver.h ++++ b/include/drm/ttm/ttm_bo_driver.h +@@ -609,6 +609,17 @@ ttm_flag_masked(uint32_t *old, uint32_t new, uint32_t mask) + } + + /** ++ * ttm_tt_create ++ * ++ * @bo: pointer to a struct ttm_buffer_object ++ * @zero_alloc: true if allocated pages needs to be zeroed ++ * ++ * Make sure we have a TTM structure allocated for the given BO. ++ * No pages are actually allocated. ++ */ ++int ttm_tt_create(struct ttm_buffer_object *bo, bool zero_alloc); ++ ++/** + * ttm_tt_init + * + * @ttm: The struct ttm_tt. +@@ -647,8 +658,8 @@ extern void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma); + * + * Bind the pages of @ttm to an aperture location identified by @bo_mem + */ +-extern int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem); +- ++extern int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem, ++ struct ttm_operation_ctx *ctx); + /** + * ttm_ttm_destroy: + * +@@ -694,6 +705,15 @@ extern int ttm_tt_swapout(struct ttm_tt *ttm, + struct file *persistent_swap_storage); + + /** ++ * ttm_tt_populate - allocate pages for a ttm ++ * ++ * @ttm: Pointer to the ttm_tt structure ++ * ++ * Calls the driver method to allocate pages for a ttm ++ */ ++extern int ttm_tt_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx); ++ ++/** + * ttm_tt_unpopulate - free pages from a ttm + * + * @ttm: Pointer to the ttm_tt structure +@@ -823,24 +843,24 @@ static inline int __ttm_bo_reserve(struct ttm_buffer_object *bo, + bool interruptible, bool no_wait, + struct ww_acquire_ctx *ticket) + { +- int ret = 0; +- +- if (no_wait) { +- bool success; +- if (WARN_ON(ticket)) +- return -EBUSY; +- +- success = ww_mutex_trylock(&bo->resv->lock); +- return success ? 0 : -EBUSY; +- } +- +- if (interruptible) +- ret = ww_mutex_lock_interruptible(&bo->resv->lock, ticket); +- else +- ret = ww_mutex_lock(&bo->resv->lock, ticket); +- if (ret == -EINTR) +- return -ERESTARTSYS; +- return ret; ++ int ret = 0; ++ ++ if (no_wait) { ++ bool success; ++ if (WARN_ON(ticket)) ++ return -EBUSY; ++ ++ success = reservation_object_trylock(bo->resv); ++ return success ? 0 : -EBUSY; ++ } ++ ++ if (interruptible) ++ ret = reservation_object_lock_interruptible(bo->resv, ticket); ++ else ++ ret = reservation_object_lock(bo->resv, ticket); ++ if (ret == -EINTR) ++ return -ERESTARTSYS; ++ return ret; + } + + /** +@@ -935,18 +955,6 @@ static inline int ttm_bo_reserve_slowpath(struct ttm_buffer_object *bo, + } + + /** +- * __ttm_bo_unreserve +- * @bo: A pointer to a struct ttm_buffer_object. +- * +- * Unreserve a previous reservation of @bo where the buffer object is +- * already on lru lists. +- */ +-static inline void __ttm_bo_unreserve(struct ttm_buffer_object *bo) +-{ +- ww_mutex_unlock(&bo->resv->lock); +-} +- +-/** + * ttm_bo_unreserve + * + * @bo: A pointer to a struct ttm_buffer_object. +@@ -955,13 +963,12 @@ static inline void __ttm_bo_unreserve(struct ttm_buffer_object *bo) + */ + static inline void ttm_bo_unreserve(struct ttm_buffer_object *bo) + { +- if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { +- spin_lock(&bo->bdev->glob->lru_lock); +- ttm_bo_add_to_lru(bo); +- spin_unlock(&bo->glob->lru_lock); +- spin_unlock(&bo->bdev->glob->lru_lock); +- } +- __ttm_bo_unreserve(bo); ++ if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { ++ spin_lock(&bo->bdev->glob->lru_lock); ++ ttm_bo_add_to_lru(bo); ++ spin_unlock(&bo->bdev->glob->lru_lock); ++ } ++ reservation_object_unlock(bo->resv); + } + + /** +@@ -1113,7 +1120,7 @@ extern const struct ttm_mem_type_manager_func ttm_bo_manager_func; + struct ttm_tt *ttm_agp_tt_create(struct ttm_buffer_object *bo, + struct agp_bridge_data *bridge, + uint32_t page_flags); +-int ttm_agp_tt_populate(struct ttm_tt *ttm); ++int ttm_agp_tt_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx); + void ttm_agp_tt_unpopulate(struct ttm_tt *ttm); + #endif + +diff --git a/include/drm/ttm/ttm_memory.h b/include/drm/ttm/ttm_memory.h +index c8885bc9..bc8b22e 100644 +--- a/include/drm/ttm/ttm_memory.h ++++ b/include/drm/ttm/ttm_memory.h +@@ -38,20 +38,6 @@ + #include "ttm_bo_api.h" + + /** +- * struct ttm_mem_shrink - callback to shrink TTM memory usage. +- * +- * @do_shrink: The callback function. +- * +- * Arguments to the do_shrink functions are intended to be passed using +- * inheritance. That is, the argument class derives from struct ttm_mem_shrink, +- * and can be accessed using container_of(). +- */ +- +-struct ttm_mem_shrink { +- int (*do_shrink) (struct ttm_mem_shrink *); +-}; +- +-/** + * struct ttm_mem_global - Global memory accounting structure. + * + * @shrink: A single callback to shrink TTM memory usage. Extend this +@@ -94,59 +80,6 @@ struct ttm_mem_global { + #endif + }; + +-/** +- * ttm_mem_init_shrink - initialize a struct ttm_mem_shrink object +- * +- * @shrink: The object to initialize. +- * @func: The callback function. +- */ +- +-static inline void ttm_mem_init_shrink(struct ttm_mem_shrink *shrink, +- int (*func) (struct ttm_mem_shrink *)) +-{ +- shrink->do_shrink = func; +-} +- +-/** +- * ttm_mem_register_shrink - register a struct ttm_mem_shrink object. +- * +- * @glob: The struct ttm_mem_global object to register with. +- * @shrink: An initialized struct ttm_mem_shrink object to register. +- * +- * Returns: +- * -EBUSY: There's already a callback registered. (May change). +- */ +- +-static inline int ttm_mem_register_shrink(struct ttm_mem_global *glob, +- struct ttm_mem_shrink *shrink) +-{ +- spin_lock(&glob->lock); +- if (glob->shrink != NULL) { +- spin_unlock(&glob->lock); +- return -EBUSY; +- } +- glob->shrink = shrink; +- spin_unlock(&glob->lock); +- return 0; +-} +- +-/** +- * ttm_mem_unregister_shrink - unregister a struct ttm_mem_shrink object. +- * +- * @glob: The struct ttm_mem_global object to unregister from. +- * @shrink: A previously registert struct ttm_mem_shrink object. +- * +- */ +- +-static inline void ttm_mem_unregister_shrink(struct ttm_mem_global *glob, +- struct ttm_mem_shrink *shrink) +-{ +- spin_lock(&glob->lock); +- BUG_ON(glob->shrink != shrink); +- glob->shrink = NULL; +- spin_unlock(&glob->lock); +-} +- + extern int ttm_mem_global_init(struct ttm_mem_global *glob); + extern void ttm_mem_global_release(struct ttm_mem_global *glob); + extern int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory, +@@ -157,7 +90,7 @@ extern int ttm_mem_global_alloc_page(struct ttm_mem_global *glob, + struct page *page, uint64_t size, + struct ttm_operation_ctx *ctx); + extern void ttm_mem_global_free_page(struct ttm_mem_global *glob, +- struct page *page); ++ struct page *page, uint64_t size); + extern size_t ttm_round_pot(size_t size); + extern uint64_t ttm_get_kernel_zone_memory_size(struct ttm_mem_global *glob); + extern bool ttm_check_under_lowerlimit(struct ttm_mem_global *glob, +diff --git a/include/drm/ttm/ttm_page_alloc.h b/include/drm/ttm/ttm_page_alloc.h +index 80aa53c..606a3ab 100644 +--- a/include/drm/ttm/ttm_page_alloc.h ++++ b/include/drm/ttm/ttm_page_alloc.h +@@ -58,7 +58,7 @@ extern int ttm_pool_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx); + */ + extern void ttm_pool_unpopulate(struct ttm_tt *ttm); + +-extern +int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt, ++extern int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt, + struct ttm_operation_ctx *ctx); + + /** +@@ -92,11 +92,6 @@ extern int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev, + extern void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev); + + /** +- * Populates and DMA maps pages to fullfil a ttm_dma_populate() request +- */ +-int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt); +- +-/** + * Unpopulates and DMA unmaps pages as part of a + * ttm_dma_unpopulate() request */ + void ttm_unmap_and_unpopulate_pages(struct device *dev, struct ttm_dma_tt *tt); +diff --git a/include/linux/reservation.h b/include/linux/reservation.h +index 21fc84d..eb444ff 100644 +--- a/include/linux/reservation.h ++++ b/include/linux/reservation.h +@@ -167,6 +167,28 @@ reservation_object_lock(struct reservation_object *obj, + } + + /** ++ * reservation_object_lock_interruptible - lock the reservation object ++ * @obj: the reservation object ++ * @ctx: the locking context ++ * ++ * Locks the reservation object interruptible for exclusive access and ++ * modification. Note, that the lock is only against other writers, readers ++ * will run concurrently with a writer under RCU. The seqlock is used to ++ * notify readers if they overlap with a writer. ++ * ++ * As the reservation object may be locked by multiple parties in an ++ * undefined order, a #ww_acquire_ctx is passed to unwind if a cycle ++ * is detected. See ww_mutex_lock() and ww_acquire_init(). A reservation ++ * object may be locked by itself by passing NULL as @ctx. ++ */ ++static inline int ++reservation_object_lock_interruptible(struct reservation_object *obj, ++ struct ww_acquire_ctx *ctx) ++{ ++ return ww_mutex_lock_interruptible(&obj->lock, ctx); ++} ++ ++/** + * reservation_object_trylock - trylock the reservation object + * @obj: the reservation object + * +-- +2.7.4 + |