aboutsummaryrefslogtreecommitdiffstats
path: root/meta-v1000/recipes-kernel/linux/linux-yocto-4.14.71/1640-drm-amdgpu-New-HSA-MMU-notifiers-to-work-under-memor.patch
diff options
context:
space:
mode:
Diffstat (limited to 'meta-v1000/recipes-kernel/linux/linux-yocto-4.14.71/1640-drm-amdgpu-New-HSA-MMU-notifiers-to-work-under-memor.patch')
-rw-r--r--meta-v1000/recipes-kernel/linux/linux-yocto-4.14.71/1640-drm-amdgpu-New-HSA-MMU-notifiers-to-work-under-memor.patch1434
1 files changed, 0 insertions, 1434 deletions
diff --git a/meta-v1000/recipes-kernel/linux/linux-yocto-4.14.71/1640-drm-amdgpu-New-HSA-MMU-notifiers-to-work-under-memor.patch b/meta-v1000/recipes-kernel/linux/linux-yocto-4.14.71/1640-drm-amdgpu-New-HSA-MMU-notifiers-to-work-under-memor.patch
deleted file mode 100644
index faa756f6..00000000
--- a/meta-v1000/recipes-kernel/linux/linux-yocto-4.14.71/1640-drm-amdgpu-New-HSA-MMU-notifiers-to-work-under-memor.patch
+++ /dev/null
@@ -1,1434 +0,0 @@
-From 8683beb2f68e2116a28f5a922ec9d7eb81c35faa Mon Sep 17 00:00:00 2001
-From: Felix Kuehling <Felix.Kuehling@amd.com>
-Date: Tue, 21 Mar 2017 11:55:47 -0400
-Subject: [PATCH 1640/4131] drm/amdgpu: New HSA MMU notifiers to work under
- memory pressure
-
-MMU notifiers can be invoked in reclaim-fs context under memory
-pressure. Any locks held in this case are subject to limitations to
-prevent potential deadlocks: memory allocations performed while
-holding those locks in any context must not cause memory reclaims.
-
-Particularly reservation objects are problematic in this respect,
-because they are locked (reserved) in too many places, potentially
-even other drivers, to enforce such limitations.
-
-This commit rewrites the HSA MMU notifier to avoid locking reservation
-objects. To allow this, the MMU notifier cannot take the current
-mapping state of a BO into account. MMU notifiers on HSA userptr
-BOs cause evictions on all GPUs. The only locks taken by the new
-MMU notifiers are the rmn->lock and the KFD DQM lock.
-
-Restore is still done by a delayed worker, but handles multiple BOs
-in a single per-process worker, rather than per-BO workers.
-
-Change-Id: I011fd11c95747caa117a592845ba0a85dc6e77be
-Signed-off-by: Felix Kuehling <Felix.Kuehling@amd.com>
----
- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 170 -----
- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 27 +-
- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 843 +++++++++++++----------
- drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c | 7 +-
- 4 files changed, 480 insertions(+), 567 deletions(-)
-
-diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
-index 3a1776b..f27d67bc 100755
---- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
-+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
-@@ -202,176 +202,6 @@ int amdgpu_amdkfd_resume(struct amdgpu_device *adev)
- return r;
- }
-
--/* Cancel any scheduled restore work or wait for it to finish. Must be
-- * called with the mem->lock held. First drop the mm reference. If the
-- * worker has already started, it will detect that mm was dropped and
-- * cancel itself.
-- *
-- * If the worker has already started, it needs to take the
-- * mem->lock. To prevent deadlocks, we need to briefly drop the lock
-- * while waiting. During that time someone else may schedule another
-- * restore. So repeat the process if necessary.
-- *
-- * mmput needs to be called without holding the lock to prevent
-- * circular lock dependencies.
-- */
--static void cancel_restore_locked(struct kgd_mem *mem)
--{
-- struct mm_struct *mm;
--
-- while (mem->mm) {
-- /* update_user_pages needs to drop the lock
-- * briefly. Therefore holding the lock is no guarantee
-- * that no restore is in progress
-- */
-- if (mem->busy) {
-- mutex_unlock(&mem->lock);
-- schedule_timeout_uninterruptible(1);
-- mutex_lock(&mem->lock);
-- continue;
-- }
--
-- mm = mem->mm;
-- mem->mm = NULL;
--
-- mutex_unlock(&mem->lock);
--
-- mmput(mm);
-- cancel_delayed_work_sync(&mem->work);
--
-- mutex_lock(&mem->lock);
-- }
--}
--
--void amdgpu_amdkfd_cancel_restore_mem(struct kgd_mem *mem)
--{
-- mutex_lock(&mem->lock);
-- cancel_restore_locked(mem);
-- mutex_unlock(&mem->lock);
--}
--
--int amdgpu_amdkfd_evict_mem(struct amdgpu_device *adev, struct kgd_mem *mem,
-- struct mm_struct *mm)
--{
-- int r;
--
-- if (!adev->kfd)
-- return -ENODEV;
--
-- mutex_lock(&mem->lock);
--
-- if (++mem->evicted > 1) {
-- /* Memory was already evicted. It may have been
-- * scheduled for restoration, but that restoration
-- * hasn't happened yet. When the worker starts it will
-- * know and abort.
-- */
-- mutex_unlock(&mem->lock);
-- return 0;
-- }
--
-- r = amdgpu_amdkfd_gpuvm_evict_mem(mem, mm);
--
-- if (r != 0)
-- /* First eviction failed, setting count back to 0 will
-- * make the corresponding restore fail gracefully */
-- mem->evicted = 0;
-- else
-- /* First eviction counts as 2. Eviction counter == 1
-- * means that restoration is scheduled. */
-- mem->evicted = 2;
--
-- mutex_unlock(&mem->lock);
--
-- return r;
--}
--
--static void amdgdu_amdkfd_restore_mem_worker(struct work_struct *work)
--{
-- struct delayed_work *dwork = to_delayed_work(work);
-- struct kgd_mem *mem = container_of(dwork, struct kgd_mem, work);
-- struct amdgpu_device *adev;
-- struct mm_struct *mm;
-- int ret = 0;
--
-- mutex_lock(&mem->lock);
--
-- adev = amdgpu_ttm_adev(mem->bo->tbo.bdev);
-- mm = mem->mm;
--
-- /* Check if restore was canceled */
-- if (!mm) {
-- mutex_unlock(&mem->lock);
-- return;
-- }
--
-- /* Only restore if no other eviction happened since restore
-- * was scheduled.
-- */
-- if (mem->evicted == 1) {
-- ret = amdgpu_amdkfd_gpuvm_restore_mem(mem, mm);
-- if (ret != -EBUSY && ret != -EDEADLK)
-- mem->evicted = 0;
-- }
--
-- /* If restore failed due to the VM being updated concurrently,
-- * reschedule restore again in a jiffie
-- */
-- if (ret == -EDEADLK && mem->evicted == 1) {
-- pr_err("Rescheduling restore\n");
-- mm = NULL;
-- schedule_delayed_work(&mem->work, 1);
-- } else {
-- BUG_ON(mem->mm != mm);
-- mem->mm = NULL;
-- }
--
-- mutex_unlock(&mem->lock);
--
-- if (mm)
-- mmput(mm);
--}
--
--int amdgpu_amdkfd_schedule_restore_mem(struct amdgpu_device *adev,
-- struct kgd_mem *mem,
-- struct mm_struct *mm,
-- unsigned long delay)
--{
-- int r = 0;
--
-- if (!adev->kfd)
-- return -ENODEV;
--
-- mutex_lock(&mem->lock);
--
-- if (mem->evicted <= 1) {
-- /* Buffer is not evicted (== 0) or its restoration is
-- * already scheduled (== 1) */
-- pr_err("Unbalanced restore of evicted buffer %p\n", mem);
-- mutex_unlock(&mem->lock);
-- return -EFAULT;
-- } else if (--mem->evicted > 1) {
-- mutex_unlock(&mem->lock);
-- return 0;
-- }
--
-- /* mem->evicted is 1 after decrementing. If a restoration was
-- * already scheduled, just let it do its job. Otherwise
-- * schedule another one.
-- */
-- if (!mem->mm) {
-- mem->mm = mm;
-- atomic_inc(&mm->mm_users);
-- INIT_DELAYED_WORK(&mem->work,
-- amdgdu_amdkfd_restore_mem_worker);
-- schedule_delayed_work(&mem->work, delay);
-- }
--
-- mutex_unlock(&mem->lock);
--
-- return r;
--}
--
- int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
- uint32_t vmid, uint64_t gpu_addr,
- uint32_t *ib_cmd, uint32_t ib_len)
-diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
-index a6a538ac..9e89aee 100644
---- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
-+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
-@@ -57,17 +57,18 @@ struct kgd_mem {
- unsigned int mapped_to_gpu_memory;
- void *kptr;
- uint64_t va;
-- unsigned int evicted; /* eviction counter */
-- struct delayed_work work; /* for restore evicted mem */
-- struct mm_struct *mm; /* for restore */
-
- uint32_t mapping_flags;
-
-+ atomic_t invalid;
-+ struct amdkfd_process_info *process_info;
-+ struct page **user_pages;
-+
-+
- /* flags bitfield */
- bool coherent : 1;
- bool no_substitute : 1;
- bool aql_queue : 1;
-- bool busy : 1;
- };
-
-
-@@ -89,6 +90,9 @@ struct amdkfd_process_info {
- struct list_head vm_list_head;
- /* List head for all KFD BOs that belong to a KFD process. */
- struct list_head kfd_bo_list;
-+ /* List of userptr BOs that are valid or invalid */
-+ struct list_head userptr_valid_list;
-+ struct list_head userptr_inval_list;
- /* Lock to protect kfd_bo_list */
- struct mutex lock;
-
-@@ -96,6 +100,11 @@ struct amdkfd_process_info {
- unsigned int n_vms;
- /* Eviction Fence */
- struct amdgpu_amdkfd_fence *eviction_fence;
-+
-+ /* MMU-notifier related fields */
-+ atomic_t evicted_bos;
-+ struct delayed_work work;
-+ struct pid *pid;
- };
-
- /* struct amdkfd_vm -
-@@ -130,13 +139,9 @@ void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev);
- void amdgpu_amdkfd_device_init(struct amdgpu_device *adev);
- void amdgpu_amdkfd_device_fini(struct amdgpu_device *adev);
-
--int amdgpu_amdkfd_evict_mem(struct amdgpu_device *adev, struct kgd_mem *mem,
-- struct mm_struct *mm);
--int amdgpu_amdkfd_schedule_restore_mem(struct amdgpu_device *adev,
-- struct kgd_mem *mem,
-- struct mm_struct *mm,
-- unsigned long delay);
--void amdgpu_amdkfd_cancel_restore_mem(struct kgd_mem *mem);
-+int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm);
-+int amdgpu_amdkfd_schedule_restore_userptr(struct kgd_mem *mem,
-+ unsigned long delay);
- int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
- uint32_t vmid, uint64_t gpu_addr,
- uint32_t *ib_cmd, uint32_t ib_len);
-diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
-index 29ca428..443348e 100644
---- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
-+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
-@@ -74,6 +74,8 @@ static const char * const domain_bit_to_string[] = {
-
- #define domain_string(domain) domain_bit_to_string[ffs(domain)-1]
-
-+static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work);
-+
-
- static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd)
- {
-@@ -389,44 +391,28 @@ static void remove_bo_from_vm(struct amdgpu_device *adev,
- static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain,
- bool wait)
- {
-- int ret = 0;
-+ int ret;
-
-- if (!amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {
-- amdgpu_ttm_placement_from_domain(bo, domain);
-+ if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm),
-+ "Called with userptr BO"))
-+ return -EINVAL;
-
-- ret = ttm_bo_validate(&bo->tbo, &bo->placement,
-- false, false);
-+ amdgpu_ttm_placement_from_domain(bo, domain);
-+
-+ ret = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
-+ if (ret)
-+ goto validate_fail;
-+ if (wait) {
-+ struct amdgpu_amdkfd_fence **ef_list;
-+ unsigned int ef_count;
-+
-+ ret = amdgpu_amdkfd_remove_eviction_fence(bo, NULL, &ef_list,
-+ &ef_count);
- if (ret)
- goto validate_fail;
-- if (wait) {
-- struct amdgpu_amdkfd_fence **ef_list;
-- unsigned int ef_count;
--
-- ret = amdgpu_amdkfd_remove_eviction_fence(bo, NULL,
-- &ef_list,
-- &ef_count);
-- if (ret)
-- goto validate_fail;
--
-- ttm_bo_wait(&bo->tbo, false, false);
-- amdgpu_amdkfd_add_eviction_fence(bo, ef_list,
-- ef_count);
-- }
-- } else {
-- amdgpu_ttm_placement_from_domain(bo, domain);
-- ret = ttm_bo_validate(&bo->tbo, &bo->placement,
-- true, false);
-- if (ret) {
-- /* Don't leak the pages. If validate failed,
-- * the pages aren't bound, and won't be
-- * released by unbind later on
-- */
-- release_pages(bo->tbo.ttm->pages,
-- bo->tbo.ttm->num_pages, 0);
-- goto validate_fail;
-- }
-- if (wait)
-- ttm_bo_wait(&bo->tbo, false, false);
-+
-+ ttm_bo_wait(&bo->tbo, false, false);
-+ amdgpu_amdkfd_add_eviction_fence(bo, ef_list, ef_count);
- }
-
- validate_fail:
-@@ -440,19 +426,6 @@ static int amdgpu_amdkfd_validate(void *param, struct amdgpu_bo *bo)
- return amdgpu_amdkfd_bo_validate(bo, p->domain, p->wait);
- }
-
--static int amdgpu_amdkfd_bo_invalidate(struct amdgpu_bo *bo)
--{
-- int ret = 0;
--
-- if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {
-- amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
-- ret = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
-- if (ret != 0)
-- pr_err("Failed to invalidate userptr BO\n");
-- }
-- return ret;
--}
--
- static int validate_pt_pd_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm)
- {
- struct amdgpu_bo *pd = vm->root.bo;
-@@ -479,7 +452,8 @@ static int validate_pt_pd_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm)
- }
-
- static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem *mem,
-- struct amdkfd_process_info *process_info)
-+ struct amdkfd_process_info *process_info,
-+ bool userptr)
- {
- struct ttm_validate_buffer *entry = &mem->validate_list;
- struct amdgpu_bo *bo = mem->bo;
-@@ -488,10 +462,97 @@ static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem *mem,
- entry->shared = true;
- entry->bo = &bo->tbo;
- mutex_lock(&process_info->lock);
-- list_add_tail(&entry->head, &process_info->kfd_bo_list);
-+ if (userptr)
-+ list_add_tail(&entry->head, &process_info->userptr_valid_list);
-+ else
-+ list_add_tail(&entry->head, &process_info->kfd_bo_list);
- mutex_unlock(&process_info->lock);
- }
-
-+/* Initializes user pages. It registers the MMU notifier and validates
-+ * the userptr BO in the GTT domain.
-+ *
-+ * The BO must already be on the userptr_valid_list. Otherwise an
-+ * eviction and restore may happen that leaves the new BO unmapped
-+ * with the user mode queues running.
-+ *
-+ * Takes the process_info->lock to protect against concurrent restore
-+ * workers.
-+ *
-+ * Returns 0 for success, negative errno for errors.
-+ */
-+static int init_user_pages(struct kgd_mem *mem, struct mm_struct *mm,
-+ uint64_t user_addr)
-+{
-+ struct amdkfd_process_info *process_info = mem->process_info;
-+ struct amdgpu_bo *bo = mem->bo;
-+ int ret = 0;
-+
-+ mutex_lock(&process_info->lock);
-+
-+ ret = amdgpu_ttm_tt_set_userptr(bo->tbo.ttm, user_addr, 0);
-+ if (ret) {
-+ pr_err("%s: Failed to set userptr: %d\n", __func__, ret);
-+ goto out;
-+ }
-+
-+ ret = amdgpu_mn_register(bo, user_addr);
-+ if (ret) {
-+ pr_err("%s: Failed to register MMU notifier: %d\n",
-+ __func__, ret);
-+ goto out;
-+ }
-+
-+ /* If no restore worker is running concurrently, user_pages
-+ * should not be allocated
-+ */
-+ WARN(mem->user_pages, "Leaking user_pages array");
-+
-+ mem->user_pages = drm_calloc_large(bo->tbo.ttm->num_pages,
-+ sizeof(struct page *));
-+ if (!mem->user_pages) {
-+ pr_err("%s: Failed to allocate pages array\n", __func__);
-+ ret = -ENOMEM;
-+ goto unregister_out;
-+ }
-+
-+ down_read(&mm->mmap_sem);
-+ ret = amdgpu_ttm_tt_get_user_pages(bo->tbo.ttm, mem->user_pages);
-+ up_read(&mm->mmap_sem);
-+ if (ret) {
-+ pr_err("%s: Failed to get user pages\n", __func__);
-+ goto free_out;
-+ }
-+
-+ memcpy(bo->tbo.ttm->pages, mem->user_pages,
-+ sizeof(struct page *) * bo->tbo.ttm->num_pages);
-+
-+ ret = amdgpu_bo_reserve(bo, true);
-+ if (ret) {
-+ pr_err("%s: Failed to reserve BO\n", __func__);
-+ goto release_out;
-+ }
-+ amdgpu_ttm_placement_from_domain(bo, mem->domain);
-+ ret = ttm_bo_validate(&bo->tbo, &bo->placement,
-+ true, false);
-+ if (ret)
-+ pr_err("%s: failed to validate BO\n", __func__);
-+ amdgpu_bo_unreserve(bo);
-+
-+release_out:
-+ if (ret)
-+ release_pages(mem->user_pages, bo->tbo.ttm->num_pages, 0);
-+free_out:
-+ drm_free_large(mem->user_pages);
-+ mem->user_pages = NULL;
-+unregister_out:
-+ if (ret)
-+ amdgpu_mn_unregister(bo);
-+out:
-+ mutex_unlock(&process_info->lock);
-+ return ret;
-+}
-+
- static int __alloc_memory_of_gpu(struct kgd_dev *kgd, uint64_t va,
- uint64_t size, void *vm, struct kgd_mem **mem,
- uint64_t *offset, void **kptr,
-@@ -578,21 +639,6 @@ static int __alloc_memory_of_gpu(struct kgd_dev *kgd, uint64_t va,
- if (userptr)
- bo->flags |= AMDGPU_AMDKFD_USERPTR_BO;
-
-- if (userptr) {
-- ret = amdgpu_ttm_tt_set_userptr(bo->tbo.ttm, user_addr, 0);
-- if (ret) {
-- pr_err("Failed to set userptr. ret %d\n", ret);
-- goto allocate_mem_set_userptr_failed;
-- }
--
-- ret = amdgpu_mn_register(bo, user_addr);
-- if (ret) {
-- pr_err("Failed to register MMU notifier %d\n",
-- ret);
-- goto allocate_mem_set_userptr_failed;
-- }
-- }
--
- if (kptr) {
- ret = amdgpu_bo_reserve(bo, true);
- if (ret) {
-@@ -621,7 +667,18 @@ static int __alloc_memory_of_gpu(struct kgd_dev *kgd, uint64_t va,
- (*mem)->va = va;
- (*mem)->domain = domain;
- (*mem)->mapped_to_gpu_memory = 0;
-- add_kgd_mem_to_kfd_bo_list(*mem, kfd_vm->process_info);
-+ (*mem)->process_info = kfd_vm->process_info;
-+ add_kgd_mem_to_kfd_bo_list(*mem, kfd_vm->process_info, userptr);
-+
-+ if (userptr) {
-+ ret = init_user_pages(*mem, current->mm, user_addr);
-+ if (ret) {
-+ mutex_lock(&kfd_vm->process_info->lock);
-+ list_del(&(*mem)->validate_list.head);
-+ mutex_unlock(&kfd_vm->process_info->lock);
-+ goto allocate_init_user_pages_failed;
-+ }
-+ }
-
- if (offset)
- *offset = amdgpu_bo_mmap_offset(bo);
-@@ -633,9 +690,8 @@ static int __alloc_memory_of_gpu(struct kgd_dev *kgd, uint64_t va,
- allocate_mem_pin_bo_failed:
- amdgpu_bo_unreserve(bo);
- allocate_mem_reserve_bo_failed:
-- if (userptr)
-- amdgpu_mn_unregister(bo);
--allocate_mem_set_userptr_failed:
-+
-+allocate_init_user_pages_failed:
- amdgpu_bo_unref(&bo);
- err_bo_create:
- kfree(*mem);
-@@ -807,90 +863,6 @@ static void unreserve_bo_and_vms(struct bo_vm_reservation_context *ctx,
- ctx->vm_pd = NULL;
- }
-
--/* Must be called with mem->lock held and a BO/VM reservation
-- * context. Temporarily drops the lock and reservation for updating
-- * user pointers, to avoid circular lock dependencies between MM locks
-- * and buffer reservations. If user pages are invalidated while the
-- * lock and reservation are dropped, try again. */
--static int update_user_pages(struct kgd_mem *mem, struct mm_struct *mm,
-- struct bo_vm_reservation_context *ctx)
--{
-- struct amdgpu_bo *bo;
-- unsigned tries = 10;
-- int ret;
--
-- bo = mem->bo;
-- if (!amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
-- return 0;
--
-- if (bo->tbo.ttm->state != tt_bound) {
-- struct page **pages;
-- int invalidated;
--
-- /* get user pages without locking the BO to avoid
-- * circular lock dependency with MMU notifier. Retry
-- * until we have the current version. */
-- ttm_eu_backoff_reservation(&ctx->ticket, &ctx->list);
-- ctx->reserved = false;
-- pages = drm_calloc_large(bo->tbo.ttm->num_pages,
-- sizeof(struct page *));
-- if (!pages)
-- return -ENOMEM;
--
-- mem->busy = true;
-- mutex_unlock(&mem->lock);
--
-- while (true) {
-- down_read(&mm->mmap_sem);
-- ret = amdgpu_ttm_tt_get_user_pages(bo->tbo.ttm, pages);
-- up_read(&mm->mmap_sem);
--
-- mutex_lock(&mem->lock);
-- mem->busy = false;
-- if (ret != 0)
-- return ret;
--
-- BUG_ON(bo != mem->bo);
--
-- ret = ttm_eu_reserve_buffers(&ctx->ticket, &ctx->list,
-- false, &ctx->duplicates);
-- if (unlikely(ret != 0)) {
-- release_pages(pages, bo->tbo.ttm->num_pages, 0);
-- drm_free_large(pages);
-- return ret;
-- }
-- ctx->reserved = true;
-- if (!amdgpu_ttm_tt_userptr_invalidated(bo->tbo.ttm,
-- &invalidated) ||
-- bo->tbo.ttm->state == tt_bound ||
-- --tries == 0)
-- break;
--
-- release_pages(pages, bo->tbo.ttm->num_pages, 0);
-- ttm_eu_backoff_reservation(&ctx->ticket, &ctx->list);
-- ctx->reserved = false;
-- mem->busy = true;
-- mutex_unlock(&mem->lock);
-- }
--
-- /* If someone else already bound it, release our pages
-- * array, otherwise copy it into the ttm BO. */
-- if (bo->tbo.ttm->state == tt_bound || tries == 0)
-- release_pages(pages, bo->tbo.ttm->num_pages, 0);
-- else
-- memcpy(bo->tbo.ttm->pages, pages,
-- sizeof(struct page *) * bo->tbo.ttm->num_pages);
-- drm_free_large(pages);
-- }
--
-- if (tries == 0) {
-- pr_err("Gave up trying to update user pages\n");
-- return -EDEADLK;
-- }
--
-- return 0;
--}
--
- static int unmap_bo_from_gpuvm(struct amdgpu_device *adev,
- struct kfd_bo_va_list *entry,
- struct amdgpu_sync *sync)
-@@ -976,7 +948,8 @@ static int update_gpuvm_pte(struct amdgpu_device *adev,
- }
-
- static int map_bo_to_gpuvm(struct amdgpu_device *adev,
-- struct kfd_bo_va_list *entry, struct amdgpu_sync *sync)
-+ struct kfd_bo_va_list *entry, struct amdgpu_sync *sync,
-+ bool no_update_pte)
- {
- int ret;
- struct amdgpu_bo *bo = entry->bo_va->bo;
-@@ -1025,6 +998,9 @@ static int map_bo_to_gpuvm(struct amdgpu_device *adev,
- /* Add the eviction fence back */
- amdgpu_bo_fence(pd, &kvm->process_info->eviction_fence->base, true);
-
-+ if (no_update_pte)
-+ return 0;
-+
- ret = update_gpuvm_pte(adev, entry, sync);
- if (ret != 0) {
- pr_err("update_gpuvm_pte() failed\n");
-@@ -1169,8 +1145,23 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
- /* lock is not needed after this, since mem is unused and will
- * be freed anyway */
-
-+ /* No more MMU notifiers */
- amdgpu_mn_unregister(mem->bo);
-- amdgpu_amdkfd_cancel_restore_mem(mem);
-+
-+ /* Make sure restore workers don't access the BO any more */
-+ bo_list_entry = &mem->validate_list;
-+ mutex_lock(&process_info->lock);
-+ list_del(&bo_list_entry->head);
-+ mutex_unlock(&process_info->lock);
-+
-+ /* Free user pages if necessary */
-+ if (mem->user_pages) {
-+ pr_debug("%s: Freeing user_pages array\n", __func__);
-+ if (mem->user_pages[0])
-+ release_pages(mem->user_pages,
-+ mem->bo->tbo.ttm->num_pages, 0);
-+ drm_free_large(mem->user_pages);
-+ }
-
- ret = reserve_bo_and_cond_vms(mem, NULL, VA_DO_NOT_CARE, &ctx);
- if (unlikely(ret != 0))
-@@ -1203,11 +1194,6 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
- }
-
- /* Free the BO*/
-- bo_list_entry = &mem->validate_list;
-- mutex_lock(&process_info->lock);
-- list_del(&bo_list_entry->head);
-- mutex_unlock(&process_info->lock);
--
- amdgpu_bo_unref(&mem->bo);
- kfree(mem);
-
-@@ -1226,14 +1212,28 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
- struct kfd_bo_va_list *bo_va_entry = NULL;
- struct kfd_bo_va_list *bo_va_entry_aql = NULL;
- struct amdkfd_vm *kfd_vm = (struct amdkfd_vm *)vm;
-- int num_to_quiesce = 0;
- unsigned long bo_size;
-+ bool is_invalid_userptr;
-
- BUG_ON(kgd == NULL);
- BUG_ON(mem == NULL);
-
- adev = get_amdgpu_device(kgd);
-
-+ /* Make sure restore is not running concurrently. Since we
-+ * don't map invalid userptr BOs, we rely on the next restore
-+ * worker to do the mapping
-+ */
-+ mutex_lock(&mem->process_info->lock);
-+
-+ /* Lock mmap-sem. If we find an invalid userptr BO, we can be
-+ * sure that the MMU notifier is no longer running
-+ * concurrently and the queues are actually stopped
-+ */
-+ down_read(&current->mm->mmap_sem);
-+ is_invalid_userptr = atomic_read(&mem->invalid);
-+ up_read(&current->mm->mmap_sem);
-+
- mutex_lock(&mem->lock);
-
- bo = mem->bo;
-@@ -1252,6 +1252,14 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
- if (unlikely(ret != 0))
- goto bo_reserve_failed;
-
-+ /* Userptr can be marked as "not invalid", but not actually be
-+ * validated yet (still in the system domain). In that case
-+ * the queues are still stopped and we can leave mapping for
-+ * the next restore worker
-+ */
-+ if (bo->tbo.mem.mem_type == TTM_PL_SYSTEM)
-+ is_invalid_userptr = true;
-+
- if (check_if_add_bo_to_vm((struct amdgpu_vm *)vm, mem)) {
- ret = add_bo_to_vm(adev, mem, (struct amdgpu_vm *)vm, false,
- &bo_va_entry);
-@@ -1265,13 +1273,8 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
- }
- }
-
-- if (mem->mapped_to_gpu_memory == 0 && !mem->evicted) {
-- ret = update_user_pages(mem, current->mm, &ctx);
-- if (ret != 0) {
-- pr_err("update_user_pages failed\n");
-- goto update_user_pages_failed;
-- }
--
-+ if (mem->mapped_to_gpu_memory == 0 &&
-+ !amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {
- /* Validate BO only once. The eviction fence gets added to BO
- * the first time it is mapped. Validate will wait for all
- * background evictions to complete.
-@@ -1285,22 +1288,12 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
-
- list_for_each_entry(entry, &mem->bo_va_list, bo_list) {
- if (entry->bo_va->vm == vm && !entry->is_mapped) {
-- if (mem->evicted) {
-- /* If the BO is evicted, just mark the
-- * mapping as mapped and the GPU's queues
-- * will be stopped later.
-- */
-- entry->is_mapped = true;
-- mem->mapped_to_gpu_memory++;
-- num_to_quiesce++;
-- continue;
-- }
--
- pr_debug("\t map VA 0x%llx - 0x%llx in entry %p\n",
- entry->va, entry->va + bo_size,
- entry);
-
-- ret = map_bo_to_gpuvm(adev, entry, &ctx.sync);
-+ ret = map_bo_to_gpuvm(adev, entry, &ctx.sync,
-+ is_invalid_userptr);
- if (ret != 0) {
- pr_err("Failed to map radeon bo to gpuvm\n");
- goto map_bo_to_gpuvm_failed;
-@@ -1318,24 +1311,11 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
- true);
- unreserve_bo_and_vms(&ctx, true);
-
-- while (num_to_quiesce--) {
-- /* Now stop the GPU's queues while bo and VMs are unreserved.
-- * quiesce_mm() is reference counted, and that is why we can
-- * call it multiple times.
-- */
-- ret = kgd2kfd->quiesce_mm(adev->kfd, current->mm);
-- if (ret != 0) {
-- pr_err("quiesce_mm() failed\n");
-- reserve_bo_and_vm(mem, vm, &ctx);
-- goto map_bo_to_gpuvm_failed;
-- }
-- }
--
-+ mutex_unlock(&mem->process_info->lock);
- mutex_unlock(&mem->lock);
- return ret;
-
- map_bo_to_gpuvm_failed:
--update_user_pages_failed:
- if (bo_va_entry_aql)
- remove_bo_from_vm(adev, bo_va_entry_aql, bo_size);
- add_bo_to_vm_failed_aql:
-@@ -1344,6 +1324,7 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
- add_bo_to_vm_failed:
- unreserve_bo_and_vms(&ctx, false);
- bo_reserve_failed:
-+ mutex_unlock(&mem->process_info->lock);
- mutex_unlock(&mem->lock);
- return ret;
- }
-@@ -1407,6 +1388,8 @@ int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, void **vm,
- mutex_init(&info->lock);
- INIT_LIST_HEAD(&info->vm_list_head);
- INIT_LIST_HEAD(&info->kfd_bo_list);
-+ INIT_LIST_HEAD(&info->userptr_valid_list);
-+ INIT_LIST_HEAD(&info->userptr_inval_list);
-
- info->eviction_fence =
- amdgpu_amdkfd_fence_create(fence_context_alloc(1),
-@@ -1416,6 +1399,12 @@ int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, void **vm,
- goto create_evict_fence_fail;
- }
-
-+ info->pid = get_task_pid(current->group_leader,
-+ PIDTYPE_PID);
-+ atomic_set(&info->evicted_bos, 0);
-+ INIT_DELAYED_WORK(&info->work,
-+ amdgpu_amdkfd_restore_userptr_worker);
-+
- *process_info = info;
- }
-
-@@ -1468,9 +1457,15 @@ void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev *kgd, void *vm)
- list_del(&kfd_vm->vm_list_node);
- mutex_unlock(&process_info->lock);
-
-- /* Release eviction fence */
-+ /* Release per-process resources */
- if (!process_info->n_vms) {
-+ WARN_ON(!list_empty(&process_info->kfd_bo_list));
-+ WARN_ON(!list_empty(&process_info->userptr_valid_list));
-+ WARN_ON(!list_empty(&process_info->userptr_inval_list));
-+
- fence_put(&process_info->eviction_fence->base);
-+ cancel_delayed_work_sync(&process_info->work);
-+ put_pid(process_info->pid);
- kfree(process_info);
- }
-
-@@ -1521,7 +1516,6 @@ int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
- int ret = 0;
- struct bo_vm_reservation_context ctx;
- struct amdkfd_process_info *process_info;
-- int num_to_resume = 0;
- unsigned long bo_size;
-
- BUG_ON(kgd == NULL);
-@@ -1561,17 +1555,6 @@ int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
-
- list_for_each_entry(entry, &mem->bo_va_list, bo_list) {
- if (entry->bo_va->vm == vm && entry->is_mapped) {
-- if (mem->evicted) {
-- /* If the BO is evicted, just mark the
-- * mapping as unmapped and the GPU's queues
-- * will be resumed later.
-- */
-- entry->is_mapped = false;
-- mem->mapped_to_gpu_memory--;
-- num_to_resume++;
-- continue;
-- }
--
- pr_debug("\t unmap VA 0x%llx - 0x%llx from entry %p\n",
- entry->va,
- entry->va + bo_size,
-@@ -1593,14 +1576,13 @@ int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
- }
-
- /* If BO is unmapped from all VMs, unfence it. It can be evicted if
-- * required. User pages of userptr BOs can be released.
-+ * required.
- */
-- if (mem->mapped_to_gpu_memory == 0) {
-+ if (mem->mapped_to_gpu_memory == 0 &&
-+ !amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm))
- amdgpu_amdkfd_remove_eviction_fence(mem->bo,
- process_info->eviction_fence,
- NULL, NULL);
-- amdgpu_amdkfd_bo_invalidate(mem->bo);
-- }
-
- if (mapped_before == mem->mapped_to_gpu_memory) {
- pr_debug("BO VA 0x%llx size 0x%lx is not mapped to vm %p\n",
-@@ -1610,21 +1592,6 @@ int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
-
- unreserve_out:
- unreserve_bo_and_vms(&ctx, false);
--
-- while (current->mm && num_to_resume--) {
-- /* Now resume GPU's queues while bo and VMs are
-- * unreserved. This function runs in a work queue
-- * during process termination. Only resume queues if
-- * we're running in process context. resume_mm() is
-- * reference counted, and that is why we can call it
-- * multiple times.
-- */
-- ret = kgd2kfd->resume_mm(adev->kfd, current->mm);
-- if (ret != 0) {
-- pr_err("resume_mm() failed.\n");
-- break;
-- }
-- }
- out:
- mutex_unlock(&mem->lock);
- return ret;
-@@ -1875,7 +1842,8 @@ int amdgpu_amdkfd_gpuvm_import_dmabuf(struct kgd_dev *kgd,
- (*mem)->domain = (bo->prefered_domains & AMDGPU_GEM_DOMAIN_VRAM) ?
- AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT;
- (*mem)->mapped_to_gpu_memory = 0;
-- add_kgd_mem_to_kfd_bo_list(*mem, kfd_vm->process_info);
-+ (*mem)->process_info = kfd_vm->process_info;
-+ add_kgd_mem_to_kfd_bo_list(*mem, kfd_vm->process_info, false);
-
- return 0;
- }
-@@ -1938,208 +1906,321 @@ static int validate_pd_pt_bos(struct amdkfd_process_info *process_info)
- return 0;
- }
-
--/* Runs out of process context. mem->lock must be held. */
--int amdgpu_amdkfd_gpuvm_evict_mem(struct kgd_mem *mem, struct mm_struct *mm)
-+/* Evict a userptr BO by stopping the queues if necessary
-+ *
-+ * Runs in MMU notifier, may be in RECLAIM_FS context. This means it
-+ * cannot do any memory allocations, and cannot take any locks that
-+ * are held elsewhere while allocating memory. Therefore this is as
-+ * simple as possible, using atomic counters.
-+ *
-+ * It doesn't do anything to the BO itself. The real work happens in
-+ * restore, where we get updated page addresses. This function only
-+ * ensures that GPU access to the BO is stopped.
-+ */
-+int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem,
-+ struct mm_struct *mm)
- {
-- struct kfd_bo_va_list *entry;
-- unsigned int n_evicted = 0, n_unmapped = 0;
-+ struct amdkfd_process_info *process_info = mem->process_info;
-+ int invalid, evicted_bos;
- int r = 0;
-- struct bo_vm_reservation_context ctx;
-
-- pr_debug("Evicting buffer %p\n", mem);
--
-- if (mem->mapped_to_gpu_memory == 0)
-- return 0;
-+ invalid = atomic_inc_return(&mem->invalid);
-+ evicted_bos = atomic_inc_return(&process_info->evicted_bos);
-+ if (evicted_bos == 1) {
-+ /* First eviction, stop the queues */
-+ r = kgd2kfd->quiesce_mm(NULL, mm);
-+ if (r != 0)
-+ pr_err("Failed to quiesce KFD\n");
-+ }
-
-- /* Remove all GPU mappings of the buffer, but don't change any
-- * of the is_mapped flags so we can restore it later. The
-- * queues of the affected GPUs are quiesced first. Count the
-- * number of evicted mappings so we can roll back if something
-- * goes wrong. */
-+ return r;
-+}
-
-- list_for_each_entry(entry, &mem->bo_va_list, bo_list) {
-- struct amdgpu_device *adev;
-+/* Update invalid userptr BOs
-+ *
-+ * Moves invalidated (evicted) userptr BOs from userptr_valid_list to
-+ * userptr_inval_list and updates user pages for all BOs that have
-+ * been invalidated since their last update.
-+ */
-+static int update_invalid_user_pages(struct amdkfd_process_info *process_info,
-+ struct mm_struct *mm)
-+{
-+ struct kgd_mem *mem, *tmp_mem;
-+ struct amdgpu_bo *bo;
-+ int invalid, ret = 0;
-
-- if (!entry->is_mapped)
-- continue;
-+ /* Move all invalidated BOs to the userptr_inval_list and
-+ * release their user pages by migration to the CPU domain
-+ */
-+ list_for_each_entry_safe(mem, tmp_mem,
-+ &process_info->userptr_valid_list,
-+ validate_list.head) {
-+ if (!atomic_read(&mem->invalid))
-+ continue; /* BO is still valid */
-
-- adev = (struct amdgpu_device *)entry->kgd_dev;
-+ bo = mem->bo;
-
-- r = kgd2kfd->quiesce_mm(adev->kfd, mm);
-- if (r != 0) {
-- pr_err("Failed to quiesce KFD\n");
-- goto fail;
-+ if (amdgpu_bo_reserve(bo, true))
-+ return -EAGAIN;
-+ amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
-+ ret = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
-+ amdgpu_bo_unreserve(bo);
-+ if (ret) {
-+ pr_err("%s: Failed to invalidate userptr BO\n",
-+ __func__);
-+ return -EAGAIN;
- }
-
-- n_evicted++;
-+ list_move_tail(&mem->validate_list.head,
-+ &process_info->userptr_inval_list);
- }
-
-- r = reserve_bo_and_cond_vms(mem, NULL, VA_MAPPED, &ctx);
-- if (unlikely(r != 0))
-- goto fail;
--
-- list_for_each_entry(entry, &mem->bo_va_list, bo_list) {
-- struct amdgpu_device *adev;
-+ if (list_empty(&process_info->userptr_inval_list))
-+ return 0; /* All evicted userptr BOs were freed */
-
-- if (!entry->is_mapped)
-+ /* Go through userptr_inval_list and update any invalid user_pages */
-+ down_read(&mm->mmap_sem);
-+ list_for_each_entry(mem, &process_info->userptr_inval_list,
-+ validate_list.head) {
-+ invalid = atomic_read(&mem->invalid);
-+ if (!invalid)
-+ /* BO hasn't been invalidated since the last
-+ * revalidation attempt. Keep its BO list.
-+ */
- continue;
-
-- adev = (struct amdgpu_device *)entry->kgd_dev;
-+ bo = mem->bo;
-+
-+ if (!mem->user_pages) {
-+ mem->user_pages =
-+ drm_calloc_large(bo->tbo.ttm->num_pages,
-+ sizeof(struct page *));
-+ if (!mem->user_pages) {
-+ ret = -ENOMEM;
-+ pr_err("%s: Failed to allocate pages array\n",
-+ __func__);
-+ goto unlock_mmap_out;
-+ }
-+ } else if (mem->user_pages[0]) {
-+ release_pages(mem->user_pages,
-+ bo->tbo.ttm->num_pages, 0);
-+ }
-
-- r = unmap_bo_from_gpuvm(adev, entry, &ctx.sync);
-- if (r != 0) {
-- pr_err("Failed unmap VA 0x%llx\n",
-- mem->va);
-- unreserve_bo_and_vms(&ctx, true);
-- goto fail;
-+ /* Get updated user pages */
-+ ret = amdgpu_ttm_tt_get_user_pages(bo->tbo.ttm,
-+ mem->user_pages);
-+ if (ret) {
-+ mem->user_pages[0] = NULL;
-+ pr_err("%s: Failed to get user pages\n", __func__);
-+ goto unlock_mmap_out;
- }
-
-- n_unmapped++;
-+ /* Mark the BO as valid unless it was invalidated
-+ * again concurrently
-+ */
-+ if (atomic_cmpxchg(&mem->invalid, invalid, 0) != invalid) {
-+ ret = -EAGAIN;
-+ goto unlock_mmap_out;
-+ }
- }
-+unlock_mmap_out:
-+ up_read(&mm->mmap_sem);
-+ return ret;
-+}
-
-- amdgpu_amdkfd_bo_invalidate(mem->bo);
--
-- unreserve_bo_and_vms(&ctx, true);
--
-- return 0;
-+/* Validate invalid userptr BOs
-+ *
-+ * Validates BOs on the userptr_inval_list, and moves them back to the
-+ * userptr_valid_list. Also updates GPUVM page tables with new page
-+ * addresses and waits for the page table updates to complete.
-+ */
-+static int validate_invalid_user_pages(struct amdkfd_process_info *process_info)
-+{
-+ struct amdgpu_bo_list_entry *pd_bo_list_entries;
-+ struct list_head resv_list, duplicates;
-+ struct ww_acquire_ctx ticket;
-+ struct amdgpu_sync sync;
-
--fail:
-- /* To avoid hangs and keep state consistent, roll back partial
-- * eviction by restoring queues and marking mappings as
-- * unmapped. Access to now unmapped buffers will fault. */
-- list_for_each_entry(entry, &mem->bo_va_list, bo_list) {
-- struct amdgpu_device *adev;
-+ struct amdkfd_vm *peer_vm;
-+ struct kgd_mem *mem, *tmp_mem;
-+ struct amdgpu_bo *bo;
-+ int i, ret;
-
-- if (n_evicted == 0)
-- break;
-- if (!entry->is_mapped)
-- continue;
-+ pd_bo_list_entries = kcalloc(process_info->n_vms,
-+ sizeof(struct amdgpu_bo_list_entry),
-+ GFP_KERNEL);
-+ if (!pd_bo_list_entries) {
-+ pr_err("%s: Failed to allocate PD BO list entries\n", __func__);
-+ return -ENOMEM;
-+ }
-
-- if (n_unmapped) {
-- entry->is_mapped = false;
-- n_unmapped--;
-- }
-+ INIT_LIST_HEAD(&resv_list);
-+ INIT_LIST_HEAD(&duplicates);
-
-- adev = (struct amdgpu_device *)entry->kgd_dev;
-- if (kgd2kfd->resume_mm(adev->kfd, mm))
-- pr_err("Failed to resume KFD\n");
-- n_evicted--;
-+ /* Get all the page directory BOs that need to be reserved */
-+ i = 0;
-+ list_for_each_entry(peer_vm, &process_info->vm_list_head,
-+ vm_list_node)
-+ amdgpu_vm_get_pd_bo(&peer_vm->base, &resv_list,
-+ &pd_bo_list_entries[i++]);
-+ /* Add the userptr_inval_list entries to resv_list */
-+ list_for_each_entry(mem, &process_info->userptr_inval_list,
-+ validate_list.head) {
-+ list_add_tail(&mem->resv_list.head, &resv_list);
-+ mem->resv_list.bo = mem->validate_list.bo;
-+ mem->resv_list.shared = mem->validate_list.shared;
- }
-
-- return r;
--}
-+ /* Reserve all BOs and page tables for validation */
-+ ret = ttm_eu_reserve_buffers(&ticket, &resv_list, false, &duplicates);
-+ WARN(!list_empty(&duplicates), "Duplicates should be empty");
-+ if (ret)
-+ goto out;
-
--/* Runs out of process context. mem->lock must be held. */
--int amdgpu_amdkfd_gpuvm_restore_mem(struct kgd_mem *mem, struct mm_struct *mm)
--{
-- struct bo_vm_reservation_context ctx;
-- struct kfd_bo_va_list *entry;
-- uint32_t domain;
-- int r, ret = 0;
-- bool have_pages = false;
-+ amdgpu_sync_create(&sync);
-
-- pr_debug("Restoring buffer %p\n", mem);
-+ ret = validate_pd_pt_bos(process_info);
-+ if (ret)
-+ goto unreserve_out;
-
-- if (mem->mapped_to_gpu_memory == 0)
-- return 0;
-+ /* Validate BOs and update GPUVM page tables */
-+ list_for_each_entry_safe(mem, tmp_mem,
-+ &process_info->userptr_inval_list,
-+ validate_list.head) {
-+ struct kfd_bo_va_list *bo_va_entry;
-
-- domain = mem->domain;
-+ bo = mem->bo;
-
-- ret = reserve_bo_and_cond_vms(mem, NULL, VA_MAPPED, &ctx);
-- if (likely(ret == 0)) {
-- ret = update_user_pages(mem, mm, &ctx);
-- have_pages = !ret;
-- if (!have_pages) {
-- unreserve_bo_and_vms(&ctx, false);
-- if (ret == -ESRCH)
-- /* process terminating, fail quiet and fast */
-- return ret;
-- else if (ret == -EDEADLK)
-- /* Someone else is still updating the
-- * VM, let's try again later
-- */
-- return ret;
-- pr_err("get_user_pages failed. Probably userptr is freed. %d\n",
-- ret);
-+ /* Copy pages array and validate the BO */
-+ memcpy(bo->tbo.ttm->pages, mem->user_pages,
-+ sizeof(struct page *) * bo->tbo.ttm->num_pages);
-+ amdgpu_ttm_placement_from_domain(bo, mem->domain);
-+ ret = ttm_bo_validate(&bo->tbo, &bo->placement,
-+ false, false);
-+ if (ret) {
-+ pr_err("%s: failed to validate BO\n", __func__);
-+ goto unreserve_out;
- }
-- /* update_user_pages drops the lock briefly. Check if
-- * someone else evicted or restored the buffer in the
-- * mean time. Return -EBUSY to let the caller know.
-+
-+ /* Validate succeeded, now the BO owns the pages, free
-+ * our copy of the pointer array. Put this BO back on
-+ * the userptr_valid_list. If we need to revalidate
-+ * it, we need to start from scratch.
- */
-- if (mem->evicted != 1) {
-- if (have_pages)
-- unreserve_bo_and_vms(&ctx, false);
-- return -EBUSY;
-- }
-- }
-+ drm_free_large(mem->user_pages);
-+ mem->user_pages = NULL;
-+ list_move_tail(&mem->validate_list.head,
-+ &process_info->userptr_valid_list);
-
-- if (have_pages) {
-- r = amdgpu_amdkfd_bo_validate(mem->bo, domain, true);
-- if (unlikely(r != 0)) {
-- pr_err("Failed to validate BO %p\n", mem);
-- have_pages = false;
-- unreserve_bo_and_vms(&ctx, false);
-+ list_for_each_entry(bo_va_entry, &mem->bo_va_list, bo_list) {
-+ if (!bo_va_entry->is_mapped)
-+ continue;
-+
-+ ret = update_gpuvm_pte((struct amdgpu_device *)
-+ bo_va_entry->kgd_dev,
-+ bo_va_entry, &sync);
-+ if (ret) {
-+ pr_err("%s: update PTE failed\n", __func__);
-+ /* make sure this gets validated again */
-+ atomic_inc(&mem->invalid);
-+ goto unreserve_out;
-+ }
- }
- }
-+unreserve_out:
-+ ttm_eu_backoff_reservation(&ticket, &resv_list);
-+ amdgpu_sync_wait(&sync);
-+ amdgpu_sync_free(&sync);
-+out:
-+ kfree(pd_bo_list_entries);
-
-- /* Try to restore all mappings. Mappings that fail to restore
-- * will be marked as unmapped. If we failed to get the user
-- * pages, all mappings will be marked as unmapped. */
-- list_for_each_entry(entry, &mem->bo_va_list, bo_list) {
-- struct amdgpu_device *adev;
--
-- if (!entry->is_mapped)
-- continue;
--
-- adev = (struct amdgpu_device *)entry->kgd_dev;
-+ return ret;
-+}
-
-- if (unlikely(!have_pages)) {
-- entry->map_fail = true;
-- continue;
-- }
-+/* Worker callback to restore evicted userptr BOs
-+ *
-+ * Tries to update and validate all userptr BOs. If successful and no
-+ * concurrent evictions happened, the queues are restarted. Otherwise,
-+ * reschedule for another attempt later.
-+ */
-+static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work)
-+{
-+ struct delayed_work *dwork = to_delayed_work(work);
-+ struct amdkfd_process_info *process_info =
-+ container_of(dwork, struct amdkfd_process_info, work);
-+ struct task_struct *usertask;
-+ struct mm_struct *mm;
-+ int evicted_bos;
-+
-+ evicted_bos = atomic_read(&process_info->evicted_bos);
-+ if (!evicted_bos)
-+ return;
-
-- r = map_bo_to_gpuvm(adev, entry, &ctx.sync);
-- if (unlikely(r != 0)) {
-- pr_err("Failed to map BO to gpuvm\n");
-- entry->map_fail = true;
-- if (ret == 0)
-- ret = r;
-- }
-+ /* Reference task and mm in case of concurrent process termination */
-+ usertask = get_pid_task(process_info->pid, PIDTYPE_PID);
-+ if (!usertask)
-+ return;
-+ mm = get_task_mm(usertask);
-+ if (!mm) {
-+ put_task_struct(usertask);
-+ return;
- }
-
-- if (have_pages)
-- unreserve_bo_and_vms(&ctx, true);
-+ mutex_lock(&process_info->lock);
-
-- /* Resume queues after unreserving the BOs and most
-- * importantly, waiting for the BO fences to guarantee that
-- * the page table updates have completed.
-+ if (update_invalid_user_pages(process_info, mm))
-+ goto unlock_out;
-+ /* userptr_inval_list can be empty if all evicted userptr BOs
-+ * have been freed. In that case there is nothing to validate
-+ * and we can just restart the queues.
- */
-- list_for_each_entry(entry, &mem->bo_va_list, bo_list) {
-- struct amdgpu_device *adev;
--
-- if (!entry->is_mapped)
-- continue;
-+ if (!list_empty(&process_info->userptr_inval_list)) {
-+ if (atomic_read(&process_info->evicted_bos) != evicted_bos)
-+ goto unlock_out; /* Concurrent eviction, try again */
-
-- /* Mapping failed. To be in a consistent state, mark the
-- * buffer as unmapped, but state of the buffer will be
-- * not evicted. A vm fault will generated if user space tries
-- * to access this buffer.
-+ if (validate_invalid_user_pages(process_info))
-+ goto unlock_out;
-+ }
-+ /* Final check for concurrent evicton and atomic update. If
-+ * another eviction happens after successful update, it will
-+ * be a first eviction that calls quiesce_mm. The eviction
-+ * reference counting inside KFD will handle this case.
-+ */
-+ if (atomic_cmpxchg(&process_info->evicted_bos, evicted_bos, 0) !=
-+ evicted_bos)
-+ goto unlock_out;
-+ evicted_bos = 0;
-+ if (kgd2kfd->resume_mm(NULL, mm)) {
-+ pr_err("%s: Failed to resume KFD\n", __func__);
-+ /* No recovery from this failure. Probably the CP is
-+ * hanging. No point trying again.
- */
-- if (entry->map_fail) {
-- entry->is_mapped = false;
-- mem->mapped_to_gpu_memory--;
-- }
-- adev = (struct amdgpu_device *)entry->kgd_dev;
--
-- r = kgd2kfd->resume_mm(adev->kfd, mm);
-- if (r != 0) {
-- pr_err("Failed to resume KFD\n");
-- if (ret == 0)
-- ret = r;
-- }
- }
-+unlock_out:
-+ mutex_unlock(&process_info->lock);
-+ mmput(mm);
-+ put_task_struct(usertask);
-
-- return ret;
-+ /* If validation failed, reschedule another attempt */
-+ if (evicted_bos)
-+ schedule_delayed_work(&process_info->work, 1);
-+}
-+
-+/* Schedule delayed restoring of userptr BOs
-+ *
-+ * This runs in an MMU notifier. See limitations above. The scheduled
-+ * worker is free of those limitations. Delaying the restore allows
-+ * multiple MMU notifiers to happen in rapid succession, for example
-+ * when fork COWs many BOs at once.
-+ */
-+int amdgpu_amdkfd_schedule_restore_userptr(struct kgd_mem *mem,
-+ unsigned long delay)
-+{
-+ struct amdkfd_process_info *process_info = mem->process_info;
-+
-+ schedule_delayed_work(&process_info->work, delay);
-+
-+ return 0;
- }
-
- /** amdgpu_amdkfd_gpuvm_restore_process_bos - Restore all BOs for the given
-diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
-index 7076d08..9d78a4f 100644
---- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
-+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
-@@ -88,8 +88,6 @@ static void amdgpu_mn_destroy(struct work_struct *work)
- list_for_each_entry_safe(bo, next_bo, &node->bos, mn_list) {
- bo->mn = NULL;
- list_del_init(&bo->mn_list);
-- if (rmn->type == AMDGPU_MN_TYPE_HSA)
-- amdgpu_amdkfd_cancel_restore_mem(bo->kfd_bo);
- }
- kfree(node);
- }
-@@ -230,7 +228,7 @@ static void amdgpu_mn_invalidate_range_start_hsa(struct mmu_notifier *mn,
-
- if (amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm,
- start, end))
-- amdgpu_amdkfd_evict_mem(amdgpu_ttm_adev(bo->tbo.bdev), mem, mm);
-+ amdgpu_amdkfd_evict_userptr(mem, mm);
- }
- }
-
-@@ -277,8 +275,7 @@ static void amdgpu_mn_invalidate_range_end_hsa(struct mmu_notifier *mn,
-
- if (amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm,
- start, end))
-- amdgpu_amdkfd_schedule_restore_mem(amdgpu_ttm_adev(bo->tbo.bdev),
-- mem, mm, 1);
-+ amdgpu_amdkfd_schedule_restore_userptr(mem, 1);
- }
- }
-
---
-2.7.4
-
generated by cgit 1.2.3-korg (git 2.39.0) at 2024-04-23 18:21:07 +0000