path: root/meta-v1000/recipes-kernel/linux/linux-yocto-4.14.71/1522-drm-amdkgd-Remove-permanent-pinning-of-KFD-BOs.patch

From b155dd247e44d18c81da06d77ab4b654d6440ba1 Mon Sep 17 00:00:00 2001
From: Harish Kasiviswanathan <Harish.Kasiviswanathan@amd.com>
Date: Thu, 7 Jul 2016 12:24:45 -0400
Subject: [PATCH 1522/4131] drm/amdkgd: Remove permanent pinning of KFD BOs

Also decouple BO validate and map functions.

Change-Id: I6defa6ae95e32155e487e4c35c67f9ddff53a29b
Signed-off-by: Harish Kasiviswanathan <Harish.Kasiviswanathan@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 136 ++++++++++++++++++-----
 1 file changed, 108 insertions(+), 28 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index eed9aca..3bf6e64 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -290,6 +290,53 @@ static int unpin_bo(struct amdgpu_bo *bo)
 	return ret;
 }
 
+static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain,
+				     bool wait)
+{
+	int ret = 0;
+
+	if (!amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {
+		amdgpu_ttm_placement_from_domain(bo, domain);
+		ret = ttm_bo_validate(&bo->tbo, &bo->placement,
+				      false, false);
+		if (ret)
+			goto validate_fail;
+		if (wait)
+			ret = ttm_bo_wait(&bo->tbo, false, false);
+	} else {
+		/* Userptrs are not pinned. Therefore we can use the
+		 * bo->pin_count for our version of pinning without conflict.
+		 */
+		if (bo->pin_count == 0) {
+			amdgpu_ttm_placement_from_domain(bo, domain);
+			ret = ttm_bo_validate(&bo->tbo, &bo->placement,
+					      true, false);
+			if (ret)
+				goto validate_fail;
+			if (wait)
+				ttm_bo_wait(&bo->tbo, false, false);
+		}
+		bo->pin_count++;
+	}
+
+validate_fail:
+	return ret;
+}
+
+static int amdgpu_amdkfd_bo_invalidate(struct amdgpu_bo *bo)
+{
+	int ret = 0;
+
+	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) &&
+		(--bo->pin_count == 0)) {
+		amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
+		ret = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
+		if (ret != 0)
+			pr_err("amdgpu: failed to invalidate userptr BO\n");
+	}
+	return ret;
+}
+
 static int try_pin_pts(struct amdgpu_vm *vm)
 {
 	int i, ret = 0;
@@ -808,24 +855,12 @@ static int update_user_pages(struct kgd_mem *mem, struct mm_struct *mm,
 }
 
 static int map_bo_to_gpuvm(struct amdgpu_device *adev, struct amdgpu_bo *bo,
-		struct amdgpu_bo_va *bo_va, uint32_t domain,
+		struct amdgpu_bo_va *bo_va,
 		struct amdgpu_sync *sync)
 {
 	struct amdgpu_vm *vm;
 	int ret;
 
-	/*
-	 * We need to pin the allocated BO and PTs not yet pinned to
-	 * create a mapping of virtual to MC address. PD is already pinned
-	 * in amdgpu_amdkfd_gpuvm_create_process_vm().
-	 */
-	/* Pin BO*/
-	ret = try_pin_bo(bo, domain);
-	if (ret != 0) {
-		pr_err("amdkfd: Failed to pin BO\n");
-		return ret;
-	}
-
 	vm = bo_va->vm;
 	/* Pin PTs */
 	ret = try_pin_pts(vm);
@@ -878,9 +913,7 @@ static int map_bo_to_gpuvm(struct amdgpu_device *adev, struct amdgpu_bo *bo,
 	amdgpu_vm_bo_update(adev, bo_va, NULL);
 	amdgpu_sync_fence(adev, sync, bo_va->last_pt_update);
 err_unpin_bo:
-	/* PTs are not needed to be unpinned*/
-	unpin_bo(bo);
-
+	amdgpu_amdkfd_bo_invalidate(bo);
 	return ret;
 }
 
@@ -1020,6 +1053,11 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
 	if (unlikely(ret != 0))
 		return ret;
 
+	/* The eviction fence should be removed by the last unmap.
+	 * TODO: Log an error condition if the bo still has the eviction fence
+	 * attached
+	 */
+	amdgpu_bo_fence(mem->data2.bo, NULL, false);
 	pr_debug("Releasing BO with VA 0x%llx, size %lu bytes\n",
 					mem->data2.va,
 					mem->data2.bo->tbo.mem.size);
@@ -1074,6 +1112,7 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
 	struct bo_vm_reservation_context ctx;
 	struct kfd_bo_va_list *bo_va_entry = NULL;
 	struct kfd_bo_va_list *bo_va_entry_aql = NULL;
+	struct amdkfd_vm *kfd_vm = (struct amdkfd_vm *)vm;
 
 	BUG_ON(kgd == NULL);
 	BUG_ON(mem == NULL);
@@ -1116,6 +1155,25 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
 			goto update_user_pages_failed;
 	}
 
+	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {
+		ret = amdgpu_amdkfd_bo_validate(bo, domain, true);
+		if (ret) {
+			pr_debug("amdkfd: userptr: Validate failed\n");
+			goto map_bo_to_gpuvm_failed;
+		}
+	} else if (mem->data2.mapped_to_gpu_memory == 0) {
+		/* Validate BO only once. The eviction fence gets added to BO
+		 * the first time it is mapped. Validate will wait for all
+		 * background evictions to complete.
+		 */
+		ret = amdgpu_amdkfd_bo_validate(bo, domain, true);
+		if (ret) {
+			pr_debug("amdkfd: Validate failed\n");
+			goto map_bo_to_gpuvm_failed;
+		}
+	}
+
+
 	list_for_each_entry(entry, &mem->data2.bo_va_list, bo_list) {
 		if (entry->bo_va->vm == vm && !entry->is_mapped) {
 			if (mem->data2.evicted) {
@@ -1134,7 +1192,7 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
 			pr_debug("amdkfd: Trying to map VA 0x%llx to vm %p\n",
 					mem->data2.va, vm);
 
-			ret = map_bo_to_gpuvm(adev, bo, entry->bo_va, domain,
+			ret = map_bo_to_gpuvm(adev, bo, entry->bo_va,
 					      &ctx.sync);
 			if (ret != 0) {
 				pr_err("amdkfd: Failed to map radeon bo to gpuvm\n");
@@ -1147,6 +1205,9 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
 		}
 	}
 
+	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) == NULL)
+		amdgpu_bo_fence(bo, &kfd_vm->master->eviction_fence->base,
+				true);
 	unreserve_bo_and_vms(&ctx, true);
 
 	mutex_unlock(&mem->data2.lock);
@@ -1314,8 +1375,7 @@ static int unmap_bo_from_gpuvm(struct amdgpu_device *adev,
 
 	amdgpu_vm_clear_invalids(adev, vm, sync);
 
-	/* Unpin BO*/
-	unpin_bo(bo);
+	amdgpu_amdkfd_bo_invalidate(bo);
 
 	return 0;
 }
@@ -1404,6 +1464,14 @@ int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
 		}
 	}
 
+	/* If BO is unmapped from all VMs, unfence it. It can be evicted if
+	 * required.
+	 * TODO: For interop this will remove fences added by graphics driver.
+	 * Remove only KFD eviction fence
+	 */
+	if (mem->data2.mapped_to_gpu_memory == 0)
+		amdgpu_bo_fence(mem->data2.bo, NULL, false);
+
 	if (mapped_before == mem->data2.mapped_to_gpu_memory) {
 		pr_debug("BO size %lu bytes at va 0x%llx is not mapped on GPU %x:%x.%x\n",
 			 mem->data2.bo->tbo.mem.size, mem->data2.va,
@@ -1800,7 +1868,16 @@ int amdgpu_amdkfd_gpuvm_restore_mem(struct kgd_mem *mem, struct mm_struct *mm)
 			continue;
 		}
 
-		r = map_bo_to_gpuvm(adev, mem->data2.bo, entry->bo_va, domain,
+		r = amdgpu_amdkfd_bo_validate(mem->data2.bo, domain, true);
+		if (unlikely(r != 0)) {
+			pr_err("Failed to validate BO\n");
+			entry->is_mapped = false;
+			if (ret == 0)
+				ret = r;
+			continue;
+		}
+
+		r = map_bo_to_gpuvm(adev, mem->data2.bo, entry->bo_va,
 				    &ctx.sync);
 		if (unlikely(r != 0)) {
 			pr_err("Failed to map BO to gpuvm\n");
@@ -1928,8 +2005,8 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *m_vm)
 	list_for_each_entry(entry, &ctx.list, tv.head) {
 		struct amdgpu_bo *bo = entry->robj;
 
-		amdgpu_ttm_placement_from_domain(bo, bo->prefered_domains);
-		ret = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
+		ret = amdgpu_amdkfd_bo_validate(bo, bo->prefered_domains,
+						false);
 		if (ret) {
 			pr_debug("Memory eviction: Validate failed. Try again\n");
 			goto validate_map_fail;
@@ -1938,8 +2015,8 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *m_vm)
 	list_for_each_entry(entry, &ctx.duplicates, tv.head) {
 		struct amdgpu_bo *bo = entry->robj;
 
-		amdgpu_ttm_placement_from_domain(bo, bo->prefered_domains);
-		ret = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
+		ret = amdgpu_amdkfd_bo_validate(bo, bo->prefered_domains,
+						false);
 		if (ret) {
 			pr_debug("Memory eviction: Validate failed. Try again\n");
 			goto validate_map_fail;
@@ -1953,7 +2030,6 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *m_vm)
 		struct amdgpu_bo *bo = entry->robj;
 
 		ttm_bo_wait(&bo->tbo, false, false);
-		amdgpu_bo_fence(bo, &master_vm->eviction_fence->base, true);
 	}
 
 
@@ -1965,18 +2041,17 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *m_vm)
 		uint32_t domain = mem->data2.domain;
 		struct kfd_bo_va_list *bo_va_entry;
 
-		amdgpu_ttm_placement_from_domain(bo, domain);
-		ret = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
+		ret = amdgpu_amdkfd_bo_validate(bo, domain, false);
 		if (ret) {
 			pr_debug("Memory eviction: Validate failed. Try again\n");
 			goto validate_map_fail;
 		}
+
 		list_for_each_entry(bo_va_entry, &mem->data2.bo_va_list,
 				    bo_list) {
 			ret = map_bo_to_gpuvm((struct amdgpu_device *)
 					      bo_va_entry->kgd_dev,
 					      bo, bo_va_entry->bo_va,
-					      domain,
 					      &ctx.sync);
 			if (ret) {
 				pr_debug("Memory eviction: Map failed. Try again\n");
@@ -1995,6 +2070,11 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *m_vm)
 		ttm_bo_wait(&bo->tbo, false, false);
 		amdgpu_bo_fence(bo, &master_vm->eviction_fence->base, true);
 	}
+	list_for_each_entry(entry, &ctx.list, tv.head) {
+		struct amdgpu_bo *bo = entry->robj;
+
+		amdgpu_bo_fence(bo, &master_vm->eviction_fence->base, true);
+	}
 validate_map_fail:
 	/* Add PDs to kfd_bo_list for unreserve */
 	list_splice_init(&ctx.list, &master_vm->kfd_bo_list);
-- 
2.7.4