path: root/common/recipes-kernel/linux/linux-yocto-4.14.71/1514-drm-amdgpu-Introduce-KFD-memory-eviction-fence.patch

From f0106f84ebb6d65c72f69f97c1339e1291031a3a Mon Sep 17 00:00:00 2001
From: Harish Kasiviswanathan <Harish.Kasiviswanathan@amd.com>
Date: Thu, 16 Jun 2016 18:37:22 -0400
Subject: [PATCH 1514/4131] drm/amdgpu: Introduce KFD memory eviction fence

Fence helper functions to deal with KFD memory eviction.

Big Idea: Since KFD submissions are done by user queues, a BO cannot be
evicted unless all the user queues for that process are quiesced.

All the BOs in a process share an eviction fence. When process X wants
to map VRAM memory but TTM can't find enough space, TTM will attempt to
evict BOs from its LRU list. TTM checks if the BO is valuable to evict
by calling ttm_bo_driver->eviction_valuable().

ttm_bo_driver->eviction_valuable() - will return false if the BO belongs
to process X. Otherwise, it will return true to indicate BO can be
evicted by TTM.

If ttm_bo_driver->eviction_valuable returns true, then TTM will continue
the evcition process for that BO by calling ttm_bo_evict -->
amdgpu_bo_move --> amdgpu_copy_buffer(). This sets up job in GPU
scheduler.

GPU Scheduler (amd_sched_main) - sets up a cb (fence_add_callback) to
nofity when the BO is free to move. fence_add_callback -->
enable_signaling --> amdgpu_amdkfd_fence.enable_signaling

amdgpu_amdkfd_fence.enable_signaling - Start a work item that will
quiesce user queues and signal fence. The work item will also start
another delayed work item to restore BOs

Change-Id: I73bf9663fca56ce22418f403407c41e21d2563da
Signed-off-by: Harish Kasiviswanathan <Harish.Kasiviswanathan@amd.com>

 Conflicts:
        drivers/gpu/drm/amd/amdgpu/Makefile
---
 drivers/gpu/drm/amd/amdgpu/Makefile              |   2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h       |  14 ++
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c | 191 +++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c |  28 +++-
 4 files changed, 228 insertions(+), 7 deletions(-)
 mode change 100755 => 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
 create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c

diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index 9540fbf..7ff4af3 100755
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -32,7 +32,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
 	amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
 	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
 	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o amdgpu_atomfirmware.o \
-	amdgpu_queue_mgr.o amdgpu_vf_error.o amdgpu_sem.o 
+	amdgpu_queue_mgr.o amdgpu_vf_error.o amdgpu_sem.o amdgpu_amdkfd_fence.o
 
 # add asic specific block
 amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
old mode 100755
new mode 100644
index 1ee31f9..83bea0e
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -72,6 +72,18 @@ struct kgd_mem {
 };
 
 
+/* KFD Memory Eviction */
+struct amdgpu_amdkfd_fence {
+	struct fence base;
+	void *mm;
+	spinlock_t lock;
+	char timeline_name[TASK_COMM_LEN];
+};
+
+struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
+						       void *mm);
+bool amd_kfd_fence_check_mm(struct fence *f, void *mm);
+
 /* struct amdkfd_vm -
  *  For Memory Eviction KGD requires a mechanism to keep track of all KFD BOs
  * belonging to a KFD process. All the VMs belonging to the same process point
@@ -98,6 +110,8 @@ struct amdkfd_vm {
 	/* Number of VMs including master VM */
 	unsigned n_vms;
 	struct amdgpu_device *adev;
+	/* Eviction Fence. Initialized only for master_vm */
+	struct amdgpu_amdkfd_fence *eviction_fence;
 };
 
 int amdgpu_amdkfd_init(void);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c
new file mode 100644
index 0000000..6fdd24c
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c
@@ -0,0 +1,191 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/fence.h>
+#include <linux/spinlock.h>
+#include <linux/atomic.h>
+#include <linux/stacktrace.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include "amdgpu_amdkfd.h"
+
+const struct fence_ops amd_kfd_fence_ops;
+static atomic_t fence_seq = ATOMIC_INIT(0);
+
+static int amd_kfd_fence_signal(struct fence *f);
+
+/* Eviction Fence
+ * Fence helper functions to deal with KFD memory eviction.
+ * Big Idea - Since KFD submissions are done by user queues, a BO cannot be
+ *  evicted unless all the user queues for that process are evicted.
+ *
+ * All the BOs in a process share an eviction fence. When process X wants
+ * to map VRAM memory but TTM can't find enough space, TTM will attempt to
+ * evict BOs from its LRU list. TTM checks if the BO is valuable to evict
+ * by calling ttm_bo_driver->eviction_valuable().
+ *
+ * ttm_bo_driver->eviction_valuable() - will return false if the BO belongs
+ *  to process X. Otherwise, it will return true to indicate BO can be
+ *  evicted by TTM.
+ *
+ * If ttm_bo_driver->eviction_valuable returns true, then TTM will continue
+ * the evcition process for that BO by calling ttm_bo_evict --> amdgpu_bo_move
+ * --> amdgpu_copy_buffer(). This sets up job in GPU scheduler.
+ *
+ * GPU Scheduler (amd_sched_main) - sets up a cb (fence_add_callback) to
+ *  nofity when the BO is free to move. fence_add_callback --> enable_signaling
+ *  --> amdgpu_amdkfd_fence.enable_signaling
+ *
+ * amdgpu_amdkfd_fence.enable_signaling - Start a work item that will quiesce
+ * user queues and signal fence. The work item will also start another delayed
+ * work item to restore BOs
+ */
+
+struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
+						       void *mm)
+{
+	struct amdgpu_amdkfd_fence *fence = NULL;
+
+	fence = kzalloc(sizeof(struct amdgpu_amdkfd_fence), GFP_KERNEL);
+	if (fence == NULL)
+		return NULL;
+
+	/* mm_struct mm is used as void pointer to identify the parent
+	 * KFD process. Don't dereference it. Fence and any threads using
+	 * mm is guranteed to be released before process termination.
+	 */
+	fence->mm = mm;
+	get_task_comm(fence->timeline_name, current);
+	spin_lock_init(&fence->lock);
+
+	fence_init(&fence->base, &amd_kfd_fence_ops, &fence->lock,
+		   context, atomic_inc_return(&fence_seq));
+
+	return fence;
+}
+
+static struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct fence *f)
+{
+	struct amdgpu_amdkfd_fence *fence;
+
+	if (!f)
+		return NULL;
+
+	fence = container_of(f, struct amdgpu_amdkfd_fence, base);
+	if (fence && f->ops == &amd_kfd_fence_ops)
+		return fence;
+
+	return NULL;
+}
+
+static const char *amd_kfd_fence_get_driver_name(struct fence *f)
+{
+	return "amdgpu_amdkfd_fence";
+}
+
+static const char *amd_kfd_fence_get_timeline_name(struct fence *f)
+{
+	struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
+
+	return fence->timeline_name;
+}
+
+/**
+ * amd_kfd_fence_enable_signaling - This gets called when TTM wants to evict
+ *  a KFD BO and schedules a job to move the BO.
+ *  If fence is already signaled return true.
+ *  If fence is not signaled schedule a evict KFD process work item.
+ */
+static bool amd_kfd_fence_enable_signaling(struct fence *f)
+{
+	if (fence_is_signaled(f))
+		return true;
+
+	/* TODO: If the fence is not signaled, call into KFD to schedule
+	 * work item that will prepare for KFD BO evictions
+	 */
+	return true;
+}
+
+static int amd_kfd_fence_signal(struct fence *f)
+{
+	unsigned long flags;
+	int ret;
+
+	spin_lock_irqsave(f->lock, flags);
+	/* Set enabled bit so cb will called */
+	set_bit(FENCE_FLAG_ENABLE_SIGNAL_BIT, &f->flags);
+	ret = fence_signal_locked(f);
+	spin_unlock_irqrestore(f->lock, flags);
+
+	return ret;
+}
+
+/**
+ * amd_kfd_fence_release - callback that fence can be freed
+ *
+ * @fence: fence
+ *
+ * This function is called when the reference count becomes zero.
+ * It just RCU schedules freeing up the fence.
+*/
+static void amd_kfd_fence_release(struct fence *f)
+{
+	struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
+	/* Unconditionally signal the fence. The process is getting
+	 * terminated.
+	 */
+	if (WARN_ON(!fence))
+		return; /* Not an amdgpu_amdkfd_fence */
+
+	amd_kfd_fence_signal(f);
+	kfree_rcu(f, rcu);
+}
+
+/**
+ * amd_kfd_fence_check_mm - Check if @mm is same as that of the fence @f
+ *  if same return TRUE else return FALSE.
+ *
+ * @f: [IN] fence
+ * @mm: [IN] mm that needs to be verified
+*/
+bool amd_kfd_fence_check_mm(struct fence *f, void *mm)
+{
+	struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
+
+	if (!fence)
+		return false;
+	else if (fence->mm == mm)
+		return true;
+
+	return false;
+}
+
+const struct fence_ops amd_kfd_fence_ops = {
+	.get_driver_name = amd_kfd_fence_get_driver_name,
+	.get_timeline_name = amd_kfd_fence_get_timeline_name,
+	.enable_signaling = amd_kfd_fence_enable_signaling,
+	.signaled = NULL,
+	.wait = fence_default_wait,
+	.release = amd_kfd_fence_release,
+};
+
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 60bbcfe..bab085c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -1081,18 +1081,23 @@ int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, void **vm,
 	if (ret != 0) {
 		pr_err("amdgpu: failed init vm ret %d\n", ret);
 		/* Undo everything related to the new VM context */
-		amdgpu_vm_fini(adev, &new_vm->base);
-		kfree(new_vm);
-		new_vm = NULL;
+		goto vm_init_fail;
 	}
 	new_vm->adev = adev;
 	mutex_init(&new_vm->lock);
 	INIT_LIST_HEAD(&new_vm->kfd_bo_list);
 	INIT_LIST_HEAD(&new_vm->kfd_vm_list);
 
-	if (master_vm == NULL)
+	if (master_vm == NULL) {
 		new_vm->master = new_vm;
-	else {
+		new_vm->eviction_fence =
+			amdgpu_amdkfd_fence_create(fence_context_alloc(1),
+						   current->mm);
+		if (new_vm->master->eviction_fence == NULL) {
+			pr_err("Failed to create eviction fence\n");
+			goto evict_fence_fail;
+		}
+	} else {
 		new_vm->master = master_vm;
 		list_add_tail(&new_vm->kfd_vm_list,
 			      &((struct amdkfd_vm *)master_vm)->kfd_vm_list);
@@ -1121,18 +1126,29 @@ int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, void **vm,
 			amdgpu_bo_gpu_offset(new_vm->base.page_directory));
 
 	return ret;
+
+evict_fence_fail:
+	amdgpu_vm_fini(adev, &new_vm->base);
+vm_init_fail:
+	kfree(new_vm);
+	return ret;
+
 }
 
 void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev *kgd, void *vm)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
-	struct amdgpu_vm *avm = (struct amdgpu_vm *) vm;
+	struct amdkfd_vm *kfd_vm = (struct amdkfd_vm *) vm;
+	struct amdgpu_vm *avm = &kfd_vm->base;
 	struct amdgpu_bo *pd;
 
 	BUG_ON(kgd == NULL);
 	BUG_ON(vm == NULL);
 
 	pr_debug("Destroying process vm with address %p\n", vm);
+	/* Release eviction fence */
+	if (kfd_vm->master == kfd_vm && kfd_vm->eviction_fence != NULL)
+		fence_put(&kfd_vm->eviction_fence->base);
 
 	/* Unpin PTs */
 	unpin_pts(avm);
-- 
2.7.4