path: root/common/recipes-kernel/linux/linux-yocto-4.14.71/5638-drm-amdkfd-CMA-Use-shadow-system-BO-for-userptr.patch

From 9559d8e4c93ecb9e2ed7f98cd1116947fc243aa4 Mon Sep 17 00:00:00 2001
From: Harish Kasiviswanathan <Harish.Kasiviswanathan@amd.com>
Date: Tue, 27 Mar 2018 14:36:18 -0400
Subject: [PATCH 5638/5725] drm/amdkfd: CMA: Use shadow system BO for userptr

userptrs BO could be evicted during CMA operations. If one of the BO
involved is a userptr, then a shadow BO is created using its underlying
pages. A sg table is created by pinning the backing system pages and
system BO is created using this sg table. This temporary BO is used for
the copy operation.

v2: get_user_pages() could return less than requrested pages. Handle
this condition

Change-Id: Ied26bb481bfa8bb5b488f46f94451477b45746e0
Signed-off-by: Harish Kasiviswanathan <Harish.Kasiviswanathan@amd.com>
---
 drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 264 ++++++++++++++++++++++++++++---
 drivers/gpu/drm/amd/amdkfd/kfd_priv.h    |  13 ++
 2 files changed, 251 insertions(+), 26 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index 8941312..4953374 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -1688,6 +1688,187 @@ static int kfd_ioctl_ipc_import_handle(struct file *filep,
 #define MAX_KMALLOC_PAGES (PAGE_SIZE * 2)
 #define MAX_PP_KMALLOC_COUNT (MAX_KMALLOC_PAGES/sizeof(struct page *))
 
+static void kfd_put_sg_table(struct sg_table *sg)
+{
+	unsigned int i;
+	struct scatterlist *s;
+
+	for_each_sg(sg->sgl, s, sg->nents, i)
+		put_page(sg_page(s));
+}
+
+
+/* Create a sg table for the given userptr BO by pinning its system pages
+ * @bo: userptr BO
+ * @offset: Offset into BO
+ * @mm/@task: mm_struct & task_struct of the process that holds the BO
+ * @size: in/out: desired size / actual size which could be smaller
+ * @sg_size: out: Size of sg table. This is ALIGN_UP(@size)
+ * @ret_sg: out sg table
+ */
+static int kfd_create_sg_table_from_userptr_bo(struct kfd_bo *bo,
+					       int64_t offset, int cma_write,
+					       struct mm_struct *mm,
+					       struct task_struct *task,
+					       uint64_t *size,
+					       uint64_t *sg_size,
+					       struct sg_table **ret_sg)
+{
+	int ret, locked = 1;
+	struct sg_table *sg = NULL;
+	unsigned int i, offset_in_page, flags = 0;
+	unsigned long nents, n;
+	unsigned long pa = (bo->cpuva + offset) & PAGE_MASK;
+	unsigned int cur_page = 0;
+	struct scatterlist *s;
+	uint64_t sz = *size;
+	struct page **process_pages;
+
+	*sg_size = 0;
+	sg = kmalloc(sizeof(*sg), GFP_KERNEL);
+	if (!sg)
+		return -ENOMEM;
+
+	offset_in_page = offset & (PAGE_SIZE - 1);
+	nents = (sz + offset_in_page + PAGE_SIZE - 1) / PAGE_SIZE;
+
+	ret = sg_alloc_table(sg, nents, GFP_KERNEL);
+	if (unlikely(ret)) {
+		ret = -ENOMEM;
+		goto sg_alloc_fail;
+	}
+	process_pages = kmalloc_array(nents, sizeof(struct pages *),
+				      GFP_KERNEL);
+	if (!process_pages) {
+		ret = -ENOMEM;
+		goto page_alloc_fail;
+	}
+
+	if (cma_write)
+		flags = FOLL_WRITE;
+	locked = 1;
+	down_read(&mm->mmap_sem);
+	n = get_user_pages_remote(task, mm, pa, nents, flags, process_pages,
+				  NULL, &locked);
+	if (locked)
+		up_read(&mm->mmap_sem);
+	if (n <= 0) {
+		pr_err("CMA: Invalid virtual address 0x%lx\n", pa);
+		ret = -EFAULT;
+		goto get_user_fail;
+	}
+	if (n != nents) {
+		/* Pages pinned < requested. Set the size accordingly */
+		*size = (n * PAGE_SIZE) - offset_in_page;
+		pr_debug("Requested %lx but pinned %lx\n", nents, n);
+	}
+
+	sz = 0;
+	for_each_sg(sg->sgl, s, n, i) {
+		sg_set_page(s, process_pages[cur_page], PAGE_SIZE,
+			    offset_in_page);
+		sg_dma_address(s) = page_to_phys(process_pages[cur_page]);
+		offset_in_page = 0;
+		cur_page++;
+		sz += PAGE_SIZE;
+	}
+	*ret_sg = sg;
+	*sg_size = sz;
+
+	kfree(process_pages);
+	return 0;
+
+get_user_fail:
+	kfree(process_pages);
+page_alloc_fail:
+	sg_free_table(sg);
+sg_alloc_fail:
+	kfree(sg);
+	return ret;
+}
+
+static void kfd_free_cma_bos(struct cma_iter *ci)
+{
+	struct cma_system_bo *cma_bo, *tmp;
+
+	list_for_each_entry_safe(cma_bo, tmp, &ci->cma_list, list) {
+		struct kfd_dev *dev = cma_bo->dev;
+
+		/* sg table is deleted by free_memory_of_gpu */
+		kfd_put_sg_table(cma_bo->sg);
+		dev->kfd2kgd->free_memory_of_gpu(dev->kgd, cma_bo->mem);
+		list_del(&cma_bo->list);
+		kfree(cma_bo);
+	}
+}
+
+/* Create a system BO by pinning underlying system pages of the given userptr
+ * BO @ubo
+ * @ubo: Userptr BO
+ * @offset: Offset into ubo
+ * @size: in/out: The size of the new BO could be less than requested if all
+ *        the pages couldn't be pinned. This would be reflected in @size
+ * @mm/@task: mm/task to which @ubo belongs to
+ * @cma_bo: out: new system BO
+ */
+static int kfd_create_cma_system_bo(struct kfd_dev *kdev, struct kfd_bo *ubo,
+				    uint64_t *size, uint64_t offset,
+				    int cma_write, struct kfd_process *p,
+				    struct mm_struct *mm,
+				    struct task_struct *task,
+				    struct cma_system_bo **cma_bo)
+{
+	int ret;
+	struct kfd_process_device *pdd = NULL;
+	struct cma_system_bo *cbo;
+	uint64_t sg_size;
+
+	uint32_t flags = ALLOC_MEM_FLAGS_GTT | ALLOC_MEM_FLAGS_WRITABLE |
+			 ALLOC_MEM_FLAGS_NO_SUBSTITUTE;
+
+	*cma_bo = NULL;
+	cbo = kzalloc(sizeof(**cma_bo), GFP_KERNEL);
+	if (!cbo)
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&cbo->list);
+	ret = kfd_create_sg_table_from_userptr_bo(ubo, offset, cma_write, mm,
+						  task, size, &sg_size,
+						  &cbo->sg);
+	if (ret) {
+		pr_err("Failed to create system BO. sg table error %d\n", ret);
+		return ret;
+	}
+
+	mutex_lock(&p->mutex);
+	pdd = kfd_get_process_device_data(kdev, p);
+	if (!pdd) {
+		pr_err("Process device data doesn't exist\n");
+		ret = -EINVAL;
+		goto pdd_fail;
+	}
+
+	ret = kdev->kfd2kgd->alloc_memory_of_gpu(kdev->kgd, 0ULL, sg_size,
+						 pdd->vm, cbo->sg,
+						 &cbo->mem, NULL, flags);
+	if (ret) {
+		pr_err("Failed to create shadow system BO %d\n", ret);
+		goto pdd_fail;
+	}
+	mutex_unlock(&p->mutex);
+	cbo->dev = kdev;
+	*cma_bo = cbo;
+
+	return ret;
+
+pdd_fail:
+	mutex_unlock(&p->mutex);
+	kfd_put_sg_table(cbo->sg);
+	sg_free_table(cbo->sg);
+	kfree(cbo->sg);
+	return ret;
+}
+
 /* Update cma_iter.cur_bo with KFD BO that is assocaited with
  * cma_iter.array.va_addr
  */
@@ -1739,30 +1920,31 @@ static int kfd_cma_iter_init(struct kfd_memory_range *arr, unsigned long segs,
 				struct kfd_process *p, struct mm_struct *mm,
 				struct task_struct *task, struct cma_iter *ci)
 {
-       int ret;
-       int nr;
-
-       if (!arr || !segs)
-               return -EINVAL;
-
-       memset(ci, 0, sizeof(*ci));
-       ci->array = arr;
-       ci->nr_segs = segs;
-       ci->p = p;
-       ci->offset = 0;
-       ci->mm = mm;
-       ci->task = task;
-       for (nr = 0; nr < segs; nr++)
-               ci->total += arr[nr].size;
-
-       /* Valid but size is 0. So copied will also be 0 */
-       if (!ci->total)
-               return 0;
-
-       ret = kfd_cma_iter_update_bo(ci);
-       if (!ret)
-               ci->bo_offset = arr->va_addr - ci->cur_bo->it.start;
-       return ret;
+	int ret;
+	int nr;
+	
+	if (!arr || !segs)
+		return -EINVAL;
+	
+	memset(ci, 0, sizeof(*ci));
+	INIT_LIST_HEAD(&ci->cma_list);
+	ci->array = arr;
+	ci->nr_segs = segs;
+	ci->p = p;
+	ci->offset = 0;
+	ci->mm = mm;
+	ci->task = task;
+	for (nr = 0; nr < segs; nr++)
+		ci->total += arr[nr].size;
+	
+	/* Valid but size is 0. So copied will also be 0 */
+	if (!ci->total)
+		return 0;
+	
+	ret = kfd_cma_iter_update_bo(ci);
+	if (!ret)
+		ci->bo_offset = arr->va_addr - ci->cur_bo->it.start;
+	return ret;
 }
 
 static bool kfd_cma_iter_end(struct cma_iter *ci)
@@ -1912,16 +2094,43 @@ static int kfd_copy_bos(struct cma_iter *si, struct cma_iter *di,
 	if (src_bo->cpuva && dst_bo->cpuva)
 		return kfd_copy_userptr_bos(si, di, cma_write, size, copied);
 
-	if (src_bo->dev->kgd != dst_bo->dev->kgd) {
+	/* If either source or dest. is userptr, create a shadow system BO
+	 * by using the underlying userptr BO pages. Then use this shadow
+	 * BO for copy. src_offset & dst_offset are adjusted because the new BO
+	 * is only created for the window (offset, size) requested.
+	 * The BOs are stored in cma_list for deferred cleanup. This minimizes
+	 * fence waiting just to the last fence.
+	 */
+	if (src_bo->cpuva) {
+		err = kfd_create_cma_system_bo(dst_bo->dev, src_bo, &size,
+					       si->bo_offset, cma_write,
+					       si->p, si->mm, si->task,
+					       &si->cma_bo);
+		src_mem = si->cma_bo->mem;
+		src_offset = si->bo_offset & (PAGE_SIZE - 1);
+		list_add_tail(&si->cma_bo->list, &si->cma_list);
+	} else if (dst_bo->cpuva) {
+		err = kfd_create_cma_system_bo(src_bo->dev, dst_bo, &size,
+					       di->bo_offset, cma_write,
+					       di->p, di->mm, di->task,
+					       &di->cma_bo);
+		dst_mem = di->cma_bo->mem;
+		dst_offset = di->bo_offset & (PAGE_SIZE - 1);
+		list_add_tail(&di->cma_bo->list, &di->cma_list);
+	} else if (src_bo->dev->kgd != dst_bo->dev->kgd) {
 		pr_err("CMA %d fail. Not same dev\n", cma_write);
 		err = -EINVAL;
 	}
 
+	if (err) {
+		pr_err("Failed to create system BO %d", err);
+		err = -EINVAL;
+	}
+
 	err = dst_bo->dev->kfd2kgd->copy_mem_to_mem(src_bo->dev->kgd, src_mem,
 						     src_offset, dst_mem,
 						     dst_offset, size, f,
 						     copied);
-
 	return err;
 }
 
@@ -2120,6 +2329,9 @@ static int kfd_ioctl_cross_memory_copy(struct file *filep,
 		dma_fence_put(lfence);
 	}
 
+	kfd_free_cma_bos(&si);
+	kfd_free_cma_bos(&di);
+
 kfd_process_fail:
 	mmput(remote_mm);
 mm_access_fail:
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 93462fa..d4b802e 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -305,6 +305,13 @@ struct kfd_bo {
 	uint64_t cpuva;
 };
 
+struct cma_system_bo {
+	struct kgd_mem *mem;
+	struct sg_table *sg;
+	struct kfd_dev *dev;
+	struct list_head list;
+};
+
 /* Similar to iov_iter */
 struct cma_iter {
 	/* points to current entry of range array */
@@ -322,6 +329,12 @@ struct cma_iter {
 	struct kfd_bo *cur_bo;
 	/* offset w.r.t cur_bo */
 	unsigned long bo_offset;
+	/* If cur_bo is a userptr BO, then a shadow system BO is created
+	 * using its underlying pages. cma_bo holds this BO. cma_list is a
+	 * list cma_bos created in one session
+	 */
+	struct cma_system_bo *cma_bo;
+	struct list_head cma_list;
 };
 
 /* KGD2KFD callbacks */
-- 
2.7.4