path: root/meta-amd-bsp/recipes-kernel/linux-6.6/linux-yocto-6.6/0003-ntb_perf-add-support-for-higher-resources.patch

From 74940974f590888e4a760dd3905ce8a6ebd35609 Mon Sep 17 00:00:00 2001
From: Raju Rangoju <Raju.Rangoju@amd.com>
Date: Fri, 9 Feb 2024 18:11:36 +0530
Subject: [PATCH 3/5] ntb_perf: add support for higher resources

Add support for up to 128 threads and 64GB Memory window.
Add the necessary support for enabling multi channels per thread.
Introduce the module param use_dma_2p to handle 2p system.

Signed-off-by: Raju Rangoju <Raju.Rangoju@amd.com>
---
 drivers/ntb/test/ntb_perf.c | 315 +++++++++++++++++++++++-------------
 1 file changed, 202 insertions(+), 113 deletions(-)

diff --git a/drivers/ntb/test/ntb_perf.c b/drivers/ntb/test/ntb_perf.c
index 553f1f46bc66..e6b01254dcce 100644
--- a/drivers/ntb/test/ntb_perf.c
+++ b/drivers/ntb/test/ntb_perf.c
@@ -92,10 +92,11 @@ MODULE_VERSION(DRIVER_VERSION);
 MODULE_AUTHOR("Dave Jiang <dave.jiang@intel.com>");
 MODULE_DESCRIPTION("PCIe NTB Performance Measurement Tool");
 
-#define MAX_THREADS_CNT		32
+#define MAX_THREADS_CNT		128
+#define MAX_DMACH_CNT		128
 #define DEF_THREADS_CNT		1
 #define MAX_CHUNK_SIZE		SZ_1M
-#define MAX_CHUNK_ORDER		20 /* no larger than 1M */
+#define MAX_CHUNK_ORDER		31 /* no larger than 2G */
 
 #define DMA_TRIES		100
 #define DMA_MDELAY		10
@@ -104,9 +105,9 @@ MODULE_DESCRIPTION("PCIe NTB Performance Measurement Tool");
 #define MSG_UDELAY_LOW		1000000
 #define MSG_UDELAY_HIGH		2000000
 
-#define PERF_BUF_LEN 1024
+#define PERF_BUF_LEN 65536
 
-static unsigned long max_mw_size;
+static unsigned long max_mw_size = (SZ_32G * 2);
 module_param(max_mw_size, ulong, 0644);
 MODULE_PARM_DESC(max_mw_size, "Upper limit of memory window size");
 
@@ -122,6 +123,11 @@ static bool use_dma; /* default to 0 */
 module_param(use_dma, bool, 0644);
 MODULE_PARM_DESC(use_dma, "Use DMA engine to measure performance");
 
+static bool use_dma_2p; /* default to 0 */
+module_param(use_dma_2p, bool, 0644);
+MODULE_PARM_DESC(use_dma_2p, "Use DMA engine also from other node to measure performance");
+
+
 /*==============================================================================
  *                         Perf driver data definition
  *==============================================================================
@@ -150,7 +156,6 @@ struct perf_peer {
 	resource_size_t outbuf_size;
 	void __iomem *outbuf;
 	phys_addr_t out_phys_addr;
-	dma_addr_t dma_dst_addr;
 	/* Inbound MW params */
 	dma_addr_t inbuf_xlat;
 	resource_size_t inbuf_size;
@@ -172,14 +177,16 @@ struct perf_thread {
 	/* DMA-based test sync parameters */
 	atomic_t dma_sync;
 	wait_queue_head_t dma_wait;
-	struct dma_chan *dma_chan;
+	struct dma_chan *dma_chan[MAX_DMACH_CNT];
 
 	/* Data source and measured statistics */
-	void *src;
+	void *src[MAX_DMACH_CNT];
+	dma_addr_t dma_src_addr[MAX_DMACH_CNT];
 	u64 copied;
 	ktime_t duration;
 	int status;
 	struct work_struct work;
+	dma_addr_t dma_dst_addr[MAX_DMACH_CNT];
 };
 #define to_thread_work(__work) \
 	container_of(__work, struct perf_thread, work)
@@ -197,6 +204,7 @@ struct perf_ctx {
 	wait_queue_head_t twait;
 	atomic_t tsync;
 	u8 tcnt;
+	u8 dch_cnt;
 	struct perf_peer *test_peer;
 	struct perf_thread threads[MAX_THREADS_CNT];
 
@@ -785,76 +793,60 @@ static void perf_dma_copy_callback(void *data)
 static int perf_copy_chunk(struct perf_thread *pthr,
 			   void __iomem *dst, void *src, size_t len)
 {
+	struct perf_ctx *perf = pthr->perf;
 	struct dma_async_tx_descriptor *tx;
-	struct dmaengine_unmap_data *unmap;
 	struct device *dma_dev;
 	int try = 0, ret = 0;
 	struct perf_peer *peer = pthr->perf->test_peer;
 	void __iomem *vbase;
 	void __iomem *dst_vaddr;
-	dma_addr_t dst_dma_addr;
+	dma_addr_t dma_dst_addr, dma_src_addr;
+	int tidx;
 
 	if (!use_dma) {
 		memcpy_toio(dst, src, len);
 		goto ret_check_tsync;
 	}
 
-	dma_dev = pthr->dma_chan->device->dev;
+	for (tidx = 0; tidx < perf->dch_cnt; tidx++) {
+		dma_dev = pthr->dma_chan[tidx]->device->dev;
 
-	if (!is_dma_copy_aligned(pthr->dma_chan->device, offset_in_page(src),
-				 offset_in_page(dst), len))
-		return -EIO;
+		if (!is_dma_copy_aligned(pthr->dma_chan[tidx]->device, offset_in_page(src),
+					 offset_in_page(dst), len))
+			return -EIO;
 
-	vbase = peer->outbuf;
-	dst_vaddr = dst;
-	dst_dma_addr = peer->dma_dst_addr + (dst_vaddr - vbase);
+		vbase = peer->outbuf;
+		dst_vaddr = dst;
+		dma_dst_addr = pthr->dma_dst_addr[tidx] + (dst_vaddr - vbase);
+		dma_src_addr = pthr->dma_src_addr[tidx] + (dst_vaddr - vbase);
 
-	unmap = dmaengine_get_unmap_data(dma_dev, 1, GFP_NOWAIT);
-	if (!unmap)
-		return -ENOMEM;
-
-	unmap->len = len;
-	unmap->addr[0] = dma_map_page(dma_dev, virt_to_page(src),
-		offset_in_page(src), len, DMA_TO_DEVICE);
-	if (dma_mapping_error(dma_dev, unmap->addr[0])) {
-		ret = -EIO;
-		goto err_free_resource;
-	}
-	unmap->to_cnt = 1;
-
-	do {
-		tx = dmaengine_prep_dma_memcpy(pthr->dma_chan, dst_dma_addr,
-			unmap->addr[0], len, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
-		if (!tx)
-			msleep(DMA_MDELAY);
-	} while (!tx && (try++ < DMA_TRIES));
-
-	if (!tx) {
-		ret = -EIO;
-		goto err_free_resource;
-	}
+		do {
+			tx = dmaengine_prep_dma_memcpy(pthr->dma_chan[tidx], dma_dst_addr,
+						       dma_src_addr, len, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+			if (!tx)
+				msleep(DMA_MDELAY);
+		} while (!tx && (try++ < DMA_TRIES));
 
-	tx->callback = perf_dma_copy_callback;
-	tx->callback_param = pthr;
-	dma_set_unmap(tx, unmap);
+		if (!tx) {
+			ret = -EIO;
+			goto err_free_resource;
+		}
 
-	ret = dma_submit_error(dmaengine_submit(tx));
-	if (ret) {
-		dmaengine_unmap_put(unmap);
-		goto err_free_resource;
-	}
+		tx->callback = perf_dma_copy_callback;
+		tx->callback_param = pthr;
 
-	dmaengine_unmap_put(unmap);
+		ret = dma_submit_error(dmaengine_submit(tx));
+		if (ret)
+			goto err_free_resource;
 
-	atomic_inc(&pthr->dma_sync);
-	dma_async_issue_pending(pthr->dma_chan);
+		atomic_inc(&pthr->dma_sync);
+		dma_async_issue_pending(pthr->dma_chan[tidx]);
+	}
 
 ret_check_tsync:
 	return likely(atomic_read(&pthr->perf->tsync) > 0) ? 0 : -EINTR;
 
 err_free_resource:
-	dmaengine_unmap_put(unmap);
-
 	return ret;
 }
 
@@ -865,7 +857,10 @@ static bool perf_dma_filter(struct dma_chan *chan, void *data)
 
 	node = dev_to_node(&perf->ntb->dev);
 
-	return node == NUMA_NO_NODE || node == dev_to_node(chan->device->dev);
+	if(!use_dma_2p)
+		return node == NUMA_NO_NODE || node == dev_to_node(chan->device->dev);
+
+	return true;
 }
 
 static int perf_init_test(struct perf_thread *pthr)
@@ -873,49 +868,77 @@ static int perf_init_test(struct perf_thread *pthr)
 	struct perf_ctx *perf = pthr->perf;
 	dma_cap_mask_t dma_mask;
 	struct perf_peer *peer = pthr->perf->test_peer;
+	u64 chunk_size;
+	int tidx, i;
 
-	pthr->src = kmalloc_node(perf->test_peer->outbuf_size, GFP_KERNEL,
-				 dev_to_node(&perf->ntb->dev));
-	if (!pthr->src)
-		return -ENOMEM;
+	chunk_size = 1ULL << chunk_order;
 
-	get_random_bytes(pthr->src, perf->test_peer->outbuf_size);
+	if(peer->outbuf_size > chunk_size) {
+		peer->outbuf_size = chunk_size;
+		pr_info("ntb_perf: outbuf size reduced to 0x%llx \n", chunk_size);
+	}
 
 	if (!use_dma)
 		return 0;
 
 	dma_cap_zero(dma_mask);
 	dma_cap_set(DMA_MEMCPY, dma_mask);
-	pthr->dma_chan = dma_request_channel(dma_mask, perf_dma_filter, perf);
-	if (!pthr->dma_chan) {
-		dev_err(&perf->ntb->dev, "%d: Failed to get DMA channel\n",
-			pthr->tidx);
-		goto err_free;
-	}
-	peer->dma_dst_addr =
-		dma_map_resource(pthr->dma_chan->device->dev,
-				 peer->out_phys_addr, peer->outbuf_size,
-				 DMA_FROM_DEVICE, 0);
-	if (dma_mapping_error(pthr->dma_chan->device->dev,
-			      peer->dma_dst_addr)) {
-		dev_err(pthr->dma_chan->device->dev, "%d: Failed to map DMA addr\n",
-			pthr->tidx);
-		peer->dma_dst_addr = 0;
-		dma_release_channel(pthr->dma_chan);
-		goto err_free;
-	}
-	dev_dbg(pthr->dma_chan->device->dev, "%d: Map MMIO %pa to DMA addr %pad\n",
-			pthr->tidx,
-			&peer->out_phys_addr,
-			&peer->dma_dst_addr);
 
+	for (tidx = 0; tidx < perf->dch_cnt; tidx++) {
+		pthr->dma_chan[tidx] = dma_request_channel(dma_mask, perf_dma_filter, perf);
+		if (!pthr->dma_chan[tidx]) {
+			dev_err(&perf->ntb->dev, "%d: Failed to get DMA channel\n",
+				pthr->tidx);
+			goto err_free;
+		}
+
+		pthr->src[tidx] = dma_alloc_coherent(pthr->dma_chan[tidx]->device->dev,
+						     perf->test_peer->outbuf_size,
+						     &pthr->dma_src_addr[tidx],
+						     GFP_KERNEL);
+
+		if (!pthr->src[tidx])
+			goto err_free;
+
+		get_random_bytes(pthr->src[tidx], perf->test_peer->outbuf_size);
+
+		pthr->dma_dst_addr[tidx] =
+			dma_map_resource(pthr->dma_chan[tidx]->device->dev,
+					 peer->out_phys_addr, peer->outbuf_size,
+					 DMA_FROM_DEVICE, 0);
+		if (dma_mapping_error(pthr->dma_chan[tidx]->device->dev,
+				      pthr->dma_dst_addr[tidx])) {
+			dev_err(pthr->dma_chan[tidx]->device->dev, "%d: Failed to map DMA addr\n",
+				pthr->tidx);
+			pthr->dma_dst_addr[tidx] = 0;
+			dma_release_channel(pthr->dma_chan[tidx]);
+			goto err_free;
+		}
+		dev_dbg(pthr->dma_chan[tidx]->device->dev, "%d: Map MMIO %pa to DMA addr %pad\n",
+				pthr->tidx,
+				&peer->out_phys_addr,
+				&pthr->dma_dst_addr[tidx]);
+	}
 	atomic_set(&pthr->dma_sync, 0);
 	return 0;
 
 err_free:
 	atomic_dec(&perf->tsync);
 	wake_up(&perf->twait);
-	kfree(pthr->src);
+
+	for (i = 0; i <= tidx; i++) {
+		if (pthr->dma_dst_addr[i])
+			dma_unmap_resource(pthr->dma_chan[i]->device->dev,
+					   pthr->dma_dst_addr[i],
+					   pthr->perf->test_peer->outbuf_size,
+					   DMA_FROM_DEVICE, 0);
+		if (pthr->dma_src_addr[i])
+			dma_free_coherent(pthr->dma_chan[i]->device->dev,
+					  perf->test_peer->outbuf_size,
+					  pthr->src[i], pthr->dma_src_addr[i]);
+		if (pthr->dma_chan[i])
+			dma_release_channel(pthr->dma_chan[i]);
+	}
 	return -ENODEV;
 }
 
@@ -932,14 +955,17 @@ static int perf_run_test(struct perf_thread *pthr)
 	chunk_size = 1ULL << chunk_order;
 	chunk_size = min_t(u64, peer->outbuf_size, chunk_size);
 
-	flt_src = pthr->src;
+	flt_src = pthr->src[0];
 	bnd_dst = peer->outbuf + peer->outbuf_size;
-	flt_dst = peer->outbuf;
 
+	flt_dst = peer->outbuf;
 	pthr->duration = ktime_get();
 
 	/* Copied field is cleared on test launch stage */
 	while (pthr->copied < total_size) {
+		if(flt_dst >= (peer->outbuf + max_mw_size) )
+			flt_dst = peer->outbuf;
+
 		ret = perf_copy_chunk(pthr, flt_dst, flt_src, chunk_size);
 		if (ret) {
 			dev_err(&perf->ntb->dev, "%d: Got error %d on test\n",
@@ -948,14 +974,12 @@ static int perf_run_test(struct perf_thread *pthr)
 		}
 
 		pthr->copied += chunk_size;
-
 		flt_dst += chunk_size;
 		flt_src += chunk_size;
 		if (flt_dst >= bnd_dst || flt_dst < peer->outbuf) {
 			flt_dst = peer->outbuf;
-			flt_src = pthr->src;
+			flt_src = pthr->src[0];
 		}
-
 		/* Give up CPU to give a chance for other threads to use it */
 		schedule();
 	}
@@ -995,27 +1019,27 @@ static int perf_sync_test(struct perf_thread *pthr)
 static void perf_clear_test(struct perf_thread *pthr)
 {
 	struct perf_ctx *perf = pthr->perf;
+	int tidx;
 
 	if (!use_dma)
-		goto no_dma_notify;
+		return;
 
 	/*
 	 * If test finished without errors, termination isn't needed.
 	 * We call it anyway just to be sure of the transfers completion.
 	 */
-	(void)dmaengine_terminate_sync(pthr->dma_chan);
-	if (pthr->perf->test_peer->dma_dst_addr)
-		dma_unmap_resource(pthr->dma_chan->device->dev,
-				   pthr->perf->test_peer->dma_dst_addr,
-				   pthr->perf->test_peer->outbuf_size,
-				   DMA_FROM_DEVICE, 0);
-
-	dma_release_channel(pthr->dma_chan);
-
-no_dma_notify:
-	atomic_dec(&perf->tsync);
-	wake_up(&perf->twait);
-	kfree(pthr->src);
+	for (tidx = 0; tidx < perf->dch_cnt; tidx++) {
+		(void)dmaengine_terminate_sync(pthr->dma_chan[tidx]);
+		if (pthr->dma_dst_addr[tidx])
+			dma_unmap_resource(pthr->dma_chan[tidx]->device->dev,
+					   pthr->dma_dst_addr[tidx],
+					   pthr->perf->test_peer->outbuf_size,
+					   DMA_FROM_DEVICE, 0);
+		dma_free_coherent(pthr->dma_chan[tidx]->device->dev,
+				  perf->test_peer->outbuf_size,
+				  pthr->src[tidx], pthr->dma_src_addr[tidx]);
+		dma_release_channel(pthr->dma_chan[tidx]);
+	}
 }
 
 static void perf_thread_work(struct work_struct *work)
@@ -1048,32 +1072,47 @@ static void perf_thread_work(struct work_struct *work)
 	perf_clear_test(pthr);
 }
 
-static int perf_set_tcnt(struct perf_ctx *perf, u8 tcnt)
+static void perf_terminate_test(struct perf_ctx *perf)
 {
-	if (tcnt == 0 || tcnt > MAX_THREADS_CNT)
+	int tidx;
+
+	atomic_set(&perf->tsync, -1);
+	wake_up(&perf->twait);
+
+	for (tidx = 0; tidx < MAX_THREADS_CNT; tidx++) {
+		wake_up(&perf->threads[tidx].dma_wait);
+		cancel_work_sync(&perf->threads[tidx].work);
+	}
+}
+
+static int perf_set_dch_cnt(struct perf_ctx *perf, u8 dch_cnt)
+{
+	if (dch_cnt == 0 || dch_cnt > MAX_THREADS_CNT)
 		return -EINVAL;
 
 	if (test_and_set_bit_lock(0, &perf->busy_flag))
 		return -EBUSY;
 
-	perf->tcnt = tcnt;
+	perf->dch_cnt = dch_cnt;
 
 	clear_bit_unlock(0, &perf->busy_flag);
 
 	return 0;
 }
 
-static void perf_terminate_test(struct perf_ctx *perf)
+static int perf_set_tcnt(struct perf_ctx *perf, u8 tcnt)
 {
-	int tidx;
+	if (tcnt == 0 || tcnt > MAX_THREADS_CNT)
+		return -EINVAL;
 
-	atomic_set(&perf->tsync, -1);
-	wake_up(&perf->twait);
+	if (test_and_set_bit_lock(0, &perf->busy_flag))
+		return -EBUSY;
 
-	for (tidx = 0; tidx < MAX_THREADS_CNT; tidx++) {
-		wake_up(&perf->threads[tidx].dma_wait);
-		cancel_work_sync(&perf->threads[tidx].work);
-	}
+	perf->tcnt = tcnt;
+
+	clear_bit_unlock(0, &perf->busy_flag);
+
+	return 0;
 }
 
 static int perf_submit_test(struct perf_peer *peer)
@@ -1118,7 +1157,7 @@ static int perf_read_stats(struct perf_ctx *perf, char *buf,
 			   size_t size, ssize_t *pos)
 {
 	struct perf_thread *pthr;
-	int tidx;
+	int tidx, didx;
 
 	if (test_and_set_bit_lock(0, &perf->busy_flag))
 		return -EBUSY;
@@ -1138,10 +1177,19 @@ static int perf_read_stats(struct perf_ctx *perf, char *buf,
 			continue;
 		}
 
+	     if (!use_dma) {
 		(*pos) += scnprintf(buf + *pos, size - *pos,
 			"%d: copied %llu bytes in %llu usecs, %llu MBytes/s\n",
 			tidx, pthr->copied, ktime_to_us(pthr->duration),
 			div64_u64(pthr->copied, ktime_to_us(pthr->duration)));
+	     } else {
+		for (didx = 0; didx < perf->dch_cnt; didx++) {
+			(*pos) += scnprintf(buf + *pos, size - *pos,
+				"%d: %s : copied %llu bytes in %llu usecs, %llu MBytes/s\n",
+				tidx, dma_chan_name(pthr->dma_chan[didx]), pthr->copied, ktime_to_us(pthr->duration),
+				div64_u64(pthr->copied, ktime_to_us(pthr->duration)));
+		}
+	     }
 	}
 
 	clear_bit_unlock(0, &perf->busy_flag);
@@ -1155,6 +1203,7 @@ static void perf_init_threads(struct perf_ctx *perf)
 	int tidx;
 
 	perf->tcnt = DEF_THREADS_CNT;
+	perf->dch_cnt = DEF_THREADS_CNT;
 	perf->test_peer = &perf->peers[0];
 	init_waitqueue_head(&perf->twait);
 
@@ -1350,6 +1399,43 @@ static const struct file_operations perf_dbgfs_tcnt = {
 	.write = perf_dbgfs_write_tcnt
 };
 
+static ssize_t perf_dbgfs_read_dch_cnt(struct file *filep, char __user *ubuf,
+				    size_t size, loff_t *offp)
+{
+	struct perf_ctx *perf = filep->private_data;
+	char buf[8];
+	ssize_t pos;
+
+	pos = scnprintf(buf, sizeof(buf), "%hhu\n", perf->tcnt);
+
+	return simple_read_from_buffer(ubuf, size, offp, buf, pos);
+}
+
+static ssize_t perf_dbgfs_write_dch_cnt(struct file *filep,
+				     const char __user *ubuf,
+				     size_t size, loff_t *offp)
+{
+	struct perf_ctx *perf = filep->private_data;
+	int ret;
+	u8 val;
+
+	ret = kstrtou8_from_user(ubuf, size, 0, &val);
+	if (ret)
+		return ret;
+
+	ret = perf_set_dch_cnt(perf, val);
+	if (ret)
+		return ret;
+
+	return size;
+}
+
+static const struct file_operations perf_dbgfs_dch_cnt = {
+	.open = simple_open,
+	.read = perf_dbgfs_read_dch_cnt,
+	.write = perf_dbgfs_write_dch_cnt
+};
+
 static void perf_setup_dbgfs(struct perf_ctx *perf)
 {
 	struct pci_dev *pdev = perf->ntb->pdev;
@@ -1369,6 +1455,9 @@ static void perf_setup_dbgfs(struct perf_ctx *perf)
 	debugfs_create_file("threads_count", 0600, perf->dbgfs_dir, perf,
 			    &perf_dbgfs_tcnt);
 
+	debugfs_create_file("dma_ch_per_thread", 0600, perf->dbgfs_dir, perf,
+			    &perf_dbgfs_dch_cnt);
+
 	/* They are made read-only for test exec safety and integrity */
 	debugfs_create_u8("chunk_order", 0500, perf->dbgfs_dir, &chunk_order);
 
-- 
2.34.1