path: root/common/recipes-kernel/linux/linux-yocto-4.14.71/1122-port-in-all-files.patch

From 2ccdb14dd9edc4d31c4e4c2932ccf498578e62a5 Mon Sep 17 00:00:00 2001
From: Chaudhary Amit Kumar <chaudharyamit.kumar@amd.com>
Date: Wed, 17 Oct 2018 19:14:21 +0530
Subject: [PATCH 1122/4131] port in all files

amdkfd source files snapshot at

commit 9918a8f15a957dff68d8bb7d88a2e6485368b626
Author: shaoyunl <Shaoyun.Liu@amd.com>
Signed-off-by: Chaudhary Amit Kumar <chaudharyamit.kumar@amd.com>
---
 drivers/gpu/drm/amd/amdkfd/Kconfig                 |    1 +
 drivers/gpu/drm/amd/amdkfd/Makefile                |    2 +-
 drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c   |   43 +-
 drivers/gpu/drm/amd/amdkfd/cik_int.h               |   22 +-
 drivers/gpu/drm/amd/amdkfd/cik_regs.h              |  175 ++-
 .../gpu/drm/amd/amdkfd/cwsr_trap_handler_carrizo.h | 1377 ++++++++++++++++++++
 drivers/gpu/drm/amd/amdkfd/kfd_chardev.c           | 1344 ++++++++++++++++---
 drivers/gpu/drm/amd/amdkfd/kfd_crat.c              | 1163 +++++++++++++++++
 drivers/gpu/drm/amd/amdkfd/kfd_crat.h              |   40 +-
 drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c            |  972 +++++++-------
 drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.h            |   66 +-
 drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.c            |  247 +++-
 drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.h            |  313 +++--
 drivers/gpu/drm/amd/amdkfd/kfd_device.c            |  283 +++-
 .../gpu/drm/amd/amdkfd/kfd_device_queue_manager.c  |  497 +++++--
 .../gpu/drm/amd/amdkfd/kfd_device_queue_manager.h  |   29 +-
 .../drm/amd/amdkfd/kfd_device_queue_manager_cik.c  |    2 +
 .../drm/amd/amdkfd/kfd_device_queue_manager_vi.c   |  106 ++
 drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c          |    3 +-
 drivers/gpu/drm/amd/amdkfd/kfd_events.c            |  522 +++++---
 drivers/gpu/drm/amd/amdkfd/kfd_events.h            |    3 +-
 drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c       |   79 +-
 drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c         |    6 +-
 drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c      |   26 +-
 drivers/gpu/drm/amd/amdkfd/kfd_module.c            |   30 +-
 drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c       |    3 +
 drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h       |    3 +-
 drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c   |   88 +-
 drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c    |  227 +++-
 drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c    |  212 ++-
 drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers.h       |  120 +-
 drivers/gpu/drm/amd/amdkfd/kfd_priv.h              |  269 +++-
 drivers/gpu/drm/amd/amdkfd/kfd_process.c           |  542 +++++++-
 .../gpu/drm/amd/amdkfd/kfd_process_queue_manager.c |  119 +-
 drivers/gpu/drm/amd/amdkfd/kfd_rdma.c              |  296 +++++
 drivers/gpu/drm/amd/amdkfd/kfd_topology.c          |  841 ++++++------
 drivers/gpu/drm/amd/amdkfd/kfd_topology.h          |   23 +-
 37 files changed, 8208 insertions(+), 1886 deletions(-)
 create mode 100644 drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_carrizo.h
 create mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_crat.c
 create mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_rdma.c

diff --git a/drivers/gpu/drm/amd/amdkfd/Kconfig b/drivers/gpu/drm/amd/amdkfd/Kconfig
index e13c67c..ac49532 100644
--- a/drivers/gpu/drm/amd/amdkfd/Kconfig
+++ b/drivers/gpu/drm/amd/amdkfd/Kconfig
@@ -5,5 +5,6 @@
 config HSA_AMD
 	tristate "HSA kernel driver for AMD GPU devices"
 	depends on (DRM_RADEON || DRM_AMDGPU) && AMD_IOMMU_V2 && X86_64
+	select DRM_AMDGPU_USERPTR
 	help
 	  Enable this if you want to use HSA features on AMD GPU devices.
diff --git a/drivers/gpu/drm/amd/amdkfd/Makefile b/drivers/gpu/drm/amd/amdkfd/Makefile
index 7bb0bc0..b5d3db9 100644
--- a/drivers/gpu/drm/amd/amdkfd/Makefile
+++ b/drivers/gpu/drm/amd/amdkfd/Makefile
@@ -15,6 +15,6 @@ amdkfd-y	:= kfd_module.o kfd_device.o kfd_chardev.o kfd_topology.o \
 		kfd_process_queue_manager.o kfd_device_queue_manager.o \
 		kfd_device_queue_manager_cik.o kfd_device_queue_manager_vi.o \
 		kfd_interrupt.o kfd_events.o cik_event_interrupt.o \
-		kfd_dbgdev.o kfd_dbgmgr.o
+		kfd_dbgdev.o kfd_dbgmgr.o kfd_flat_memory.o kfd_crat.o kfd_rdma.o
 
 obj-$(CONFIG_HSA_AMD)	+= amdkfd.o
diff --git a/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c b/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c
index 211fc48..02a9082 100644
--- a/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c
+++ b/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c
@@ -24,40 +24,59 @@
 #include "kfd_events.h"
 #include "cik_int.h"
 
-static bool cik_event_interrupt_isr(struct kfd_dev *dev,
+static bool is_cpc_vm_fault(struct kfd_dev *dev,
 					const uint32_t *ih_ring_entry)
 {
-	unsigned int pasid;
 	const struct cik_ih_ring_entry *ihre =
 			(const struct cik_ih_ring_entry *)ih_ring_entry;
 
-	pasid = (ihre->ring_id & 0xffff0000) >> 16;
+	if ((ihre->source_id == CIK_INTSRC_GFX_PAGE_INV_FAULT ||
+		ihre->source_id == CIK_INTSRC_GFX_MEM_PROT_FAULT) &&
+	    ihre->vmid >= dev->vm_info.first_vmid_kfd &&
+	    ihre->vmid <= dev->vm_info.last_vmid_kfd)
+		return true;
+	return false;
+}
+static bool cik_event_interrupt_isr(struct kfd_dev *dev,
+					const uint32_t *ih_ring_entry)
+{
+	const struct cik_ih_ring_entry *ihre =
+			(const struct cik_ih_ring_entry *)ih_ring_entry;
 
 	/* Do not process in ISR, just request it to be forwarded to WQ. */
-	return (pasid != 0) &&
+	return (ihre->pasid != 0) &&
 		(ihre->source_id == CIK_INTSRC_CP_END_OF_PIPE ||
 		ihre->source_id == CIK_INTSRC_SQ_INTERRUPT_MSG ||
-		ihre->source_id == CIK_INTSRC_CP_BAD_OPCODE);
+		ihre->source_id == CIK_INTSRC_CP_BAD_OPCODE ||
+		is_cpc_vm_fault(dev, ih_ring_entry));
 }
 
 static void cik_event_interrupt_wq(struct kfd_dev *dev,
 					const uint32_t *ih_ring_entry)
 {
-	unsigned int pasid;
 	const struct cik_ih_ring_entry *ihre =
 			(const struct cik_ih_ring_entry *)ih_ring_entry;
 
-	pasid = (ihre->ring_id & 0xffff0000) >> 16;
-
-	if (pasid == 0)
+	if (ihre->pasid == 0)
 		return;
 
 	if (ihre->source_id == CIK_INTSRC_CP_END_OF_PIPE)
-		kfd_signal_event_interrupt(pasid, 0, 0);
+		kfd_signal_event_interrupt(ihre->pasid, 0, 0);
 	else if (ihre->source_id == CIK_INTSRC_SQ_INTERRUPT_MSG)
-		kfd_signal_event_interrupt(pasid, ihre->data & 0xFF, 8);
+		kfd_signal_event_interrupt(ihre->pasid, ihre->data & 0xFF, 8);
 	else if (ihre->source_id == CIK_INTSRC_CP_BAD_OPCODE)
-		kfd_signal_hw_exception_event(pasid);
+		kfd_signal_hw_exception_event(ihre->pasid);
+	else if (ihre->source_id == CIK_INTSRC_GFX_PAGE_INV_FAULT ||
+		ihre->source_id == CIK_INTSRC_GFX_MEM_PROT_FAULT) {
+		struct kfd_vm_fault_info info;
+
+		dev->kfd2kgd->get_vm_fault_info(dev->kgd, &info);
+		kfd_process_vm_fault(dev->dqm, ihre->pasid);
+		if (info.vmid == ihre->vmid)
+			kfd_signal_vm_fault_event(dev, ihre->pasid, &info);
+		else
+			kfd_signal_vm_fault_event(dev, ihre->pasid, NULL);
+	}
 }
 
 const struct kfd_event_interrupt_class event_interrupt_class_cik = {
diff --git a/drivers/gpu/drm/amd/amdkfd/cik_int.h b/drivers/gpu/drm/amd/amdkfd/cik_int.h
index 79a16d2..feb3c24 100644
--- a/drivers/gpu/drm/amd/amdkfd/cik_int.h
+++ b/drivers/gpu/drm/amd/amdkfd/cik_int.h
@@ -26,16 +26,30 @@
 #include <linux/types.h>
 
 struct cik_ih_ring_entry {
-	uint32_t source_id;
-	uint32_t data;
-	uint32_t ring_id;
-	uint32_t reserved;
+	uint32_t source_id:8;
+	uint32_t reserved1:8;
+	uint32_t reserved2:16;
+
+	uint32_t data:28;
+	uint32_t reserved3:4;
+
+	/* pipeid, meid and unused3 are officially called RINGID,
+	 * but for our purposes, they always decode into pipe and ME. */
+	uint32_t pipeid:2;
+	uint32_t meid:2;
+	uint32_t reserved4:4;
+	uint32_t vmid:8;
+	uint32_t pasid:16;
+
+	uint32_t reserved5;
 };
 
 #define CIK_INTSRC_DEQUEUE_COMPLETE	0xC6
 #define CIK_INTSRC_CP_END_OF_PIPE	0xB5
 #define CIK_INTSRC_CP_BAD_OPCODE	0xB7
 #define CIK_INTSRC_SQ_INTERRUPT_MSG	0xEF
+#define CIK_INTSRC_GFX_PAGE_INV_FAULT	0x92
+#define CIK_INTSRC_GFX_MEM_PROT_FAULT	0x93
 
 #endif
 
diff --git a/drivers/gpu/drm/amd/amdkfd/cik_regs.h b/drivers/gpu/drm/amd/amdkfd/cik_regs.h
index 48769d1..607fc5c 100644
--- a/drivers/gpu/drm/amd/amdkfd/cik_regs.h
+++ b/drivers/gpu/drm/amd/amdkfd/cik_regs.h
@@ -23,11 +23,33 @@
 #ifndef CIK_REGS_H
 #define CIK_REGS_H
 
+#define IH_VMID_0_LUT					0x3D40u
+
+#define BIF_DOORBELL_CNTL				0x530Cu
+
+#define	SRBM_GFX_CNTL					0xE44
+#define	PIPEID(x)					((x) << 0)
+#define	MEID(x)						((x) << 2)
+#define	VMID(x)						((x) << 4)
+#define	QUEUEID(x)					((x) << 8)
+
+#define	SQ_CONFIG					0x8C00
+
+#define	SH_MEM_BASES					0x8C28
 /* if PTR32, these are the bases for scratch and lds */
 #define	PRIVATE_BASE(x)					((x) << 0) /* scratch */
 #define	SHARED_BASE(x)					((x) << 16) /* LDS */
+#define	SH_MEM_APE1_BASE				0x8C2C
+/* if PTR32, this is the base location of GPUVM */
+#define	SH_MEM_APE1_LIMIT				0x8C30
+/* if PTR32, this is the upper limit of GPUVM */
+#define	SH_MEM_CONFIG					0x8C34
 #define	PTR32						(1 << 0)
+#define PRIVATE_ATC					(1 << 1)
 #define	ALIGNMENT_MODE(x)				((x) << 2)
+#define	SH_MEM_ALIGNMENT_MODE_DWORD			0
+#define	SH_MEM_ALIGNMENT_MODE_DWORD_STRICT		1
+#define	SH_MEM_ALIGNMENT_MODE_STRICT			2
 #define	SH_MEM_ALIGNMENT_MODE_UNALIGNED			3
 #define	DEFAULT_MTYPE(x)				((x) << 4)
 #define	APE1_MTYPE(x)					((x) << 7)
@@ -36,37 +58,164 @@
 #define	MTYPE_CACHED					0
 #define	MTYPE_NONCACHED					3
 
+
+#define SH_STATIC_MEM_CONFIG				0x9604u
+
+#define	TC_CFG_L1_LOAD_POLICY0				0xAC68
+#define	TC_CFG_L1_LOAD_POLICY1				0xAC6C
+#define	TC_CFG_L1_STORE_POLICY				0xAC70
+#define	TC_CFG_L2_LOAD_POLICY0				0xAC74
+#define	TC_CFG_L2_LOAD_POLICY1				0xAC78
+#define	TC_CFG_L2_STORE_POLICY0				0xAC7C
+#define	TC_CFG_L2_STORE_POLICY1				0xAC80
+#define	TC_CFG_L2_ATOMIC_POLICY				0xAC84
+#define	TC_CFG_L1_VOLATILE				0xAC88
+#define	TC_CFG_L2_VOLATILE				0xAC8C
+
+#define CP_PQ_WPTR_POLL_CNTL				0xC20C
+#define	WPTR_POLL_EN					(1 << 31)
+
+#define CPC_INT_CNTL					0xC2D0
+#define CP_ME1_PIPE0_INT_CNTL				0xC214
+#define CP_ME1_PIPE1_INT_CNTL				0xC218
+#define CP_ME1_PIPE2_INT_CNTL				0xC21C
+#define CP_ME1_PIPE3_INT_CNTL				0xC220
+#define CP_ME2_PIPE0_INT_CNTL				0xC224
+#define CP_ME2_PIPE1_INT_CNTL				0xC228
+#define CP_ME2_PIPE2_INT_CNTL				0xC22C
+#define CP_ME2_PIPE3_INT_CNTL				0xC230
+#define DEQUEUE_REQUEST_INT_ENABLE			(1 << 13)
+#define WRM_POLL_TIMEOUT_INT_ENABLE			(1 << 17)
+#define PRIV_REG_INT_ENABLE				(1 << 23)
+#define TIME_STAMP_INT_ENABLE				(1 << 26)
+#define GENERIC2_INT_ENABLE				(1 << 29)
+#define GENERIC1_INT_ENABLE				(1 << 30)
+#define GENERIC0_INT_ENABLE				(1 << 31)
+#define CP_ME1_PIPE0_INT_STATUS				0xC214
+#define CP_ME1_PIPE1_INT_STATUS				0xC218
+#define CP_ME1_PIPE2_INT_STATUS				0xC21C
+#define CP_ME1_PIPE3_INT_STATUS				0xC220
+#define CP_ME2_PIPE0_INT_STATUS				0xC224
+#define CP_ME2_PIPE1_INT_STATUS				0xC228
+#define CP_ME2_PIPE2_INT_STATUS				0xC22C
+#define CP_ME2_PIPE3_INT_STATUS				0xC230
+#define DEQUEUE_REQUEST_INT_STATUS			(1 << 13)
+#define WRM_POLL_TIMEOUT_INT_STATUS			(1 << 17)
+#define PRIV_REG_INT_STATUS				(1 << 23)
+#define TIME_STAMP_INT_STATUS				(1 << 26)
+#define GENERIC2_INT_STATUS				(1 << 29)
+#define GENERIC1_INT_STATUS				(1 << 30)
+#define GENERIC0_INT_STATUS				(1 << 31)
+
+#define CP_HPD_EOP_BASE_ADDR				0xC904
+#define CP_HPD_EOP_BASE_ADDR_HI				0xC908
+#define CP_HPD_EOP_VMID					0xC90C
+#define CP_HPD_EOP_CONTROL				0xC910
+#define	EOP_SIZE(x)					((x) << 0)
+#define	EOP_SIZE_MASK					(0x3f << 0)
+#define CP_MQD_BASE_ADDR				0xC914
+#define CP_MQD_BASE_ADDR_HI				0xC918
+#define CP_HQD_ACTIVE					0xC91C
+#define CP_HQD_VMID					0xC920
+
+#define CP_HQD_PERSISTENT_STATE				0xC924u
 #define	DEFAULT_CP_HQD_PERSISTENT_STATE			(0x33U << 8)
 #define	PRELOAD_REQ					(1 << 0)
 
-#define	MQD_CONTROL_PRIV_STATE_EN			(1U << 8)
-
-#define	DEFAULT_MIN_IB_AVAIL_SIZE			(3U << 20)
-
-#define	IB_ATC_EN					(1U << 23)
-
+#define CP_HQD_PIPE_PRIORITY				0xC928u
+#define CP_HQD_QUEUE_PRIORITY				0xC92Cu
+#define CP_HQD_QUANTUM					0xC930u
 #define	QUANTUM_EN					1U
 #define	QUANTUM_SCALE_1MS				(1U << 4)
 #define	QUANTUM_DURATION(x)				((x) << 8)
 
+#define CP_HQD_PQ_BASE					0xC934
+#define CP_HQD_PQ_BASE_HI				0xC938
+#define CP_HQD_PQ_RPTR					0xC93C
+#define CP_HQD_PQ_RPTR_REPORT_ADDR			0xC940
+#define CP_HQD_PQ_RPTR_REPORT_ADDR_HI			0xC944
+#define CP_HQD_PQ_WPTR_POLL_ADDR			0xC948
+#define CP_HQD_PQ_WPTR_POLL_ADDR_HI			0xC94C
+#define CP_HQD_PQ_DOORBELL_CONTROL			0xC950
+#define	DOORBELL_OFFSET(x)				((x) << 2)
+#define	DOORBELL_OFFSET_MASK				(0x1fffff << 2)
+#define	DOORBELL_SOURCE					(1 << 28)
+#define	DOORBELL_SCHD_HIT				(1 << 29)
+#define	DOORBELL_EN					(1 << 30)
+#define	DOORBELL_HIT					(1 << 31)
+#define CP_HQD_PQ_WPTR					0xC954
+#define CP_HQD_PQ_CONTROL				0xC958
+#define	QUEUE_SIZE(x)					((x) << 0)
+#define	QUEUE_SIZE_MASK					(0x3f << 0)
 #define	RPTR_BLOCK_SIZE(x)				((x) << 8)
+#define	RPTR_BLOCK_SIZE_MASK				(0x3f << 8)
 #define	MIN_AVAIL_SIZE(x)				((x) << 20)
+#define	PQ_ATC_EN					(1 << 23)
+#define	PQ_VOLATILE					(1 << 26)
+#define	NO_UPDATE_RPTR					(1 << 27)
+#define	UNORD_DISPATCH					(1 << 28)
+#define	ROQ_PQ_IB_FLIP					(1 << 29)
+#define	PRIV_STATE					(1 << 30)
+#define	KMD_QUEUE					(1 << 31)
+
 #define	DEFAULT_RPTR_BLOCK_SIZE				RPTR_BLOCK_SIZE(5)
 #define	DEFAULT_MIN_AVAIL_SIZE				MIN_AVAIL_SIZE(3)
 
-#define	PQ_ATC_EN					(1 << 23)
-#define	NO_UPDATE_RPTR					(1 << 27)
+#define CP_HQD_IB_BASE_ADDR				0xC95Cu
+#define CP_HQD_IB_BASE_ADDR_HI				0xC960u
+#define CP_HQD_IB_RPTR					0xC964u
+#define CP_HQD_IB_CONTROL				0xC968u
+#define	IB_ATC_EN					(1U << 23)
+#define	DEFAULT_MIN_IB_AVAIL_SIZE			(3U << 20)
 
-#define	DOORBELL_OFFSET(x)				((x) << 2)
-#define	DOORBELL_EN					(1 << 30)
+#define CP_HQD_DEQUEUE_REQUEST				0xC974
+#define	DEQUEUE_REQUEST_DRAIN				1
+#define DEQUEUE_REQUEST_RESET				2
+#define		DEQUEUE_INT					(1U << 8)
 
-#define	PRIV_STATE					(1 << 30)
-#define	KMD_QUEUE					(1 << 31)
+#define CP_HQD_SEMA_CMD					0xC97Cu
+#define CP_HQD_MSG_TYPE					0xC980u
+#define CP_HQD_ATOMIC0_PREOP_LO				0xC984u
+#define CP_HQD_ATOMIC0_PREOP_HI				0xC988u
+#define CP_HQD_ATOMIC1_PREOP_LO				0xC98Cu
+#define CP_HQD_ATOMIC1_PREOP_HI				0xC990u
+#define CP_HQD_HQ_SCHEDULER0				0xC994u
+#define CP_HQD_HQ_SCHEDULER1				0xC998u
 
-#define	AQL_ENABLE					1
+
+#define CP_MQD_CONTROL					0xC99C
+#define	MQD_VMID(x)					((x) << 0)
+#define	MQD_VMID_MASK					(0xf << 0)
+#define	MQD_CONTROL_PRIV_STATE_EN			(1U << 8)
 
 #define GRBM_GFX_INDEX					0x30800
+#define	INSTANCE_INDEX(x)				((x) << 0)
+#define	SH_INDEX(x)					((x) << 8)
+#define	SE_INDEX(x)					((x) << 16)
+#define	SH_BROADCAST_WRITES				(1 << 29)
+#define	INSTANCE_BROADCAST_WRITES			(1 << 30)
+#define	SE_BROADCAST_WRITES				(1 << 31)
 
+#define SQC_CACHES					0x30d20
+#define SQC_POLICY					0x8C38u
+#define SQC_VOLATILE					0x8C3Cu
+
+#define CP_PERFMON_CNTL					0x36020
+
+#define ATC_VMID0_PASID_MAPPING				0x339Cu
+#define	ATC_VMID_PASID_MAPPING_UPDATE_STATUS		0x3398u
 #define	ATC_VMID_PASID_MAPPING_VALID			(1U << 31)
 
+#define ATC_VM_APERTURE0_CNTL				0x3310u
+#define	ATS_ACCESS_MODE_NEVER				0
+#define	ATS_ACCESS_MODE_ALWAYS				1
+
+#define ATC_VM_APERTURE0_CNTL2				0x3318u
+#define ATC_VM_APERTURE0_HIGH_ADDR			0x3308u
+#define ATC_VM_APERTURE0_LOW_ADDR			0x3300u
+#define ATC_VM_APERTURE1_CNTL				0x3314u
+#define ATC_VM_APERTURE1_CNTL2				0x331Cu
+#define ATC_VM_APERTURE1_HIGH_ADDR			0x330Cu
+#define ATC_VM_APERTURE1_LOW_ADDR			0x3304u
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_carrizo.h b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_carrizo.h
new file mode 100644
index 0000000..1880dc0
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_carrizo.h
@@ -0,0 +1,1377 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if 0
+  HW (CARRIZO) source code for CWSR trap handler
+
+var G8SR_WDMEM_HWREG_OFFSET = 0
+var G8SR_WDMEM_SGPR_OFFSET  = 128  // in bytes
+
+// Keep definition same as the app shader, These 2 time stamps are part of the app shader... Should before any Save and after restore.
+
+var G8SR_DEBUG_TIMESTAMP = 0
+var G8SR_DEBUG_TS_SAVE_D_OFFSET = 40*4  // ts_save_d timestamp offset relative to SGPR_SR_memory_offset
+var s_g8sr_ts_save_s    = s[34:35]   // save start
+var s_g8sr_ts_sq_save_msg  = s[36:37]   // The save shader send SAVEWAVE msg to spi
+var s_g8sr_ts_spi_wrexec   = s[38:39]   // the SPI write the sr address to SQ
+var s_g8sr_ts_save_d    = s[40:41]   // save end
+var s_g8sr_ts_restore_s = s[42:43]   // restore start
+var s_g8sr_ts_restore_d = s[44:45]   // restore end
+
+var G8SR_VGPR_SR_IN_DWX4 = 0
+var G8SR_SAVE_BUF_RSRC_WORD1_STRIDE_DWx4 = 0x00100000    // DWx4 stride is 4*4Bytes
+var G8SR_RESTORE_BUF_RSRC_WORD1_STRIDE_DWx4  = G8SR_SAVE_BUF_RSRC_WORD1_STRIDE_DWx4
+
+
+/*************************************************************************/
+/*					control on how to run the shader					 */
+/*************************************************************************/
+//any hack that needs to be made to run this code in EMU (either becasue various EMU code are not ready or no compute save & restore in EMU run)
+var EMU_RUN_HACK					=	0
+var EMU_RUN_HACK_RESTORE_NORMAL		=	0
+var EMU_RUN_HACK_SAVE_NORMAL_EXIT	=	0
+var	EMU_RUN_HACK_SAVE_SINGLE_WAVE	=	0
+var EMU_RUN_HACK_SAVE_FIRST_TIME	= 	0					//for interrupted restore in which the first save is through EMU_RUN_HACK
+var EMU_RUN_HACK_SAVE_FIRST_TIME_TBA_LO	= 	0					//for interrupted restore in which the first save is through EMU_RUN_HACK
+var EMU_RUN_HACK_SAVE_FIRST_TIME_TBA_HI	= 	0					//for interrupted restore in which the first save is through EMU_RUN_HACK
+var SAVE_LDS						= 	1
+var WG_BASE_ADDR_LO					=   0x9000a000
+var WG_BASE_ADDR_HI					=	0x0
+var WAVE_SPACE						=	0x5000				//memory size that each wave occupies in workgroup state mem
+var CTX_SAVE_CONTROL				=	0x0
+var CTX_RESTORE_CONTROL				=	CTX_SAVE_CONTROL
+var SIM_RUN_HACK					=	0					//any hack that needs to be made to run this code in SIM (either becasue various RTL code are not ready or no compute save & restore in RTL run)
+var	SGPR_SAVE_USE_SQC				=	1					//use SQC D$ to do the write
+var USE_MTBUF_INSTEAD_OF_MUBUF		=	0					//becasue TC EMU curently asserts on 0 of // overload DFMT field to carry 4 more bits of stride for MUBUF opcodes
+var SWIZZLE_EN						=	0					//whether we use swizzled buffer addressing
+
+/**************************************************************************/
+/*                     	variables							              */
+/**************************************************************************/
+var SQ_WAVE_STATUS_INST_ATC_SHIFT  = 23
+var SQ_WAVE_STATUS_INST_ATC_MASK   = 0x00800000
+
+var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT	= 12
+var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE		= 9
+var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT	= 8
+var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE	= 6
+var SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT	= 24
+var SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE	= 3						//FIXME	 sq.blk still has 4 bits at this time while SQ programming guide has 3 bits
+
+var	SQ_WAVE_TRAPSTS_SAVECTX_MASK	=	0x400
+var SQ_WAVE_TRAPSTS_EXCE_MASK       =   0x1FF          			// Exception mask					
+var	SQ_WAVE_TRAPSTS_SAVECTX_SHIFT	=	10					
+var	SQ_WAVE_TRAPSTS_MEM_VIOL_MASK	=	0x100					
+var	SQ_WAVE_TRAPSTS_MEM_VIOL_SHIFT	=	8		
+var	SQ_WAVE_TRAPSTS_PRE_SAVECTX_MASK 	=	0x3FF
+var	SQ_WAVE_TRAPSTS_PRE_SAVECTX_SHIFT 	=	0x0
+var	SQ_WAVE_TRAPSTS_PRE_SAVECTX_SIZE 	=	10
+var	SQ_WAVE_TRAPSTS_POST_SAVECTX_MASK 	=	0xFFFFF800	
+var	SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT 	=	11
+var	SQ_WAVE_TRAPSTS_POST_SAVECTX_SIZE 	=	21	
+
+var SQ_WAVE_IB_STS_RCNT_SHIFT			=	16					//FIXME
+var SQ_WAVE_IB_STS_RCNT_SIZE			=	4					//FIXME
+var SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT	=	15					//FIXME
+var SQ_WAVE_IB_STS_FIRST_REPLAY_SIZE	=	1					//FIXME
+var SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK_NEG	= 0x00007FFF	//FIXME
+ 
+var	SQ_BUF_RSRC_WORD1_ATC_SHIFT		=	24
+var	SQ_BUF_RSRC_WORD3_MTYPE_SHIFT	=	27
+
+
+/*      Save        */
+var	S_SAVE_BUF_RSRC_WORD1_STRIDE		=	0x00040000  		//stride is 4 bytes 
+var	S_SAVE_BUF_RSRC_WORD3_MISC			= 	0x00807FAC			//SQ_SEL_X/Y/Z/W, BUF_NUM_FORMAT_FLOAT, (0 for MUBUF stride[17:14] when ADD_TID_ENABLE and BUF_DATA_FORMAT_32 for MTBUF), ADD_TID_ENABLE			
+
+var	S_SAVE_SPI_INIT_ATC_MASK			=	0x08000000			//bit[27]: ATC bit
+var	S_SAVE_SPI_INIT_ATC_SHIFT			=	27
+var	S_SAVE_SPI_INIT_MTYPE_MASK			=	0x70000000			//bit[30:28]: Mtype
+var	S_SAVE_SPI_INIT_MTYPE_SHIFT			=	28
+var	S_SAVE_SPI_INIT_FIRST_WAVE_MASK		=	0x04000000			//bit[26]: FirstWaveInTG
+var	S_SAVE_SPI_INIT_FIRST_WAVE_SHIFT	=	26
+
+var S_SAVE_PC_HI_RCNT_SHIFT				=	28					//FIXME	 check with Brian to ensure all fields other than PC[47:0] can be used
+var S_SAVE_PC_HI_RCNT_MASK				=   0xF0000000			//FIXME
+var S_SAVE_PC_HI_FIRST_REPLAY_SHIFT		=	27					//FIXME
+var S_SAVE_PC_HI_FIRST_REPLAY_MASK		=	0x08000000			//FIXME
+
+var	s_save_spi_init_lo				=	exec_lo
+var s_save_spi_init_hi				=	exec_hi
+
+												//tba_lo and tba_hi need to be saved/restored
+var	s_save_pc_lo			=	ttmp0			//{TTMP1, TTMP0} = {3��h0,pc_rewind[3:0], HT[0],trapID[7:0], PC[47:0]}
+var	s_save_pc_hi			=	ttmp1			
+var s_save_exec_lo			=	ttmp2
+var s_save_exec_hi			= 	ttmp3			
+var	s_save_status			=	ttmp4			
+var	s_save_trapsts			=	ttmp5			//not really used until the end of the SAVE routine
+var s_save_xnack_mask_lo	=	ttmp6
+var s_save_xnack_mask_hi	=	ttmp7
+var	s_save_buf_rsrc0		=	ttmp8
+var	s_save_buf_rsrc1		=	ttmp9
+var	s_save_buf_rsrc2		=	ttmp10
+var	s_save_buf_rsrc3		=	ttmp11
+
+var s_save_mem_offset		= 	tma_lo				
+var s_save_alloc_size		=	s_save_trapsts			//conflict
+var s_save_tmp              =   s_save_buf_rsrc2       	//shared with s_save_buf_rsrc2  (conflict: should not use mem access with s_save_tmp at the same time)
+var s_save_m0				=	tma_hi					
+
+/*      Restore     */
+var	S_RESTORE_BUF_RSRC_WORD1_STRIDE			=	S_SAVE_BUF_RSRC_WORD1_STRIDE 
+var	S_RESTORE_BUF_RSRC_WORD3_MISC			= 	S_SAVE_BUF_RSRC_WORD3_MISC		 
+
+var	S_RESTORE_SPI_INIT_ATC_MASK			    =	0x08000000			//bit[27]: ATC bit
+var	S_RESTORE_SPI_INIT_ATC_SHIFT			=	27
+var	S_RESTORE_SPI_INIT_MTYPE_MASK			=	0x70000000			//bit[30:28]: Mtype
+var	S_RESTORE_SPI_INIT_MTYPE_SHIFT			=	28
+var	S_RESTORE_SPI_INIT_FIRST_WAVE_MASK		=	0x04000000			//bit[26]: FirstWaveInTG
+var	S_RESTORE_SPI_INIT_FIRST_WAVE_SHIFT	    =	26
+
+var S_RESTORE_PC_HI_RCNT_SHIFT				=	S_SAVE_PC_HI_RCNT_SHIFT
+var S_RESTORE_PC_HI_RCNT_MASK				=   S_SAVE_PC_HI_RCNT_MASK
+var S_RESTORE_PC_HI_FIRST_REPLAY_SHIFT		=	S_SAVE_PC_HI_FIRST_REPLAY_SHIFT
+var S_RESTORE_PC_HI_FIRST_REPLAY_MASK		=	S_SAVE_PC_HI_FIRST_REPLAY_MASK
+
+var s_restore_spi_init_lo                   =   exec_lo
+var s_restore_spi_init_hi                   =   exec_hi
+
+var s_restore_mem_offset		= 	ttmp2
+var s_restore_alloc_size		=	ttmp3
+var s_restore_tmp           	=   ttmp6				//tba_lo/hi need to be restored
+var s_restore_mem_offset_save	= 	s_restore_tmp 		//no conflict
+
+var s_restore_m0			=	s_restore_alloc_size	//no conflict			
+
+var s_restore_mode			=  	ttmp7
+
+var	s_restore_pc_lo		    =	ttmp0			
+var	s_restore_pc_hi		    =	ttmp1
+var s_restore_exec_lo		=	tma_lo					//no conflict
+var s_restore_exec_hi		= 	tma_hi					//no conflict
+var	s_restore_status	    =	ttmp4			
+var	s_restore_trapsts	    =	ttmp5
+var s_restore_xnack_mask_lo	=	xnack_mask_lo
+var s_restore_xnack_mask_hi	=	xnack_mask_hi
+var	s_restore_buf_rsrc0		=	ttmp8
+var	s_restore_buf_rsrc1		=	ttmp9
+var	s_restore_buf_rsrc2		=	ttmp10
+var	s_restore_buf_rsrc3		=	ttmp11
+
+/**************************************************************************/
+/*                     	trap handler entry points			              */
+/**************************************************************************/
+/* Shader Main*/
+
+shader main
+  asic(CARRIZO)
+  type(CS)
+
+
+    if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_RESTORE_NORMAL)) 					//hack to use trap_id for determining save/restore
+		//FIXME VCCZ un-init assertion s_getreg_b32  	s_save_status, hwreg(HW_REG_STATUS)			//save STATUS since we will change SCC
+		s_and_b32 s_save_tmp, s_save_pc_hi, 0xffff0000 				//change SCC
+    	s_cmp_eq_u32 s_save_tmp, 0x007e0000  						//Save: trap_id = 0x7e. Restore: trap_id = 0x7f.  
+    	s_cbranch_scc0 L_JUMP_TO_RESTORE							//do not need to recover STATUS here  since we are going to RESTORE
+		//FIXME  s_setreg_b32 	hwreg(HW_REG_STATUS), 	s_save_status		//need to recover STATUS since we are going to SAVE	
+		s_branch L_SKIP_RESTORE 									//NOT restore, SAVE actually
+	else	
+		s_branch L_SKIP_RESTORE 									//NOT restore. might be a regular trap or save
+    end
+
+L_JUMP_TO_RESTORE:
+    s_branch L_RESTORE												//restore
+
+L_SKIP_RESTORE:
+	
+	s_getreg_b32  	s_save_status, hwreg(HW_REG_STATUS)								//save STATUS since we will change SCC
+	s_getreg_b32  	s_save_trapsts, hwreg(HW_REG_TRAPSTS)    		 				
+	s_and_b32		s_save_trapsts, s_save_trapsts, SQ_WAVE_TRAPSTS_SAVECTX_MASK	//check whether this is for save  
+	s_cbranch_scc1	L_SAVE		     	  						//this is the operation for save
+
+    // *********    Handle non-CWSR traps       *******************
+if (!EMU_RUN_HACK)
+    /* read tba and tma for next level trap handler, ttmp4 is used as s_save_status */
+	s_load_dwordx4  [ttmp8,ttmp9,ttmp10, ttmp11], [tma_lo,tma_hi], 0 
+	s_waitcnt lgkmcnt(0)
+	s_or_b32        ttmp7, ttmp8, ttmp9
+	s_cbranch_scc0  L_NO_NEXT_TRAP //next level trap handler not been set
+	s_mov_b32       tma_lo, ttmp10  //set tma_lo/hi for next level trap handler
+	s_mov_b32       tma_hi, ttmp11 
+	s_setreg_b32    hwreg(HW_REG_STATUS), s_save_status //restore HW status(SCC)
+	s_setpc_b64     [ttmp8,ttmp9] //jump to next level trap handler 
+
+L_NO_NEXT_TRAP:
+	s_getreg_b32    s_save_trapsts, hwreg(HW_REG_TRAPSTS)
+	s_and_b32       s_save_trapsts, s_save_trapsts, SQ_WAVE_TRAPSTS_EXCE_MASK // Check whether it is an exception
+	s_cbranch_scc1  L_EXCP_CASE   // Exception, jump back to the shader program directly.
+	s_add_u32       ttmp0, ttmp0, 4   // S_TRAP case, add 4 to ttmp0
+	s_addc_u32	ttmp1, ttmp1, 0
+L_EXCP_CASE:
+	s_and_b32	ttmp1, ttmp1, 0xFFFF
+	s_setreg_b32    hwreg(HW_REG_STATUS), s_save_status //restore HW status(SCC)   
+	s_rfe_b64    	[ttmp0, ttmp1]
+end
+    // *********        End handling of non-CWSR traps   *******************
+
+/**************************************************************************/
+/*                     	save routine						              */
+/**************************************************************************/
+
+L_SAVE:	
+
+if G8SR_DEBUG_TIMESTAMP
+        s_memrealtime	s_g8sr_ts_save_s
+        s_waitcnt lgkmcnt(0)         //FIXME, will cause xnack??
+end
+
+	//check whether there is mem_viol
+	s_getreg_b32  	s_save_trapsts, hwreg(HW_REG_TRAPSTS)    		 				
+	s_and_b32	s_save_trapsts, s_save_trapsts, SQ_WAVE_TRAPSTS_MEM_VIOL_MASK			
+	s_cbranch_scc0	L_NO_PC_REWIND
+    
+	//if so, need rewind PC assuming GDS operation gets NACKed
+	s_mov_b32       s_save_tmp, 0															//clear mem_viol bit
+	s_setreg_b32	hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_MEM_VIOL_SHIFT, 1), s_save_tmp	//clear mem_viol bit 
+	s_and_b32 		s_save_pc_hi, s_save_pc_hi, 0x0000ffff    //pc[47:32]
+	s_sub_u32 		s_save_pc_lo, s_save_pc_lo, 8             //pc[31:0]-8
+	s_subb_u32 		s_save_pc_hi, s_save_pc_hi, 0x0			  // -scc
+
+L_NO_PC_REWIND:
+    s_mov_b32       s_save_tmp, 0															//clear saveCtx bit
+	s_setreg_b32	hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_SAVECTX_SHIFT, 1), s_save_tmp		//clear saveCtx bit   
+
+	s_mov_b32		s_save_xnack_mask_lo,	xnack_mask_lo									//save XNACK_MASK  
+	s_mov_b32		s_save_xnack_mask_hi,	xnack_mask_hi    //save XNACK must before any memory operation
+	s_getreg_b32	s_save_tmp, hwreg(HW_REG_IB_STS, SQ_WAVE_IB_STS_RCNT_SHIFT, SQ_WAVE_IB_STS_RCNT_SIZE)					//save RCNT
+	s_lshl_b32		s_save_tmp, s_save_tmp, S_SAVE_PC_HI_RCNT_SHIFT
+	s_or_b32		s_save_pc_hi, s_save_pc_hi, s_save_tmp
+	s_getreg_b32	s_save_tmp, hwreg(HW_REG_IB_STS, SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT, SQ_WAVE_IB_STS_FIRST_REPLAY_SIZE)	//save FIRST_REPLAY
+	s_lshl_b32		s_save_tmp, s_save_tmp, S_SAVE_PC_HI_FIRST_REPLAY_SHIFT
+	s_or_b32		s_save_pc_hi, s_save_pc_hi, s_save_tmp
+	s_getreg_b32	s_save_tmp, hwreg(HW_REG_IB_STS)										//clear RCNT and FIRST_REPLAY in IB_STS
+	s_and_b32		s_save_tmp, s_save_tmp, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK_NEG
+
+	s_setreg_b32	hwreg(HW_REG_IB_STS), s_save_tmp
+    
+	/*		inform SPI the readiness and wait for SPI's go signal */
+	s_mov_b32		s_save_exec_lo,	exec_lo													//save EXEC and use EXEC for the go signal from SPI
+	s_mov_b32		s_save_exec_hi,	exec_hi
+	s_mov_b64		exec, 	0x0																//clear EXEC to get ready to receive
+
+if G8SR_DEBUG_TIMESTAMP
+        s_memrealtime  s_g8sr_ts_sq_save_msg
+        s_waitcnt lgkmcnt(0)
+end
+
+	if (EMU_RUN_HACK)
+	
+	else
+		s_sendmsg	sendmsg(MSG_SAVEWAVE)  //send SPI a message and wait for SPI's write to EXEC  
+	end
+
+  L_SLEEP:		
+	s_sleep 0x2                // sleep 1 (64clk) is not enough for 8 waves per SIMD, which will cause SQ hang, since the 7,8th wave could not get arbit to exec inst, while other waves are stuck into the sleep-loop and waiting for wrexec!=0
+	
+	if (EMU_RUN_HACK)
+																							
+	else
+		s_cbranch_execz	L_SLEEP                                                         
+	end
+
+if G8SR_DEBUG_TIMESTAMP
+        s_memrealtime  s_g8sr_ts_spi_wrexec
+        s_waitcnt lgkmcnt(0)
+end
+
+	/*      setup Resource Contants    */
+	if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_SAVE_SINGLE_WAVE))	
+		//calculate wd_addr using absolute thread id 
+		v_readlane_b32 s_save_tmp, v9, 0
+		s_lshr_b32 s_save_tmp, s_save_tmp, 6
+		s_mul_i32 s_save_tmp, s_save_tmp, WAVE_SPACE
+		s_add_i32 s_save_spi_init_lo, s_save_tmp, WG_BASE_ADDR_LO
+		s_mov_b32 s_save_spi_init_hi, WG_BASE_ADDR_HI
+		s_and_b32 s_save_spi_init_hi, s_save_spi_init_hi, CTX_SAVE_CONTROL		
+	else
+	end
+	if ((EMU_RUN_HACK) && (EMU_RUN_HACK_SAVE_SINGLE_WAVE))
+		s_add_i32 s_save_spi_init_lo, s_save_tmp, WG_BASE_ADDR_LO
+		s_mov_b32 s_save_spi_init_hi, WG_BASE_ADDR_HI
+		s_and_b32 s_save_spi_init_hi, s_save_spi_init_hi, CTX_SAVE_CONTROL		
+	else
+	end
+	
+	
+	s_mov_b32		s_save_buf_rsrc0, 	s_save_spi_init_lo														//base_addr_lo
+	s_and_b32		s_save_buf_rsrc1, 	s_save_spi_init_hi, 0x0000FFFF											//base_addr_hi
+	s_or_b32		s_save_buf_rsrc1, 	s_save_buf_rsrc1,  S_SAVE_BUF_RSRC_WORD1_STRIDE
+    s_mov_b32       s_save_buf_rsrc2,   0                                               						//NUM_RECORDS initial value = 0 (in bytes) although not neccessarily inited
+	s_mov_b32		s_save_buf_rsrc3, 	S_SAVE_BUF_RSRC_WORD3_MISC
+	s_and_b32		s_save_tmp,         s_save_spi_init_hi, S_SAVE_SPI_INIT_ATC_MASK		
+	s_lshr_b32		s_save_tmp,  		s_save_tmp, (S_SAVE_SPI_INIT_ATC_SHIFT-SQ_BUF_RSRC_WORD1_ATC_SHIFT)			//get ATC bit into position
+	s_or_b32		s_save_buf_rsrc3, 	s_save_buf_rsrc3,  s_save_tmp											//or ATC
+	s_and_b32		s_save_tmp,         s_save_spi_init_hi, S_SAVE_SPI_INIT_MTYPE_MASK		
+	s_lshr_b32		s_save_tmp,  		s_save_tmp, (S_SAVE_SPI_INIT_MTYPE_SHIFT-SQ_BUF_RSRC_WORD3_MTYPE_SHIFT)		//get MTYPE bits into position
+	s_or_b32		s_save_buf_rsrc3, 	s_save_buf_rsrc3,  s_save_tmp											//or MTYPE	
+	
+	//FIXME  right now s_save_m0/s_save_mem_offset use tma_lo/tma_hi  (might need to save them before using them?)
+	s_mov_b32		s_save_m0,			m0																	//save M0
+	
+	/* 		global mem offset			*/
+	s_mov_b32		s_save_mem_offset, 	0x0																		//mem offset initial value = 0
+
+
+
+
+	/* 		save HW registers	*/
+	//////////////////////////////
+
+  L_SAVE_HWREG:
+        // HWREG SR memory offset : size(VGPR)+size(SGPR)
+       get_vgpr_size_bytes(s_save_mem_offset)
+       get_sgpr_size_bytes(s_save_tmp)
+       s_add_u32 s_save_mem_offset, s_save_mem_offset, s_save_tmp
+
+
+    s_mov_b32		s_save_buf_rsrc2, 0x4								//NUM_RECORDS	in bytes
+	if (SWIZZLE_EN)
+		s_add_u32		s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0						//FIXME need to use swizzle to enable bounds checking?
+	else
+		s_mov_b32		s_save_buf_rsrc2,  0x1000000								//NUM_RECORDS in bytes
+	end
+
+	
+	write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)					//M0
+
+	if ((EMU_RUN_HACK) && (EMU_RUN_HACK_SAVE_FIRST_TIME))      
+		s_add_u32 s_save_pc_lo, s_save_pc_lo, 4             //pc[31:0]+4
+		s_addc_u32 s_save_pc_hi, s_save_pc_hi, 0x0			//carry bit over
+		s_mov_b32	tba_lo, EMU_RUN_HACK_SAVE_FIRST_TIME_TBA_LO
+	    s_mov_b32	tba_hi, EMU_RUN_HACK_SAVE_FIRST_TIME_TBA_HI	
+	end
+
+	write_hwreg_to_mem(s_save_pc_lo, s_save_buf_rsrc0, s_save_mem_offset)					//PC
+	write_hwreg_to_mem(s_save_pc_hi, s_save_buf_rsrc0, s_save_mem_offset)
+	write_hwreg_to_mem(s_save_exec_lo, s_save_buf_rsrc0, s_save_mem_offset)				//EXEC
+	write_hwreg_to_mem(s_save_exec_hi, s_save_buf_rsrc0, s_save_mem_offset)
+ 	// Save the tma_lo and tma_hi content from exec_lo and ttmp5            
+	s_mov_b32          s_save_exec_lo, exec_lo 
+	s_mov_b32          s_save_exec_hi, ttmp5   
+	write_hwreg_to_mem(s_save_status, s_save_buf_rsrc0, s_save_mem_offset)				//STATUS 
+	
+	//s_save_trapsts conflicts with s_save_alloc_size
+	s_getreg_b32    s_save_trapsts, hwreg(HW_REG_TRAPSTS)
+	write_hwreg_to_mem(s_save_trapsts, s_save_buf_rsrc0, s_save_mem_offset)				//TRAPSTS
+	
+	write_hwreg_to_mem(s_save_xnack_mask_lo, s_save_buf_rsrc0, s_save_mem_offset)			//XNACK_MASK_LO
+	write_hwreg_to_mem(s_save_xnack_mask_hi, s_save_buf_rsrc0, s_save_mem_offset)			//XNACK_MASK_HI
+	
+	//use s_save_tmp would introduce conflict here between s_save_tmp and s_save_buf_rsrc2
+	s_getreg_b32 	s_save_m0, hwreg(HW_REG_MODE)                                                   //MODE
+	write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)
+	write_hwreg_to_mem(tba_lo, s_save_buf_rsrc0, s_save_mem_offset)						//TBA_LO
+	write_hwreg_to_mem(tba_hi, s_save_buf_rsrc0, s_save_mem_offset)						//TBA_HI
+	write_hwreg_to_mem(s_save_exec_lo, s_save_buf_rsrc0, s_save_mem_offset)				//TMA_LO 
+	write_hwreg_to_mem(s_save_exec_hi, s_save_buf_rsrc0, s_save_mem_offset)				//TMA_HI 
+
+	/*      the first wave in the threadgroup    */
+        // save fist_wave bits in tba_hi unused bit.26
+	s_and_b32		s_save_tmp, s_save_spi_init_hi, S_SAVE_SPI_INIT_FIRST_WAVE_MASK     // extract fisrt wave bit
+    //s_or_b32        tba_hi, s_save_tmp, tba_hi                                        // save first wave bit to tba_hi.bits[26]
+    s_mov_b32        s_save_exec_hi, 0x0                                
+    s_or_b32         s_save_exec_hi, s_save_tmp, s_save_exec_hi                          // save first wave bit to s_save_exec_hi.bits[26]
+
+
+	/*      	save SGPRs	    */
+        // Save SGPR before LDS save, then the s0 to s4 can be used during LDS save...
+	//////////////////////////////
+
+    // SGPR SR memory offset : size(VGPR)	
+    get_vgpr_size_bytes(s_save_mem_offset)
+    // TODO, change RSRC word to rearrange memory layout for SGPRS
+
+	s_getreg_b32 	s_save_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE) 				//spgr_size
+	s_add_u32 		s_save_alloc_size, s_save_alloc_size, 1
+	s_lshl_b32 		s_save_alloc_size, s_save_alloc_size, 4 						//Number of SGPRs = (sgpr_size + 1) * 16   (non-zero value) 
+	
+	if (SGPR_SAVE_USE_SQC)
+		s_lshl_b32		s_save_buf_rsrc2,	s_save_alloc_size, 2					//NUM_RECORDS in bytes 
+	else
+		s_lshl_b32		s_save_buf_rsrc2,	s_save_alloc_size, 8					//NUM_RECORDS in bytes (64 threads)
+	end
+	
+	if (SWIZZLE_EN)
+		s_add_u32		s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0						//FIXME need to use swizzle to enable bounds checking?
+	else
+		s_mov_b32		s_save_buf_rsrc2,  0x1000000								//NUM_RECORDS in bytes
+	end
+
+
+	// backup s_save_buf_rsrc0,1 to s_save_pc_lo/hi, since write_16sgpr_to_mem function will change the rsrc0
+    //s_mov_b64 s_save_pc_lo, s_save_buf_rsrc0
+    s_mov_b64 s_save_xnack_mask_lo, s_save_buf_rsrc0
+    s_add_u32 s_save_buf_rsrc0, s_save_buf_rsrc0, s_save_mem_offset
+
+	s_mov_b32 		m0, 0x0 						//SGPR initial index value =0		
+  L_SAVE_SGPR_LOOP: 					
+    // SGPR is allocated in 16 SGPR granularity					
+	s_movrels_b64 	s0, s0     //s0 = s[0+m0], s1 = s[1+m0]
+    s_movrels_b64   s2, s2     //s2 = s[2+m0], s3 = s[3+m0]
+	s_movrels_b64 	s4, s4     //s4 = s[4+m0], s5 = s[5+m0]
+    s_movrels_b64   s6, s6     //s6 = s[6+m0], s7 = s[7+m0]
+	s_movrels_b64 	s8, s8     //s8 = s[8+m0], s9 = s[9+m0]
+    s_movrels_b64   s10, s10   //s10 = s[10+m0], s11 = s[11+m0]
+	s_movrels_b64 	s12, s12   //s12 = s[12+m0], s13 = s[13+m0]
+    s_movrels_b64   s14, s14   //s14 = s[14+m0], s15 = s[15+m0]
+
+	write_16sgpr_to_mem(s0, s_save_buf_rsrc0, s_save_mem_offset) //PV: the best performance should be using s_buffer_store_dwordx4
+	s_add_u32		m0, m0, 16														//next sgpr index
+	s_cmp_lt_u32 	m0, s_save_alloc_size 											//scc = (m0 < s_save_alloc_size) ? 1 : 0
+	s_cbranch_scc1 	L_SAVE_SGPR_LOOP									//SGPR save is complete?
+    // restore s_save_buf_rsrc0,1
+    //s_mov_b64 s_save_buf_rsrc0, s_save_pc_lo
+    s_mov_b64 s_save_buf_rsrc0, s_save_xnack_mask_lo
+
+
+
+
+	/*      	save first 4 VGPR, then LDS save could use   */
+        // each wave will alloc 4 vgprs at least...
+	/////////////////////////////////////////////////////////////////////////////////////
+
+    s_mov_b32       s_save_mem_offset, 0
+ 	s_mov_b32		exec_lo, 0xFFFFFFFF 											//need every thread from now on
+	s_mov_b32		exec_hi, 0xFFFFFFFF
+	
+	if (SWIZZLE_EN)
+		s_add_u32		s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0						//FIXME need to use swizzle to enable bounds checking?
+	else
+		s_mov_b32		s_save_buf_rsrc2,  0x1000000								//NUM_RECORDS in bytes
+	end
+
+
+	// VGPR Allocated in 4-GPR granularity
+
+if G8SR_VGPR_SR_IN_DWX4
+        // the const stride for DWx4 is 4*4 bytes
+        s_and_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0x0000FFFF   // reset const stride to 0
+        s_or_b32  s_save_buf_rsrc1, s_save_buf_rsrc1, G8SR_SAVE_BUF_RSRC_WORD1_STRIDE_DWx4  // const stride to 4*4 bytes
+        
+        buffer_store_dwordx4 v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
+
+        s_and_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0x0000FFFF   // reset const stride to 0
+        s_or_b32  s_save_buf_rsrc1, s_save_buf_rsrc1, S_SAVE_BUF_RSRC_WORD1_STRIDE  // reset const stride to 4 bytes
+else
+		buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
+		buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1  offset:256
+		buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1  offset:256*2
+		buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1  offset:256*3
+end
+
+
+
+	/*      	save LDS	    */
+	//////////////////////////////
+	
+  L_SAVE_LDS:
+
+        // Change EXEC to all threads...	
+	s_mov_b32		exec_lo, 0xFFFFFFFF   //need every thread from now on   
+	s_mov_b32		exec_hi, 0xFFFFFFFF
+	
+	s_getreg_b32 	s_save_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE) 			//lds_size
+	s_and_b32		s_save_alloc_size, s_save_alloc_size, 0xFFFFFFFF				//lds_size is zero?
+	s_cbranch_scc0	L_SAVE_LDS_DONE                                                                            //no lds used? jump to L_SAVE_DONE
+
+	s_barrier               //LDS is used? wait for other waves in the same TG 
+	//s_and_b32		s_save_tmp, tba_hi, S_SAVE_SPI_INIT_FIRST_WAVE_MASK                //exec is still used here     
+    s_and_b32		s_save_tmp, s_save_exec_hi, S_SAVE_SPI_INIT_FIRST_WAVE_MASK                //exec is still used here
+	s_cbranch_scc0	L_SAVE_LDS_DONE
+
+        // first wave do LDS save;
+        
+	s_lshl_b32 		s_save_alloc_size, s_save_alloc_size, 6 						//LDS size in dwords = lds_size * 64dw
+	s_lshl_b32 		s_save_alloc_size, s_save_alloc_size, 2 						//LDS size in bytes
+	s_mov_b32		s_save_buf_rsrc2,  s_save_alloc_size  							//NUM_RECORDS in bytes
+
+    // LDS at offset: size(VGPR)+SIZE(SGPR)+SIZE(HWREG)
+    // 
+    get_vgpr_size_bytes(s_save_mem_offset)
+    get_sgpr_size_bytes(s_save_tmp)
+    s_add_u32  s_save_mem_offset, s_save_mem_offset, s_save_tmp
+    s_add_u32 s_save_mem_offset, s_save_mem_offset, get_hwreg_size_bytes()
+
+
+	if (SWIZZLE_EN)
+		s_add_u32		s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0	      //FIXME need to use swizzle to enable bounds checking?
+	else
+		s_mov_b32		s_save_buf_rsrc2,  0x1000000                  //NUM_RECORDS in bytes
+	end
+
+	s_mov_b32 		m0, 0x0                                               //lds_offset initial value = 0
+
+
+var LDS_DMA_ENABLE = 0
+var UNROLL = 0
+if UNROLL==0 && LDS_DMA_ENABLE==1
+        s_mov_b32  s3, 256*2
+        s_nop 0
+        s_nop 0
+        s_nop 0
+  L_SAVE_LDS_LOOP:
+        //TODO: looks the 2 buffer_store/load clause for s/r will hurt performance.???
+	if (SAVE_LDS)     //SPI always alloc LDS space in 128DW granularity
+            buffer_store_lds_dword s_save_buf_rsrc0, s_save_mem_offset lds:1            // first 64DW
+            buffer_store_lds_dword s_save_buf_rsrc0, s_save_mem_offset lds:1 offset:256 // second 64DW
+	end
+
+	s_add_u32		m0, m0, s3											//every buffer_store_lds does 256 bytes
+	s_add_u32		s_save_mem_offset, s_save_mem_offset, s3							//mem offset increased by 256 bytes
+	s_cmp_lt_u32	m0, s_save_alloc_size												//scc=(m0 < s_save_alloc_size) ? 1 : 0
+	s_cbranch_scc1  L_SAVE_LDS_LOOP														//LDS save is complete?
+
+elsif LDS_DMA_ENABLE==1 && UNROLL==1 // UNROOL  , has ichace miss
+      // store from higest LDS address to lowest
+      s_mov_b32  s3, 256*2
+      s_sub_u32  m0, s_save_alloc_size, s3
+      s_add_u32 s_save_mem_offset, s_save_mem_offset, m0
+      s_lshr_b32 s_save_alloc_size, s_save_alloc_size, 9   // how many 128 trunks...
+      s_sub_u32 s_save_alloc_size, 128, s_save_alloc_size   // store from higheset addr to lowest
+      s_mul_i32 s_save_alloc_size, s_save_alloc_size, 6*4   // PC offset increment,  each LDS save block cost 6*4 Bytes instruction
+      s_add_u32 s_save_alloc_size, s_save_alloc_size, 3*4   //2is the below 2 inst...//s_addc and s_setpc
+      s_nop 0
+      s_nop 0
+      s_nop 0   //pad 3 dw to let LDS_DMA align with 64Bytes
+      s_getpc_b64 s[0:1]                              // reuse s[0:1], since s[0:1] already saved
+      s_add_u32   s0, s0,s_save_alloc_size
+      s_addc_u32  s1, s1, 0
+      s_setpc_b64 s[0:1]
+      
+
+       for var i =0; i< 128; i++    
+            // be careful to make here a 64Byte aligned address, which could improve performance...
+            buffer_store_lds_dword s_save_buf_rsrc0, s_save_mem_offset lds:1 offset:0           // first 64DW
+            buffer_store_lds_dword s_save_buf_rsrc0, s_save_mem_offset lds:1 offset:256           // second 64DW
+
+	    if i!=127
+		s_sub_u32  m0, m0, s3      // use a sgpr to shrink 2DW-inst to 1DW inst to improve performance , i.e.  pack more LDS_DMA inst to one Cacheline
+	    	s_sub_u32  s_save_mem_offset, s_save_mem_offset,  s3
+            end
+       end
+
+else   // BUFFER_STORE
+      v_mbcnt_lo_u32_b32 v2, 0xffffffff, 0x0
+      v_mbcnt_hi_u32_b32 v3, 0xffffffff, v2     // tid
+      v_mul_i32_i24 v2, v3, 8   // tid*8
+      v_mov_b32 v3, 256*2
+      s_mov_b32 m0, 0x10000
+      s_mov_b32 s0, s_save_buf_rsrc3
+      s_and_b32 s_save_buf_rsrc3, s_save_buf_rsrc3, 0xFF7FFFFF    // disable add_tid 
+      s_or_b32 s_save_buf_rsrc3, s_save_buf_rsrc3, 0x58000   //DFMT
+
+L_SAVE_LDS_LOOP_VECTOR:
+      ds_read_b64 v[0:1], v2    //x =LDS[a], byte address
+      s_waitcnt lgkmcnt(0)
+      buffer_store_dwordx2  v[0:1], v2, s_save_buf_rsrc0, s_save_mem_offset offen:1  glc:1  slc:1
+//      s_waitcnt vmcnt(0)
+      v_add_u32 v2, vcc[0:1], v2, v3
+      v_cmp_lt_u32 vcc[0:1], v2, s_save_alloc_size
+      s_cbranch_vccnz L_SAVE_LDS_LOOP_VECTOR
+
+      // restore rsrc3
+      s_mov_b32 s_save_buf_rsrc3, s0
+      
+end
+
+L_SAVE_LDS_DONE:	
+
+
+	/*      	save VGPRs  - set the Rest VGPRs	    */
+	//////////////////////////////////////////////////////////////////////////////////////
+  L_SAVE_VGPR:
+    // VGPR SR memory offset: 0
+    // TODO rearrange the RSRC words to use swizzle for VGPR save...
+  
+    s_mov_b32       s_save_mem_offset, (0+256*4)                                    // for the rest VGPRs
+ 	s_mov_b32		exec_lo, 0xFFFFFFFF 											//need every thread from now on
+	s_mov_b32		exec_hi, 0xFFFFFFFF
+	
+	s_getreg_b32 	s_save_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE) 					//vpgr_size
+	s_add_u32 		s_save_alloc_size, s_save_alloc_size, 1
+	s_lshl_b32 		s_save_alloc_size, s_save_alloc_size, 2 						//Number of VGPRs = (vgpr_size + 1) * 4    (non-zero value)   //FIXME for GFX, zero is possible 
+	s_lshl_b32		s_save_buf_rsrc2,  s_save_alloc_size, 8							//NUM_RECORDS in bytes (64 threads*4)
+	if (SWIZZLE_EN)
+		s_add_u32		s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0						//FIXME need to use swizzle to enable bounds checking?
+	else
+		s_mov_b32		s_save_buf_rsrc2,  0x1000000								//NUM_RECORDS in bytes
+	end
+
+
+	// VGPR Allocated in 4-GPR granularity
+
+if G8SR_VGPR_SR_IN_DWX4
+        // the const stride for DWx4 is 4*4 bytes
+        s_and_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0x0000FFFF   // reset const stride to 0
+        s_or_b32  s_save_buf_rsrc1, s_save_buf_rsrc1, G8SR_SAVE_BUF_RSRC_WORD1_STRIDE_DWx4  // const stride to 4*4 bytes
+        
+        s_mov_b32         m0, 4     // skip first 4 VGPRs
+        s_cmp_lt_u32      m0, s_save_alloc_size
+        s_cbranch_scc0    L_SAVE_VGPR_LOOP_END      // no more vgprs
+
+        s_set_gpr_idx_on  m0, 0x1   // This will change M0
+        s_add_u32         s_save_alloc_size, s_save_alloc_size, 0x1000  // because above inst change m0
+L_SAVE_VGPR_LOOP:
+        v_mov_b32         v0, v0   // v0 = v[0+m0]
+        v_mov_b32         v1, v1
+        v_mov_b32         v2, v2
+        v_mov_b32         v3, v3
+
+
+        buffer_store_dwordx4 v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
+        s_add_u32         m0, m0, 4
+        s_add_u32         s_save_mem_offset, s_save_mem_offset, 256*4
+        s_cmp_lt_u32      m0, s_save_alloc_size
+	s_cbranch_scc1 	L_SAVE_VGPR_LOOP												//VGPR save is complete?
+	s_set_gpr_idx_off
+L_SAVE_VGPR_LOOP_END:
+
+        s_and_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0x0000FFFF   // reset const stride to 0
+        s_or_b32  s_save_buf_rsrc1, s_save_buf_rsrc1, S_SAVE_BUF_RSRC_WORD1_STRIDE  // reset const stride to 4 bytes
+else
+    // VGPR store using dw burst	
+    s_mov_b32 		  m0, 0x4   //VGPR initial index value =0
+    s_cmp_lt_u32      m0, s_save_alloc_size
+    s_cbranch_scc0    L_SAVE_VGPR_END
+
+
+    s_set_gpr_idx_on    m0, 0x1 //M0[7:0] = M0[7:0] and M0[15:12] = 0x1
+    s_add_u32		s_save_alloc_size, s_save_alloc_size, 0x1000					//add 0x1000 since we compare m0 against it later	
+
+  L_SAVE_VGPR_LOOP: 										
+	v_mov_b32 		v0, v0				//v0 = v[0+m0]	
+	v_mov_b32 		v1, v1				//v0 = v[0+m0]	
+	v_mov_b32 		v2, v2				//v0 = v[0+m0]	
+	v_mov_b32 		v3, v3				//v0 = v[0+m0]	
+	    
+    if(USE_MTBUF_INSTEAD_OF_MUBUF)       
+		tbuffer_store_format_x v0, v0, s_save_buf_rsrc0, s_save_mem_offset format:BUF_NUM_FORMAT_FLOAT format: BUF_DATA_FORMAT_32 slc:1 glc:1
+    else
+		buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
+		buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1  offset:256
+		buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1  offset:256*2
+		buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1  offset:256*3
+	end
+
+    s_add_u32		m0, m0, 4														//next vgpr index
+	s_add_u32		s_save_mem_offset, s_save_mem_offset, 256*4						//every buffer_store_dword does 256 bytes
+	s_cmp_lt_u32 	m0,	s_save_alloc_size 											//scc = (m0 < s_save_alloc_size) ? 1 : 0
+	s_cbranch_scc1 	L_SAVE_VGPR_LOOP												//VGPR save is complete?
+	s_set_gpr_idx_off
+end
+	
+L_SAVE_VGPR_END:
+
+
+
+
+
+	
+	/*     S_PGM_END_SAVED  */    							//FIXME  graphics ONLY
+	if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_SAVE_NORMAL_EXIT))	
+		s_and_b32 s_save_pc_hi, s_save_pc_hi, 0x0000ffff    //pc[47:32]
+		s_add_u32 s_save_pc_lo, s_save_pc_lo, 4             //pc[31:0]+4
+		s_addc_u32 s_save_pc_hi, s_save_pc_hi, 0x0			//carry bit over
+		s_rfe_b64 s_save_pc_lo                              //Return to the main shader program
+	else
+	end
+
+// Save Done timestamp 
+if G8SR_DEBUG_TIMESTAMP
+        s_memrealtime	s_g8sr_ts_save_d
+        // SGPR SR memory offset : size(VGPR)	
+        get_vgpr_size_bytes(s_save_mem_offset)
+        s_add_u32 s_save_mem_offset, s_save_mem_offset, G8SR_DEBUG_TS_SAVE_D_OFFSET
+        s_waitcnt lgkmcnt(0)         //FIXME, will cause xnack??
+        // Need reset rsrc2??
+        s_mov_b32 m0, s_save_mem_offset
+        s_mov_b32 s_save_buf_rsrc2,  0x1000000                                  //NUM_RECORDS in bytes
+        s_buffer_store_dwordx2 s_g8sr_ts_save_d, s_save_buf_rsrc0, m0		glc:1	
+end
+
+	
+    s_branch	L_END_PGM
+	
+
+				
+/**************************************************************************/
+/*                     	restore routine						              */
+/**************************************************************************/
+
+L_RESTORE:
+    /*      Setup Resource Contants    */
+    if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_RESTORE_NORMAL))
+		//calculate wd_addr using absolute thread id
+		v_readlane_b32 s_restore_tmp, v9, 0
+		s_lshr_b32 s_restore_tmp, s_restore_tmp, 6
+		s_mul_i32 s_restore_tmp, s_restore_tmp, WAVE_SPACE
+		s_add_i32 s_restore_spi_init_lo, s_restore_tmp, WG_BASE_ADDR_LO
+		s_mov_b32 s_restore_spi_init_hi, WG_BASE_ADDR_HI
+		s_and_b32 s_restore_spi_init_hi, s_restore_spi_init_hi, CTX_RESTORE_CONTROL	
+	else
+	end
+
+if G8SR_DEBUG_TIMESTAMP
+        s_memrealtime	s_g8sr_ts_restore_s
+        s_waitcnt lgkmcnt(0)         //FIXME, will cause xnack??
+        // tma_lo/hi are sgpr 110, 111, which will not used for 112 SGPR allocated case...
+        s_mov_b32 s_restore_pc_lo, s_g8sr_ts_restore_s[0]
+        s_mov_b32 s_restore_pc_hi, s_g8sr_ts_restore_s[1]   //backup ts to ttmp0/1, sicne exec will be finally restored..
+end
+
+
+	
+    s_mov_b32		s_restore_buf_rsrc0, 	s_restore_spi_init_lo															//base_addr_lo
+	s_and_b32		s_restore_buf_rsrc1, 	s_restore_spi_init_hi, 0x0000FFFF												//base_addr_hi
+	s_or_b32		s_restore_buf_rsrc1, 	s_restore_buf_rsrc1,  S_RESTORE_BUF_RSRC_WORD1_STRIDE
+    s_mov_b32       s_restore_buf_rsrc2,   	0                                               								//NUM_RECORDS initial value = 0 (in bytes)
+	s_mov_b32		s_restore_buf_rsrc3, 	S_RESTORE_BUF_RSRC_WORD3_MISC
+	s_and_b32		s_restore_tmp,         	s_restore_spi_init_hi, S_RESTORE_SPI_INIT_ATC_MASK		
+	s_lshr_b32		s_restore_tmp,  		s_restore_tmp, (S_RESTORE_SPI_INIT_ATC_SHIFT-SQ_BUF_RSRC_WORD1_ATC_SHIFT)		//get ATC bit into position
+	s_or_b32		s_restore_buf_rsrc3, 	s_restore_buf_rsrc3,  s_restore_tmp												//or ATC
+	s_and_b32		s_restore_tmp,         	s_restore_spi_init_hi, S_RESTORE_SPI_INIT_MTYPE_MASK		
+	s_lshr_b32		s_restore_tmp,  		s_restore_tmp, (S_RESTORE_SPI_INIT_MTYPE_SHIFT-SQ_BUF_RSRC_WORD3_MTYPE_SHIFT)	//get MTYPE bits into position
+	s_or_b32		s_restore_buf_rsrc3, 	s_restore_buf_rsrc3,  s_restore_tmp												//or MTYPE
+	
+	/* 		global mem offset			*/
+//	s_mov_b32		s_restore_mem_offset, 0x0								//mem offset initial value = 0
+	
+	/*      the first wave in the threadgroup    */
+	s_and_b32		s_restore_tmp, s_restore_spi_init_hi, S_RESTORE_SPI_INIT_FIRST_WAVE_MASK			
+	s_cbranch_scc0	L_RESTORE_VGPR
+
+    /*      	restore LDS	    */
+	//////////////////////////////
+  L_RESTORE_LDS:
+
+	s_mov_b32		exec_lo, 0xFFFFFFFF 													//need every thread from now on   //be consistent with SAVE although can be moved ahead
+	s_mov_b32		exec_hi, 0xFFFFFFFF
+	
+	s_getreg_b32 	s_restore_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE) 				//lds_size
+	s_and_b32		s_restore_alloc_size, s_restore_alloc_size, 0xFFFFFFFF					//lds_size is zero?
+	s_cbranch_scc0	L_RESTORE_VGPR															//no lds used? jump to L_RESTORE_VGPR
+	s_lshl_b32 		s_restore_alloc_size, s_restore_alloc_size, 6 							//LDS size in dwords = lds_size * 64dw
+	s_lshl_b32 		s_restore_alloc_size, s_restore_alloc_size, 2 							//LDS size in bytes
+	s_mov_b32		s_restore_buf_rsrc2,	s_restore_alloc_size							//NUM_RECORDS in bytes
+
+    // LDS at offset: size(VGPR)+SIZE(SGPR)+SIZE(HWREG)
+    // 
+    get_vgpr_size_bytes(s_restore_mem_offset)
+    get_sgpr_size_bytes(s_restore_tmp)
+    s_add_u32  s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp
+    s_add_u32  s_restore_mem_offset, s_restore_mem_offset, get_hwreg_size_bytes()            //FIXME, Check if offset overflow???
+
+
+	if (SWIZZLE_EN)
+		s_add_u32		s_restore_buf_rsrc2, s_restore_buf_rsrc2, 0x0						//FIXME need to use swizzle to enable bounds checking?
+	else
+		s_mov_b32		s_restore_buf_rsrc2,  0x1000000										//NUM_RECORDS in bytes
+	end
+	s_mov_b32 		m0, 0x0 																//lds_offset initial value = 0
+
+  L_RESTORE_LDS_LOOP:									
+	if (SAVE_LDS)
+	    buffer_load_dword	v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset lds:1                    // first 64DW
+	    buffer_load_dword	v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset lds:1 offset:256         // second 64DW
+	end
+    s_add_u32		m0, m0, 256*2					                            // 128 DW
+	s_add_u32		s_restore_mem_offset, s_restore_mem_offset, 256*2           //mem offset increased by 128DW
+	s_cmp_lt_u32	m0, s_restore_alloc_size                                    //scc=(m0 < s_restore_alloc_size) ? 1 : 0
+	s_cbranch_scc1  L_RESTORE_LDS_LOOP														//LDS restore is complete?
+
+    
+    /*      	restore VGPRs	    */
+	//////////////////////////////
+  L_RESTORE_VGPR:
+        // VGPR SR memory offset : 0	
+	s_mov_b32		s_restore_mem_offset, 0x0
+ 	s_mov_b32		exec_lo, 0xFFFFFFFF 													//need every thread from now on   //be consistent with SAVE although can be moved ahead
+	s_mov_b32		exec_hi, 0xFFFFFFFF
+	
+	s_getreg_b32 	s_restore_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE) 	//vpgr_size
+	s_add_u32 		s_restore_alloc_size, s_restore_alloc_size, 1
+	s_lshl_b32 		s_restore_alloc_size, s_restore_alloc_size, 2 							//Number of VGPRs = (vgpr_size + 1) * 4    (non-zero value)
+    s_lshl_b32		s_restore_buf_rsrc2,  s_restore_alloc_size, 8						    //NUM_RECORDS in bytes (64 threads*4)
+	if (SWIZZLE_EN)
+		s_add_u32		s_restore_buf_rsrc2, s_restore_buf_rsrc2, 0x0						//FIXME need to use swizzle to enable bounds checking?
+	else
+		s_mov_b32		s_restore_buf_rsrc2,  0x1000000										//NUM_RECORDS in bytes
+	end
+
+if G8SR_VGPR_SR_IN_DWX4
+     get_vgpr_size_bytes(s_restore_mem_offset)
+     s_sub_u32         s_restore_mem_offset, s_restore_mem_offset, 256*4
+
+     // the const stride for DWx4 is 4*4 bytes
+     s_and_b32 s_restore_buf_rsrc1, s_restore_buf_rsrc1, 0x0000FFFF   // reset const stride to 0
+     s_or_b32  s_restore_buf_rsrc1, s_restore_buf_rsrc1, G8SR_RESTORE_BUF_RSRC_WORD1_STRIDE_DWx4  // const stride to 4*4 bytes
+
+     s_mov_b32         m0, s_restore_alloc_size
+     s_set_gpr_idx_on  m0, 0x8    // Note.. This will change m0
+
+L_RESTORE_VGPR_LOOP:
+     buffer_load_dwordx4 v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1
+     s_waitcnt vmcnt(0)
+     s_sub_u32         m0, m0, 4
+     v_mov_b32         v0, v0   // v[0+m0] = v0
+     v_mov_b32         v1, v1
+     v_mov_b32         v2, v2
+     v_mov_b32         v3, v3
+     s_sub_u32         s_restore_mem_offset, s_restore_mem_offset, 256*4
+     s_cmp_eq_u32      m0, 0x8000
+     s_cbranch_scc0    L_RESTORE_VGPR_LOOP
+     s_set_gpr_idx_off
+
+     s_and_b32 s_restore_buf_rsrc1, s_restore_buf_rsrc1, 0x0000FFFF   // reset const stride to 0
+     s_or_b32  s_restore_buf_rsrc1, s_restore_buf_rsrc1, S_RESTORE_BUF_RSRC_WORD1_STRIDE  // const stride to 4*4 bytes
+
+else
+	// VGPR load using dw burst
+	s_mov_b32		s_restore_mem_offset_save, s_restore_mem_offset		// restore start with v1, v0 will be the last
+	s_add_u32		s_restore_mem_offset, s_restore_mem_offset, 256*4
+    s_mov_b32 		m0, 4 			     				//VGPR initial index value = 1
+	s_set_gpr_idx_on  m0, 0x8						//M0[7:0] = M0[7:0] and M0[15:12] = 0x8
+    s_add_u32		s_restore_alloc_size, s_restore_alloc_size, 0x8000						//add 0x8000 since we compare m0 against it later	
+
+  L_RESTORE_VGPR_LOOP: 										
+    if(USE_MTBUF_INSTEAD_OF_MUBUF)       
+		tbuffer_load_format_x v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset format:BUF_NUM_FORMAT_FLOAT format: BUF_DATA_FORMAT_32 slc:1 glc:1
+    else
+		buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset	slc:1 glc:1	
+		buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset	slc:1 glc:1	offset:256
+		buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset	slc:1 glc:1	offset:256*2
+		buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset	slc:1 glc:1	offset:256*3
+	end
+	s_waitcnt		vmcnt(0)																//ensure data ready
+	v_mov_b32		v0, v0																	//v[0+m0] = v0
+    v_mov_b32       v1, v1
+    v_mov_b32       v2, v2
+    v_mov_b32       v3, v3
+    s_add_u32		m0, m0, 4																//next vgpr index
+	s_add_u32		s_restore_mem_offset, s_restore_mem_offset, 256*4							//every buffer_load_dword does 256 bytes
+	s_cmp_lt_u32 	m0,	s_restore_alloc_size 												//scc = (m0 < s_restore_alloc_size) ? 1 : 0
+	s_cbranch_scc1 	L_RESTORE_VGPR_LOOP														//VGPR restore (except v0) is complete?
+	s_set_gpr_idx_off
+																							/* VGPR restore on v0 */
+    if(USE_MTBUF_INSTEAD_OF_MUBUF)       
+		tbuffer_load_format_x v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save format:BUF_NUM_FORMAT_FLOAT format: BUF_DATA_FORMAT_32 slc:1 glc:1
+    else
+		buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save	slc:1 glc:1	
+		buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save	slc:1 glc:1	offset:256
+		buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save	slc:1 glc:1	offset:256*2
+		buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save	slc:1 glc:1	offset:256*3
+	end
+
+end
+	
+    /*      	restore SGPRs	    */
+	//////////////////////////////
+
+    // SGPR SR memory offset : size(VGPR)	
+    get_vgpr_size_bytes(s_restore_mem_offset)
+    get_sgpr_size_bytes(s_restore_tmp)
+    s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp
+    s_sub_u32 s_restore_mem_offset, s_restore_mem_offset, 16*4     // restore SGPR from S[n] to S[0], by 16 sgprs group
+    // TODO, change RSRC word to rearrange memory layout for SGPRS
+
+	s_getreg_b32 	s_restore_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE) 				//spgr_size
+	s_add_u32 		s_restore_alloc_size, s_restore_alloc_size, 1
+	s_lshl_b32 		s_restore_alloc_size, s_restore_alloc_size, 4 							//Number of SGPRs = (sgpr_size + 1) * 16   (non-zero value)
+
+	if (SGPR_SAVE_USE_SQC)
+		s_lshl_b32		s_restore_buf_rsrc2,	s_restore_alloc_size, 2						//NUM_RECORDS in bytes 
+	else
+		s_lshl_b32		s_restore_buf_rsrc2,	s_restore_alloc_size, 8						//NUM_RECORDS in bytes (64 threads)
+	end
+	if (SWIZZLE_EN)
+		s_add_u32		s_restore_buf_rsrc2, s_restore_buf_rsrc2, 0x0						//FIXME need to use swizzle to enable bounds checking?
+	else
+		s_mov_b32		s_restore_buf_rsrc2,  0x1000000										//NUM_RECORDS in bytes
+	end
+
+    /* If 112 SGPRs ar allocated, 4 sgprs are not used TBA(108,109),TMA(110,111),
+       However, we are safe to restore these 4 SGPRs anyway, since TBA,TMA will later be restored by HWREG
+    */
+    s_mov_b32 m0, s_restore_alloc_size
+
+ L_RESTORE_SGPR_LOOP: 
+	read_16sgpr_from_mem(s0, s_restore_buf_rsrc0, s_restore_mem_offset)  //PV: further performance improvement can be made
+	s_waitcnt		lgkmcnt(0)																//ensure data ready
+
+    s_sub_u32 m0, m0, 16    // Restore from S[n] to S[0]
+
+	s_movreld_b64 	s0, s0 		//s[0+m0] = s0
+    s_movreld_b64 	s2, s2
+    s_movreld_b64 	s4, s4
+    s_movreld_b64 	s6, s6
+    s_movreld_b64 	s8, s8
+    s_movreld_b64 	s10, s10
+    s_movreld_b64 	s12, s12
+    s_movreld_b64 	s14, s14
+
+	s_cmp_eq_u32 	m0, 0				//scc = (m0 < s_restore_alloc_size) ? 1 : 0
+	s_cbranch_scc0 	L_RESTORE_SGPR_LOOP             //SGPR restore (except s0) is complete?
+	
+    /* 		restore HW registers	*/
+	//////////////////////////////
+  L_RESTORE_HWREG:
+
+
+if G8SR_DEBUG_TIMESTAMP
+      s_mov_b32 s_g8sr_ts_restore_s[0], s_restore_pc_lo
+      s_mov_b32 s_g8sr_ts_restore_s[1], s_restore_pc_hi
+end
+
+    // HWREG SR memory offset : size(VGPR)+size(SGPR)
+    get_vgpr_size_bytes(s_restore_mem_offset)
+    get_sgpr_size_bytes(s_restore_tmp)
+    s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp
+
+
+    s_mov_b32		s_restore_buf_rsrc2, 0x4												//NUM_RECORDS	in bytes
+	if (SWIZZLE_EN)
+		s_add_u32		s_restore_buf_rsrc2, s_restore_buf_rsrc2, 0x0						//FIXME need to use swizzle to enable bounds checking?
+	else
+		s_mov_b32		s_restore_buf_rsrc2,  0x1000000										//NUM_RECORDS in bytes
+	end
+
+	read_hwreg_from_mem(s_restore_m0, s_restore_buf_rsrc0, s_restore_mem_offset)					//M0
+	read_hwreg_from_mem(s_restore_pc_lo, s_restore_buf_rsrc0, s_restore_mem_offset)				//PC
+	read_hwreg_from_mem(s_restore_pc_hi, s_restore_buf_rsrc0, s_restore_mem_offset)
+	read_hwreg_from_mem(s_restore_exec_lo, s_restore_buf_rsrc0, s_restore_mem_offset)				//EXEC
+	read_hwreg_from_mem(s_restore_exec_hi, s_restore_buf_rsrc0, s_restore_mem_offset)
+	read_hwreg_from_mem(s_restore_status, s_restore_buf_rsrc0, s_restore_mem_offset)				//STATUS
+	read_hwreg_from_mem(s_restore_trapsts, s_restore_buf_rsrc0, s_restore_mem_offset)				//TRAPSTS
+    read_hwreg_from_mem(xnack_mask_lo, s_restore_buf_rsrc0, s_restore_mem_offset)					//XNACK_MASK_LO
+	read_hwreg_from_mem(xnack_mask_hi, s_restore_buf_rsrc0, s_restore_mem_offset)					//XNACK_MASK_HI
+	read_hwreg_from_mem(s_restore_mode, s_restore_buf_rsrc0, s_restore_mem_offset)				//MODE
+	read_hwreg_from_mem(tba_lo, s_restore_buf_rsrc0, s_restore_mem_offset)						//TBA_LO
+	read_hwreg_from_mem(tba_hi, s_restore_buf_rsrc0, s_restore_mem_offset)						//TBA_HI
+	
+	s_waitcnt		lgkmcnt(0)																						//from now on, it is safe to restore STATUS and IB_STS
+
+	s_and_b32 s_restore_pc_hi, s_restore_pc_hi, 0x0000ffff    	//pc[47:32]        //Do it here in order not to affect STATUS
+
+	//for normal save & restore, the saved PC points to the next inst to execute, no adjustment needs to be made, otherwise:
+	if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_RESTORE_NORMAL))
+		s_add_u32 s_restore_pc_lo, s_restore_pc_lo, 8            //pc[31:0]+8	  //two back-to-back s_trap are used (first for save and second for restore)
+		s_addc_u32	s_restore_pc_hi, s_restore_pc_hi, 0x0		 //carry bit over
+	end	
+	if ((EMU_RUN_HACK) && (EMU_RUN_HACK_RESTORE_NORMAL))	      
+		s_add_u32 s_restore_pc_lo, s_restore_pc_lo, 4            //pc[31:0]+4     // save is hack through s_trap but restore is normal
+		s_addc_u32	s_restore_pc_hi, s_restore_pc_hi, 0x0		 //carry bit over
+	end
+	
+	s_mov_b32 		m0, 		s_restore_m0
+	s_mov_b32 		exec_lo, 	s_restore_exec_lo
+	s_mov_b32 		exec_hi, 	s_restore_exec_hi
+	
+	read_hwreg_from_mem(tma_lo, s_restore_buf_rsrc0, s_restore_mem_offset)	    //tma_lo
+	read_hwreg_from_mem(tma_hi, s_restore_buf_rsrc0, s_restore_mem_offset)	    //tma_hi
+	s_waitcnt		lgkmcnt(0)		//from now on, it is safe to restore STATUS and IB_STS
+	s_and_b32		s_restore_m0, SQ_WAVE_TRAPSTS_PRE_SAVECTX_MASK, s_restore_trapsts
+	s_setreg_b32	hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_PRE_SAVECTX_SHIFT, SQ_WAVE_TRAPSTS_PRE_SAVECTX_SIZE), s_restore_m0
+	s_and_b32		s_restore_m0, SQ_WAVE_TRAPSTS_POST_SAVECTX_MASK, s_restore_trapsts
+	s_lshr_b32		s_restore_m0, s_restore_m0, SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT
+	s_setreg_b32	hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT, SQ_WAVE_TRAPSTS_POST_SAVECTX_SIZE), s_restore_m0
+	//s_setreg_b32 	hwreg(HW_REG_TRAPSTS), 	s_restore_trapsts      //don't overwrite SAVECTX bit as it may be set through external SAVECTX during restore
+	s_setreg_b32 	hwreg(HW_REG_MODE), 	s_restore_mode
+	//reuse s_restore_m0 as a temp register
+	s_and_b32		s_restore_m0, s_restore_pc_hi, S_SAVE_PC_HI_RCNT_MASK
+	s_lshr_b32		s_restore_m0, s_restore_m0, S_SAVE_PC_HI_RCNT_SHIFT
+	s_lshl_b32		s_restore_m0, s_restore_m0, SQ_WAVE_IB_STS_RCNT_SHIFT
+	s_mov_b32		s_restore_tmp, 0x0																				//IB_STS is zero
+	s_or_b32		s_restore_tmp, s_restore_tmp, s_restore_m0
+	s_and_b32		s_restore_m0, s_restore_pc_hi, S_SAVE_PC_HI_FIRST_REPLAY_MASK
+	s_lshr_b32		s_restore_m0, s_restore_m0, S_SAVE_PC_HI_FIRST_REPLAY_SHIFT
+	s_lshl_b32		s_restore_m0, s_restore_m0, SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT
+	s_or_b32		s_restore_tmp, s_restore_tmp, s_restore_m0
+    s_and_b32       s_restore_m0, s_restore_status, SQ_WAVE_STATUS_INST_ATC_MASK 
+    s_lshr_b32		s_restore_m0, s_restore_m0, SQ_WAVE_STATUS_INST_ATC_SHIFT    
+    s_setreg_b32 	hwreg(HW_REG_IB_STS), 	s_restore_tmp
+
+	s_and_b64    exec, exec, exec  // Restore STATUS.EXECZ, not writable by s_setreg_b32
+	s_and_b64    vcc, vcc, vcc  // Restore STATUS.VCCZ, not writable by s_setreg_b32
+	s_setreg_b32 	hwreg(HW_REG_STATUS), 	s_restore_status     // SCC is included, which is changed by previous salu
+	
+	s_barrier													//barrier to ensure the readiness of LDS before access attemps from any other wave in the same TG //FIXME not performance-optimal at this time
+
+if G8SR_DEBUG_TIMESTAMP
+	s_memrealtime s_g8sr_ts_restore_d
+	s_waitcnt lgkmcnt(0)
+end	
+	
+//	s_rfe_b64 s_restore_pc_lo                              		//Return to the main shader program and resume execution
+    s_rfe_restore_b64  s_restore_pc_lo, s_restore_m0            // s_restore_m0[0] is used to set STATUS.inst_atc 
+
+
+/**************************************************************************/
+/*                     	the END								              */
+/**************************************************************************/	
+L_END_PGM:	
+	s_endpgm
+	
+end	
+
+
+/**************************************************************************/
+/*                     	the helper functions							  */
+/**************************************************************************/
+
+//Only for save hwreg to mem
+function write_hwreg_to_mem(s, s_rsrc, s_mem_offset)
+		s_mov_b32 exec_lo, m0					//assuming exec_lo is not needed anymore from this point on
+		s_mov_b32 m0, s_mem_offset
+		s_buffer_store_dword s, s_rsrc, m0		glc:0	
+		s_add_u32		s_mem_offset, s_mem_offset, 4
+		s_mov_b32	m0, exec_lo
+end
+
+//Only for save hwreg to mem 
+function write_tma_to_mem(s, s_rsrc, offset_imm) 
+        s_mov_b32 exec_lo, m0					//assuming exec_lo is not needed anymore from this point on 
+        s_mov_b32 m0, offset_imm 
+        s_buffer_store_dword s, s_rsrc, m0		glc:0	 
+        s_mov_b32	m0, exec_lo 
+end 
+
+// HWREG are saved before SGPRs, so all HWREG could be use.
+function write_16sgpr_to_mem(s, s_rsrc, s_mem_offset)
+
+		s_buffer_store_dwordx4 s[0], s_rsrc, 0  glc:0	
+		s_buffer_store_dwordx4 s[4], s_rsrc, 16  glc:0	
+		s_buffer_store_dwordx4 s[8], s_rsrc, 32  glc:0	
+		s_buffer_store_dwordx4 s[12], s_rsrc, 48 glc:0	
+        s_add_u32       s_rsrc[0], s_rsrc[0], 4*16
+        s_addc_u32 		s_rsrc[1], s_rsrc[1], 0x0			  // +scc
+end
+
+
+function read_hwreg_from_mem(s, s_rsrc, s_mem_offset)
+	s_buffer_load_dword s, s_rsrc, s_mem_offset		glc:1
+	s_add_u32		s_mem_offset, s_mem_offset, 4
+end
+
+function read_16sgpr_from_mem(s, s_rsrc, s_mem_offset)
+	s_buffer_load_dwordx16 s, s_rsrc, s_mem_offset		glc:1
+	s_sub_u32		s_mem_offset, s_mem_offset, 4*16
+end
+
+
+
+function get_lds_size_bytes(s_lds_size_byte)
+    // SQ LDS granularity is 64DW, while PGM_RSRC2.lds_size is in granularity 128DW	 
+    s_getreg_b32   s_lds_size_byte, hwreg(HW_REG_LDS_ALLOC, SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT, SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE) 			// lds_size
+    s_lshl_b32     s_lds_size_byte, s_lds_size_byte, 8 						//LDS size in dwords = lds_size * 64 *4Bytes    // granularity 64DW
+end
+
+function get_vgpr_size_bytes(s_vgpr_size_byte)
+    s_getreg_b32   s_vgpr_size_byte, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE)  //vpgr_size
+    s_add_u32      s_vgpr_size_byte, s_vgpr_size_byte, 1
+    s_lshl_b32     s_vgpr_size_byte, s_vgpr_size_byte, (2+8) //Number of VGPRs = (vgpr_size + 1) * 4 * 64 * 4   (non-zero value)   //FIXME for GFX, zero is possible 
+end
+
+function get_sgpr_size_bytes(s_sgpr_size_byte)
+    s_getreg_b32   s_sgpr_size_byte, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE)  //spgr_size
+    s_add_u32      s_sgpr_size_byte, s_sgpr_size_byte, 1
+    s_lshl_b32     s_sgpr_size_byte, s_sgpr_size_byte, 6 //Number of SGPRs = (sgpr_size + 1) * 16 *4   (non-zero value) 
+end
+
+function get_hwreg_size_bytes
+    return 128 //HWREG size 128 bytes
+end
+
+#endif
+
+static const uint32_t cwsr_trap_carrizo_hex[] = {
+	0xbf820001, 0xbf820131,
+	0xb8f4f802, 0xb8f5f803,
+	0x8675ff75, 0x00000400,
+	0xbf850013, 0xc00a1e37,
+	0x00000000, 0xbf8c007f,
+	0x87777978, 0xbf840004,
+	0xbeee007a, 0xbeef007b,
+	0xb974f802, 0xbe801d78,
+	0xb8f5f803, 0x8675ff75,
+	0x000001ff, 0xbf850002,
+	0x80708470, 0x82718071,
+	0x8671ff71, 0x0000ffff,
+	0xb974f802, 0xbe801f70,
+	0xb8f5f803, 0x8675ff75,
+	0x00000100, 0xbf840006,
+	0xbefa0080, 0xb97a0203,
+	0x8671ff71, 0x0000ffff,
+	0x80f08870, 0x82f18071,
+	0xbefa0080, 0xb97a0283,
+	0xbef60068, 0xbef70069,
+	0xb8fa1c07, 0x8e7a9c7a,
+	0x87717a71, 0xb8fa03c7,
+	0x8e7a9b7a, 0x87717a71,
+	0xb8faf807, 0x867aff7a,
+	0x00007fff, 0xb97af807,
+	0xbef2007e, 0xbef3007f,
+	0xbefe0180, 0xbf900004,
+	0xbf8e0002, 0xbf88fffe,
+	0xbef8007e, 0x8679ff7f,
+	0x0000ffff, 0x8779ff79,
+	0x00040000, 0xbefa0080,
+	0xbefb00ff, 0x00807fac,
+	0x867aff7f, 0x08000000,
+	0x8f7a837a, 0x877b7a7b,
+	0x867aff7f, 0x70000000,
+	0x8f7a817a, 0x877b7a7b,
+	0xbeef007c, 0xbeee0080,
+	0xb8ee2a05, 0x806e816e,
+	0x8e6e8a6e, 0xb8fa1605,
+	0x807a817a, 0x8e7a867a,
+	0x806e7a6e, 0xbefa0084,
+	0xbefa00ff, 0x01000000,
+	0xbefe007c, 0xbefc006e,
+	0xc0601bfc, 0x0000007c,
+	0x806e846e, 0xbefc007e,
+	0xbefe007c, 0xbefc006e,
+	0xc0601c3c, 0x0000007c,
+	0x806e846e, 0xbefc007e,
+	0xbefe007c, 0xbefc006e,
+	0xc0601c7c, 0x0000007c,
+	0x806e846e, 0xbefc007e,
+	0xbefe007c, 0xbefc006e,
+	0xc0601cbc, 0x0000007c,
+	0x806e846e, 0xbefc007e,
+	0xbefe007c, 0xbefc006e,
+	0xc0601cfc, 0x0000007c,
+	0x806e846e, 0xbefc007e,
+	0xbef2007e, 0xbef30075,
+	0xbefe007c, 0xbefc006e,
+	0xc0601d3c, 0x0000007c,
+	0x806e846e, 0xbefc007e,
+	0xb8f5f803, 0xbefe007c,
+	0xbefc006e, 0xc0601d7c,
+	0x0000007c, 0x806e846e,
+	0xbefc007e, 0xbefe007c,
+	0xbefc006e, 0xc0601dbc,
+	0x0000007c, 0x806e846e,
+	0xbefc007e, 0xbefe007c,
+	0xbefc006e, 0xc0601dfc,
+	0x0000007c, 0x806e846e,
+	0xbefc007e, 0xb8eff801,
+	0xbefe007c, 0xbefc006e,
+	0xc0601bfc, 0x0000007c,
+	0x806e846e, 0xbefc007e,
+	0xbefe007c, 0xbefc006e,
+	0xc0601b3c, 0x0000007c,
+	0x806e846e, 0xbefc007e,
+	0xbefe007c, 0xbefc006e,
+	0xc0601b7c, 0x0000007c,
+	0x806e846e, 0xbefc007e,
+	0xbefe007c, 0xbefc006e,
+	0xc0601cbc, 0x0000007c,
+	0x806e846e, 0xbefc007e,
+	0xbefe007c, 0xbefc006e,
+	0xc0601cfc, 0x0000007c,
+	0x806e846e, 0xbefc007e,
+	0x867aff7f, 0x04000000,
+	0xbef30080, 0x8773737a,
+	0xb8ee2a05, 0x806e816e,
+	0x8e6e8a6e, 0xb8f51605,
+	0x80758175, 0x8e758475,
+	0x8e7a8275, 0xbefa00ff,
+	0x01000000, 0xbef60178,
+	0x80786e78, 0xbefc0080,
+	0xbe802b00, 0xbe822b02,
+	0xbe842b04, 0xbe862b06,
+	0xbe882b08, 0xbe8a2b0a,
+	0xbe8c2b0c, 0xbe8e2b0e,
+	0xc06a003c, 0x00000000,
+	0xc06a013c, 0x00000010,
+	0xc06a023c, 0x00000020,
+	0xc06a033c, 0x00000030,
+	0x8078c078, 0x82798079,
+	0x807c907c, 0xbf0a757c,
+	0xbf85ffeb, 0xbef80176,
+	0xbeee0080, 0xbefe00c1,
+	0xbeff00c1, 0xbefa00ff,
+	0x01000000, 0xe0724000,
+	0x6e1e0000, 0xe0724100,
+	0x6e1e0100, 0xe0724200,
+	0x6e1e0200, 0xe0724300,
+	0x6e1e0300, 0xbefe00c1,
+	0xbeff00c1, 0xb8f54306,
+	0x8675c175, 0xbf84002c,
+	0xbf8a0000, 0x867aff73,
+	0x04000000, 0xbf840028,
+	0x8e758675, 0x8e758275,
+	0xbefa0075, 0xb8ee2a05,
+	0x806e816e, 0x8e6e8a6e,
+	0xb8fa1605, 0x807a817a,
+	0x8e7a867a, 0x806e7a6e,
+	0x806eff6e, 0x00000080,
+	0xbefa00ff, 0x01000000,
+	0xbefc0080, 0xd28c0002,
+	0x000100c1, 0xd28d0003,
+	0x000204c1, 0xd1060002,
+	0x00011103, 0x7e0602ff,
+	0x00000200, 0xbefc00ff,
+	0x00010000, 0xbe80007b,
+	0x867bff7b, 0xff7fffff,
+	0x877bff7b, 0x00058000,
+	0xd8ec0000, 0x00000002,
+	0xbf8c007f, 0xe0765000,
+	0x6e1e0002, 0x32040702,
+	0xd0c9006a, 0x0000eb02,
+	0xbf87fff7, 0xbefb0000,
+	0xbeee00ff, 0x00000400,
+	0xbefe00c1, 0xbeff00c1,
+	0xb8f52a05, 0x80758175,
+	0x8e758275, 0x8e7a8875,
+	0xbefa00ff, 0x01000000,
+	0xbefc0084, 0xbf0a757c,
+	0xbf840015, 0xbf11017c,
+	0x8075ff75, 0x00001000,
+	0x7e000300, 0x7e020301,
+	0x7e040302, 0x7e060303,
+	0xe0724000, 0x6e1e0000,
+	0xe0724100, 0x6e1e0100,
+	0xe0724200, 0x6e1e0200,
+	0xe0724300, 0x6e1e0300,
+	0x807c847c, 0x806eff6e,
+	0x00000400, 0xbf0a757c,
+	0xbf85ffef, 0xbf9c0000,
+	0xbf8200d1, 0xbef8007e,
+	0x8679ff7f, 0x0000ffff,
+	0x8779ff79, 0x00040000,
+	0xbefa0080, 0xbefb00ff,
+	0x00807fac, 0x8676ff7f,
+	0x08000000, 0x8f768376,
+	0x877b767b, 0x8676ff7f,
+	0x70000000, 0x8f768176,
+	0x877b767b, 0x8676ff7f,
+	0x04000000, 0xbf84001e,
+	0xbefe00c1, 0xbeff00c1,
+	0xb8f34306, 0x8673c173,
+	0xbf840019, 0x8e738673,
+	0x8e738273, 0xbefa0073,
+	0xb8f22a05, 0x80728172,
+	0x8e728a72, 0xb8f61605,
+	0x80768176, 0x8e768676,
+	0x80727672, 0x8072ff72,
+	0x00000080, 0xbefa00ff,
+	0x01000000, 0xbefc0080,
+	0xe0510000, 0x721e0000,
+	0xe0510100, 0x721e0000,
+	0x807cff7c, 0x00000200,
+	0x8072ff72, 0x00000200,
+	0xbf0a737c, 0xbf85fff6,
+	0xbef20080, 0xbefe00c1,
+	0xbeff00c1, 0xb8f32a05,
+	0x80738173, 0x8e738273,
+	0x8e7a8873, 0xbefa00ff,
+	0x01000000, 0xbef60072,
+	0x8072ff72, 0x00000400,
+	0xbefc0084, 0xbf11087c,
+	0x8073ff73, 0x00008000,
+	0xe0524000, 0x721e0000,
+	0xe0524100, 0x721e0100,
+	0xe0524200, 0x721e0200,
+	0xe0524300, 0x721e0300,
+	0xbf8c0f70, 0x7e000300,
+	0x7e020301, 0x7e040302,
+	0x7e060303, 0x807c847c,
+	0x8072ff72, 0x00000400,
+	0xbf0a737c, 0xbf85ffee,
+	0xbf9c0000, 0xe0524000,
+	0x761e0000, 0xe0524100,
+	0x761e0100, 0xe0524200,
+	0x761e0200, 0xe0524300,
+	0x761e0300, 0xb8f22a05,
+	0x80728172, 0x8e728a72,
+	0xb8f61605, 0x80768176,
+	0x8e768676, 0x80727672,
+	0x80f2c072, 0xb8f31605,
+	0x80738173, 0x8e738473,
+	0x8e7a8273, 0xbefa00ff,
+	0x01000000, 0xbefc0073,
+	0xc031003c, 0x00000072,
+	0x80f2c072, 0xbf8c007f,
+	0x80fc907c, 0xbe802d00,
+	0xbe822d02, 0xbe842d04,
+	0xbe862d06, 0xbe882d08,
+	0xbe8a2d0a, 0xbe8c2d0c,
+	0xbe8e2d0e, 0xbf06807c,
+	0xbf84fff1, 0xb8f22a05,
+	0x80728172, 0x8e728a72,
+	0xb8f61605, 0x80768176,
+	0x8e768676, 0x80727672,
+	0xbefa0084, 0xbefa00ff,
+	0x01000000, 0xc0211cfc,
+	0x00000072, 0x80728472,
+	0xc0211c3c, 0x00000072,
+	0x80728472, 0xc0211c7c,
+	0x00000072, 0x80728472,
+	0xc0211bbc, 0x00000072,
+	0x80728472, 0xc0211bfc,
+	0x00000072, 0x80728472,
+	0xc0211d3c, 0x00000072,
+	0x80728472, 0xc0211d7c,
+	0x00000072, 0x80728472,
+	0xc0211a3c, 0x00000072,
+	0x80728472, 0xc0211a7c,
+	0x00000072, 0x80728472,
+	0xc0211dfc, 0x00000072,
+	0x80728472, 0xc0211b3c,
+	0x00000072, 0x80728472,
+	0xc0211b7c, 0x00000072,
+	0x80728472, 0xbf8c007f,
+	0x8671ff71, 0x0000ffff,
+	0xbefc0073, 0xbefe006e,
+	0xbeff006f, 0xc0211bbc,
+	0x00000072, 0x80728472,
+	0xc0211bfc, 0x00000072,
+	0x80728472, 0xbf8c007f,
+	0x867375ff, 0x000003ff,
+	0xb9734803, 0x867375ff,
+	0xfffff800, 0x8f738b73,
+	0xb973a2c3, 0xb977f801,
+	0x8673ff71, 0xf0000000,
+	0x8f739c73, 0x8e739073,
+	0xbef60080, 0x87767376,
+	0x8673ff71, 0x08000000,
+	0x8f739b73, 0x8e738f73,
+	0x87767376, 0x8673ff74,
+	0x00800000, 0x8f739773,
+	0xb976f807, 0x86fe7e7e,
+	0x86ea6a6a, 0xb974f802,
+	0xbf8a0000, 0x95807370,
+	0xbf810000, 0x00000000,
+};
+
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index 3016098..595640a 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -31,16 +31,23 @@
 #include <uapi/linux/kfd_ioctl.h>
 #include <linux/time.h>
 #include <linux/mm.h>
-#include <linux/mman.h>
+#include <uapi/asm-generic/mman-common.h>
 #include <asm/processor.h>
+
 #include "kfd_priv.h"
 #include "kfd_device_queue_manager.h"
 #include "kfd_dbgmgr.h"
+#include "cik_regs.h"
 
 static long kfd_ioctl(struct file *, unsigned int, unsigned long);
 static int kfd_open(struct inode *, struct file *);
 static int kfd_mmap(struct file *, struct vm_area_struct *);
+static uint32_t kfd_convert_user_mem_alloction_flags(
+		struct kfd_dev *dev,
+		uint32_t userspace_flags);
+static bool kfd_is_large_bar(struct kfd_dev *dev);
 
+static int kfd_evict(struct file *filep, struct kfd_process *p, void *data);
 static const char kfd_dev_name[] = "kfd";
 
 static const struct file_operations kfd_fops = {
@@ -117,7 +124,7 @@ static int kfd_open(struct inode *inode, struct file *filep)
 		return -EPERM;
 	}
 
-	process = kfd_create_process(current);
+	process = kfd_create_process(filep);
 	if (IS_ERR(process))
 		return PTR_ERR(process);
 
@@ -206,6 +213,7 @@ static int set_queue_properties_from_user(struct queue_properties *q_properties,
 	q_properties->ctx_save_restore_area_address =
 			args->ctx_save_restore_address;
 	q_properties->ctx_save_restore_area_size = args->ctx_save_restore_size;
+	q_properties->ctl_stack_size = args->ctl_stack_size;
 	if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE ||
 		args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
 		q_properties->type = KFD_QUEUE_TYPE_COMPUTE;
@@ -270,7 +278,7 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
 		return -EINVAL;
 	}
 
-	mutex_lock(&p->mutex);
+	down_write(&p->lock);
 
 	pdd = kfd_bind_process_to_device(dev, p);
 	if (IS_ERR(pdd)) {
@@ -282,8 +290,7 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
 			p->pasid,
 			dev->id);
 
-	err = pqm_create_queue(&p->pqm, dev, filep, &q_properties,
-				0, q_properties.type, &queue_id);
+	err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, &queue_id);
 	if (err != 0)
 		goto err_create_queue;
 
@@ -291,10 +298,10 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
 
 
 	/* Return gpu_id as doorbell offset for mmap usage */
-	args->doorbell_offset = (KFD_MMAP_DOORBELL_MASK | args->gpu_id);
+	args->doorbell_offset = (KFD_MMAP_TYPE_DOORBELL | args->gpu_id);
 	args->doorbell_offset <<= PAGE_SHIFT;
 
-	mutex_unlock(&p->mutex);
+	up_write(&p->lock);
 
 	pr_debug("kfd: queue id %d was created successfully\n", args->queue_id);
 
@@ -311,7 +318,7 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
 
 err_create_queue:
 err_bind_process:
-	mutex_unlock(&p->mutex);
+	up_write(&p->lock);
 	return err;
 }
 
@@ -325,11 +332,11 @@ static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p,
 				args->queue_id,
 				p->pasid);
 
-	mutex_lock(&p->mutex);
+	down_write(&p->lock);
 
 	retval = pqm_destroy_queue(&p->pqm, args->queue_id);
 
-	mutex_unlock(&p->mutex);
+	up_write(&p->lock);
 	return retval;
 }
 
@@ -371,11 +378,33 @@ static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p,
 	pr_debug("kfd: updating queue id %d for PASID %d\n",
 			args->queue_id, p->pasid);
 
-	mutex_lock(&p->mutex);
+	down_write(&p->lock);
 
 	retval = pqm_update_queue(&p->pqm, args->queue_id, &properties);
 
-	mutex_unlock(&p->mutex);
+	up_write(&p->lock);
+
+	return retval;
+}
+
+static int kfd_ioctl_set_cu_mask(struct file *filp, struct kfd_process *p,
+					void *data)
+{
+	int retval;
+	struct kfd_ioctl_set_cu_mask_args *args = data;
+	struct queue_properties properties;
+	uint32_t __user *cu_mask_ptr = (uint32_t __user *)args->cu_mask_ptr;
+
+	if (get_user(properties.cu_mask, cu_mask_ptr))
+		return -EFAULT;
+	if (properties.cu_mask == 0)
+		return 0;
+
+	down_write(&p->lock);
+
+	retval = pqm_set_cu_mask(&p->pqm, args->queue_id, &properties);
+
+	up_write(&p->lock);
 
 	return retval;
 }
@@ -403,7 +432,7 @@ static int kfd_ioctl_set_memory_policy(struct file *filep,
 	if (dev == NULL)
 		return -EINVAL;
 
-	mutex_lock(&p->mutex);
+	down_write(&p->lock);
 
 	pdd = kfd_bind_process_to_device(dev, p);
 	if (IS_ERR(pdd)) {
@@ -427,46 +456,80 @@ static int kfd_ioctl_set_memory_policy(struct file *filep,
 		err = -EINVAL;
 
 out:
-	mutex_unlock(&p->mutex);
+	up_write(&p->lock);
 
 	return err;
 }
 
-static int kfd_ioctl_dbg_register(struct file *filep,
-				struct kfd_process *p, void *data)
+static int kfd_ioctl_set_trap_handler(struct file *filep,
+					struct kfd_process *p, void *data)
 {
-	struct kfd_ioctl_dbg_register_args *args = data;
+	struct kfd_ioctl_set_trap_handler_args *args = data;
 	struct kfd_dev *dev;
-	struct kfd_dbgmgr *dbgmgr_ptr;
+	int err = 0;
 	struct kfd_process_device *pdd;
-	bool create_ok;
-	long status = 0;
 
 	dev = kfd_device_by_id(args->gpu_id);
 	if (dev == NULL)
 		return -EINVAL;
 
-	if (dev->device_info->asic_family == CHIP_CARRIZO) {
-		pr_debug("kfd_ioctl_dbg_register not supported on CZ\n");
-		return -EINVAL;
+	down_write(&p->lock);
+
+	pdd = kfd_bind_process_to_device(dev, p);
+	if (IS_ERR(pdd)) {
+		err = -ESRCH;
+		goto out;
+	}
+	if (!dev->cwsr_enabled || !pdd->qpd.cwsr_kaddr) {
+		pr_err("kfd: CWSR is not enabled, can't set trap handler.\n");
+		err = -EINVAL;
+		goto out;
 	}
 
-	mutex_lock(kfd_get_dbgmgr_mutex());
-	mutex_lock(&p->mutex);
+	if (dev->dqm->ops.set_trap_handler(dev->dqm,
+					&pdd->qpd,
+					args->tba_addr,
+					args->tma_addr))
+		err = -EINVAL;
 
-	/*
-	 * make sure that we have pdd, if this the first queue created for
-	 * this process
-	 */
+out:
+	up_write(&p->lock);
+
+	return err;
+}
+
+static int
+kfd_ioctl_dbg_register(struct file *filep, struct kfd_process *p, void *data)
+{
+	long status = -EFAULT;
+	struct kfd_ioctl_dbg_register_args *args = data;
+	struct kfd_dev *dev;
+	struct kfd_dbgmgr *dbgmgr_ptr;
+	struct kfd_process_device *pdd;
+	bool create_ok = false;
+
+	pr_debug("kfd:dbg: %s\n", __func__);
+
+	dev = kfd_device_by_id(args->gpu_id);
+	if (!dev) {
+		dev_info(NULL, "Error! kfd: In func %s >> getting device by id failed\n", __func__);
+		return status;
+	}
+
+	down_write(&p->lock);
+	mutex_lock(get_dbgmgr_mutex());
+
+	/* make sure that we have pdd, if this the first queue created for this process */
 	pdd = kfd_bind_process_to_device(dev, p);
-	if (IS_ERR(pdd)) {
-		mutex_unlock(&p->mutex);
-		mutex_unlock(kfd_get_dbgmgr_mutex());
+	if (IS_ERR(pdd) < 0) {
+		mutex_unlock(get_dbgmgr_mutex());
+		up_write(&p->lock);
 		return PTR_ERR(pdd);
 	}
 
 	if (dev->dbgmgr == NULL) {
 		/* In case of a legal call, we have no dbgmgr yet */
+
 		create_ok = kfd_dbgmgr_create(&dbgmgr_ptr, dev);
 		if (create_ok) {
 			status = kfd_dbgmgr_register(dbgmgr_ptr, p);
@@ -475,13 +538,10 @@ static int kfd_ioctl_dbg_register(struct file *filep,
 			else
 				dev->dbgmgr = dbgmgr_ptr;
 		}
-	} else {
-		pr_debug("debugger already registered\n");
-		status = -EINVAL;
 	}
 
-	mutex_unlock(&p->mutex);
-	mutex_unlock(kfd_get_dbgmgr_mutex());
+	mutex_unlock(get_dbgmgr_mutex());
+	up_write(&p->lock);
 
 	return status;
 }
@@ -489,9 +549,9 @@ static int kfd_ioctl_dbg_register(struct file *filep,
 static int kfd_ioctl_dbg_unregister(struct file *filep,
 				struct kfd_process *p, void *data)
 {
+	long status = -EFAULT;
 	struct kfd_ioctl_dbg_unregister_args *args = data;
 	struct kfd_dev *dev;
-	long status;
 
 	dev = kfd_device_by_id(args->gpu_id);
 	if (dev == NULL)
@@ -502,7 +562,7 @@ static int kfd_ioctl_dbg_unregister(struct file *filep,
 		return -EINVAL;
 	}
 
-	mutex_lock(kfd_get_dbgmgr_mutex());
+	mutex_lock(get_dbgmgr_mutex());
 
 	status = kfd_dbgmgr_unregister(dev->dbgmgr, p);
 	if (status == 0) {
@@ -510,7 +570,7 @@ static int kfd_ioctl_dbg_unregister(struct file *filep,
 		dev->dbgmgr = NULL;
 	}
 
-	mutex_unlock(kfd_get_dbgmgr_mutex());
+	mutex_unlock(get_dbgmgr_mutex());
 
 	return status;
 }
@@ -519,125 +579,144 @@ static int kfd_ioctl_dbg_unregister(struct file *filep,
  * Parse and generate variable size data structure for address watch.
  * Total size of the buffer and # watch points is limited in order
  * to prevent kernel abuse. (no bearing to the much smaller HW limitation
- * which is enforced by dbgdev module)
+ * which is enforced by dbgdev module.
  * please also note that the watch address itself are not "copied from user",
  * since it be set into the HW in user mode values.
  *
  */
-static int kfd_ioctl_dbg_address_watch(struct file *filep,
-					struct kfd_process *p, void *data)
+
+static int
+kfd_ioctl_dbg_address_watch(struct file *filep,
+		struct kfd_process *p,
+		void *data)
 {
+	long status = -EFAULT;
 	struct kfd_ioctl_dbg_address_watch_args *args = data;
 	struct kfd_dev *dev;
 	struct dbg_address_watch_info aw_info;
-	unsigned char *args_buff;
-	long status;
-	void __user *cmd_from_user;
-	uint64_t watch_mask_value = 0;
+	unsigned char *args_buff = NULL;
 	unsigned int args_idx = 0;
+	uint64_t watch_mask_value = 0;
 
 	memset((void *) &aw_info, 0, sizeof(struct dbg_address_watch_info));
 
-	dev = kfd_device_by_id(args->gpu_id);
-	if (dev == NULL)
-		return -EINVAL;
+	do {
+		dev = kfd_device_by_id(args->gpu_id);
+		if (!dev) {
+			dev_info(NULL,
+			"Error! kfd: In func %s >> get device by id failed\n",
+			__func__);
+			break;
+		}
 
-	if (dev->device_info->asic_family == CHIP_CARRIZO) {
-		pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n");
-		return -EINVAL;
-	}
+		if (args->buf_size_in_bytes > MAX_ALLOWED_AW_BUFF_SIZE) {
+			status = -EINVAL;
+			break;
+		}
 
-	cmd_from_user = (void __user *) args->content_ptr;
+		if (args->buf_size_in_bytes <= sizeof(*args)) {
+			status = -EINVAL;
+			break;
+		}
 
-	/* Validate arguments */
+		/* this is the actual buffer to work with */
 
-	if ((args->buf_size_in_bytes > MAX_ALLOWED_AW_BUFF_SIZE) ||
-		(args->buf_size_in_bytes <= sizeof(*args) + sizeof(int) * 2) ||
-		(cmd_from_user == NULL))
-		return -EINVAL;
+		args_buff = kzalloc(args->buf_size_in_bytes -
+						sizeof(*args), GFP_KERNEL);
+		if (args_buff == NULL) {
+			status = -ENOMEM;
+			break;
+		}
 
-	/* this is the actual buffer to work with */
-	args_buff = memdup_user(cmd_from_user,
-				args->buf_size_in_bytes - sizeof(*args));
-	if (IS_ERR(args_buff))
-		return PTR_ERR(args_buff);
+		/* this is the actual buffer to work with */
+		args_buff = memdup_user(cmd_from_user,
+					args->buf_size_in_bytes - sizeof(*args));
+		if (IS_ERR(args_buff))
+			return PTR_ERR(args_buff);
 
-	aw_info.process = p;
+		aw_info.process = p;
 
-	aw_info.num_watch_points = *((uint32_t *)(&args_buff[args_idx]));
-	args_idx += sizeof(aw_info.num_watch_points);
+		aw_info.num_watch_points = *((uint32_t *)(&args_buff[args_idx]));
+		args_idx += sizeof(aw_info.num_watch_points);
 
-	aw_info.watch_mode = (enum HSA_DBG_WATCH_MODE *) &args_buff[args_idx];
-	args_idx += sizeof(enum HSA_DBG_WATCH_MODE) * aw_info.num_watch_points;
+		aw_info.watch_mode = (HSA_DBG_WATCH_MODE *) &args_buff[args_idx];
+		args_idx += sizeof(HSA_DBG_WATCH_MODE) * aw_info.num_watch_points;
 
-	/*
-	 * set watch address base pointer to point on the array base
-	 * within args_buff
-	 */
-	aw_info.watch_address = (uint64_t *) &args_buff[args_idx];
+		/* set watch address base pointer to point on the array base within args_buff */
 
-	/* skip over the addresses buffer */
-	args_idx += sizeof(aw_info.watch_address) * aw_info.num_watch_points;
+		aw_info.watch_address = (uint64_t *) &args_buff[args_idx];
 
-	if (args_idx >= args->buf_size_in_bytes - sizeof(*args)) {
-		kfree(args_buff);
-		return -EINVAL;
-	}
+		/*skip over the addresses buffer */
+		args_idx += sizeof(aw_info.watch_address) * aw_info.num_watch_points;
 
-	watch_mask_value = (uint64_t) args_buff[args_idx];
+		if (args_idx >= args->buf_size_in_bytes) {
+			status = -EINVAL;
+			break;
+		}
 
-	if (watch_mask_value > 0) {
-		/*
-		 * There is an array of masks.
-		 * set watch mask base pointer to point on the array base
-		 * within args_buff
-		 */
-		aw_info.watch_mask = (uint64_t *) &args_buff[args_idx];
+		watch_mask_value = (uint64_t) args_buff[args_idx];
 
-		/* skip over the masks buffer */
-		args_idx += sizeof(aw_info.watch_mask) *
-				aw_info.num_watch_points;
-	} else {
-		/* just the NULL mask, set to NULL and skip over it */
-		aw_info.watch_mask = NULL;
-		args_idx += sizeof(aw_info.watch_mask);
-	}
+		if (watch_mask_value > 0) {
+			/* there is an array of masks */
 
-	if (args_idx >= args->buf_size_in_bytes - sizeof(args)) {
-		kfree(args_buff);
-		return -EINVAL;
-	}
+			/* set watch mask base pointer to point on the array base within args_buff */
+			aw_info.watch_mask = (uint64_t *) &args_buff[args_idx];
+
+			/*skip over the masks buffer */
+			args_idx += sizeof(aw_info.watch_mask) * aw_info.num_watch_points;
+		}
+
+		else
+			/* just the NULL mask, set to NULL and skip over it */
+		{
+			aw_info.watch_mask = NULL;
+			args_idx += sizeof(aw_info.watch_mask);
+		}
 
-	/* Currently HSA Event is not supported for DBG */
-	aw_info.watch_event = NULL;
+		if (args_idx > args->buf_size_in_bytes) {
+			status = -EINVAL;
+			break;
+		}
 
-	mutex_lock(kfd_get_dbgmgr_mutex());
+		aw_info.watch_event = NULL;	/* Currently HSA Event is not supported for DBG */
+		status = 0;
+
+	} while (0);
+
+	if (status == 0) {
+		mutex_lock(get_dbgmgr_mutex());
 
-	status = kfd_dbgmgr_address_watch(dev->dbgmgr, &aw_info);
+		status = kfd_dbgmgr_address_watch(dev->dbgmgr, &aw_info);
 
-	mutex_unlock(kfd_get_dbgmgr_mutex());
+		mutex_unlock(get_dbgmgr_mutex());
+
+	}
 
 	kfree(args_buff);
 
 	return status;
 }
 
-/* Parse and generate fixed size data structure for wave control */
-static int kfd_ioctl_dbg_wave_control(struct file *filep,
-					struct kfd_process *p, void *data)
+/*
+ * Parse and generate fixed size data structure for wave control.
+ * Buffer is generated in a "packed" form, for avoiding structure packing/pending dependencies.
+ */
+
+static int
+kfd_ioctl_dbg_wave_control(struct file *filep, struct kfd_process *p, void *data)
 {
+	long status = -EFAULT;
 	struct kfd_ioctl_dbg_wave_control_args *args = data;
 	struct kfd_dev *dev;
 	struct dbg_wave_control_info wac_info;
-	unsigned char *args_buff;
-	uint32_t computed_buff_size;
-	long status;
-	void __user *cmd_from_user;
+	unsigned char *args_buff = NULL;
 	unsigned int args_idx = 0;
+	uint32_t computed_buff_size;
 
 	memset((void *) &wac_info, 0, sizeof(struct dbg_wave_control_info));
 
 	/* we use compact form, independent of the packing attribute value */
+
 	computed_buff_size = sizeof(*args) +
 				sizeof(wac_info.mode) +
 				sizeof(wac_info.operand) +
@@ -645,26 +724,25 @@ static int kfd_ioctl_dbg_wave_control(struct file *filep,
 				sizeof(wac_info.dbgWave_msg.MemoryVA) +
 				sizeof(wac_info.trapId);
 
-	dev = kfd_device_by_id(args->gpu_id);
-	if (dev == NULL)
-		return -EINVAL;
 
-	if (dev->device_info->asic_family == CHIP_CARRIZO) {
-		pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n");
-		return -EINVAL;
-	}
+	dev_info(NULL, "kfd: In func %s - start\n", __func__);
 
-	/* input size must match the computed "compact" size */
-	if (args->buf_size_in_bytes != computed_buff_size) {
-		pr_debug("size mismatch, computed : actual %u : %u\n",
-				args->buf_size_in_bytes, computed_buff_size);
-		return -EINVAL;
-	}
+	do {
+		dev = kfd_device_by_id(args->gpu_id);
+		if (!dev) {
+			dev_info(NULL, "Error! kfd: In func %s >> getting device by id failed\n", __func__);
+			break;
+		}
 
-	cmd_from_user = (void __user *) args->content_ptr;
+		/* input size must match the computed "compact" size */
 
-	if (cmd_from_user == NULL)
-		return -EINVAL;
+		if (args->buf_size_in_bytes != computed_buff_size) {
+			dev_info(NULL,
+					 "Error! kfd: In func %s >> size mismatch, computed : actual %u : %u\n",
+					__func__, args->buf_size_in_bytes, computed_buff_size);
+			status = -EINVAL;
+			break;
+		}
 
 	/* copy the entire buffer from user */
 
@@ -673,34 +751,51 @@ static int kfd_ioctl_dbg_wave_control(struct file *filep,
 	if (IS_ERR(args_buff))
 		return PTR_ERR(args_buff);
 
-	/* move ptr to the start of the "pay-load" area */
-	wac_info.process = p;
+		if (copy_from_user(args_buff,
+				(void __user *) args->content_ptr,
+				args->buf_size_in_bytes - sizeof(*args))) {
+			dev_info(NULL,
+			"Error! kfd: In func %s >> copy_from_user failed\n",
+			 __func__);
+			break;
+		}
+
+		/* move ptr to the start of the "pay-load" area */
+
+
+		wac_info.process = p;
+
+		wac_info.operand = (HSA_DBG_WAVEOP) *((HSA_DBG_WAVEOP *)(&args_buff[args_idx]));
+		args_idx += sizeof(wac_info.operand);
 
-	wac_info.operand = *((enum HSA_DBG_WAVEOP *)(&args_buff[args_idx]));
-	args_idx += sizeof(wac_info.operand);
+		wac_info.mode = (HSA_DBG_WAVEMODE) *((HSA_DBG_WAVEMODE *)(&args_buff[args_idx]));
+		args_idx += sizeof(wac_info.mode);
 
-	wac_info.mode = *((enum HSA_DBG_WAVEMODE *)(&args_buff[args_idx]));
-	args_idx += sizeof(wac_info.mode);
+		wac_info.trapId = (uint32_t) *((uint32_t *)(&args_buff[args_idx]));
+		args_idx += sizeof(wac_info.trapId);
 
-	wac_info.trapId = *((uint32_t *)(&args_buff[args_idx]));
-	args_idx += sizeof(wac_info.trapId);
+		wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value = *((uint32_t *)(&args_buff[args_idx]));
+		wac_info.dbgWave_msg.MemoryVA = NULL;
 
-	wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value =
-					*((uint32_t *)(&args_buff[args_idx]));
-	wac_info.dbgWave_msg.MemoryVA = NULL;
 
-	mutex_lock(kfd_get_dbgmgr_mutex());
+		status = 0;
+
+	} while (0);
+	if (status == 0) {
+		mutex_lock(get_dbgmgr_mutex());
+
+		dev_info(NULL,
+				"kfd: In func %s >> calling dbg manager process %p, operand %u, mode %u, trapId %u, message %u\n",
+				__func__, wac_info.process, wac_info.operand, wac_info.mode, wac_info.trapId,
+				wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value);
 
-	pr_debug("Calling dbg manager process %p, operand %u, mode %u, trapId %u, message %u\n",
-			wac_info.process, wac_info.operand,
-			wac_info.mode, wac_info.trapId,
-			wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value);
+		status = kfd_dbgmgr_wave_control(dev->dbgmgr, &wac_info);
 
-	status = kfd_dbgmgr_wave_control(dev->dbgmgr, &wac_info);
+		dev_info(NULL, "kfd: In func %s >> returned status of dbg manager is %ld\n", __func__, status);
 
-	pr_debug("Returned status of dbg manager is %ld\n", status);
+		mutex_unlock(get_dbgmgr_mutex());
 
-	mutex_unlock(kfd_get_dbgmgr_mutex());
+	}
 
 	kfree(args_buff);
 
@@ -748,7 +843,7 @@ static int kfd_ioctl_get_process_apertures(struct file *filp,
 
 	args->num_of_nodes = 0;
 
-	mutex_lock(&p->mutex);
+	down_write(&p->lock);
 
 	/*if the process-device list isn't empty*/
 	if (kfd_has_process_device_data(p)) {
@@ -787,52 +882,180 @@ static int kfd_ioctl_get_process_apertures(struct file *filp,
 				(args->num_of_nodes < NUM_OF_SUPPORTED_GPUS));
 	}
 
-	mutex_unlock(&p->mutex);
+	up_write(&p->lock);
 
 	return 0;
 }
 
-static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p,
-					void *data)
+static int kfd_ioctl_get_process_apertures_new(struct file *filp,
+				struct kfd_process *p, void *data)
+{
+	struct kfd_ioctl_get_process_apertures_new_args *args = data;
+	struct kfd_process_device_apertures *pa;
+	struct kfd_process_device *pdd;
+	uint32_t nodes = 0;
+	int ret;
+
+	dev_dbg(kfd_device, "get apertures for PASID %d", p->pasid);
+
+	if (args->num_of_nodes == 0) {
+		/* Return number of nodes, so that user space can alloacate
+		* sufficient memory */
+		down_write(&p->lock);
+
+		if (!kfd_has_process_device_data(p)) {
+			up_write(&p->lock);
+			return 0;
+		}
+
+		/* Run over all pdd of the process */
+		pdd = kfd_get_first_process_device_data(p);
+		do {
+			args->num_of_nodes++;
+		} while ((pdd =
+			kfd_get_next_process_device_data(p, pdd)) != NULL);
+
+		up_write(&p->lock);
+		return 0;
+	}
+
+	/* Fill in process-aperture information for all available
+	 * nodes, but not more than args->num_of_nodes as that is
+	 * the amount of memory allocated by user */
+	pa = kzalloc((sizeof(struct kfd_process_device_apertures) *
+				args->num_of_nodes), GFP_KERNEL);
+	if (!pa)
+		return -ENOMEM;
+
+	down_write(&p->lock);
+
+	if (!kfd_has_process_device_data(p)) {
+		up_write(&p->lock);
+		args->num_of_nodes = 0;
+		kfree(pa);
+		return 0;
+	}
+
+	/* Run over all pdd of the process */
+	pdd = kfd_get_first_process_device_data(p);
+	do {
+		pa[nodes].gpu_id = pdd->dev->id;
+		pa[nodes].lds_base = pdd->lds_base;
+		pa[nodes].lds_limit = pdd->lds_limit;
+		pa[nodes].gpuvm_base = pdd->gpuvm_base;
+		pa[nodes].gpuvm_limit = pdd->gpuvm_limit;
+		pa[nodes].scratch_base = pdd->scratch_base;
+		pa[nodes].scratch_limit = pdd->scratch_limit;
+
+		dev_dbg(kfd_device,
+			"gpu id %u\n", pdd->dev->id);
+		dev_dbg(kfd_device,
+			"lds_base %llX\n", pdd->lds_base);
+		dev_dbg(kfd_device,
+			"lds_limit %llX\n", pdd->lds_limit);
+		dev_dbg(kfd_device,
+			"gpuvm_base %llX\n", pdd->gpuvm_base);
+		dev_dbg(kfd_device,
+			"gpuvm_limit %llX\n", pdd->gpuvm_limit);
+		dev_dbg(kfd_device,
+			"scratch_base %llX\n", pdd->scratch_base);
+		dev_dbg(kfd_device,
+			"scratch_limit %llX\n", pdd->scratch_limit);
+		nodes++;
+	} while (
+		(pdd = kfd_get_next_process_device_data(p, pdd)) != NULL &&
+		(nodes < args->num_of_nodes));
+	up_write(&p->lock);
+
+	args->num_of_nodes = nodes;
+	ret = copy_to_user(
+			(void __user *)args->kfd_process_device_apertures_ptr,
+			pa,
+			(nodes * sizeof(struct kfd_process_device_apertures)));
+	kfree(pa);
+	return ret ? -EFAULT : 0;
+}
+
+static int
+kfd_ioctl_create_event(struct file *filp, struct kfd_process *p, void *data)
 {
 	struct kfd_ioctl_create_event_args *args = data;
-	int err;
+	struct kfd_dev *kfd;
+	struct kfd_process_device *pdd;
+	int err = -EINVAL;
+	void *mem, *kern_addr = NULL;
+
+	pr_debug("amdkfd: Event page offset 0x%llx\n", args->event_page_offset);
+
+	if (args->event_page_offset) {
+		kfd = kfd_device_by_id(GET_GPU_ID(args->event_page_offset));
+		if (!kfd) {
+			pr_err("amdkfd: can't find kfd device\n");
+			return -EFAULT;
+		}
+		if (KFD_IS_DGPU(kfd->device_info->asic_family)) {
+			down_write(&p->lock);
+			pdd = kfd_bind_process_to_device(kfd, p);
+			if (IS_ERR(pdd) < 0) {
+				err = PTR_ERR(pdd);
+				up_write(&p->lock);
+				return -EFAULT;
+			}
+			mem = kfd_process_device_translate_handle(pdd,
+				GET_IDR_HANDLE(args->event_page_offset));
+			if (!mem) {
+				pr_err("amdkfd: can't find BO offset is 0x%llx\n",
+						args->event_page_offset);
+				up_write(&p->lock);
+				return -EFAULT;
+			}
+			up_write(&p->lock);
 
-	err = kfd_event_create(filp, p, args->event_type,
-				args->auto_reset != 0, args->node_id,
-				&args->event_id, &args->event_trigger_data,
-				&args->event_page_offset,
-				&args->event_slot_index);
+			/* Map dGPU gtt BO to kernel */
+			kfd->kfd2kgd->map_gtt_bo_to_kernel(kfd->kgd,
+					mem, &kern_addr);
+		}
+	}
+
+	err = kfd_event_create(filp, p,
+			args->event_type,
+			args->auto_reset != 0,
+			args->node_id,
+			&args->event_id,
+			&args->event_trigger_data,
+			&args->event_page_offset,
+			&args->event_slot_index,
+			kern_addr);
 
 	return err;
 }
 
-static int kfd_ioctl_destroy_event(struct file *filp, struct kfd_process *p,
-					void *data)
+static int
+kfd_ioctl_destroy_event(struct file *filp, struct kfd_process *p, void *data)
 {
 	struct kfd_ioctl_destroy_event_args *args = data;
 
 	return kfd_event_destroy(p, args->event_id);
 }
 
-static int kfd_ioctl_set_event(struct file *filp, struct kfd_process *p,
-				void *data)
+static int
+kfd_ioctl_set_event(struct file *filp, struct kfd_process *p, void *data)
 {
 	struct kfd_ioctl_set_event_args *args = data;
 
 	return kfd_set_event(p, args->event_id);
 }
 
-static int kfd_ioctl_reset_event(struct file *filp, struct kfd_process *p,
-				void *data)
+static int
+kfd_ioctl_reset_event(struct file *filp, struct kfd_process *p, void *data)
 {
 	struct kfd_ioctl_reset_event_args *args = data;
 
 	return kfd_reset_event(p, args->event_id);
 }
 
-static int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p,
-				void *data)
+static int
+kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p, void *data)
 {
 	struct kfd_ioctl_wait_events_args *args = data;
 	enum kfd_event_wait_result wait_result;
@@ -847,6 +1070,711 @@ static int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p,
 
 	return err;
 }
+static int kfd_ioctl_alloc_scratch_memory(struct file *filep,
+					struct kfd_process *p, void *data)
+{
+	struct kfd_ioctl_alloc_memory_of_gpu_args *args =
+			(struct kfd_ioctl_alloc_memory_of_gpu_args *)data;
+	struct kfd_process_device *pdd;
+	struct kfd_dev *dev;
+	long err;
+
+	if (args->size == 0)
+		return -EINVAL;
+
+	dev = kfd_device_by_id(args->gpu_id);
+	if (dev == NULL)
+		return -EINVAL;
+
+	down_write(&p->lock);
+
+	pdd = kfd_bind_process_to_device(dev, p);
+	if (IS_ERR(pdd) < 0) {
+		err = PTR_ERR(pdd);
+		goto bind_process_to_device_fail;
+	}
+
+	pdd->sh_hidden_private_base_vmid = args->va_addr;
+	pdd->qpd.sh_hidden_private_base = args->va_addr;
+
+	up_write(&p->lock);
+
+	if (sched_policy == KFD_SCHED_POLICY_NO_HWS && pdd->qpd.vmid != 0) {
+		err = dev->kfd2kgd->alloc_memory_of_scratch(
+			dev->kgd, args->va_addr, pdd->qpd.vmid);
+		if (err != 0)
+			goto alloc_memory_of_scratch_failed;
+	}
+
+	return 0;
+
+bind_process_to_device_fail:
+	up_write(&p->lock);
+alloc_memory_of_scratch_failed:
+	return -EFAULT;
+}
+
+static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep,
+					struct kfd_process *p, void *data)
+{
+	struct kfd_ioctl_alloc_memory_of_gpu_args *args = data;
+	struct kfd_process_device *pdd;
+	void *mem;
+	struct kfd_dev *dev;
+	int idr_handle;
+	long err;
+
+	if (args->size == 0)
+		return -EINVAL;
+
+	dev = kfd_device_by_id(args->gpu_id);
+	if (dev == NULL)
+		return -EINVAL;
+
+	down_write(&p->lock);
+	pdd = kfd_bind_process_to_device(dev, p);
+	up_write(&p->lock);
+	if (IS_ERR(pdd) < 0)
+		return PTR_ERR(pdd);
+
+	err = dev->kfd2kgd->alloc_memory_of_gpu(
+		dev->kgd, args->va_addr, args->size,
+		pdd->vm, (struct kgd_mem **) &mem, NULL, NULL, pdd, 0);
+
+	if (err != 0)
+		return err;
+
+	down_write(&p->lock);
+	idr_handle = kfd_process_device_create_obj_handle(pdd, mem,
+			args->va_addr, args->size);
+	up_write(&p->lock);
+	if (idr_handle < 0) {
+		dev->kfd2kgd->free_memory_of_gpu(dev->kgd,
+						 (struct kgd_mem *) mem);
+		return -EFAULT;
+	}
+
+	args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
+
+	return 0;
+}
+
+bool kfd_is_large_bar(struct kfd_dev *dev)
+{
+	struct kfd_local_mem_info mem_info;
+
+	if (debug_largebar) {
+		pr_debug("amdkfd: simulate large-bar allocation on non large-bar machine\n");
+		return true;
+	}
+
+	if (!KFD_IS_DGPU(dev->device_info->asic_family))
+		return false;
+
+	dev->kfd2kgd->get_local_mem_info(dev->kgd, &mem_info);
+	if (mem_info.local_mem_size_private == 0 &&
+			mem_info.local_mem_size_public > 0)
+		return true;
+	return false;
+}
+
+static uint32_t kfd_convert_user_mem_alloction_flags(
+		struct kfd_dev *dev,
+		uint32_t userspace_flags)
+{
+	uint32_t kernel_allocation_flags;
+
+	kernel_allocation_flags = 0;
+
+	/* Allocate VRAM bo */
+	if ((userspace_flags & KFD_IOC_ALLOC_MEM_FLAGS_DGPU_DEVICE) ||
+		(userspace_flags & KFD_IOC_ALLOC_MEM_FLAGS_APU_DEVICE)) {
+		kernel_allocation_flags = ALLOC_MEM_FLAGS_VRAM;
+		if ((userspace_flags & KFD_IOC_ALLOC_MEM_FLAGS_DGPU_DEVICE) &&
+				kfd_is_large_bar(dev))
+			kernel_allocation_flags |= ALLOC_MEM_FLAGS_PUBLIC;
+		goto out;
+	}
+	/*
+	 * Since currently user space library doesn't uses scratch
+	 * allocation flag I route it to VRAM
+	 */
+	if ((userspace_flags & KFD_IOC_ALLOC_MEM_FLAGS_DGPU_SCRATCH) ||
+		(userspace_flags & KFD_IOC_ALLOC_MEM_FLAGS_APU_SCRATCH)) {
+		kernel_allocation_flags = ALLOC_MEM_FLAGS_VRAM;
+		goto out;
+	}
+	/*
+	 * The current usage for *_HOST allocation flags are for GTT memory
+	 * Need to verify if we're node zero or we want to allocate bo on
+	 * public domain for P2P buffers.
+	 */
+	if (userspace_flags & KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST) {
+		kernel_allocation_flags = ALLOC_MEM_FLAGS_GTT;
+		goto out;
+	}
+	/* Allocate userptr BO */
+	if (userspace_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
+		kernel_allocation_flags = ALLOC_MEM_FLAGS_USERPTR;
+		goto out;
+	}
+
+out:
+	if (userspace_flags & KFD_IOC_ALLOC_MEM_FLAGS_DGPU_AQL_QUEUE_MEM)
+		kernel_allocation_flags |= ALLOC_MEM_FLAGS_AQL_QUEUE_MEM;
+	/* Current HW doesn't support non paged memory */
+	kernel_allocation_flags |= ALLOC_MEM_FLAGS_NONPAGED;
+	/*
+	 *  Set by default execute access as this buffer might be allocated
+	 * for CP's ring buffer
+	 */
+	kernel_allocation_flags |= ALLOC_MEM_FLAGS_EXECUTE_ACCESS;
+	kernel_allocation_flags |= ALLOC_MEM_FLAGS_NO_SUBSTITUTE;
+
+	pr_debug("amdkfd: user allocation flags 0x%x kernel allocation flags: 0x%x\n",
+			userspace_flags, kernel_allocation_flags);
+
+	return kernel_allocation_flags;
+}
+
+static int kfd_ioctl_alloc_memory_of_gpu_new(struct file *filep,
+					struct kfd_process *p, void *data)
+{
+	struct kfd_ioctl_alloc_memory_of_gpu_new_args *args = data;
+	struct kfd_process_device *pdd;
+	void *mem;
+	struct kfd_dev *dev;
+	int idr_handle;
+	long err;
+	uint64_t offset;
+
+	if (args->size == 0)
+		return -EINVAL;
+
+	dev = kfd_device_by_id(args->gpu_id);
+	if (dev == NULL)
+		return -EINVAL;
+
+	down_write(&p->lock);
+	pdd = kfd_bind_process_to_device(dev, p);
+	up_write(&p->lock);
+	if (IS_ERR(pdd) < 0)
+		return PTR_ERR(pdd);
+
+	offset = args->mmap_offset;
+	err = dev->kfd2kgd->alloc_memory_of_gpu(
+		dev->kgd, args->va_addr, args->size,
+		pdd->vm, (struct kgd_mem **) &mem, &offset,
+		NULL, pdd,
+		kfd_convert_user_mem_alloction_flags(dev, args->flags));
+
+	if (err != 0)
+		return err;
+
+	down_write(&p->lock);
+	idr_handle = kfd_process_device_create_obj_handle(pdd, mem,
+			args->va_addr, args->size);
+	up_write(&p->lock);
+	if (idr_handle < 0) {
+		dev->kfd2kgd->free_memory_of_gpu(dev->kgd,
+						 (struct kgd_mem *) mem);
+		return -EFAULT;
+	}
+
+	args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
+	if ((args->flags & KFD_IOC_ALLOC_MEM_FLAGS_DGPU_DEVICE) != 0 &&
+			!kfd_is_large_bar(dev)) {
+		args->mmap_offset = 0;
+	} else {
+		args->mmap_offset = KFD_MMAP_TYPE_MAP_BO;
+		args->mmap_offset |= KFD_MMAP_GPU_ID(args->gpu_id);
+		args->mmap_offset <<= PAGE_SHIFT;
+		args->mmap_offset |= offset;
+	}
+
+	return 0;
+}
+
+static int kfd_ioctl_free_memory_of_gpu(struct file *filep,
+					struct kfd_process *p, void *data)
+{
+	struct kfd_ioctl_free_memory_of_gpu_args *args = data;
+	struct kfd_process_device *pdd;
+	struct kfd_bo *buf_obj;
+	struct kfd_dev *dev;
+	int ret;
+
+	dev = kfd_device_by_id(GET_GPU_ID(args->handle));
+	if (dev == NULL)
+		return -EINVAL;
+
+	down_write(&p->lock);
+
+	pdd = kfd_get_process_device_data(dev, p);
+	if (!pdd) {
+		pr_err("Process device data doesn't exist\n");
+		ret = -EINVAL;
+		goto err_unlock;
+	}
+
+	buf_obj = kfd_process_device_find_bo(pdd,
+					GET_IDR_HANDLE(args->handle));
+	if (buf_obj == NULL) {
+		ret = -EINVAL;
+		goto err_unlock;
+	}
+	run_rdma_free_callback(buf_obj);
+
+	up_write(&p->lock);
+
+	ret = dev->kfd2kgd->free_memory_of_gpu(dev->kgd, buf_obj->mem);
+
+	/* If freeing the buffer failed, leave the handle in place for
+	 * clean-up during process tear-down. */
+	if (ret == 0) {
+		down_write(&p->lock);
+		kfd_process_device_remove_obj_handle(
+			pdd, GET_IDR_HANDLE(args->handle));
+		up_write(&p->lock);
+	}
+
+	return ret;
+
+err_unlock:
+	up_write(&p->lock);
+	return ret;
+}
+
+int kfd_map_memory_to_gpu(struct kfd_dev *dev, void *mem,
+		struct kfd_process *p, struct kfd_process_device *pdd)
+{
+	int err;
+
+	BUG_ON(!dev);
+	BUG_ON(!pdd);
+
+	err = dev->kfd2kgd->map_memory_to_gpu(
+		dev->kgd, (struct kgd_mem *) mem, pdd->vm);
+
+	if (err != 0)
+		return err;
+
+	radeon_flush_tlb(dev, p->pasid);
+
+	err = dev->dqm->ops.set_page_directory_base(dev->dqm, &pdd->qpd);
+	if (err != 0) {
+		dev->kfd2kgd->unmap_memory_to_gpu(dev->kgd,
+				(struct kgd_mem *) mem, pdd->vm);
+		return err;
+	}
+
+	return 0;
+}
+
+static int kfd_ioctl_map_memory_to_gpu(struct file *filep,
+					struct kfd_process *p, void *data)
+{
+	struct kfd_ioctl_map_memory_to_gpu_new_args *args = data;
+	struct kfd_process_device *pdd, *peer_pdd;
+	void *mem;
+	struct kfd_dev *dev, *peer;
+	long err = 0;
+	int i, num_dev;
+	uint32_t *devices_arr = NULL;
+	int bo_size;
+
+	dev = kfd_device_by_id(GET_GPU_ID(args->handle));
+	if (dev == NULL)
+		return -EINVAL;
+
+	if (args->device_ids_array_size > 0 &&
+			(args->device_ids_array_size < sizeof(uint32_t))) {
+		pr_err("amdkfd: err node IDs array size %u\n",
+				args->device_ids_array_size);
+		return -EFAULT;
+	}
+
+	if (args->device_ids_array_size > 0) {
+		devices_arr = kmalloc(args->device_ids_array_size, GFP_KERNEL);
+		if (!devices_arr)
+			return -ENOMEM;
+
+		err = copy_from_user(devices_arr,
+				(void __user *)args->device_ids_array,
+				args->device_ids_array_size);
+		if (err != 0) {
+			err = -EFAULT;
+			goto copy_from_user_failed;
+		}
+	}
+
+	down_write(&p->lock);
+
+	pdd = kfd_bind_process_to_device(dev, p);
+	if (IS_ERR(pdd) < 0) {
+		err = PTR_ERR(pdd);
+		goto bind_process_to_device_failed;
+	}
+
+	mem = kfd_process_device_translate_handle(pdd,
+						GET_IDR_HANDLE(args->handle));
+	up_write(&p->lock);
+
+	if (mem == NULL) {
+		err = PTR_ERR(mem);
+		goto get_mem_obj_from_handle_failed;
+	}
+
+	if (args->device_ids_array_size > 0) {
+		num_dev = args->device_ids_array_size / sizeof(uint32_t);
+		for (i = 0 ; i < num_dev; i++) {
+			peer = kfd_device_by_id(devices_arr[i]);
+			if (!peer) {
+				pr_err("amdkfd: didn't found kfd-dev for 0x%x\n",
+						devices_arr[i]);
+				err = -EFAULT;
+				goto get_mem_obj_from_handle_failed;
+			}
+			down_write(&p->lock);
+			peer_pdd = kfd_bind_process_to_device(peer, p);
+			up_write(&p->lock);
+			if (!peer_pdd) {
+				err = -EFAULT;
+				goto get_mem_obj_from_handle_failed;
+			}
+			err = kfd_map_memory_to_gpu(peer, mem, p, peer_pdd);
+			if (err != 0)
+				pr_err("amdkfd: failed to map\n");
+		}
+	} else {
+		err = kfd_map_memory_to_gpu(dev, mem, p, pdd);
+		if (err != 0)
+			pr_err("amdkfd: failed to map\n");
+	}
+
+	bo_size = dev->kfd2kgd->return_bo_size(dev->kgd, mem);
+	down_write(&p->lock);
+	pdd->mapped_size += bo_size;
+	up_write(&p->lock);
+
+	if (args->device_ids_array_size > 0 && devices_arr)
+		kfree(devices_arr);
+
+	return err;
+
+bind_process_to_device_failed:
+	up_write(&p->lock);
+get_mem_obj_from_handle_failed:
+copy_from_user_failed:
+	kfree(devices_arr);
+	return err;
+}
+
+static int kfd_ioctl_map_memory_to_gpu_wrapper(struct file *filep,
+					struct kfd_process *p, void *data)
+{
+	struct kfd_ioctl_map_memory_to_gpu_args *args = data;
+	struct kfd_ioctl_map_memory_to_gpu_new_args new_args;
+
+	new_args.handle = args->handle;
+	new_args.device_ids_array = NULL;
+	new_args.device_ids_array_size = 0;
+
+	return kfd_ioctl_map_memory_to_gpu(filep, p, &new_args);
+}
+
+static int kfd_ioctl_unmap_memory_from_gpu(struct file *filep,
+					struct kfd_process *p, void *data)
+{
+	struct kfd_ioctl_unmap_memory_from_gpu_new_args *args = data;
+	struct kfd_process_device *pdd, *peer_pdd;
+	void *mem;
+	struct kfd_dev *dev, *peer;
+	long err = 0;
+	uint32_t *devices_arr = NULL, num_dev, i;
+	int bo_size;
+
+	dev = kfd_device_by_id(GET_GPU_ID(args->handle));
+	if (dev == NULL)
+		return -EINVAL;
+
+	if (args->device_ids_array_size > 0 &&
+			(args->device_ids_array_size < sizeof(uint32_t))) {
+		pr_err("amdkfd: err node IDs array size %u\n",
+				args->device_ids_array_size);
+		return -EFAULT;
+	}
+
+	if (args->device_ids_array_size > 0) {
+		devices_arr = kmalloc(args->device_ids_array_size, GFP_KERNEL);
+		if (!devices_arr)
+			return -ENOMEM;
+
+		err = copy_from_user(devices_arr,
+				(void __user *)args->device_ids_array,
+				args->device_ids_array_size);
+		if (err != 0) {
+			err = -EFAULT;
+			goto copy_from_user_failed;
+		}
+	}
+
+	down_write(&p->lock);
+
+	pdd = kfd_get_process_device_data(dev, p);
+	if (!pdd) {
+		pr_err("Process device data doesn't exist\n");
+		err = PTR_ERR(pdd);
+		goto bind_process_to_device_failed;
+	}
+
+	mem = kfd_process_device_translate_handle(pdd,
+						GET_IDR_HANDLE(args->handle));
+	up_write(&p->lock);
+
+	if (mem == NULL) {
+		err = PTR_ERR(mem);
+		goto get_mem_obj_from_handle_failed;
+	}
+
+	if (args->device_ids_array_size > 0) {
+		num_dev = args->device_ids_array_size / sizeof(uint32_t);
+		for (i = 0 ; i < num_dev; i++) {
+			peer = kfd_device_by_id(devices_arr[i]);
+			if (!peer) {
+				err = -EFAULT;
+				goto get_mem_obj_from_handle_failed;
+			}
+			down_write(&p->lock);
+			peer_pdd = kfd_get_process_device_data(peer, p);
+			up_write(&p->lock);
+			if (!peer_pdd) {
+				err = -EFAULT;
+				goto get_mem_obj_from_handle_failed;
+			}
+			peer->kfd2kgd->unmap_memory_to_gpu(peer->kgd,
+					mem, peer_pdd->vm);
+			radeon_flush_tlb(peer, p->pasid);
+		}
+	} else {
+		dev->kfd2kgd->unmap_memory_to_gpu(dev->kgd, mem, pdd->vm);
+		radeon_flush_tlb(dev, p->pasid);
+	}
+
+	bo_size = dev->kfd2kgd->return_bo_size(dev->kgd, mem);
+	down_write(&p->lock);
+	pdd->mapped_size -= bo_size;
+	up_write(&p->lock);
+
+	return 0;
+
+bind_process_to_device_failed:
+	up_write(&p->lock);
+get_mem_obj_from_handle_failed:
+copy_from_user_failed:
+	kfree(devices_arr);
+	return err;
+}
+
+static int kfd_ioctl_unmap_memory_from_gpu_wrapper(struct file *filep,
+					struct kfd_process *p, void *data)
+{
+	struct kfd_ioctl_unmap_memory_from_gpu_args *args = data;
+	struct kfd_ioctl_unmap_memory_from_gpu_new_args new_args;
+
+	new_args.handle = args->handle;
+	new_args.device_ids_array = NULL;
+	new_args.device_ids_array_size = 0;
+
+	return kfd_ioctl_unmap_memory_from_gpu(filep, p, &new_args);
+}
+
+static int kfd_ioctl_open_graphic_handle(struct file *filep,
+					struct kfd_process *p,
+					void *data)
+{
+	struct kfd_ioctl_open_graphic_handle_args *args = data;
+	struct kfd_dev *dev;
+	struct kfd_process_device *pdd;
+	void *mem;
+	int idr_handle;
+	long err;
+
+	dev = kfd_device_by_id(args->gpu_id);
+	if (dev == NULL)
+		return -EINVAL;
+
+	if (dev->device_info->asic_family != CHIP_KAVERI) {
+		pr_debug("kfd_ioctl_open_graphic_handle only supported on KV\n");
+		return -EINVAL;
+	}
+
+	down_write(&p->lock);
+	pdd = kfd_bind_process_to_device(dev, p);
+	up_write(&p->lock);
+	if (IS_ERR(pdd) < 0)
+		return PTR_ERR(pdd);
+
+	err = dev->kfd2kgd->open_graphic_handle(dev->kgd,
+			args->va_addr,
+			(struct kgd_vm *) pdd->vm,
+			args->graphic_device_fd,
+			args->graphic_handle,
+			(struct kgd_mem **) &mem);
+
+	if (err != 0)
+		return err;
+
+	down_write(&p->lock);
+	/*TODO: When open_graphic_handle is implemented, we need to create
+	* the corresponding interval tree. We need to know the size of
+	* the buffer through open_graphic_handle(). We use 1 for now.*/
+	idr_handle = kfd_process_device_create_obj_handle(pdd, mem,
+			args->va_addr, 1);
+	up_write(&p->lock);
+	if (idr_handle < 0) {
+		/* FIXME: destroy_process_gpumem doesn't seem to be
+		 * implemented anywhere */
+		dev->kfd2kgd->destroy_process_gpumem(dev->kgd, mem);
+		return -EFAULT;
+	}
+
+	args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
+
+	return 0;
+}
+
+static int kfd_ioctl_set_process_dgpu_aperture(struct file *filep,
+		struct kfd_process *p, void *data)
+{
+	struct kfd_ioctl_set_process_dgpu_aperture_args *args = data;
+	struct kfd_dev *dev;
+	struct kfd_process_device *pdd;
+	long err;
+
+	dev = kfd_device_by_id(args->gpu_id);
+	if (dev == NULL)
+		return -EINVAL;
+
+	down_write(&p->lock);
+
+	pdd = kfd_bind_process_to_device(dev, p);
+	if (IS_ERR(pdd) < 0) {
+		err = PTR_ERR(pdd);
+		goto exit;
+	}
+
+	err = kfd_set_process_dgpu_aperture(pdd, args->dgpu_base,
+			args->dgpu_limit);
+
+exit:
+	up_write(&p->lock);
+	return err;
+}
+
+static int kfd_ioctl_get_dmabuf_info(struct file *filep,
+		struct kfd_process *p, void *data)
+{
+	struct kfd_ioctl_get_dmabuf_info_args *args = data;
+	struct kfd_dev *dev = NULL;
+	struct kgd_dev *dma_buf_kgd;
+	void *metadata_buffer = NULL;
+	uint32_t flags;
+	unsigned i;
+	int r;
+
+	/* Find a KFD GPU device that supports the get_dmabuf_info query */
+	for (i = 0; kfd_topology_enum_kfd_devices(i, &dev) == 0; i++)
+		if (dev && dev->kfd2kgd->get_dmabuf_info)
+			break;
+	if (!dev)
+		return -EINVAL;
+
+	if (args->metadata_ptr) {
+		metadata_buffer = kzalloc(args->metadata_size, GFP_KERNEL);
+		if (!metadata_buffer)
+			return -ENOMEM;
+	}
+
+	/* Get dmabuf info from KGD */
+	r = dev->kfd2kgd->get_dmabuf_info(dev->kgd, args->dmabuf_fd,
+					  &dma_buf_kgd, &args->size,
+					  metadata_buffer, args->metadata_size,
+					  &args->metadata_size, &flags);
+	if (r)
+		goto exit;
+
+	/* Reverse-lookup gpu_id from kgd pointer */
+	dev = kfd_device_by_kgd(dma_buf_kgd);
+	if (!dev) {
+		r = -EINVAL;
+		goto exit;
+	}
+	args->gpu_id = kfd_get_gpu_id(dev);
+
+	/* Translate flags */
+	if (flags & ALLOC_MEM_FLAGS_VRAM) {
+		args->flags = KFD_IS_DGPU(dev->device_info->asic_family) ?
+			KFD_IOC_ALLOC_MEM_FLAGS_DGPU_DEVICE :
+			KFD_IOC_ALLOC_MEM_FLAGS_APU_DEVICE;
+	} else
+		args->flags = KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST;
+
+	/* Copy metadata buffer to user mode */
+	if (metadata_buffer) {
+		r = copy_to_user((void __user *)args->metadata_ptr,
+				 metadata_buffer, args->metadata_size);
+		if (r != 0)
+			r = -EFAULT;
+	}
+
+exit:
+	kfree(metadata_buffer);
+
+	return r;
+}
+
+static int kfd_ioctl_import_dmabuf(struct file *filep,
+		struct kfd_process *p, void *data)
+{
+	struct kfd_ioctl_import_dmabuf_args *args = data;
+	struct kfd_dev *dev;
+	struct kfd_process_device *pdd;
+	void *mem;
+	uint64_t size;
+	int idr_handle;
+	int r;
+
+	dev = kfd_device_by_id(args->gpu_id);
+	if (!dev || !dev->kfd2kgd->import_dmabuf)
+		return -EINVAL;
+
+	down_write(&p->lock);
+	pdd = kfd_bind_process_to_device(dev, p);
+	up_write(&p->lock);
+	if (IS_ERR(pdd) < 0)
+		return PTR_ERR(pdd);
+
+	r = dev->kfd2kgd->import_dmabuf(dev->kgd, args->dmabuf_fd,
+					args->va_addr, pdd->vm,
+					(struct kgd_mem **)&mem, &size);
+	if (r)
+		return r;
+
+	down_write(&p->lock);
+	idr_handle = kfd_process_device_create_obj_handle(pdd, mem,
+			args->va_addr, size);
+	up_write(&p->lock);
+	if (idr_handle < 0) {
+		dev->kfd2kgd->free_memory_of_gpu(dev->kgd,
+						 (struct kgd_mem *)mem);
+		return -EFAULT;
+	}
+
+	args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
+
+	return 0;
+}
 
 #define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \
 	[_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, .cmd_drv = 0, .name = #ioctl}
@@ -900,10 +1828,65 @@ static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
 
 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_WAVE_CONTROL,
 			kfd_ioctl_dbg_wave_control, 0),
+
+	AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_MEMORY_OF_GPU,
+			kfd_ioctl_alloc_memory_of_gpu, 0),
+
+	AMDKFD_IOCTL_DEF(AMDKFD_IOC_FREE_MEMORY_OF_GPU,
+			kfd_ioctl_free_memory_of_gpu, 0),
+
+	AMDKFD_IOCTL_DEF(AMDKFD_IOC_MAP_MEMORY_TO_GPU,
+			kfd_ioctl_map_memory_to_gpu_wrapper, 0),
+
+	AMDKFD_IOCTL_DEF(AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU,
+			kfd_ioctl_unmap_memory_from_gpu_wrapper, 0),
+
+	AMDKFD_IOCTL_DEF(AMDKFD_IOC_OPEN_GRAPHIC_HANDLE,
+			kfd_ioctl_open_graphic_handle, 0),
+
+	AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_MEMORY_OF_SCRATCH,
+			kfd_ioctl_alloc_scratch_memory, 0),
+
+	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_CU_MASK,
+			kfd_ioctl_set_cu_mask, 0),
+
+	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_PROCESS_DGPU_APERTURE,
+			kfd_ioctl_set_process_dgpu_aperture, 0),
+
+	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_TRAP_HANDLER,
+			kfd_ioctl_set_trap_handler, 0),
+
+	AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_MEMORY_OF_GPU_NEW,
+				kfd_ioctl_alloc_memory_of_gpu_new, 0),
+
+	AMDKFD_IOCTL_DEF(AMDKFD_IOC_MAP_MEMORY_TO_GPU_NEW,
+				kfd_ioctl_map_memory_to_gpu, 0),
+
+	AMDKFD_IOCTL_DEF(AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU_NEW,
+				kfd_ioctl_unmap_memory_from_gpu, 0),
+
+	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES_NEW,
+				kfd_ioctl_get_process_apertures_new, 0),
+
+	AMDKFD_IOCTL_DEF(AMDKFD_IOC_EVICT_MEMORY,
+				kfd_evict, 0),
+
+	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_DMABUF_INFO,
+				kfd_ioctl_get_dmabuf_info, 0),
+
+	AMDKFD_IOCTL_DEF(AMDKFD_IOC_IMPORT_DMABUF,
+				kfd_ioctl_import_dmabuf, 0)
 };
 
 #define AMDKFD_CORE_IOCTL_COUNT	ARRAY_SIZE(amdkfd_ioctls)
 
+static int kfd_evict(struct file *filep, struct kfd_process *p, void *data)
+{
+	struct kfd_ioctl_eviction_args *args = data;
+
+	return evict_size(p, args->size, args->type);
+
+}
 static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
 {
 	struct kfd_process *process;
@@ -995,20 +1978,37 @@ static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
 static int kfd_mmap(struct file *filp, struct vm_area_struct *vma)
 {
 	struct kfd_process *process;
+	struct kfd_dev *kfd;
+	unsigned long vm_pgoff;
+	int retval;
 
 	process = kfd_get_process(current);
 	if (IS_ERR(process))
 		return PTR_ERR(process);
 
-	if ((vma->vm_pgoff & KFD_MMAP_DOORBELL_MASK) ==
-			KFD_MMAP_DOORBELL_MASK) {
-		vma->vm_pgoff = vma->vm_pgoff ^ KFD_MMAP_DOORBELL_MASK;
+	vm_pgoff = vma->vm_pgoff;
+	vma->vm_pgoff = KFD_MMAP_OFFSET_VALUE_GET(vma->vm_pgoff);
+
+	switch (vm_pgoff & KFD_MMAP_TYPE_MASK) {
+	case KFD_MMAP_TYPE_DOORBELL:
 		return kfd_doorbell_mmap(process, vma);
-	} else if ((vma->vm_pgoff & KFD_MMAP_EVENTS_MASK) ==
-			KFD_MMAP_EVENTS_MASK) {
-		vma->vm_pgoff = vma->vm_pgoff ^ KFD_MMAP_EVENTS_MASK;
+
+	case KFD_MMAP_TYPE_EVENTS:
 		return kfd_event_mmap(process, vma);
+
+	case KFD_MMAP_TYPE_MAP_BO:
+		kfd = kfd_device_by_id(KFD_MMAP_GPU_ID_GET(vm_pgoff));
+		if (!kfd)
+			return -EFAULT;
+		retval = kfd->kfd2kgd->mmap_bo(kfd->kgd, vma);
+		return retval;
+
+	case KFD_MMAP_TYPE_RESERVED_MEM:
+		return kfd_reserved_mem_mmap(process, vma);
+
 	}
 
 	return -EFAULT;
 }
+
+
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
new file mode 100644
index 0000000..b3d4a50
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
@@ -0,0 +1,1163 @@
+#include <linux/kernel.h>
+#include <linux/acpi.h>
+#include <linux/mm.h>
+#include <linux/amd-iommu.h>
+#include <linux/pci.h>
+#include "kfd_crat.h"
+#include "kfd_priv.h"
+#include "kfd_topology.h"
+
+/* GPU Processor ID base for dGPUs for which VCRAT needs to be created.
+ * GPU processor ID are expressed with Bit[31]=1.
+ * The base is set to 0x8000_0000 + 0x1000 to avoid collision with GPU IDs
+ * used in the CRAT. */
+static uint32_t gpu_processor_id_low = 0x80001000;
+
+/* Return the next available gpu_processor_id and increment it for next GPU
+ * @total_cu_count - Total CUs present in the GPU including ones masked off
+ */
+static inline unsigned int get_and_inc_gpu_processor_id(
+				unsigned int total_cu_count)
+{
+	int current_id = gpu_processor_id_low;
+
+	gpu_processor_id_low += total_cu_count;
+	return current_id;
+}
+
+/* Static table to describe GPU Cache information */
+struct kfd_gpu_cache_info {
+	uint32_t	cache_size;
+	uint32_t	cache_level;
+	uint32_t	flags;
+	/* Indicates how many Compute Units share this cache
+	 * Value = 1 indicates the cache is not shared */
+	uint32_t	num_cu_shared;
+};
+
+static struct kfd_gpu_cache_info kaveri_cache_info[] = {
+	{
+		/* TCP L1 Cache per CU */
+		.cache_size = 16,
+		.cache_level = 1,
+		.flags = (CRAT_CACHE_FLAGS_ENABLED |
+				CRAT_CACHE_FLAGS_DATA_CACHE |
+				CRAT_CACHE_FLAGS_SIMD_CACHE),
+		.num_cu_shared = 1,
+
+	},
+	{
+		/* Scalar L1 Instruction Cache (in SQC module) per bank */
+		.cache_size = 16,
+		.cache_level = 1,
+		.flags = (CRAT_CACHE_FLAGS_ENABLED |
+				CRAT_CACHE_FLAGS_INST_CACHE |
+				CRAT_CACHE_FLAGS_SIMD_CACHE),
+		.num_cu_shared = 2,
+	},
+	{
+		/* Scalar L1 Data Cache (in SQC module) per bank */
+		.cache_size = 8,
+		.cache_level = 1,
+		.flags = (CRAT_CACHE_FLAGS_ENABLED |
+				CRAT_CACHE_FLAGS_DATA_CACHE |
+				CRAT_CACHE_FLAGS_SIMD_CACHE),
+		.num_cu_shared = 2,
+	},
+
+	/* TODO: Add L2 Cache information */
+};
+
+
+static struct kfd_gpu_cache_info carrizo_cache_info[] = {
+	{
+		/* TCP L1 Cache per CU */
+		.cache_size = 16,
+		.cache_level = 1,
+		.flags = (CRAT_CACHE_FLAGS_ENABLED |
+				CRAT_CACHE_FLAGS_DATA_CACHE |
+				CRAT_CACHE_FLAGS_SIMD_CACHE),
+		.num_cu_shared = 1,
+	},
+	{
+		/* Scalar L1 Instruction Cache (in SQC module) per bank */
+		.cache_size = 8,
+		.cache_level = 1,
+		.flags = (CRAT_CACHE_FLAGS_ENABLED |
+				CRAT_CACHE_FLAGS_INST_CACHE |
+				CRAT_CACHE_FLAGS_SIMD_CACHE),
+		.num_cu_shared = 4,
+	},
+	{
+		/* Scalar L1 Data Cache (in SQC module) per bank. */
+		.cache_size = 4,
+		.cache_level = 1,
+		.flags = (CRAT_CACHE_FLAGS_ENABLED |
+				CRAT_CACHE_FLAGS_DATA_CACHE |
+				CRAT_CACHE_FLAGS_SIMD_CACHE),
+		.num_cu_shared = 4,
+	},
+
+	/* TODO: Add L2 Cache information */
+};
+
+/* NOTE: In future if more information is added to struct kfd_gpu_cache_info
+ *	the following ASICs may need a separate table. */
+#define tonga_cache_info carrizo_cache_info
+#define fiji_cache_info  carrizo_cache_info
+
+static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev,
+		struct crat_subtype_computeunit *cu)
+{
+	BUG_ON(!dev);
+	BUG_ON(!cu);
+
+	dev->node_props.cpu_cores_count = cu->num_cpu_cores;
+	dev->node_props.cpu_core_id_base = cu->processor_id_low;
+	if (cu->hsa_capability & CRAT_CU_FLAGS_IOMMU_PRESENT)
+		dev->node_props.capability |= HSA_CAP_ATS_PRESENT;
+
+	pr_debug("CU CPU: cores=%d id_base=%d\n", cu->num_cpu_cores,
+			cu->processor_id_low);
+}
+
+static void kfd_populated_cu_info_gpu(struct kfd_topology_device *dev,
+		struct crat_subtype_computeunit *cu)
+{
+	BUG_ON(!dev);
+	BUG_ON(!cu);
+
+	dev->node_props.simd_id_base = cu->processor_id_low;
+	dev->node_props.simd_count = cu->num_simd_cores;
+	dev->node_props.lds_size_in_kb = cu->lds_size_in_kb;
+	dev->node_props.max_waves_per_simd = cu->max_waves_simd;
+	dev->node_props.wave_front_size = cu->wave_front_size;
+	dev->node_props.array_count = cu->array_count;
+	dev->node_props.cu_per_simd_array = cu->num_cu_per_array;
+	dev->node_props.simd_per_cu = cu->num_simd_per_cu;
+	dev->node_props.max_slots_scratch_cu = cu->max_slots_scatch_cu;
+	if (cu->hsa_capability & CRAT_CU_FLAGS_HOT_PLUGGABLE)
+		dev->node_props.capability |= HSA_CAP_HOT_PLUGGABLE;
+	pr_debug("CU GPU: id_base=%d\n", cu->processor_id_low);
+}
+
+/* kfd_parse_subtype_cu - parse compute unit subtypes and attach it to correct
+ *		topology device present in the device_list
+ */
+static int kfd_parse_subtype_cu(struct crat_subtype_computeunit *cu,
+				struct list_head *device_list)
+{
+	struct kfd_topology_device *dev;
+
+	BUG_ON(!cu);
+
+	pr_debug("Found CU entry in CRAT table with proximity_domain=%d caps=%x\n",
+			cu->proximity_domain, cu->hsa_capability);
+	list_for_each_entry(dev, device_list, list) {
+		if (cu->proximity_domain == dev->proximity_domain) {
+			if (cu->flags & CRAT_CU_FLAGS_CPU_PRESENT)
+				kfd_populated_cu_info_cpu(dev, cu);
+
+			if (cu->flags & CRAT_CU_FLAGS_GPU_PRESENT)
+				kfd_populated_cu_info_gpu(dev, cu);
+			break;
+		}
+	}
+
+	return 0;
+}
+
+/* kfd_parse_subtype_mem - parse memory subtypes and attach it to correct
+ *		topology device present in the device_list
+ */
+static int kfd_parse_subtype_mem(struct crat_subtype_memory *mem,
+				struct list_head *device_list)
+{
+	struct kfd_mem_properties *props;
+	struct kfd_topology_device *dev;
+
+	BUG_ON(!mem);
+
+	pr_debug("Found memory entry in CRAT table with proximity_domain=%d\n",
+			mem->proximity_domain);
+	list_for_each_entry(dev, device_list, list) {
+		if (mem->proximity_domain == dev->proximity_domain) {
+			props = kfd_alloc_struct(props);
+			if (props == NULL)
+				return -ENOMEM;
+
+			/*
+			 * We're on GPU node
+			 */
+			if (dev->node_props.cpu_cores_count == 0) {
+				/* APU */
+				if (mem->visibility_type == 0)
+					props->heap_type =
+						HSA_MEM_HEAP_TYPE_FB_PRIVATE;
+				/* dGPU */
+				else
+					props->heap_type = mem->visibility_type;
+			}
+			else
+				props->heap_type = HSA_MEM_HEAP_TYPE_SYSTEM;
+
+			if (mem->flags & CRAT_MEM_FLAGS_HOT_PLUGGABLE)
+				props->flags |= HSA_MEM_FLAGS_HOT_PLUGGABLE;
+			if (mem->flags & CRAT_MEM_FLAGS_NON_VOLATILE)
+				props->flags |= HSA_MEM_FLAGS_NON_VOLATILE;
+
+			props->size_in_bytes =
+				((uint64_t)mem->length_high << 32) +
+							mem->length_low;
+			props->width = mem->width;
+
+			dev->node_props.mem_banks_count++;
+			list_add_tail(&props->list, &dev->mem_props);
+
+			break;
+		}
+	}
+
+	return 0;
+}
+
+/* kfd_parse_subtype_cache - parse cache subtypes and attach it to correct
+ *		topology device present in the device_list
+ */
+static int kfd_parse_subtype_cache(struct crat_subtype_cache *cache,
+			struct list_head *device_list)
+{
+	struct kfd_cache_properties *props;
+	struct kfd_topology_device *dev;
+	uint32_t id;
+	uint32_t total_num_of_cu;
+
+	BUG_ON(!cache);
+
+	id = cache->processor_id_low;
+
+	list_for_each_entry(dev, device_list, list) {
+		total_num_of_cu = (dev->node_props.array_count *
+					dev->node_props.cu_per_simd_array);
+
+		/* Cache infomration in CRAT doesn't have proximity_domain
+		 * information as it is associated with a CPU core or GPU
+		 * Compute Unit. So map the cache using CPU core Id or SIMD
+		 * (GPU) ID.
+		 * TODO: This works because currently we can safely assume that
+		 *  Compute Units are parsed before caches are parsed. In future
+		 *  remove this dependency
+		 */
+		if ((id >= dev->node_props.cpu_core_id_base &&
+			id <= dev->node_props.cpu_core_id_base +
+				dev->node_props.cpu_cores_count) ||
+			(id >= dev->node_props.simd_id_base &&
+			id < dev->node_props.simd_id_base +
+				total_num_of_cu)) {
+			props = kfd_alloc_struct(props);
+			if (props == NULL)
+				return -ENOMEM;
+
+			props->processor_id_low = id;
+			props->cache_level = cache->cache_level;
+			props->cache_size = cache->cache_size;
+			props->cacheline_size = cache->cache_line_size;
+			props->cachelines_per_tag = cache->lines_per_tag;
+			props->cache_assoc = cache->associativity;
+			props->cache_latency = cache->cache_latency;
+			memcpy(props->sibling_map, cache->sibling_map,
+					sizeof(props->sibling_map));
+
+			if (cache->flags & CRAT_CACHE_FLAGS_DATA_CACHE)
+				props->cache_type |= HSA_CACHE_TYPE_DATA;
+			if (cache->flags & CRAT_CACHE_FLAGS_INST_CACHE)
+				props->cache_type |= HSA_CACHE_TYPE_INSTRUCTION;
+			if (cache->flags & CRAT_CACHE_FLAGS_CPU_CACHE)
+				props->cache_type |= HSA_CACHE_TYPE_CPU;
+			if (cache->flags & CRAT_CACHE_FLAGS_SIMD_CACHE)
+				props->cache_type |= HSA_CACHE_TYPE_HSACU;
+
+			dev->cache_count++;
+			dev->node_props.caches_count++;
+			list_add_tail(&props->list, &dev->cache_props);
+
+			break;
+		}
+	}
+
+	return 0;
+}
+
+/* kfd_parse_subtype_iolink - parse iolink subtypes and attach it to correct
+ *		topology device present in the device_list
+ */
+static int kfd_parse_subtype_iolink(struct crat_subtype_iolink *iolink,
+					struct list_head *device_list)
+{
+	struct kfd_iolink_properties *props;
+	struct kfd_topology_device *dev;
+	uint32_t i = 0;
+	uint32_t id_from;
+	uint32_t id_to;
+
+	BUG_ON(!iolink);
+
+	id_from = iolink->proximity_domain_from;
+	id_to = iolink->proximity_domain_to;
+
+	pr_debug("Found IO link entry in CRAT table with id_from=%d\n", id_from);
+	list_for_each_entry(dev, device_list, list) {
+		if (id_from == dev->proximity_domain) {
+			props = kfd_alloc_struct(props);
+			if (props == NULL)
+				return -ENOMEM;
+
+			props->node_from = id_from;
+			props->node_to = id_to;
+			props->ver_maj = iolink->version_major;
+			props->ver_min = iolink->version_minor;
+			props->iolink_type = iolink->io_interface_type;
+
+			/*
+			 * weight factor (derived from CDIR), currently always 1
+			 */
+			props->weight = 1;
+
+			props->min_latency = iolink->minimum_latency;
+			props->max_latency = iolink->maximum_latency;
+			props->min_bandwidth = iolink->minimum_bandwidth_mbs;
+			props->max_bandwidth = iolink->maximum_bandwidth_mbs;
+			props->rec_transfer_size =
+					iolink->recommended_transfer_size;
+
+			dev->io_link_count++;
+			dev->node_props.io_links_count++;
+			list_add_tail(&props->list, &dev->io_link_props);
+
+			break;
+		}
+		i++;
+	}
+
+	return 0;
+}
+
+/* kfd_parse_subtype - parse subtypes and attach it to correct topology device
+ *		present in the device_list
+ *	@sub_type_hdr - subtype section of crat_image
+ *	@device_list - list of topology devices present in this crat_image
+ */
+static int kfd_parse_subtype(struct crat_subtype_generic *sub_type_hdr,
+				struct list_head *device_list)
+{
+	struct crat_subtype_computeunit *cu;
+	struct crat_subtype_memory *mem;
+	struct crat_subtype_cache *cache;
+	struct crat_subtype_iolink *iolink;
+	int ret = 0;
+
+	BUG_ON(!sub_type_hdr);
+
+	switch (sub_type_hdr->type) {
+	case CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY:
+		cu = (struct crat_subtype_computeunit *)sub_type_hdr;
+		ret = kfd_parse_subtype_cu(cu, device_list);
+		break;
+	case CRAT_SUBTYPE_MEMORY_AFFINITY:
+		mem = (struct crat_subtype_memory *)sub_type_hdr;
+		ret = kfd_parse_subtype_mem(mem, device_list);
+		break;
+	case CRAT_SUBTYPE_CACHE_AFFINITY:
+		cache = (struct crat_subtype_cache *)sub_type_hdr;
+		ret = kfd_parse_subtype_cache(cache, device_list);
+		break;
+	case CRAT_SUBTYPE_TLB_AFFINITY:
+		/*
+		 * For now, nothing to do here
+		 */
+		pr_debug("Found TLB entry in CRAT table (not processing)\n");
+		break;
+	case CRAT_SUBTYPE_CCOMPUTE_AFFINITY:
+		/*
+		 * For now, nothing to do here
+		 */
+		pr_debug("Found CCOMPUTE entry in CRAT table (not processing)\n");
+		break;
+	case CRAT_SUBTYPE_IOLINK_AFFINITY:
+		iolink = (struct crat_subtype_iolink *)sub_type_hdr;
+		ret = kfd_parse_subtype_iolink(iolink, device_list);
+		break;
+	default:
+		pr_warn("Unknown subtype (%d) in CRAT\n",
+				sub_type_hdr->type);
+	}
+
+	return ret;
+}
+
+/* kfd_parse_crat_table - parse CRAT table. For each node present in CRAT
+ *		create a kfd_topology_device and add in to device_list. Also parse
+ *		CRAT subtypes and attach it to appropriate kfd_topology_device
+ *	@crat_image - input image containing CRAT
+ *	@device_list - [OUT] list of kfd_topology_device generated after parsing
+ *				   crat_image
+ *	@proximity_domain - Proximity domain of the first device in the table
+ *	Return - 0 if successful else -ve value
+ */
+int kfd_parse_crat_table(void *crat_image,
+				struct list_head *device_list,
+				uint32_t proximity_domain)
+{
+	struct kfd_topology_device *top_dev = NULL;
+	struct crat_subtype_generic *sub_type_hdr;
+	uint16_t node_id;
+	int ret;
+	struct crat_header *crat_table = (struct crat_header *)crat_image;
+	uint16_t num_nodes;
+	uint32_t image_len;
+	uint32_t last_header_type, last_header_length;
+
+	if (!crat_image)
+		return -EINVAL;
+
+	if (!list_empty(device_list)) {
+		pr_warn("Error device list should be empty\n");
+	}
+
+	num_nodes = crat_table->num_domains;
+	image_len = crat_table->length;
+
+	pr_info("Parsing CRAT table with %d nodes\n", num_nodes);
+
+	for (node_id = 0; node_id < num_nodes; node_id++) {
+		top_dev = kfd_create_topology_device(device_list);
+		if (!top_dev)
+			break;
+		top_dev->proximity_domain = proximity_domain++;
+	}
+
+	if (!top_dev)
+		return -ENOMEM;
+
+	memcpy(top_dev->oem_id, crat_table->oem_id, CRAT_OEMID_LENGTH);
+	memcpy(top_dev->oem_table_id, crat_table->oem_table_id, CRAT_OEMTABLEID_LENGTH);
+	top_dev->oem_revision = crat_table->oem_revision;
+
+	last_header_type = last_header_length = 0;
+	sub_type_hdr = (struct crat_subtype_generic *)(crat_table+1);
+	while ((char *)sub_type_hdr + sizeof(struct crat_subtype_generic) <
+			((char *)crat_image) + image_len) {
+		pr_debug("kfd parsing crat sub type header %p enabled: %s type: 0x%x length %d\n",
+				sub_type_hdr,
+				(sub_type_hdr->flags &
+					CRAT_SUBTYPE_FLAGS_ENABLED)
+					? "true" : "false",
+				sub_type_hdr->type,
+				sub_type_hdr->length);
+
+		if (sub_type_hdr->length == 0) {
+			pr_err("amdkfd: Parsing wrong CRAT's sub header last header type: %d last header len %d\n",
+				last_header_type, last_header_type);
+			pr_err("amdkfd: Current header type %d length %d\n",
+				sub_type_hdr->type, sub_type_hdr->length);
+			break;
+		}
+
+		if (sub_type_hdr->flags & CRAT_SUBTYPE_FLAGS_ENABLED) {
+			ret = kfd_parse_subtype(sub_type_hdr, device_list);
+			if (ret != 0)
+				return ret;
+		}
+
+		last_header_type = sub_type_hdr->type;
+		last_header_length = sub_type_hdr->length;
+		sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
+				sub_type_hdr->length);
+	}
+
+	return 0;
+}
+
+/* Helper function. See kfd_fill_gpu_cache_info for parameter description */
+static int fill_in_pcache(struct crat_subtype_cache *pcache,
+				struct kfd_gpu_cache_info *pcache_info,
+				struct kfd_cu_info *cu_info,
+				int mem_available,
+				int cu_bitmask,
+				int cache_type, unsigned int cu_processor_id,
+				int cu_block)
+{
+	unsigned int cu_sibling_map_mask;
+	int first_active_cu;
+
+	/* First check if enough memory is available */
+	if (mem_available - sizeof(struct crat_subtype_cache) < 0)
+		return -ENOMEM;
+
+	cu_sibling_map_mask = cu_bitmask;
+	cu_sibling_map_mask >>= cu_block;
+	cu_sibling_map_mask &=
+		((1 << pcache_info[cache_type].num_cu_shared) - 1);
+	first_active_cu = ffs(cu_sibling_map_mask);
+
+	/* CU could be inactive. In case of shared cache find the first active
+	 * CU. and incase of non-shared cache check if the CU is inactive. If
+	 * inactive active skip it*/
+	if (first_active_cu) {
+		memset(pcache, 0, sizeof(struct crat_subtype_cache));
+		pcache->type = CRAT_SUBTYPE_CACHE_AFFINITY;
+		pcache->length = sizeof(struct crat_subtype_cache);
+		pcache->flags = pcache_info[cache_type].flags;
+		pcache->processor_id_low = cu_processor_id
+					 + (first_active_cu - 1);
+		pcache->cache_level = pcache_info[cache_type].cache_level;
+		pcache->cache_size = pcache_info[cache_type].cache_size;
+
+		/* Sibling map is w.r.t processor_id_low, so shift out
+		 * inactive CU */
+		cu_sibling_map_mask =
+			cu_sibling_map_mask >> (first_active_cu - 1);
+
+		pcache->sibling_map[0] = (uint8_t)(cu_sibling_map_mask & 0xFF);
+		pcache->sibling_map[1] =
+				(uint8_t)((cu_sibling_map_mask >> 8) & 0xFF);
+		pcache->sibling_map[2] =
+				(uint8_t)((cu_sibling_map_mask >> 16) & 0xFF);
+		pcache->sibling_map[3] =
+				(uint8_t)((cu_sibling_map_mask >> 24) & 0xFF);
+		return 0;
+	}
+	return 1;
+}
+
+/* kfd_fill_gpu_cache_info - Fill GPU cache info using kfd_gpu_cache_info tables
+ *	@kdev - [IN] GPU device
+ *	@gpu_processor_id - [IN] GPU processor ID to which these caches associate
+ *	@available_size - [IN] Amount of memory available in pcache
+ *	@cu_info - [IN] Compute Unit info obtained from KGD
+ *	@pcache - [OUT] memory into which cache data is to be filled in.
+ *	@size_filled - [OUT] amount of data used up in pcache.
+ *	@num_of_entries - [OUT] number of caches added
+ */
+static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev,
+			int gpu_processor_id,
+			int available_size,
+			struct kfd_cu_info *cu_info,
+			struct crat_subtype_cache *pcache,
+			int *size_filled,
+			int *num_of_entries)
+{
+	struct kfd_gpu_cache_info *pcache_info;
+	int num_of_cache_types = 0;
+	int i, j, k;
+	int ct = 0;
+	int mem_available = available_size;
+	unsigned int cu_processor_id;
+	int ret;
+
+	switch (kdev->device_info->asic_family) {
+	case CHIP_KAVERI:
+		pcache_info = kaveri_cache_info;
+		num_of_cache_types = ARRAY_SIZE(kaveri_cache_info);
+		break;
+	case CHIP_CARRIZO:
+		pcache_info = carrizo_cache_info;
+		num_of_cache_types = ARRAY_SIZE(carrizo_cache_info);
+		break;
+	case CHIP_TONGA:
+		pcache_info = tonga_cache_info;
+		num_of_cache_types = ARRAY_SIZE(tonga_cache_info);
+		break;
+	case CHIP_FIJI:
+		pcache_info = fiji_cache_info;
+		num_of_cache_types = ARRAY_SIZE(fiji_cache_info);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	*size_filled = 0;
+	*num_of_entries = 0;
+
+	/* For each type of cache listed in the kfd_gpu_cache_info table,
+	 * go through all available Compute Units.
+	 * The [i,j,k] loop will
+	 *		if kfd_gpu_cache_info.num_cu_shared = 1
+	 *			will parse through all available CU
+	 *		If (kfd_gpu_cache_info.num_cu_shared != 1)
+	 *			then it will consider only one CU from
+	 *			the shared unit
+	 */
+
+	for (ct = 0; ct < num_of_cache_types; ct++) {
+		cu_processor_id = gpu_processor_id;
+		for (i = 0; i < cu_info->num_shader_engines; i++) {
+			for (j = 0; j < cu_info->num_shader_arrays_per_engine;
+				j++) {
+				for (k = 0; k < cu_info->num_cu_per_sh;
+					k += pcache_info[ct].num_cu_shared) {
+
+					ret = fill_in_pcache(pcache,
+						pcache_info,
+						cu_info,
+						mem_available,
+						cu_info->cu_bitmap[i][j],
+						ct,
+						cu_processor_id,
+						k);
+
+					if (ret < 0)
+						break;
+
+					if (!ret) {
+						pcache++;
+						(*num_of_entries)++;
+						mem_available -=
+							sizeof(*pcache);
+						(*size_filled) +=
+							sizeof(*pcache);
+					}
+
+					/* Move to next CU block */
+					cu_processor_id +=
+						pcache_info[ct].num_cu_shared;
+				}
+			}
+		}
+	}
+
+	pr_debug("Added [%d] GPU cache entries\n", *num_of_entries);
+
+	return 0;
+}
+
+/*
+ * kfd_create_crat_image_acpi - Allocates memory for CRAT image and
+ *		copies CRAT from ACPI (if available).
+ *
+ * NOTE: Call kfd_destroy_crat_image to free CRAT image memory
+ *
+ * @crat_image: CRAT read from ACPI. If no CRAT in ACPI then
+ *				*crat_image will be NULL
+ * @size: [OUT] size of crat_image
+ *
+ * Return 0 if successful else return -ve value
+ */
+int kfd_create_crat_image_acpi(void **crat_image, size_t *size)
+{
+	struct acpi_table_header *crat_table;
+	acpi_status status;
+	void *pcrat_image;
+
+	if (!crat_image)
+		return -EINVAL;
+
+	*crat_image = NULL;
+
+	/*
+	 * Fetch the CRAT table from ACPI
+	 */
+	status = acpi_get_table(CRAT_SIGNATURE, 0, &crat_table);
+	if (status == AE_NOT_FOUND) {
+		pr_warn("CRAT table not found\n");
+		return -ENODATA;
+	} else if (ACPI_FAILURE(status)) {
+		const char *err = acpi_format_exception(status);
+		pr_err("CRAT table error: %s\n", err);
+		return -EINVAL;
+	}
+
+	pcrat_image = kmalloc(crat_table->length, GFP_KERNEL);
+	if (!pcrat_image) {
+		pr_err("No memory for allocating CRAT image\n");
+		return -ENOMEM;
+	}
+
+	memcpy(pcrat_image, crat_table, crat_table->length);
+
+	*crat_image = pcrat_image;
+	*size = crat_table->length;
+
+	return 0;
+}
+
+/* Memory required to create Virtual CRAT.
+ * Since there is no easy way to predict the amount of memory required, the
+ * following amount are allocated for CPU and GPU Virtual CRAT. This is
+ * expected to cover all known conditions. But to be safe additional check
+ * is put in the code to ensure we don't overwrite.
+ */
+#define VCRAT_SIZE_FOR_CPU	PAGE_SIZE
+#define VCRAT_SIZE_FOR_GPU	(3 * PAGE_SIZE)
+
+/* kfd_fill_cu_for_cpu - Fill in Compute info for the given CPU NUMA node
+ *
+ *  @numa_node_id: CPU NUMA node id
+ *  @avail_size: Available size in the memory
+ *  @sub_type_hdr: Memory into which compute info will be filled in
+ *
+ *  Return 0 if successful else return -ve value
+ */
+static int kfd_fill_cu_for_cpu(int numa_node_id, int *avail_size,
+				int proximity_domain,
+				struct crat_subtype_computeunit *sub_type_hdr)
+{
+	const struct cpumask *cpumask;
+
+	*avail_size -= sizeof(struct crat_subtype_computeunit);
+	if (*avail_size < 0)
+		return -ENOMEM;
+
+	memset(sub_type_hdr, 0, sizeof(struct crat_subtype_computeunit));
+
+	/* Fill in subtype header data */
+	sub_type_hdr->type = CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY;
+	sub_type_hdr->length = sizeof(struct crat_subtype_computeunit);
+	sub_type_hdr->flags = CRAT_SUBTYPE_FLAGS_ENABLED;
+
+	cpumask = cpumask_of_node(numa_node_id);
+
+	/* Fill in CU data */
+	sub_type_hdr->flags |= CRAT_CU_FLAGS_CPU_PRESENT;
+	sub_type_hdr->proximity_domain = proximity_domain;
+	sub_type_hdr->processor_id_low = kfd_numa_node_to_apic_id(numa_node_id);
+	if (sub_type_hdr->processor_id_low == -1)
+		return -EINVAL;
+
+	sub_type_hdr->num_cpu_cores = cpumask_weight(cpumask);
+
+	return 0;
+}
+
+/* kfd_fill_mem_info_for_cpu - Fill in Memory info for the given CPU NUMA node
+ *
+ *  @numa_node_id: CPU NUMA node id
+ *  @avail_size: Available size in the memory
+ *  @sub_type_hdr: Memory into which compute info will be filled in
+ *
+ *  Return 0 if successful else return -ve value
+ */
+static int kfd_fill_mem_info_for_cpu(int numa_node_id, int *avail_size,
+			int proximity_domain,
+			struct crat_subtype_memory *sub_type_hdr)
+{
+	uint64_t mem_in_bytes = 0;
+	pg_data_t *pgdat;
+	int zone_type;
+
+	*avail_size -= sizeof(struct crat_subtype_computeunit);
+	if (*avail_size < 0)
+		return -ENOMEM;
+
+	memset(sub_type_hdr, 0, sizeof(struct crat_subtype_memory));
+
+	/* Fill in subtype header data */
+	sub_type_hdr->type = CRAT_SUBTYPE_MEMORY_AFFINITY;
+	sub_type_hdr->length = sizeof(struct crat_subtype_memory);
+	sub_type_hdr->flags = CRAT_SUBTYPE_FLAGS_ENABLED;
+
+	/* Fill in Memory Subunit data */
+
+	/* Unlike si_meminfo, si_meminfo_node is not exported. So
+	 * the following lines are duplicated from si_meminfo_node
+	 * function */
+	pgdat = NODE_DATA(numa_node_id);
+	for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++)
+		mem_in_bytes += pgdat->node_zones[zone_type].managed_pages;
+	mem_in_bytes <<= PAGE_SHIFT;
+
+	sub_type_hdr->length_low = lower_32_bits(mem_in_bytes);
+	sub_type_hdr->length_high = upper_32_bits(mem_in_bytes);
+	sub_type_hdr->proximity_domain = proximity_domain;
+
+	return 0;
+}
+
+/* kfd_create_vcrat_image_cpu - Create Virtual CRAT for CPU
+ *
+ *	@pcrat_image: Fill in VCRAT for CPU
+ *	@size:	[IN] allocated size of crat_image.
+ *			[OUT] actual size of data filled in crat_image
+ */
+static int kfd_create_vcrat_image_cpu(void *pcrat_image, size_t *size)
+{
+	struct crat_header *crat_table = (struct crat_header *)pcrat_image;
+	struct acpi_table_header *acpi_table;
+	acpi_status status;
+	struct crat_subtype_generic *sub_type_hdr;
+	int avail_size = *size;
+	int numa_node_id;
+	int ret = 0;
+
+	if (pcrat_image == NULL || avail_size < VCRAT_SIZE_FOR_CPU)
+		return -EINVAL;
+
+	/* Fill in CRAT Header.
+	 * Modify length and total_entries as subunits are added.
+	 */
+	avail_size -= sizeof(struct crat_header);
+	if (avail_size < 0)
+		return -ENOMEM;
+
+	memset(crat_table, 0, sizeof(struct crat_header));
+	memcpy(&crat_table->signature, CRAT_SIGNATURE, sizeof(crat_table->signature));
+	crat_table->length = sizeof(struct crat_header);
+
+	status = acpi_get_table("DSDT", 0, &acpi_table);
+	if (status == AE_NOT_FOUND)
+		pr_warn("DSDT table not found for OEM information\n");
+	else {
+		crat_table->oem_revision = acpi_table->revision;
+		memcpy(crat_table->oem_id, acpi_table->oem_id, CRAT_OEMID_LENGTH);
+		memcpy(crat_table->oem_table_id, acpi_table->oem_table_id, CRAT_OEMTABLEID_LENGTH);
+	}
+	crat_table->total_entries = 0;
+	crat_table->num_domains = 0;
+
+	sub_type_hdr = (struct crat_subtype_generic *)(crat_table+1);
+
+	for_each_online_node(numa_node_id) {
+		/* Fill in Subtype: Compute Unit */
+		ret = kfd_fill_cu_for_cpu(numa_node_id, &avail_size,
+			crat_table->num_domains,
+			(struct crat_subtype_computeunit *)sub_type_hdr);
+		if (ret < 0)
+			return ret;
+		crat_table->length += sub_type_hdr->length;
+		crat_table->total_entries++;
+
+		sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
+			sub_type_hdr->length);
+
+		/* Fill in Subtype: Memory */
+		ret = kfd_fill_mem_info_for_cpu(numa_node_id, &avail_size,
+			crat_table->num_domains,
+			(struct crat_subtype_memory *)sub_type_hdr);
+		if (ret < 0)
+			return ret;
+		crat_table->length += sub_type_hdr->length;
+		crat_table->total_entries++;
+
+		sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
+			sub_type_hdr->length);
+
+		crat_table->num_domains++;
+	}
+
+	/* TODO: Add cache Subtype for CPU.
+	 * Currently, CPU cache information is available in function
+	 * detect_cache_attributes(cpu) defined in the file
+	 * ./arch/x86/kernel/cpu/intel_cacheinfo.c. This function is not exported
+	 * and to get the same information the code needs to be duplicated.
+	 */
+
+	*size = crat_table->length;
+	pr_info("Virtual CRAT table created for CPU\n");
+
+	return 0;
+}
+
+static int kfd_fill_gpu_memory_affinity(int *avail_size,
+		struct kfd_dev *kdev, uint8_t type, uint64_t size,
+		struct crat_subtype_memory *sub_type_hdr,
+		uint32_t proximity_domain,
+		const struct kfd_local_mem_info *local_mem_info)
+{
+	*avail_size -= sizeof(struct crat_subtype_memory);
+	if (*avail_size < 0)
+		return -ENOMEM;
+
+	memset((void *)sub_type_hdr, 0, sizeof(struct crat_subtype_memory));
+	sub_type_hdr->type = CRAT_SUBTYPE_MEMORY_AFFINITY;
+	sub_type_hdr->length = sizeof(struct crat_subtype_memory);
+	sub_type_hdr->flags |= CRAT_SUBTYPE_FLAGS_ENABLED;
+
+	sub_type_hdr->proximity_domain = proximity_domain;
+
+	pr_debug("amdkfd: fill gpu memory affinity - type 0x%x size 0x%llx\n",
+			type, size);
+
+	sub_type_hdr->length_low = lower_32_bits(size);
+	sub_type_hdr->length_high = upper_32_bits(size);
+
+	sub_type_hdr->width = local_mem_info->vram_width;
+	sub_type_hdr->visibility_type = type;
+
+	return 0;
+}
+
+/* kfd_fill_gpu_direct_io_link - Fill in direct io link from GPU
+ *	to its NUMA node
+ *
+ *  @avail_size: Available size in the memory
+ *  @kdev - [IN] GPU device
+ *  @sub_type_hdr: Memory into which io link info will be filled in
+ *  @proximity_domain - proximity domain of the GPU node
+ *
+ *  Return 0 if successful else return -ve value
+ */
+static int kfd_fill_gpu_direct_io_link(int *avail_size,
+			struct kfd_dev *kdev,
+			struct crat_subtype_iolink *sub_type_hdr,
+			uint32_t proximity_domain)
+{
+	int proximity_domain_to;
+	*avail_size -= sizeof(struct crat_subtype_iolink);
+	if (*avail_size < 0)
+		return -ENOMEM;
+
+	memset((void *)sub_type_hdr, 0, sizeof(struct crat_subtype_iolink));
+
+	/* Fill in subtype header data */
+	sub_type_hdr->type = CRAT_SUBTYPE_IOLINK_AFFINITY;
+	sub_type_hdr->length = sizeof(struct crat_subtype_iolink);
+	sub_type_hdr->flags |= CRAT_SUBTYPE_FLAGS_ENABLED;
+
+	/* Fill in IOLINK subtype.
+	 * TODO: Fill-in other fields of iolink subtype */
+	sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_PCIEXPRESS;
+	sub_type_hdr->proximity_domain_from = proximity_domain;
+	proximity_domain_to =
+		kfd_get_proximity_domain(kdev->pdev->bus);
+	if (proximity_domain_to == -1)
+		return -EINVAL;
+
+	sub_type_hdr->proximity_domain_to = proximity_domain_to;
+	return 0;
+}
+
+/* kfd_create_vcrat_image_gpu - Create Virtual CRAT for CPU
+ *
+ *	@pcrat_image: Fill in VCRAT for GPU
+ *	@size:	[IN] allocated size of crat_image.
+ *		[OUT] actual size of data filled in crat_image
+ */
+static int kfd_create_vcrat_image_gpu(void *pcrat_image,
+			size_t *size, struct kfd_dev *kdev,
+			uint32_t proximity_domain)
+{
+	struct crat_header *crat_table = (struct crat_header *)pcrat_image;
+	struct crat_subtype_generic *sub_type_hdr;
+	struct crat_subtype_computeunit *cu;
+	struct kfd_cu_info cu_info;
+	struct amd_iommu_device_info iommu_info;
+	int avail_size = *size;
+	uint32_t total_num_of_cu;
+	int num_of_cache_entries = 0;
+	int cache_mem_filled = 0;
+	int ret = 0;
+	const u32 required_iommu_flags = AMD_IOMMU_DEVICE_FLAG_ATS_SUP |
+								AMD_IOMMU_DEVICE_FLAG_PRI_SUP |
+								AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
+	struct kfd_local_mem_info local_mem_info;
+
+	if (pcrat_image == NULL || avail_size < VCRAT_SIZE_FOR_GPU)
+		return -EINVAL;
+
+	/* Fill the CRAT Header.
+	 * Modify length and total_entries as subunits are added.
+	 */
+	avail_size -= sizeof(struct crat_header);
+	if (avail_size < 0)
+		return -ENOMEM;
+
+	memset(crat_table, 0, sizeof(struct crat_header));
+
+	memcpy(&crat_table->signature, CRAT_SIGNATURE, sizeof(crat_table->signature));
+	crat_table->length = sizeof(struct crat_header); /* Change length as we add more subtypes*/
+	crat_table->num_domains = 1;
+	crat_table->total_entries = 0;
+
+	/* Fill in Subtype: Compute Unit
+	 * First fill in the sub type header and then sub type data
+	 */
+	avail_size -= sizeof(struct crat_subtype_computeunit);
+	if (avail_size < 0)
+		return -ENOMEM;
+
+	sub_type_hdr = (struct crat_subtype_generic *)(crat_table + 1);
+	memset(sub_type_hdr, 0, sizeof(struct crat_subtype_computeunit));
+
+	sub_type_hdr->type = CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY;
+	sub_type_hdr->length = sizeof(struct crat_subtype_computeunit);
+	sub_type_hdr->flags = CRAT_SUBTYPE_FLAGS_ENABLED;
+
+	/* Fill CU subtype data */
+	cu = (struct crat_subtype_computeunit *)sub_type_hdr;
+	cu->flags |= CRAT_CU_FLAGS_GPU_PRESENT;
+	cu->proximity_domain = proximity_domain;
+
+	kdev->kfd2kgd->get_cu_info(kdev->kgd, &cu_info);
+	cu->num_simd_per_cu = cu_info.simd_per_cu;
+	cu->num_simd_cores = cu_info.simd_per_cu * cu_info.cu_active_number;
+	cu->max_waves_simd = cu_info.max_waves_per_simd;
+
+	cu->wave_front_size = cu_info.wave_front_size;
+	cu->array_count = cu_info.num_shader_arrays_per_engine *
+		cu_info.num_shader_engines;
+	total_num_of_cu = (cu->array_count * cu_info.num_cu_per_sh);
+	cu->processor_id_low = get_and_inc_gpu_processor_id(total_num_of_cu);
+	cu->num_cu_per_array = cu_info.num_cu_per_sh;
+	cu->max_slots_scatch_cu = cu_info.max_scratch_slots_per_cu;
+	cu->num_banks = cu_info.num_shader_engines;
+	cu->lds_size_in_kb = cu_info.lds_size;
+
+	cu->hsa_capability = 0;
+
+	/* Check if this node supports IOMMU. During parsing this flag will
+	 * translate to HSA_CAP_ATS_PRESENT */
+	iommu_info.flags = 0;
+	if (0 == amd_iommu_device_info(kdev->pdev, &iommu_info)) {
+		if ((iommu_info.flags & required_iommu_flags) == required_iommu_flags)
+			cu->hsa_capability |= CRAT_CU_FLAGS_IOMMU_PRESENT;
+	}
+
+	crat_table->length += sub_type_hdr->length;
+	crat_table->total_entries++;
+
+	/* Fill in Subtype: Memory. Only on systems with large BAR (no
+	 * private FB), report memory as public. On other systems
+	 * report the total FB size (public+private) as a single
+	 * private heap. */
+	kdev->kfd2kgd->get_local_mem_info(kdev->kgd, &local_mem_info);
+	sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
+			sub_type_hdr->length);
+
+	if (local_mem_info.local_mem_size_private == 0)
+		ret = kfd_fill_gpu_memory_affinity(&avail_size,
+				kdev, HSA_MEM_HEAP_TYPE_FB_PUBLIC,
+				local_mem_info.local_mem_size_public,
+				(struct crat_subtype_memory *)sub_type_hdr,
+				proximity_domain,
+				&local_mem_info);
+	else
+		ret = kfd_fill_gpu_memory_affinity(&avail_size,
+				kdev, HSA_MEM_HEAP_TYPE_FB_PRIVATE,
+				local_mem_info.local_mem_size_public +
+				local_mem_info.local_mem_size_private,
+				(struct crat_subtype_memory *)sub_type_hdr,
+				proximity_domain,
+				&local_mem_info);
+	if (ret < 0)
+		return ret;
+
+	crat_table->length += sizeof(struct crat_subtype_memory);
+	crat_table->total_entries++;
+
+	/* TODO: Fill in cache information. This information is NOT readily
+	 * available in KGD */
+	sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
+		sub_type_hdr->length);
+	ret = kfd_fill_gpu_cache_info(kdev, cu->processor_id_low,
+				avail_size,
+				&cu_info,
+				(struct crat_subtype_cache *)sub_type_hdr,
+				&cache_mem_filled,
+				&num_of_cache_entries);
+
+	if (ret < 0)
+		return ret;
+
+	crat_table->length += cache_mem_filled;
+	crat_table->total_entries += num_of_cache_entries;
+	avail_size -= cache_mem_filled;
+
+	/* Fill in Subtype: IO_LINKS
+	 *  Only direct links are added here which is Link from GPU to
+	 *  to its NUMA node. Indirect links are added by userspace.
+	 */
+	sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
+		cache_mem_filled);
+	ret = kfd_fill_gpu_direct_io_link(&avail_size, kdev,
+		(struct crat_subtype_iolink *)sub_type_hdr, proximity_domain);
+
+	if (ret < 0)
+		return ret;
+
+	crat_table->length += sub_type_hdr->length;
+	crat_table->total_entries++;
+
+	*size = crat_table->length;
+	pr_info("Virtual CRAT table created for GPU\n");
+
+	return ret;
+}
+
+/* kfd_create_crat_image_virtual - Allocates memory for CRAT image and
+ *		creates a Virtual CRAT (VCRAT) image
+ *
+ * NOTE: Call kfd_destroy_crat_image to free CRAT image memory
+ *
+ *	@crat_image: VCRAT image created because ACPI does not have a
+ *		CRAT for this device
+ *	@size: [OUT] size of virtual crat_image
+ *	@flags:	COMPUTE_UNIT_CPU - Create VCRAT for CPU device
+ *			COMPUTE_UNIT_GPU - Create VCRAT for GPU
+ *			(COMPUTE_UNIT_CPU | COMPUTE_UNIT_GPU) - Create VCRAT for APU
+ *				-- this option is not currently implemented. The assumption
+ *				   is that all AMD APUs will have CRAT
+ *	@kdev: Valid kfd_device required if flags contain COMPUTE_UNIT_GPU
+ *
+ * Return 0 if successful else return -ve value
+*/
+int kfd_create_crat_image_virtual(void **crat_image, size_t *size,
+		int flags, struct kfd_dev *kdev, uint32_t proximity_domain)
+{
+	void *pcrat_image;
+	int ret = 0;
+
+	if (!crat_image)
+		return -EINVAL;
+
+	*crat_image = NULL;
+
+	/* Allocate one VCRAT_SIZE_FOR_CPU for CPU virtual CRAT image and
+	 * VCRAT_SIZE_FOR_GPU for GPU virtual CRAT image. This should cover
+	 * all the current conditions. A check is put not to overwrite beyond
+	 * allocated size
+	 */
+	switch (flags) {
+	case COMPUTE_UNIT_CPU:
+		pcrat_image = kmalloc(VCRAT_SIZE_FOR_CPU, GFP_KERNEL);
+		if (!pcrat_image)
+			return -ENOMEM;
+		*size = VCRAT_SIZE_FOR_CPU;
+		ret = kfd_create_vcrat_image_cpu(pcrat_image, size);
+		break;
+	case COMPUTE_UNIT_GPU:
+		if (!kdev)
+			return -EINVAL;
+		pcrat_image = kmalloc(VCRAT_SIZE_FOR_GPU, GFP_KERNEL);
+		if (!pcrat_image)
+			return -ENOMEM;
+		*size = VCRAT_SIZE_FOR_GPU;
+		ret = kfd_create_vcrat_image_gpu(pcrat_image, size,
+				kdev, proximity_domain);
+		break;
+	case (COMPUTE_UNIT_CPU | COMPUTE_UNIT_GPU) :
+		/*TODO:*/
+		ret = -EINVAL;
+		pr_err("VCRAT not implemented for APU\n");
+		break;
+	default:
+		ret = -EINVAL;
+	}
+
+	if (ret == 0)
+		*crat_image = pcrat_image;
+
+	return ret;
+}
+
+
+/* kfd_destroy_crat_image
+ *
+ * @crat_image: [IN] - crat_image from kfd_create_crat_image_xxx(..)
+ *
+ */
+void kfd_destroy_crat_image(void *crat_image)
+{
+	if (crat_image)
+		kfree(crat_image);
+	return;
+}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.h b/drivers/gpu/drm/amd/amdkfd/kfd_crat.h
index a374fa3..9af3745 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.h
@@ -24,6 +24,7 @@
 #define KFD_CRAT_H_INCLUDED
 
 #include <linux/types.h>
+#include "kfd_priv.h"
 
 #pragma pack(1)
 
@@ -44,6 +45,10 @@
 
 #define CRAT_OEMID_64BIT_MASK ((1ULL << (CRAT_OEMID_LENGTH * 8)) - 1)
 
+/* Compute Unit flags */
+#define COMPUTE_UNIT_CPU	(1 << 0)	/* Create Virtual CRAT for CPU */
+#define COMPUTE_UNIT_GPU	(1 << 1)	/* Create Virtual CRAT for GPU */
+
 struct crat_header {
 	uint32_t	signature;
 	uint32_t	length;
@@ -105,7 +110,7 @@ struct crat_subtype_computeunit {
 	uint8_t		wave_front_size;
 	uint8_t		num_banks;
 	uint16_t	micro_engine_id;
-	uint8_t		num_arrays;
+	uint8_t		array_count;
 	uint8_t		num_cu_per_array;
 	uint8_t		num_simd_per_cu;
 	uint8_t		max_slots_scatch_cu;
@@ -127,13 +132,14 @@ struct crat_subtype_memory {
 	uint8_t		length;
 	uint16_t	reserved;
 	uint32_t	flags;
-	uint32_t	promixity_domain;
+	uint32_t	proximity_domain;
 	uint32_t	base_addr_low;
 	uint32_t	base_addr_high;
 	uint32_t	length_low;
 	uint32_t	length_high;
 	uint32_t	width;
-	uint8_t		reserved2[CRAT_MEMORY_RESERVED_LENGTH];
+	uint8_t		visibility_type; /* for virtual (dGPU) CRAT */
+	uint8_t		reserved2[CRAT_MEMORY_RESERVED_LENGTH - 1];
 };
 
 /*
@@ -222,9 +228,12 @@ struct crat_subtype_ccompute {
 /*
  * HSA IO Link Affinity structure and definitions
  */
-#define CRAT_IOLINK_FLAGS_ENABLED	0x00000001
-#define CRAT_IOLINK_FLAGS_COHERENCY	0x00000002
-#define CRAT_IOLINK_FLAGS_RESERVED	0xfffffffc
+#define CRAT_IOLINK_FLAGS_ENABLED	(1 << 0)
+#define CRAT_IOLINK_FLAGS_NON_COHERENT	(1 << 1)
+#define CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT (1 << 2)
+#define CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT (1 << 3)
+#define CRAT_IOLINK_FLAGS_NO_PEER_TO_PEER_DMA (1 << 4)
+#define CRAT_IOLINK_FLAGS_RESERVED_MASK 0xffffffe0
 
 /*
  * IO interface types
@@ -232,8 +241,16 @@ struct crat_subtype_ccompute {
 #define CRAT_IOLINK_TYPE_UNDEFINED	0
 #define CRAT_IOLINK_TYPE_HYPERTRANSPORT	1
 #define CRAT_IOLINK_TYPE_PCIEXPRESS	2
-#define CRAT_IOLINK_TYPE_OTHER		3
-#define CRAT_IOLINK_TYPE_MAX		255
+#define CRAT_IOLINK_TYPE_AMBA 3
+#define CRAT_IOLINK_TYPE_MIPI 4
+#define CRAT_IOLINK_TYPE_QPI_1_1 5
+#define CRAT_IOLINK_TYPE_RESERVED1 6
+#define CRAT_IOLINK_TYPE_RESERVED2 7
+#define CRAT_IOLINK_TYPE_RAPID_IO 8
+#define CRAT_IOLINK_TYPE_INFINIBAND 9
+#define CRAT_IOLINK_TYPE_RESERVED3 10
+#define CRAT_IOLINK_TYPE_OTHER 11
+#define CRAT_IOLINK_TYPE_MAX 255
 
 #define CRAT_IOLINK_RESERVED_LENGTH 24
 
@@ -291,4 +308,11 @@ struct cdit_header {
 
 #pragma pack()
 
+int kfd_create_crat_image_acpi(void **crat_image, size_t *size);
+void kfd_destroy_crat_image(void *crat_image);
+int kfd_parse_crat_table(void *crat_image,
+		struct list_head *device_list,
+		uint32_t proximity_domain);
+int kfd_create_crat_image_virtual(void **crat_image, size_t *size,
+		int flags, struct kfd_dev *kdev, uint32_t proximity_domain);
 #endif /* KFD_CRAT_H_INCLUDED */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c b/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c
index d5e19b5..4f2311e 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c
@@ -42,8 +42,6 @@
 
 static void dbgdev_address_watch_disable_nodiq(struct kfd_dev *dev)
 {
-	BUG_ON(!dev || !dev->kfd2kgd);
-
 	dev->kfd2kgd->address_watch_disable(dev->kgd);
 }
 
@@ -51,129 +49,118 @@ static int dbgdev_diq_submit_ib(struct kfd_dbgdev *dbgdev,
 				unsigned int pasid, uint64_t vmid0_address,
 				uint32_t *packet_buff, size_t size_in_bytes)
 {
+	int status = 0;
+	unsigned int *ib_packet_buff = NULL;
 	struct pm4__release_mem *rm_packet;
 	struct pm4__indirect_buffer_pasid *ib_packet;
+	struct kernel_queue *kq = dbgdev->kq;
+	size_t pq_packets_size_in_bytes = sizeof(struct pm4__release_mem) + sizeof(struct pm4__indirect_buffer_pasid);
 	struct kfd_mem_obj *mem_obj;
-	size_t pq_packets_size_in_bytes;
+
+	uint64_t *rm_state = NULL;
+
 	union ULARGE_INTEGER *largep;
 	union ULARGE_INTEGER addr;
-	struct kernel_queue *kq;
-	uint64_t *rm_state;
-	unsigned int *ib_packet_buff;
-	int status;
-
-	BUG_ON(!dbgdev || !dbgdev->kq || !packet_buff || !size_in_bytes);
-
-	kq = dbgdev->kq;
-
-	pq_packets_size_in_bytes = sizeof(struct pm4__release_mem) +
-				sizeof(struct pm4__indirect_buffer_pasid);
-
-	/*
-	 * We acquire a buffer from DIQ
-	 * The receive packet buff will be sitting on the Indirect Buffer
-	 * and in the PQ we put the IB packet + sync packet(s).
-	 */
-	status = kq->ops.acquire_packet_buffer(kq,
-				pq_packets_size_in_bytes / sizeof(uint32_t),
-				&ib_packet_buff);
-	if (status != 0) {
-		pr_err("amdkfd: acquire_packet_buffer failed\n");
-		return status;
-	}
 
-	memset(ib_packet_buff, 0, pq_packets_size_in_bytes);
+	do {
+		if ((kq == NULL) || (packet_buff == NULL) || (size_in_bytes == 0)) {
+			pr_debug("Error! kfd: In func %s >> Illegal packet parameters\n", __func__);
+			status = -EINVAL;
+			break;
+		}
+		/* todo - enter proper locking to be multithreaded safe */
 
-	ib_packet = (struct pm4__indirect_buffer_pasid *) (ib_packet_buff);
+		/* We acquire a buffer from DIQ
+		 * The receive packet buff will be sitting on the Indirect Buffer
+		 * and in the PQ we put the IB packet + sync packet(s).
+		 */
+		status = kq->ops.acquire_packet_buffer(kq, pq_packets_size_in_bytes / sizeof(uint32_t), &ib_packet_buff);
+		if (status != 0) {
+			pr_debug("Error! kfd: In func %s >> acquire_packet_buffer failed\n", __func__);
+			break;
+		}
 
-	ib_packet->header.count = 3;
-	ib_packet->header.opcode = IT_INDIRECT_BUFFER_PASID;
-	ib_packet->header.type = PM4_TYPE_3;
+		memset(ib_packet_buff, 0, pq_packets_size_in_bytes);
 
-	largep = (union ULARGE_INTEGER *) &vmid0_address;
+		ib_packet = (struct pm4__indirect_buffer_pasid *) (ib_packet_buff);
 
-	ib_packet->bitfields2.ib_base_lo = largep->u.low_part >> 2;
-	ib_packet->bitfields3.ib_base_hi = largep->u.high_part;
+		ib_packet->header.count = 3;
+		ib_packet->header.opcode = IT_INDIRECT_BUFFER_PASID;
+		ib_packet->header.type = PM4_TYPE_3;
 
-	ib_packet->control = (1 << 23) | (1 << 31) |
-			((size_in_bytes / sizeof(uint32_t)) & 0xfffff);
+		largep = (union ULARGE_INTEGER *) &vmid0_address;
 
-	ib_packet->bitfields5.pasid = pasid;
+		ib_packet->bitfields2.ib_base_lo = largep->u.low_part >> 2;
+		ib_packet->bitfields3.ib_base_hi = largep->u.high_part;
 
-	/*
-	 * for now we use release mem for GPU-CPU synchronization
-	 * Consider WaitRegMem + WriteData as a better alternative
-	 * we get a GART allocations ( gpu/cpu mapping),
-	 * for the sync variable, and wait until:
-	 * (a) Sync with HW
-	 * (b) Sync var is written by CP to mem.
-	 */
-	rm_packet = (struct pm4__release_mem *) (ib_packet_buff +
-			(sizeof(struct pm4__indirect_buffer_pasid) /
-					sizeof(unsigned int)));
+		ib_packet->control = (1 << 23) | (1 << 31) |
+				((size_in_bytes / sizeof(uint32_t)) & 0xfffff);
 
-	status = kfd_gtt_sa_allocate(dbgdev->dev, sizeof(uint64_t),
-					&mem_obj);
+		ib_packet->bitfields5.pasid = pasid;
 
-	if (status != 0) {
-		pr_err("amdkfd: Failed to allocate GART memory\n");
-		kq->ops.rollback_packet(kq);
-		return status;
-	}
+		/*
+		 * for now we use release mem for GPU-CPU synchronization
+		 * Consider WaitRegMem + WriteData as a better alternative
+		 * we get a GART allocations ( gpu/cpu mapping),
+		 * for the sync variable, and wait until:
+		 * (a) Sync with HW
+		 * (b) Sync var is written by CP to mem.
+		 */
+		rm_packet = (struct pm4__release_mem *) (ib_packet_buff +
+				(sizeof(struct pm4__indirect_buffer_pasid) / sizeof(unsigned int)));
 
-	rm_state = (uint64_t *) mem_obj->cpu_ptr;
+		status = kfd_gtt_sa_allocate(dbgdev->dev, sizeof(uint64_t),
+						&mem_obj);
 
-	*rm_state = QUEUESTATE__ACTIVE_COMPLETION_PENDING;
+		if (status == 0) {
 
-	rm_packet->header.opcode = IT_RELEASE_MEM;
-	rm_packet->header.type = PM4_TYPE_3;
-	rm_packet->header.count = sizeof(struct pm4__release_mem) /
-					sizeof(unsigned int) - 2;
+			rm_state = (uint64_t *) mem_obj->cpu_ptr;
 
-	rm_packet->bitfields2.event_type = CACHE_FLUSH_AND_INV_TS_EVENT;
-	rm_packet->bitfields2.event_index =
-				event_index___release_mem__end_of_pipe;
+			*rm_state = QUEUESTATE__ACTIVE_COMPLETION_PENDING;
 
-	rm_packet->bitfields2.cache_policy = cache_policy___release_mem__lru;
-	rm_packet->bitfields2.atc = 0;
-	rm_packet->bitfields2.tc_wb_action_ena = 1;
+			rm_packet->header.opcode = IT_RELEASE_MEM;
+			rm_packet->header.type = PM4_TYPE_3;
+			rm_packet->header.count = sizeof(struct pm4__release_mem) / sizeof(unsigned int) - 2;
 
-	addr.quad_part = mem_obj->gpu_addr;
+			rm_packet->bitfields2.event_type = CACHE_FLUSH_AND_INV_TS_EVENT;
+			rm_packet->bitfields2.event_index = event_index___release_mem__end_of_pipe;
+			rm_packet->bitfields2.cache_policy = cache_policy___release_mem__lru;
+			rm_packet->bitfields2.atc = 0;
+			rm_packet->bitfields2.tc_wb_action_ena = 1;
 
-	rm_packet->bitfields4.address_lo_32b = addr.u.low_part >> 2;
-	rm_packet->address_hi = addr.u.high_part;
+			addr.quad_part = mem_obj->gpu_addr;
 
-	rm_packet->bitfields3.data_sel =
-				data_sel___release_mem__send_64_bit_data;
+			rm_packet->bitfields4.address_lo_32b = addr.u.low_part >> 2;
+			rm_packet->address_hi = addr.u.high_part;
 
-	rm_packet->bitfields3.int_sel =
-			int_sel___release_mem__send_data_after_write_confirm;
+			rm_packet->bitfields3.data_sel = data_sel___release_mem__send_64_bit_data;
+			rm_packet->bitfields3.int_sel = int_sel___release_mem__send_data_after_write_confirm;
+			rm_packet->bitfields3.dst_sel = dst_sel___release_mem__memory_controller;
 
-	rm_packet->bitfields3.dst_sel =
-			dst_sel___release_mem__memory_controller;
+			rm_packet->data_lo = QUEUESTATE__ACTIVE;
 
-	rm_packet->data_lo = QUEUESTATE__ACTIVE;
+			kq->ops.submit_packet(kq);
 
-	kq->ops.submit_packet(kq);
+			/* Wait till CP writes sync code: */
 
-	/* Wait till CP writes sync code: */
-	status = amdkfd_fence_wait_timeout(
-			(unsigned int *) rm_state,
-			QUEUESTATE__ACTIVE, 1500);
+			status = amdkfd_fence_wait_timeout(
+					(unsigned int *) rm_state,
+					QUEUESTATE__ACTIVE, 1500);
+
+		} else {
+			pr_debug("Error! kfd: In func %s >> failed to allocate GART memory\n", __func__);
+		}
+	} while (false);
 
-	kfd_gtt_sa_free(dbgdev->dev, mem_obj);
+	if (rm_state != NULL)
+		kfd_gtt_sa_free(dbgdev->dev, mem_obj);
 
 	return status;
 }
 
 static int dbgdev_register_nodiq(struct kfd_dbgdev *dbgdev)
 {
-	BUG_ON(!dbgdev);
-
-	/*
-	 * no action is needed in this case,
-	 * just make sure diq will not be used
-	 */
+	/* no action is needed in this case, just make sure diq will not be used */
 
 	dbgdev->kq = NULL;
 
@@ -182,57 +169,68 @@ static int dbgdev_register_nodiq(struct kfd_dbgdev *dbgdev)
 
 static int dbgdev_register_diq(struct kfd_dbgdev *dbgdev)
 {
+
+	int status = 0;
+	struct kernel_queue *kq = NULL;
 	struct queue_properties properties;
 	unsigned int qid;
-	struct kernel_queue *kq = NULL;
-	int status;
+	struct process_queue_manager *pqm = dbgdev->pqm;
 
-	BUG_ON(!dbgdev || !dbgdev->pqm || !dbgdev->dev);
+	do {
 
-	status = pqm_create_queue(dbgdev->pqm, dbgdev->dev, NULL,
-				&properties, 0, KFD_QUEUE_TYPE_DIQ,
-				&qid);
+		if (!pqm) {
+			pr_debug("Error! kfd: In func %s >> No PQM\n", __func__);
+			status = -EFAULT;
+			break;
+		}
 
-	if (status) {
-		pr_err("amdkfd: Failed to create DIQ\n");
-		return status;
-	}
+		properties.type = KFD_QUEUE_TYPE_DIQ;
 
-	pr_debug("DIQ Created with queue id: %d\n", qid);
+		status = pqm_create_queue(dbgdev->pqm, dbgdev->dev, NULL,
+				&properties, &qid);
 
-	kq = pqm_get_kernel_queue(dbgdev->pqm, qid);
+		if (status != 0) {
+			pr_debug("Error! kfd: In func %s >> Create Queue failed\n", __func__);
+			break;
+		}
 
-	if (kq == NULL) {
-		pr_err("amdkfd: Error getting DIQ\n");
-		pqm_destroy_queue(dbgdev->pqm, qid);
-		return -EFAULT;
-	}
+		pr_debug("kfd: DIQ Created with queue id: %d\n", qid);
+
+		kq = pqm_get_kernel_queue(dbgdev->pqm, qid);
+
+		if (kq == NULL) {
+			pr_debug("Error! kfd: In func %s >> Error getting Kernel Queue\n", __func__);
+			status = -ENOMEM;
+			break;
+		}
+
+		dbgdev->kq = kq;
 
-	dbgdev->kq = kq;
+	} while (false);
 
 	return status;
 }
 
 static int dbgdev_unregister_nodiq(struct kfd_dbgdev *dbgdev)
 {
-	BUG_ON(!dbgdev || !dbgdev->dev);
-
 	/* disable watch address */
+
 	dbgdev_address_watch_disable_nodiq(dbgdev->dev);
 	return 0;
 }
 
 static int dbgdev_unregister_diq(struct kfd_dbgdev *dbgdev)
 {
-	/* todo - disable address watch */
-	int status;
-
-	BUG_ON(!dbgdev || !dbgdev->pqm || !dbgdev->kq);
-
-	status = pqm_destroy_queue(dbgdev->pqm,
-			dbgdev->kq->queue->properties.queue_id);
-	dbgdev->kq = NULL;
-
+	/* todo - if needed, kill wavefronts and disable watch */
+	int status = 0;
+	if ((dbgdev == NULL) || (dbgdev->pqm == NULL) || (dbgdev->kq == NULL)) {
+		pr_debug("kfd Err:In func %s >> can't destroy diq\n", __func__);
+		status = -EFAULT;
+	} else {
+		pqm_destroy_queue(dbgdev->pqm,
+				dbgdev->kq->queue->properties.queue_id);
+		dbgdev->kq = NULL;
+	}
 	return status;
 }
 
@@ -241,341 +239,350 @@ static void dbgdev_address_watch_set_registers(
 			union TCP_WATCH_ADDR_H_BITS *addrHi,
 			union TCP_WATCH_ADDR_L_BITS *addrLo,
 			union TCP_WATCH_CNTL_BITS *cntl,
-			unsigned int index, unsigned int vmid)
+			unsigned int index, unsigned int vmid,
+			unsigned int asic_family)
 {
 	union ULARGE_INTEGER addr;
 
-	BUG_ON(!adw_info || !addrHi || !addrLo || !cntl);
-
 	addr.quad_part = 0;
 	addrHi->u32All = 0;
 	addrLo->u32All = 0;
 	cntl->u32All = 0;
 
 	if (adw_info->watch_mask != NULL)
-		cntl->bitfields.mask =
-			(uint32_t) (adw_info->watch_mask[index] &
-					ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK);
+		cntl->bitfields.mask = (uint32_t) (adw_info->watch_mask[index] & ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK);
 	else
 		cntl->bitfields.mask = ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK;
 
 	addr.quad_part = (unsigned long long) adw_info->watch_address[index];
 
-	addrHi->bitfields.addr = addr.u.high_part &
-					ADDRESS_WATCH_REG_ADDHIGH_MASK;
+	addrHi->bitfields.addr = addr.u.high_part & ADDRESS_WATCH_REG_ADDHIGH_MASK;
 	addrLo->bitfields.addr =
 			(addr.u.low_part >> ADDRESS_WATCH_REG_ADDLOW_SHIFT);
 
 	cntl->bitfields.mode = adw_info->watch_mode[index];
 	cntl->bitfields.vmid = (uint32_t) vmid;
-	/* for now assume it is an ATC address */
-	cntl->u32All |= ADDRESS_WATCH_REG_CNTL_ATC_BIT;
-
+	/*  for APU assume it is an ATC address.  */
+	if (KFD_IS_DGPU(asic_family) == false)
+		cntl->u32All |= ADDRESS_WATCH_REG_CNTL_ATC_BIT;
 	pr_debug("\t\t%20s %08x\n", "set reg mask :", cntl->bitfields.mask);
-	pr_debug("\t\t%20s %08x\n", "set reg add high :",
-			addrHi->bitfields.addr);
-	pr_debug("\t\t%20s %08x\n", "set reg add low :",
-			addrLo->bitfields.addr);
+	pr_debug("\t\t%20s %08x\n", "set reg add high :", addrHi->bitfields.addr);
+	pr_debug("\t\t%20s %08x\n", "set reg add low :", addrLo->bitfields.addr);
+
 }
 
 static int dbgdev_address_watch_nodiq(struct kfd_dbgdev *dbgdev,
 					struct dbg_address_watch_info *adw_info)
 {
+
+	int status = 0;
+
 	union TCP_WATCH_ADDR_H_BITS addrHi;
 	union TCP_WATCH_ADDR_L_BITS addrLo;
 	union TCP_WATCH_CNTL_BITS cntl;
-	struct kfd_process_device *pdd;
+
+	unsigned int vmid;
 	unsigned int i;
 
-	BUG_ON(!dbgdev || !dbgdev->dev || !adw_info);
+	struct kfd_process_device *pdd;
 
-	/* taking the vmid for that process on the safe way using pdd */
-	pdd = kfd_get_process_device_data(dbgdev->dev,
-					adw_info->process);
-	if (!pdd) {
-		pr_err("amdkfd: Failed to get pdd for wave control no DIQ\n");
-		return -EFAULT;
-	}
+	do {
+		/* taking the vmid for that process on the safe way using pdd */
+		pdd = kfd_get_process_device_data(dbgdev->dev,
+						adw_info->process);
+		if (!pdd) {
+			pr_debug("Error! kfd: In func %s >> no PDD available\n", __func__);
+			status = -EFAULT;
+			break;
+		}
 
-	addrHi.u32All = 0;
-	addrLo.u32All = 0;
-	cntl.u32All = 0;
+		addrHi.u32All = 0;
+		addrLo.u32All = 0;
+		cntl.u32All = 0;
 
-	if ((adw_info->num_watch_points > MAX_WATCH_ADDRESSES) ||
-			(adw_info->num_watch_points == 0)) {
-		pr_err("amdkfd: num_watch_points is invalid\n");
-		return -EINVAL;
-	}
+		vmid = pdd->qpd.vmid;
 
-	if ((adw_info->watch_mode == NULL) ||
-		(adw_info->watch_address == NULL)) {
-		pr_err("amdkfd: adw_info fields are not valid\n");
-		return -EINVAL;
-	}
+		if ((adw_info->num_watch_points > MAX_WATCH_ADDRESSES)
+		    || (adw_info->num_watch_points == 0)) {
+			status = -EINVAL;
+			break;
+		}
 
-	for (i = 0 ; i < adw_info->num_watch_points ; i++) {
-		dbgdev_address_watch_set_registers(adw_info, &addrHi, &addrLo,
-						&cntl, i, pdd->qpd.vmid);
-
-		pr_debug("\t\t%30s\n", "* * * * * * * * * * * * * * * * * *");
-		pr_debug("\t\t%20s %08x\n", "register index :", i);
-		pr_debug("\t\t%20s %08x\n", "vmid is :", pdd->qpd.vmid);
-		pr_debug("\t\t%20s %08x\n", "Address Low is :",
-				addrLo.bitfields.addr);
-		pr_debug("\t\t%20s %08x\n", "Address high is :",
-				addrHi.bitfields.addr);
-		pr_debug("\t\t%20s %08x\n", "Address high is :",
-				addrHi.bitfields.addr);
-		pr_debug("\t\t%20s %08x\n", "Control Mask is :",
-				cntl.bitfields.mask);
-		pr_debug("\t\t%20s %08x\n", "Control Mode is :",
-				cntl.bitfields.mode);
-		pr_debug("\t\t%20s %08x\n", "Control Vmid is :",
-				cntl.bitfields.vmid);
-		pr_debug("\t\t%20s %08x\n", "Control atc  is :",
-				cntl.bitfields.atc);
-		pr_debug("\t\t%30s\n", "* * * * * * * * * * * * * * * * * *");
-
-		pdd->dev->kfd2kgd->address_watch_execute(
-						dbgdev->dev->kgd,
-						i,
-						cntl.u32All,
-						addrHi.u32All,
-						addrLo.u32All);
-	}
+		if ((adw_info->watch_mode == NULL) || (adw_info->watch_address == NULL)) {
+			status = -EINVAL;
+			break;
+		}
 
-	return 0;
+		for (i = 0; i < adw_info->num_watch_points; i++) {
+
+			dbgdev_address_watch_set_registers(
+				adw_info,
+				&addrHi,
+				&addrLo,
+				&cntl,
+				i,
+				vmid,
+				dbgdev->dev->device_info->asic_family
+				);
+
+			pr_debug("\t\t%30s\n", "* * * * * * * * * * * * * * * * * *");
+			pr_debug("\t\t%20s %08x\n", "register index :", i);
+			pr_debug("\t\t%20s %08x\n", "vmid is :", vmid);
+			pr_debug("\t\t%20s %08x\n", "Address Low is :", addrLo.bitfields.addr);
+			pr_debug("\t\t%20s %08x\n", "Address high is :", addrHi.bitfields.addr);
+			pr_debug("\t\t%20s %08x\n", "Address high is :", addrHi.bitfields.addr);
+			pr_debug("\t\t%20s %08x\n", "Control Mask is :", cntl.bitfields.mask);
+			pr_debug("\t\t%20s %08x\n", "Control Mode is :", cntl.bitfields.mode);
+			pr_debug("\t\t%20s %08x\n", "Control Vmid is :", cntl.bitfields.vmid);
+			pr_debug("\t\t%20s %08x\n", "Control atc  is :", cntl.bitfields.atc);
+			pr_debug("\t\t%30s\n", "* * * * * * * * * * * * * * * * * *");
+
+			pdd->dev->kfd2kgd->address_watch_execute(
+							dbgdev->dev->kgd,
+							i,
+							cntl.u32All,
+							addrHi.u32All,
+							addrLo.u32All);
+		}
+
+	} while (false);
+
+	return status;
 }
 
 static int dbgdev_address_watch_diq(struct kfd_dbgdev *dbgdev,
 					struct dbg_address_watch_info *adw_info)
 {
-	struct pm4__set_config_reg *packets_vec;
+
+	int status = 0;
+	unsigned int i = 0;
 	union TCP_WATCH_ADDR_H_BITS addrHi;
 	union TCP_WATCH_ADDR_L_BITS addrLo;
 	union TCP_WATCH_CNTL_BITS cntl;
-	struct kfd_mem_obj *mem_obj;
-	unsigned int aw_reg_add_dword;
-	uint32_t *packet_buff_uint;
-	unsigned int i;
-	int status;
-	size_t ib_size = sizeof(struct pm4__set_config_reg) * 4;
+
 	/* we do not control the vmid in DIQ mode, just a place holder */
 	unsigned int vmid = 0;
 
-	BUG_ON(!dbgdev || !dbgdev->dev || !adw_info);
+	struct kfd_mem_obj *mem_obj;
+	uint32_t *packet_buff_uint = NULL;
+
+	struct pm4__set_config_reg *packets_vec = NULL;
+
+	size_t ib_size = sizeof(struct pm4__set_config_reg) * 4;
+
+	unsigned int aw_reg_add_dword;
 
 	addrHi.u32All = 0;
 	addrLo.u32All = 0;
 	cntl.u32All = 0;
 
-	if ((adw_info->num_watch_points > MAX_WATCH_ADDRESSES) ||
-			(adw_info->num_watch_points == 0)) {
-		pr_err("amdkfd: num_watch_points is invalid\n");
-		return -EINVAL;
-	}
+	do {
 
-	if ((NULL == adw_info->watch_mode) ||
-			(NULL == adw_info->watch_address)) {
-		pr_err("amdkfd: adw_info fields are not valid\n");
-		return -EINVAL;
-	}
+		if ((adw_info->num_watch_points > MAX_WATCH_ADDRESSES) || (adw_info->num_watch_points == 0)) {
+			status = -EINVAL;
+			break;
+		}
+
+		if ((NULL == adw_info->watch_mode) || (NULL == adw_info->watch_address)) {
+			status = -EINVAL;
+			break;
+		}
 
-	status = kfd_gtt_sa_allocate(dbgdev->dev, ib_size, &mem_obj);
+		status = kfd_gtt_sa_allocate(dbgdev->dev, ib_size, &mem_obj);
 
-	if (status != 0) {
-		pr_err("amdkfd: Failed to allocate GART memory\n");
-		return status;
-	}
+		if (status != 0)
+			break;
 
-	packet_buff_uint = mem_obj->cpu_ptr;
-
-	memset(packet_buff_uint, 0, ib_size);
-
-	packets_vec = (struct pm4__set_config_reg *) (packet_buff_uint);
-
-	packets_vec[0].header.count = 1;
-	packets_vec[0].header.opcode = IT_SET_CONFIG_REG;
-	packets_vec[0].header.type = PM4_TYPE_3;
-	packets_vec[0].bitfields2.vmid_shift = ADDRESS_WATCH_CNTL_OFFSET;
-	packets_vec[0].bitfields2.insert_vmid = 1;
-	packets_vec[1].ordinal1 = packets_vec[0].ordinal1;
-	packets_vec[1].bitfields2.insert_vmid = 0;
-	packets_vec[2].ordinal1 = packets_vec[0].ordinal1;
-	packets_vec[2].bitfields2.insert_vmid = 0;
-	packets_vec[3].ordinal1 = packets_vec[0].ordinal1;
-	packets_vec[3].bitfields2.vmid_shift = ADDRESS_WATCH_CNTL_OFFSET;
-	packets_vec[3].bitfields2.insert_vmid = 1;
-
-	for (i = 0; i < adw_info->num_watch_points; i++) {
-		dbgdev_address_watch_set_registers(adw_info,
-						&addrHi,
-						&addrLo,
-						&cntl,
-						i,
-						vmid);
-
-		pr_debug("\t\t%30s\n", "* * * * * * * * * * * * * * * * * *");
-		pr_debug("\t\t%20s %08x\n", "register index :", i);
-		pr_debug("\t\t%20s %08x\n", "vmid is :", vmid);
-		pr_debug("\t\t%20s %p\n", "Add ptr is :",
-				adw_info->watch_address);
-		pr_debug("\t\t%20s %08llx\n", "Add     is :",
-				adw_info->watch_address[i]);
-		pr_debug("\t\t%20s %08x\n", "Address Low is :",
-				addrLo.bitfields.addr);
-		pr_debug("\t\t%20s %08x\n", "Address high is :",
-				addrHi.bitfields.addr);
-		pr_debug("\t\t%20s %08x\n", "Control Mask is :",
-				cntl.bitfields.mask);
-		pr_debug("\t\t%20s %08x\n", "Control Mode is :",
-				cntl.bitfields.mode);
-		pr_debug("\t\t%20s %08x\n", "Control Vmid is :",
-				cntl.bitfields.vmid);
-		pr_debug("\t\t%20s %08x\n", "Control atc  is :",
-				cntl.bitfields.atc);
-		pr_debug("\t\t%30s\n", "* * * * * * * * * * * * * * * * * *");
-
-		aw_reg_add_dword =
-				dbgdev->dev->kfd2kgd->address_watch_get_offset(
-					dbgdev->dev->kgd,
-					i,
-					ADDRESS_WATCH_REG_CNTL);
+		packet_buff_uint = mem_obj->cpu_ptr;
+
+		memset(packet_buff_uint, 0, ib_size);
 
-		aw_reg_add_dword /= sizeof(uint32_t);
+		packets_vec = (struct pm4__set_config_reg *) (packet_buff_uint);
 
-		packets_vec[0].bitfields2.reg_offset =
-					aw_reg_add_dword - AMD_CONFIG_REG_BASE;
+		packets_vec[0].header.count = 1;
+		packets_vec[0].header.opcode = IT_SET_CONFIG_REG;
+		packets_vec[0].header.type = PM4_TYPE_3;
+		packets_vec[0].bitfields2.vmid_shift = ADDRESS_WATCH_CNTL_OFFSET;
+		packets_vec[0].bitfields2.insert_vmid = 1;
+		packets_vec[1].ordinal1 = packets_vec[0].ordinal1;
+		packets_vec[1].bitfields2.insert_vmid = 0;
+		packets_vec[2].ordinal1 = packets_vec[0].ordinal1;
+		packets_vec[2].bitfields2.insert_vmid = 0;
+		packets_vec[3].ordinal1 = packets_vec[0].ordinal1;
+		packets_vec[3].bitfields2.vmid_shift = ADDRESS_WATCH_CNTL_OFFSET;
+		packets_vec[3].bitfields2.insert_vmid = 1;
 
-		packets_vec[0].reg_data[0] = cntl.u32All;
+		for (i = 0; i < adw_info->num_watch_points; i++) {
 
-		aw_reg_add_dword =
-				dbgdev->dev->kfd2kgd->address_watch_get_offset(
-					dbgdev->dev->kgd,
+			dbgdev_address_watch_set_registers(
+					adw_info,
+					&addrHi,
+					&addrLo,
+					&cntl,
 					i,
-					ADDRESS_WATCH_REG_ADDR_HI);
+					vmid,
+					dbgdev->dev->device_info->asic_family
+					);
+
+			pr_debug("\t\t%30s\n", "* * * * * * * * * * * * * * * * * *");
+			pr_debug("\t\t%20s %08x\n", "register index :", i);
+			pr_debug("\t\t%20s %08x\n", "vmid is :", vmid);
+			pr_debug("\t\t%20s %p\n", "Add ptr is :", adw_info->watch_address);
+			pr_debug("\t\t%20s %08llx\n", "Add     is :", adw_info->watch_address[i]);
+			pr_debug("\t\t%20s %08x\n", "Address Low is :", addrLo.bitfields.addr);
+			pr_debug("\t\t%20s %08x\n", "Address high is :", addrHi.bitfields.addr);
+			pr_debug("\t\t%20s %08x\n", "Control Mask is :", cntl.bitfields.mask);
+			pr_debug("\t\t%20s %08x\n", "Control Mode is :", cntl.bitfields.mode);
+			pr_debug("\t\t%20s %08x\n", "Control Vmid is :", cntl.bitfields.vmid);
+			pr_debug("\t\t%20s %08x\n", "Control atc  is :", cntl.bitfields.atc);
+			pr_debug("\t\t%30s\n", "* * * * * * * * * * * * * * * * * *");
+
+			aw_reg_add_dword =
+					dbgdev->dev->kfd2kgd
+					->address_watch_get_offset(
+						dbgdev->dev->kgd,
+						i,
+						ADDRESS_WATCH_REG_CNTL);
+
+			packets_vec[0].bitfields2.reg_offset = aw_reg_add_dword - CONFIG_REG_BASE;
+			packets_vec[0].reg_data[0] = cntl.u32All;
 
-		aw_reg_add_dword /= sizeof(uint32_t);
+			aw_reg_add_dword =
+					dbgdev->dev->kfd2kgd
+					->address_watch_get_offset(
+						dbgdev->dev->kgd,
+						i,
+						ADDRESS_WATCH_REG_ADDR_HI);
 
-		packets_vec[1].bitfields2.reg_offset =
-					aw_reg_add_dword - AMD_CONFIG_REG_BASE;
-		packets_vec[1].reg_data[0] = addrHi.u32All;
 
-		aw_reg_add_dword =
-				dbgdev->dev->kfd2kgd->address_watch_get_offset(
-					dbgdev->dev->kgd,
-					i,
-					ADDRESS_WATCH_REG_ADDR_LO);
+			packets_vec[1].bitfields2.reg_offset = aw_reg_add_dword - CONFIG_REG_BASE;
+			packets_vec[1].reg_data[0] = addrHi.u32All;
 
-		aw_reg_add_dword /= sizeof(uint32_t);
+			aw_reg_add_dword =
+					dbgdev->dev->kfd2kgd
+					->address_watch_get_offset(
+						dbgdev->dev->kgd,
+						i,
+						ADDRESS_WATCH_REG_ADDR_LO);
 
-		packets_vec[2].bitfields2.reg_offset =
-				aw_reg_add_dword - AMD_CONFIG_REG_BASE;
-		packets_vec[2].reg_data[0] = addrLo.u32All;
 
-		/* enable watch flag if address is not zero*/
-		if (adw_info->watch_address[i] > 0)
-			cntl.bitfields.valid = 1;
-		else
-			cntl.bitfields.valid = 0;
+			packets_vec[2].bitfields2.reg_offset = aw_reg_add_dword - CONFIG_REG_BASE;
+			packets_vec[2].reg_data[0] = addrLo.u32All;
 
-		aw_reg_add_dword =
-				dbgdev->dev->kfd2kgd->address_watch_get_offset(
-					dbgdev->dev->kgd,
-					i,
-					ADDRESS_WATCH_REG_CNTL);
+			/* enable watch flag if address is not zero*/
+			if (adw_info->watch_address[i] > 0)
+				cntl.bitfields.valid = 1;
+			else
+				cntl.bitfields.valid = 0;
 
-		aw_reg_add_dword /= sizeof(uint32_t);
+			aw_reg_add_dword =
+					dbgdev->dev->kfd2kgd
+					->address_watch_get_offset(
+						dbgdev->dev->kgd,
+						i,
+						ADDRESS_WATCH_REG_CNTL);
 
-		packets_vec[3].bitfields2.reg_offset =
-					aw_reg_add_dword - AMD_CONFIG_REG_BASE;
-		packets_vec[3].reg_data[0] = cntl.u32All;
 
-		status = dbgdev_diq_submit_ib(
-					dbgdev,
-					adw_info->process->pasid,
-					mem_obj->gpu_addr,
-					packet_buff_uint,
-					ib_size);
+			packets_vec[3].bitfields2.reg_offset = aw_reg_add_dword - CONFIG_REG_BASE;
+			packets_vec[3].reg_data[0] = cntl.u32All;
+
+			status = dbgdev_diq_submit_ib(
+						dbgdev,
+						adw_info->process->pasid,
+						mem_obj->gpu_addr,
+						packet_buff_uint,
+						ib_size);
+
+			if (status != 0) {
+				pr_debug("Error! kfd: In func %s >> failed to submit DIQ packet\n", __func__);
+				break;
+			}
 
-		if (status != 0) {
-			pr_err("amdkfd: Failed to submit IB to DIQ\n");
-			break;
 		}
-	}
 
-	kfd_gtt_sa_free(dbgdev->dev, mem_obj);
+	} while (false);
+	if (packet_buff_uint != NULL)
+		kfd_gtt_sa_free(dbgdev->dev, mem_obj);
+
 	return status;
+
 }
 
 static int dbgdev_wave_control_set_registers(
 				struct dbg_wave_control_info *wac_info,
 				union SQ_CMD_BITS *in_reg_sq_cmd,
-				union GRBM_GFX_INDEX_BITS *in_reg_gfx_index)
+				union GRBM_GFX_INDEX_BITS *in_reg_gfx_index,
+				unsigned int asic_family)
 {
 	int status = 0;
 	union SQ_CMD_BITS reg_sq_cmd;
 	union GRBM_GFX_INDEX_BITS reg_gfx_index;
-	struct HsaDbgWaveMsgAMDGen2 *pMsg;
-
-	BUG_ON(!wac_info || !in_reg_sq_cmd || !in_reg_gfx_index);
 
 	reg_sq_cmd.u32All = 0;
+
 	reg_gfx_index.u32All = 0;
-	pMsg = &wac_info->dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2;
 
 	switch (wac_info->mode) {
-	/* Send command to single wave */
-	case HSA_DBG_WAVEMODE_SINGLE:
-		/*
-		 * Limit access to the process waves only,
-		 * by setting vmid check
-		 */
+	case HSA_DBG_WAVEMODE_SINGLE:	/*  Send command to single wave  */
+		/*limit access to the process waves only,by setting vmid check */
 		reg_sq_cmd.bits.check_vmid = 1;
-		reg_sq_cmd.bits.simd_id = pMsg->ui32.SIMD;
-		reg_sq_cmd.bits.wave_id = pMsg->ui32.WaveId;
+		reg_sq_cmd.bits.simd_id = wac_info->dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.ui32.SIMD;
+		reg_sq_cmd.bits.wave_id = wac_info->dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.ui32.WaveId;
 		reg_sq_cmd.bits.mode = SQ_IND_CMD_MODE_SINGLE;
 
-		reg_gfx_index.bits.sh_index = pMsg->ui32.ShaderArray;
-		reg_gfx_index.bits.se_index = pMsg->ui32.ShaderEngine;
-		reg_gfx_index.bits.instance_index = pMsg->ui32.HSACU;
+		reg_gfx_index.bits.sh_index = wac_info->dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.ui32.ShaderArray;
+		reg_gfx_index.bits.se_index = wac_info->dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.ui32.ShaderEngine;
+		reg_gfx_index.bits.instance_index = wac_info->dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.ui32.HSACU;
 
 		break;
 
-	/* Send command to all waves with matching VMID */
-	case HSA_DBG_WAVEMODE_BROADCAST_PROCESS:
+	case HSA_DBG_WAVEMODE_BROADCAST_PROCESS:	/*  Send command to all waves with matching VMID  */
+
 
 		reg_gfx_index.bits.sh_broadcast_writes = 1;
 		reg_gfx_index.bits.se_broadcast_writes = 1;
 		reg_gfx_index.bits.instance_broadcast_writes = 1;
 
 		reg_sq_cmd.bits.mode = SQ_IND_CMD_MODE_BROADCAST;
-
 		break;
 
-	/* Send command to all CU waves with matching VMID */
-	case HSA_DBG_WAVEMODE_BROADCAST_PROCESS_CU:
+	case HSA_DBG_WAVEMODE_BROADCAST_PROCESS_CU:	/*  Send command to all CU waves with matching VMID  */
 
 		reg_sq_cmd.bits.check_vmid = 1;
 		reg_sq_cmd.bits.mode = SQ_IND_CMD_MODE_BROADCAST;
 
-		reg_gfx_index.bits.sh_index = pMsg->ui32.ShaderArray;
-		reg_gfx_index.bits.se_index = pMsg->ui32.ShaderEngine;
-		reg_gfx_index.bits.instance_index = pMsg->ui32.HSACU;
+		reg_gfx_index.bits.sh_index = wac_info->dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.ui32.ShaderArray;
+		reg_gfx_index.bits.se_index = wac_info->dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.ui32.ShaderEngine;
+		reg_gfx_index.bits.instance_index = wac_info->dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.ui32.HSACU;
 
 		break;
 
 	default:
-		return -EINVAL;
+		status = -EINVAL;
+		break;
 	}
 
 	switch (wac_info->operand) {
 	case HSA_DBG_WAVEOP_HALT:
-		reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_HALT;
+		if (asic_family == CHIP_KAVERI) {
+			reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_HALT;
+			pr_debug("kfd:dbgdev: halting KV\n");
+		} else {
+			reg_sq_cmd.bits_sethalt.cmd  = SQ_IND_CMD_NEW_SETHALT;
+			reg_sq_cmd.bits_sethalt.data = SQ_IND_CMD_DATA_HALT;
+			pr_debug("kfd:dbgdev: halting CZ\n");
+		}
 		break;
 
 	case HSA_DBG_WAVEOP_RESUME:
-		reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_RESUME;
+		if (asic_family == CHIP_KAVERI) {
+			reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_RESUME;
+			pr_debug("kfd:dbgdev: resuming KV\n");
+		} else {
+			reg_sq_cmd.bits_sethalt.cmd  = SQ_IND_CMD_NEW_SETHALT;
+			reg_sq_cmd.bits_sethalt.data = SQ_IND_CMD_DATA_RESUME;
+			pr_debug("kfd:dbgdev: resuming CZ\n");
+		}
 		break;
 
 	case HSA_DBG_WAVEOP_KILL:
@@ -601,128 +608,114 @@ static int dbgdev_wave_control_set_registers(
 	}
 
 	if (status == 0) {
-		*in_reg_sq_cmd = reg_sq_cmd;
+		*in_reg_sq_cmd    = reg_sq_cmd;
 		*in_reg_gfx_index = reg_gfx_index;
 	}
-
 	return status;
+
 }
 
 static int dbgdev_wave_control_diq(struct kfd_dbgdev *dbgdev,
 					struct dbg_wave_control_info *wac_info)
 {
 
-	int status;
+	int status = 0;
 	union SQ_CMD_BITS reg_sq_cmd;
 	union GRBM_GFX_INDEX_BITS reg_gfx_index;
 	struct kfd_mem_obj *mem_obj;
-	uint32_t *packet_buff_uint;
-	struct pm4__set_config_reg *packets_vec;
+	uint32_t *packet_buff_uint = NULL;
+	struct pm4__set_config_reg *packets_vec = NULL;
 	size_t ib_size = sizeof(struct pm4__set_config_reg) * 3;
 
-	BUG_ON(!dbgdev || !wac_info);
-
 	reg_sq_cmd.u32All = 0;
+	do {
 
-	status = dbgdev_wave_control_set_registers(wac_info, &reg_sq_cmd,
-							&reg_gfx_index);
-	if (status) {
-		pr_err("amdkfd: Failed to set wave control registers\n");
-		return status;
-	}
-
-	/* we do not control the VMID in DIQ,so reset it to a known value */
-	reg_sq_cmd.bits.vm_id = 0;
-
-	pr_debug("\t\t %30s\n", "* * * * * * * * * * * * * * * * * *");
-
-	pr_debug("\t\t mode      is: %u\n", wac_info->mode);
-	pr_debug("\t\t operand   is: %u\n", wac_info->operand);
-	pr_debug("\t\t trap id   is: %u\n", wac_info->trapId);
-	pr_debug("\t\t msg value is: %u\n",
-			wac_info->dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value);
-	pr_debug("\t\t vmid      is: N/A\n");
-
-	pr_debug("\t\t chk_vmid  is : %u\n", reg_sq_cmd.bitfields.check_vmid);
-	pr_debug("\t\t command   is : %u\n", reg_sq_cmd.bitfields.cmd);
-	pr_debug("\t\t queue id  is : %u\n", reg_sq_cmd.bitfields.queue_id);
-	pr_debug("\t\t simd id   is : %u\n", reg_sq_cmd.bitfields.simd_id);
-	pr_debug("\t\t mode      is : %u\n", reg_sq_cmd.bitfields.mode);
-	pr_debug("\t\t vm_id     is : %u\n", reg_sq_cmd.bitfields.vm_id);
-	pr_debug("\t\t wave_id   is : %u\n", reg_sq_cmd.bitfields.wave_id);
-
-	pr_debug("\t\t ibw       is : %u\n",
-			reg_gfx_index.bitfields.instance_broadcast_writes);
-	pr_debug("\t\t ii        is : %u\n",
-			reg_gfx_index.bitfields.instance_index);
-	pr_debug("\t\t sebw      is : %u\n",
-			reg_gfx_index.bitfields.se_broadcast_writes);
-	pr_debug("\t\t se_ind    is : %u\n", reg_gfx_index.bitfields.se_index);
-	pr_debug("\t\t sh_ind    is : %u\n", reg_gfx_index.bitfields.sh_index);
-	pr_debug("\t\t sbw       is : %u\n",
-			reg_gfx_index.bitfields.sh_broadcast_writes);
-
-	pr_debug("\t\t %30s\n", "* * * * * * * * * * * * * * * * * *");
-
-	status = kfd_gtt_sa_allocate(dbgdev->dev, ib_size, &mem_obj);
-
-	if (status != 0) {
-		pr_err("amdkfd: Failed to allocate GART memory\n");
-		return status;
-	}
-
-	packet_buff_uint = mem_obj->cpu_ptr;
+		status = dbgdev_wave_control_set_registers(wac_info,
+				&reg_sq_cmd,
+				&reg_gfx_index,
+				dbgdev->dev->device_info->asic_family);
 
-	memset(packet_buff_uint, 0, ib_size);
+		/* we do not control the VMID in DIQ,so reset it to a known value */
+		reg_sq_cmd.bits.vm_id = 0;
+		if (status != 0)
+			break;
+		pr_debug("\t\t %30s\n", "* * * * * * * * * * * * * * * * * *");
+
+		pr_debug("\t\t mode      is: %u\n", wac_info->mode);
+		pr_debug("\t\t operand   is: %u\n", wac_info->operand);
+		pr_debug("\t\t trap id   is: %u\n", wac_info->trapId);
+		pr_debug("\t\t msg value is: %u\n", wac_info->dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value);
+		pr_debug("\t\t vmid      is: N/A\n");
+
+		pr_debug("\t\t chk_vmid  is : %u\n", reg_sq_cmd.bitfields.check_vmid);
+		pr_debug("\t\t command   is : %u\n", reg_sq_cmd.bitfields.cmd);
+		pr_debug("\t\t queue id  is : %u\n", reg_sq_cmd.bitfields.queue_id);
+		pr_debug("\t\t simd id   is : %u\n", reg_sq_cmd.bitfields.simd_id);
+		pr_debug("\t\t mode      is : %u\n", reg_sq_cmd.bitfields.mode);
+		pr_debug("\t\t vm_id     is : %u\n", reg_sq_cmd.bitfields.vm_id);
+		pr_debug("\t\t wave_id   is : %u\n", reg_sq_cmd.bitfields.wave_id);
+
+		pr_debug("\t\t ibw       is : %u\n", reg_gfx_index.bitfields.instance_broadcast_writes);
+		pr_debug("\t\t ii        is : %u\n", reg_gfx_index.bitfields.instance_index);
+		pr_debug("\t\t sebw      is : %u\n", reg_gfx_index.bitfields.se_broadcast_writes);
+		pr_debug("\t\t se_ind    is : %u\n", reg_gfx_index.bitfields.se_index);
+		pr_debug("\t\t sh_ind    is : %u\n", reg_gfx_index.bitfields.sh_index);
+		pr_debug("\t\t sbw       is : %u\n", reg_gfx_index.bitfields.sh_broadcast_writes);
+
+		pr_debug("\t\t %30s\n", "* * * * * * * * * * * * * * * * * *");
+
+		status = kfd_gtt_sa_allocate(dbgdev->dev, ib_size, &mem_obj);
+
+		if (status != 0)
+			break;
 
-	packets_vec =  (struct pm4__set_config_reg *) packet_buff_uint;
-	packets_vec[0].header.count = 1;
-	packets_vec[0].header.opcode = IT_SET_UCONFIG_REG;
-	packets_vec[0].header.type = PM4_TYPE_3;
-	packets_vec[0].bitfields2.reg_offset =
-			GRBM_GFX_INDEX / (sizeof(uint32_t)) -
-				USERCONFIG_REG_BASE;
+		packet_buff_uint = mem_obj->cpu_ptr;
 
-	packets_vec[0].bitfields2.insert_vmid = 0;
-	packets_vec[0].reg_data[0] = reg_gfx_index.u32All;
+		memset(packet_buff_uint, 0, ib_size);
 
-	packets_vec[1].header.count = 1;
-	packets_vec[1].header.opcode = IT_SET_CONFIG_REG;
-	packets_vec[1].header.type = PM4_TYPE_3;
-	packets_vec[1].bitfields2.reg_offset = SQ_CMD / (sizeof(uint32_t)) -
-						AMD_CONFIG_REG_BASE;
+		packets_vec =  (struct pm4__set_config_reg *) packet_buff_uint;
+		packets_vec[0].header.count = 1;
+		packets_vec[0].header.opcode = IT_SET_UCONFIG_REG;
+		packets_vec[0].header.type = PM4_TYPE_3;
+		packets_vec[0].bitfields2.reg_offset = GRBM_GFX_INDEX / (sizeof(uint32_t)) - USERCONFIG_REG_BASE;
+		packets_vec[0].bitfields2.insert_vmid = 0;
+		packets_vec[0].reg_data[0] = reg_gfx_index.u32All;
 
-	packets_vec[1].bitfields2.vmid_shift = SQ_CMD_VMID_OFFSET;
-	packets_vec[1].bitfields2.insert_vmid = 1;
-	packets_vec[1].reg_data[0] = reg_sq_cmd.u32All;
+		packets_vec[1].header.count = 1;
+		packets_vec[1].header.opcode = IT_SET_CONFIG_REG;
+		packets_vec[1].header.type = PM4_TYPE_3;
+		packets_vec[1].bitfields2.reg_offset = SQ_CMD / (sizeof(uint32_t)) - CONFIG_REG_BASE;
+		packets_vec[1].bitfields2.vmid_shift = SQ_CMD_VMID_OFFSET;
+		packets_vec[1].bitfields2.insert_vmid = 1;
+		packets_vec[1].reg_data[0] = reg_sq_cmd.u32All;
 
-	/* Restore the GRBM_GFX_INDEX register */
+		/* Restore the GRBM_GFX_INDEX register */
 
-	reg_gfx_index.u32All = 0;
-	reg_gfx_index.bits.sh_broadcast_writes = 1;
-	reg_gfx_index.bits.instance_broadcast_writes = 1;
-	reg_gfx_index.bits.se_broadcast_writes = 1;
+		reg_gfx_index.u32All = 0;
+		reg_gfx_index.bits.sh_broadcast_writes = 1;
+		reg_gfx_index.bits.instance_broadcast_writes = 1;
+		reg_gfx_index.bits.se_broadcast_writes = 1;
 
 
-	packets_vec[2].ordinal1 = packets_vec[0].ordinal1;
-	packets_vec[2].bitfields2.reg_offset =
-				GRBM_GFX_INDEX / (sizeof(uint32_t)) -
-					USERCONFIG_REG_BASE;
+		packets_vec[2].ordinal1 = packets_vec[0].ordinal1;
+		packets_vec[2].bitfields2.reg_offset = GRBM_GFX_INDEX / (sizeof(uint32_t)) - USERCONFIG_REG_BASE;
+		packets_vec[2].bitfields2.insert_vmid = 0;
+		packets_vec[2].reg_data[0] = reg_gfx_index.u32All;
 
-	packets_vec[2].bitfields2.insert_vmid = 0;
-	packets_vec[2].reg_data[0] = reg_gfx_index.u32All;
+		status = dbgdev_diq_submit_ib(
+				dbgdev,
+				wac_info->process->pasid,
+				mem_obj->gpu_addr,
+				packet_buff_uint,
+				ib_size);
 
-	status = dbgdev_diq_submit_ib(
-			dbgdev,
-			wac_info->process->pasid,
-			mem_obj->gpu_addr,
-			packet_buff_uint,
-			ib_size);
+		if (status != 0)
+			pr_debug("%s\n", " Critical Error ! Submit diq packet failed ");
 
-	if (status != 0)
-		pr_err("amdkfd: Failed to submit IB to DIQ\n");
+	} while (false);
 
-	kfd_gtt_sa_free(dbgdev->dev, mem_obj);
+	if (packet_buff_uint != NULL)
+		kfd_gtt_sa_free(dbgdev->dev, mem_obj);
 
 	return status;
 }
@@ -730,66 +723,69 @@ static int dbgdev_wave_control_diq(struct kfd_dbgdev *dbgdev,
 static int dbgdev_wave_control_nodiq(struct kfd_dbgdev *dbgdev,
 					struct dbg_wave_control_info *wac_info)
 {
-	int status;
+	int status = 0;
+	unsigned int vmid = 0xffff;
 	union SQ_CMD_BITS reg_sq_cmd;
 	union GRBM_GFX_INDEX_BITS reg_gfx_index;
-	struct kfd_process_device *pdd;
 
-	BUG_ON(!dbgdev || !dbgdev->dev || !wac_info);
+	struct kfd_process_device *pdd = NULL;
 
 	reg_sq_cmd.u32All = 0;
+	status = 0;
 
 	/* taking the VMID for that process on the safe way using PDD */
 	pdd = kfd_get_process_device_data(dbgdev->dev, wac_info->process);
 
-	if (!pdd) {
-		pr_err("amdkfd: Failed to get pdd for wave control no DIQ\n");
-		return -EFAULT;
-	}
-	status = dbgdev_wave_control_set_registers(wac_info, &reg_sq_cmd,
-							&reg_gfx_index);
-	if (status) {
-		pr_err("amdkfd: Failed to set wave control registers\n");
-		return status;
+	if (pdd) {
+		status = dbgdev_wave_control_set_registers(wac_info,
+				&reg_sq_cmd,
+				&reg_gfx_index,
+				dbgdev->dev->device_info->asic_family);
+		if (status == 0) {
+
+			/* for non DIQ we need to patch the VMID: */
+
+			vmid = pdd->qpd.vmid;
+			reg_sq_cmd.bits.vm_id = vmid;
+
+			pr_debug("\t\t %30s\n", "* * * * * * * * * * * * * * * * * *");
+
+			pr_debug("\t\t mode      is: %u\n", wac_info->mode);
+			pr_debug("\t\t operand   is: %u\n", wac_info->operand);
+			pr_debug("\t\t trap id   is: %u\n", wac_info->trapId);
+			pr_debug("\t\t msg value is: %u\n", wac_info->dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value);
+			pr_debug("\t\t vmid      is: %u\n", vmid);
+
+			pr_debug("\t\t chk_vmid  is : %u\n", reg_sq_cmd.bitfields.check_vmid);
+			pr_debug("\t\t command   is : %u\n", reg_sq_cmd.bitfields.cmd);
+			pr_debug("\t\t queue id  is : %u\n", reg_sq_cmd.bitfields.queue_id);
+			pr_debug("\t\t simd id   is : %u\n", reg_sq_cmd.bitfields.simd_id);
+			pr_debug("\t\t mode      is : %u\n", reg_sq_cmd.bitfields.mode);
+			pr_debug("\t\t vm_id     is : %u\n", reg_sq_cmd.bitfields.vm_id);
+			pr_debug("\t\t wave_id   is : %u\n", reg_sq_cmd.bitfields.wave_id);
+
+			pr_debug("\t\t ibw       is : %u\n", reg_gfx_index.bitfields.instance_broadcast_writes);
+			pr_debug("\t\t ii        is : %u\n", reg_gfx_index.bitfields.instance_index);
+			pr_debug("\t\t sebw      is : %u\n", reg_gfx_index.bitfields.se_broadcast_writes);
+			pr_debug("\t\t se_ind    is : %u\n", reg_gfx_index.bitfields.se_index);
+			pr_debug("\t\t sh_ind    is : %u\n", reg_gfx_index.bitfields.sh_index);
+			pr_debug("\t\t sbw       is : %u\n", reg_gfx_index.bitfields.sh_broadcast_writes);
+
+			pr_debug("\t\t %30s\n", "* * * * * * * * * * * * * * * * * *");
+
+			dbgdev->dev->kfd2kgd
+				->wave_control_execute(dbgdev->dev->kgd,
+							reg_gfx_index.u32All,
+							reg_sq_cmd.u32All);
+		} else {
+			status = -EINVAL;
+		}
+	} else {
+		status = -EFAULT;
 	}
 
-	/* for non DIQ we need to patch the VMID: */
+	return status;
 
-	reg_sq_cmd.bits.vm_id = pdd->qpd.vmid;
-
-	pr_debug("\t\t %30s\n", "* * * * * * * * * * * * * * * * * *");
-
-	pr_debug("\t\t mode      is: %u\n", wac_info->mode);
-	pr_debug("\t\t operand   is: %u\n", wac_info->operand);
-	pr_debug("\t\t trap id   is: %u\n", wac_info->trapId);
-	pr_debug("\t\t msg value is: %u\n",
-			wac_info->dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value);
-	pr_debug("\t\t vmid      is: %u\n", pdd->qpd.vmid);
-
-	pr_debug("\t\t chk_vmid  is : %u\n", reg_sq_cmd.bitfields.check_vmid);
-	pr_debug("\t\t command   is : %u\n", reg_sq_cmd.bitfields.cmd);
-	pr_debug("\t\t queue id  is : %u\n", reg_sq_cmd.bitfields.queue_id);
-	pr_debug("\t\t simd id   is : %u\n", reg_sq_cmd.bitfields.simd_id);
-	pr_debug("\t\t mode      is : %u\n", reg_sq_cmd.bitfields.mode);
-	pr_debug("\t\t vm_id     is : %u\n", reg_sq_cmd.bitfields.vm_id);
-	pr_debug("\t\t wave_id   is : %u\n", reg_sq_cmd.bitfields.wave_id);
-
-	pr_debug("\t\t ibw       is : %u\n",
-			reg_gfx_index.bitfields.instance_broadcast_writes);
-	pr_debug("\t\t ii        is : %u\n",
-			reg_gfx_index.bitfields.instance_index);
-	pr_debug("\t\t sebw      is : %u\n",
-			reg_gfx_index.bitfields.se_broadcast_writes);
-	pr_debug("\t\t se_ind    is : %u\n", reg_gfx_index.bitfields.se_index);
-	pr_debug("\t\t sh_ind    is : %u\n", reg_gfx_index.bitfields.sh_index);
-	pr_debug("\t\t sbw       is : %u\n",
-			reg_gfx_index.bitfields.sh_broadcast_writes);
-
-	pr_debug("\t\t %30s\n", "* * * * * * * * * * * * * * * * * *");
-
-	return dbgdev->dev->kfd2kgd->wave_control_execute(dbgdev->dev->kgd,
-							reg_gfx_index.u32All,
-							reg_sq_cmd.u32All);
 }
 
 int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p)
@@ -800,13 +796,8 @@ int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p)
 	union GRBM_GFX_INDEX_BITS reg_gfx_index;
 	struct kfd_process_device *pdd;
 	struct dbg_wave_control_info wac_info;
-	int temp;
-	int first_vmid_to_scan = 8;
-	int last_vmid_to_scan = 15;
-
-	first_vmid_to_scan = ffs(dev->shared_resources.compute_vmid_bitmap) - 1;
-	temp = dev->shared_resources.compute_vmid_bitmap >> first_vmid_to_scan;
-	last_vmid_to_scan = first_vmid_to_scan + ffz(temp);
+	int first_vmid_to_scan = dev->vm_info.first_vmid_kfd;
+	int last_vmid_to_scan = dev->vm_info.last_vmid_kfd;
 
 	reg_sq_cmd.u32All = 0;
 	status = 0;
@@ -823,7 +814,7 @@ int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p)
 	for (vmid = first_vmid_to_scan; vmid <= last_vmid_to_scan; vmid++) {
 		if (dev->kfd2kgd->get_atc_vmid_pasid_mapping_valid
 				(dev->kgd, vmid)) {
-			if (dev->kfd2kgd->get_atc_vmid_pasid_mapping_valid
+			if (dev->kfd2kgd->get_atc_vmid_pasid_mapping_pasid
 					(dev->kgd, vmid) == p->pasid) {
 				pr_debug("Killing wave fronts of vmid %d and pasid %d\n",
 						vmid, p->pasid);
@@ -833,7 +824,7 @@ int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p)
 	}
 
 	if (vmid > last_vmid_to_scan) {
-		pr_err("amdkfd: didn't found vmid for pasid (%d)\n", p->pasid);
+		pr_err("amdkfd: didn't find vmid for pasid (%d)\n", p->pasid);
 		return -EFAULT;
 	}
 
@@ -843,7 +834,7 @@ int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p)
 		return -EFAULT;
 
 	status = dbgdev_wave_control_set_registers(&wac_info, &reg_sq_cmd,
-			&reg_gfx_index);
+			&reg_gfx_index, dev->device_info->asic_family);
 	if (status != 0)
 		return -EINVAL;
 
@@ -858,15 +849,12 @@ int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p)
 }
 
 void kfd_dbgdev_init(struct kfd_dbgdev *pdbgdev, struct kfd_dev *pdev,
-			enum DBGDEV_TYPE type)
+			DBGDEV_TYPE type)
 {
-	BUG_ON(!pdbgdev || !pdev);
-
 	pdbgdev->dev = pdev;
 	pdbgdev->kq = NULL;
 	pdbgdev->type = type;
 	pdbgdev->pqm = NULL;
-
 	switch (type) {
 	case DBGDEV_TYPE_NODIQ:
 		pdbgdev->dbgdev_register = dbgdev_register_nodiq;
@@ -876,10 +864,12 @@ void kfd_dbgdev_init(struct kfd_dbgdev *pdbgdev, struct kfd_dev *pdev,
 		break;
 	case DBGDEV_TYPE_DIQ:
 	default:
+
 		pdbgdev->dbgdev_register = dbgdev_register_diq;
 		pdbgdev->dbgdev_unregister = dbgdev_unregister_diq;
 		pdbgdev->dbgdev_wave_control =  dbgdev_wave_control_diq;
 		pdbgdev->dbgdev_address_watch = dbgdev_address_watch_diq;
+
 		break;
 	}
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.h b/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.h
index 03424c2..82f48ff 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.h
@@ -23,6 +23,10 @@
 #ifndef KFD_DBGDEV_H_
 #define KFD_DBGDEV_H_
 
+/*
+ * SQ_IND_CMD_CMD enum
+ */
+
 enum {
 	SQ_CMD_VMID_OFFSET = 28,
 	ADDRESS_WATCH_CNTL_OFFSET = 24
@@ -48,9 +52,9 @@ enum {
 
 /* CONFIG reg space definition */
 enum {
-	AMD_CONFIG_REG_BASE = 0x2000,	/* in dwords */
-	AMD_CONFIG_REG_END = 0x2B00,
-	AMD_CONFIG_REG_SIZE = AMD_CONFIG_REG_END - AMD_CONFIG_REG_BASE
+	CONFIG_REG_BASE = 0x2000,	/* in dwords */
+	CONFIG_REG_END = 0x2B00,
+	CONFIG_REG_SIZE = CONFIG_REG_END - CONFIG_REG_BASE
 };
 
 /* SH reg space definition */
@@ -60,22 +64,43 @@ enum {
 	SH_REG_SIZE = SH_REG_END - SH_REG_BASE
 };
 
+/* SQ_CMD definitions */
+
+enum {
+	SQ_IND_CMD_DATA_RESUME = 0,
+	SQ_IND_CMD_DATA_HALT = 1
+};
+
+enum SQ_IND_CMD_NEW {
+	SQ_IND_CMD_NEW_NULL = 0x00000000,
+	SQ_IND_CMD_NEW_SETHALT = 0x00000001,
+	SQ_IND_CMD_NEW_SAVECTX = 0x00000002,
+	SQ_IND_CMD_NEW_KILL = 0x00000003,
+	SQ_IND_CMD_NEW_DEBUG = 0x00000004,
+	SQ_IND_CMD_NEW_TRAP = 0x00000005,
+	SQ_IND_CMD_NEW_SET_PRIO = 0x00000006
+
+};
+
 enum SQ_IND_CMD_CMD {
 	SQ_IND_CMD_CMD_NULL = 0x00000000,
 	SQ_IND_CMD_CMD_HALT = 0x00000001,
 	SQ_IND_CMD_CMD_RESUME = 0x00000002,
 	SQ_IND_CMD_CMD_KILL = 0x00000003,
 	SQ_IND_CMD_CMD_DEBUG = 0x00000004,
-	SQ_IND_CMD_CMD_TRAP = 0x00000005,
+	SQ_IND_CMD_CMD_TRAP = 0x00000005
 };
+/*
+ * SQ_IND_CMD_MODE enum
+ */
 
-enum SQ_IND_CMD_MODE {
+typedef enum SQ_IND_CMD_MODE {
 	SQ_IND_CMD_MODE_SINGLE = 0x00000000,
 	SQ_IND_CMD_MODE_BROADCAST = 0x00000001,
 	SQ_IND_CMD_MODE_BROADCAST_QUEUE = 0x00000002,
 	SQ_IND_CMD_MODE_BROADCAST_PIPE = 0x00000003,
 	SQ_IND_CMD_MODE_BROADCAST_ME = 0x00000004,
-};
+} SQ_IND_CMD_MODE;
 
 union SQ_IND_INDEX_BITS {
 	struct {
@@ -106,18 +131,32 @@ union SQ_IND_CMD_BITS {
 union SQ_CMD_BITS {
 	struct {
 		uint32_t cmd:3;
-		 uint32_t:1;
+		uint32_t:1;
 		uint32_t mode:3;
 		uint32_t check_vmid:1;
 		uint32_t trap_id:3;
-		 uint32_t:5;
+		uint32_t:5;
 		uint32_t wave_id:4;
 		uint32_t simd_id:2;
-		 uint32_t:2;
+		uint32_t:2;
 		uint32_t queue_id:3;
-		 uint32_t:1;
+		uint32_t:1;
 		uint32_t vm_id:4;
 	} bitfields, bits;
+	struct {
+		uint32_t cmd:3;
+		uint32_t:1;
+		uint32_t mode:3;
+		uint32_t check_vmid:1;
+		uint32_t data:3;
+		uint32_t:5;
+		uint32_t wave_id:4;
+		uint32_t simd_id:2;
+		uint32_t:2;
+		uint32_t queue_id:3;
+		uint32_t:1;
+		uint32_t vm_id:4;
+	} bitfields_sethalt, bits_sethalt;
 	uint32_t u32All;
 	signed int i32All;
 	float f32All;
@@ -169,7 +208,7 @@ union TCP_WATCH_ADDR_L_BITS {
 };
 
 enum {
-	QUEUESTATE__INVALID = 0, /* so by default we'll get invalid state */
+	QUEUESTATE__INVALID = 0,	/* so by default we'll get invalid state */
 	QUEUESTATE__ACTIVE_COMPLETION_PENDING,
 	QUEUESTATE__ACTIVE
 };
@@ -187,7 +226,6 @@ union ULARGE_INTEGER {
 #define KFD_CIK_VMID_END_OFFSET (KFD_CIK_VMID_START_OFFSET + (8))
 
 
-void kfd_dbgdev_init(struct kfd_dbgdev *pdbgdev, struct kfd_dev *pdev,
-			enum DBGDEV_TYPE type);
+void kfd_dbgdev_init(struct kfd_dbgdev *pdbgdev, struct kfd_dev *pdev, DBGDEV_TYPE type);
 
-#endif	/* KFD_DBGDEV_H_ */
+#endif				/* KFD_DBGDEV_H_ */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.c b/drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.c
index 56d6763..5d269ea 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.c
@@ -36,42 +36,50 @@
 
 static DEFINE_MUTEX(kfd_dbgmgr_mutex);
 
-struct mutex *kfd_get_dbgmgr_mutex(void)
+struct mutex *
+get_dbgmgr_mutex(void)
 {
 	return &kfd_dbgmgr_mutex;
 }
 
+/*===========================================================================*/
 
-static void kfd_dbgmgr_uninitialize(struct kfd_dbgmgr *pmgr)
+static void
+kfd_dbgmgr_uninitialize(struct kfd_dbgmgr *pmgr)
 {
-	BUG_ON(!pmgr);
-
 	kfree(pmgr->dbgdev);
-
 	pmgr->dbgdev = NULL;
 	pmgr->pasid = 0;
 	pmgr->dev = NULL;
 }
 
-void kfd_dbgmgr_destroy(struct kfd_dbgmgr *pmgr)
+/*===========================================================================*/
+
+void
+kfd_dbgmgr_destroy(struct kfd_dbgmgr *pmgr)
 {
 	if (pmgr != NULL) {
 		kfd_dbgmgr_uninitialize(pmgr);
 		kfree(pmgr);
+		pmgr = NULL;
 	}
 }
 
-bool kfd_dbgmgr_create(struct kfd_dbgmgr **ppmgr, struct kfd_dev *pdev)
+/*===========================================================================*/
+
+bool
+kfd_dbgmgr_create(struct kfd_dbgmgr **ppmgr, struct kfd_dev *pdev)
 {
-	enum DBGDEV_TYPE type = DBGDEV_TYPE_DIQ;
+	DBGDEV_TYPE  type = DBGDEV_TYPE_DIQ;
 	struct kfd_dbgmgr *new_buff;
 
 	BUG_ON(pdev == NULL);
 	BUG_ON(!pdev->init_complete);
 
 	new_buff = kfd_alloc_struct(new_buff);
-	if (!new_buff) {
-		pr_err("amdkfd: Failed to allocate dbgmgr instance\n");
+	if (!new_buff)
+	{
+		dev_err(NULL, "Error! kfd: In func %s >> failed to allocate dbgmgr instance\n", __func__);
 		return false;
 	}
 
@@ -79,7 +87,7 @@ bool kfd_dbgmgr_create(struct kfd_dbgmgr **ppmgr, struct kfd_dev *pdev)
 	new_buff->dev = pdev;
 	new_buff->dbgdev = kfd_alloc_struct(new_buff->dbgdev);
 	if (!new_buff->dbgdev) {
-		pr_err("amdkfd: Failed to allocate dbgdev instance\n");
+		dev_err(NULL, "Error! kfd: In func %s >> failed to allocate dbgdev\n", __func__);
 		kfree(new_buff);
 		return false;
 	}
@@ -94,75 +102,200 @@ bool kfd_dbgmgr_create(struct kfd_dbgmgr **ppmgr, struct kfd_dev *pdev)
 	return true;
 }
 
-long kfd_dbgmgr_register(struct kfd_dbgmgr *pmgr, struct kfd_process *p)
+/*===========================================================================*/
+
+long
+kfd_dbgmgr_register(struct kfd_dbgmgr *pmgr, struct kfd_process *p)
 {
-	BUG_ON(!p || !pmgr || !pmgr->dbgdev);
+	long status = 0;
 
-	if (pmgr->pasid != 0) {
-		pr_debug("H/W debugger is already active using pasid %d\n",
-				pmgr->pasid);
-		return -EBUSY;
-	}
+	do {
+
+		if ((pmgr == NULL) || (pmgr->dev == NULL) || (pmgr->dbgdev == NULL)) {
+			dev_info(NULL, "Error! kfd: In func %s >> Illegal pointers\n", __func__);
+			/*  Invalid Pointer.  */
+			status = -EINVAL;
+			break;
+		}
+		if (pmgr->pasid != 0) {
+			/*  HW debugger is already active.  */
+			status = -EBUSY;
+			break;
+		}
+
+		/* remember pasid */
+
+		pmgr->pasid = p->pasid;
+
+		/* provide the pqm for diq generation */
 
-	/* remember pasid */
-	pmgr->pasid = p->pasid;
+		pmgr->dbgdev->pqm = &p->pqm;
 
-	/* provide the pqm for diq generation */
-	pmgr->dbgdev->pqm = &p->pqm;
+		/* activate the actual registering */
+		/* todo: you should lock with the process mutex here */
+		pmgr->dbgdev->dbgdev_register(pmgr->dbgdev);
+		/* todo: you should unlock with the process mutex here  */
 
-	/* activate the actual registering */
-	pmgr->dbgdev->dbgdev_register(pmgr->dbgdev);
+	} while (false);
 
-	return 0;
+	return status;
 }
 
-long kfd_dbgmgr_unregister(struct kfd_dbgmgr *pmgr, struct kfd_process *p)
+/* ========================================================================== */
+
+long
+kfd_dbgmgr_unregister(struct kfd_dbgmgr *pmgr, struct kfd_process *p)
 {
-	BUG_ON(!p || !pmgr || !pmgr->dbgdev);
 
-	/* Is the requests coming from the already registered process? */
-	if (pmgr->pasid != p->pasid) {
-		pr_debug("H/W debugger is not registered by calling pasid %d\n",
-				p->pasid);
-		return -EINVAL;
-	}
+	long status = 0;
 
-	pmgr->dbgdev->dbgdev_unregister(pmgr->dbgdev);
+	do {
 
-	pmgr->pasid = 0;
+		if ((pmgr == NULL) || (pmgr->dev == NULL)
+				|| (pmgr->dbgdev == NULL) || (p == NULL)) {
+			dev_info(NULL, "Error! kfd: In func %s >> Illegal pointers\n", __func__);
+			/* Invalid Pointer */
+			status = -EINVAL;
+			break;
+		}
+		if (pmgr->pasid != p->pasid) {
+			/* Is the requests coming from the already registered process? */
+			status = -EINVAL;
+			break;
+		}
+
+		/* todo: you should lock with the process mutex here */
+
+		pmgr->dbgdev->dbgdev_unregister(pmgr->dbgdev);
 
-	return 0;
+		/* todo: you should unlock with the process mutex here  */
+
+		pmgr->pasid = 0;
+
+	} while (false);
+
+	return status;
 }
 
-long kfd_dbgmgr_wave_control(struct kfd_dbgmgr *pmgr,
-				struct dbg_wave_control_info *wac_info)
+/* =========================================================================== */
+
+long
+kfd_dbgmgr_wave_control(struct kfd_dbgmgr *pmgr, struct dbg_wave_control_info *wac_info)
 {
-	BUG_ON(!pmgr || !pmgr->dbgdev || !wac_info);
+	long status = 0;
 
-	/* Is the requests coming from the already registered process? */
-	if (pmgr->pasid != wac_info->process->pasid) {
-		pr_debug("H/W debugger support was not registered for requester pasid %d\n",
-				wac_info->process->pasid);
-		return -EINVAL;
-	}
+	dev_info(NULL, "kfd: In func %s\n", __func__);
+
+	do {
+
+		if ((pmgr == NULL) || (pmgr->dev == NULL) || (pmgr->dbgdev == NULL) || (wac_info == NULL)
+		    || (wac_info->process == NULL)) {
+			/* Invalid Pointer */
+			dev_info(NULL, "Error! kfd: In func %s >> Illegal pointers\n", __func__);
+			status = -EINVAL;
+			break;
+		}
+		/* Is the requests coming from the already registered process? */
+		if (pmgr->pasid != wac_info->process->pasid) {
+			/* HW debugger support was not registered for requester process */
+			status = -EINVAL;
+			break;
+		}
+
+		status = (long) pmgr->dbgdev->dbgdev_wave_control(pmgr->dbgdev, wac_info);
+
+	} while (false);
+
+	return status;
 
-	return (long) pmgr->dbgdev->dbgdev_wave_control(pmgr->dbgdev, wac_info);
 }
 
-long kfd_dbgmgr_address_watch(struct kfd_dbgmgr *pmgr,
-				struct dbg_address_watch_info *adw_info)
+/* =========================================================================== */
+
+long
+kfd_dbgmgr_address_watch(struct kfd_dbgmgr *pmgr, struct dbg_address_watch_info *adw_info)
 {
-	BUG_ON(!pmgr || !pmgr->dbgdev || !adw_info);
+	long status = 0;
 
+	dev_info(NULL, "kfd: In func %s\n", __func__);
 
-	/* Is the requests coming from the already registered process? */
-	if (pmgr->pasid != adw_info->process->pasid) {
-		pr_debug("H/W debugger support was not registered for requester pasid %d\n",
-				adw_info->process->pasid);
-		return -EINVAL;
-	}
+	do {
+
+		if ((pmgr == NULL) || (pmgr->dev == NULL) || (pmgr->dbgdev == NULL) || (adw_info == NULL)
+		    || (adw_info->process == NULL)) {
+			/* Invalid Pointer */
+			dev_info(NULL, "Error! kfd: In func %s >> Illegal pointers\n", __func__);
+			status = -EINVAL;
+			break;
+		}
+		/* Is the requests coming from the already registered process? */
+		if (pmgr->pasid != adw_info->process->pasid) {
+			/* HW debugger support was not registered for requester process */
+			status = -EINVAL;
+			break;
+		}
+
+		status = (long) pmgr->dbgdev->dbgdev_address_watch(pmgr->dbgdev, adw_info);
+
+	} while (false);
+
+	return status;
 
-	return (long) pmgr->dbgdev->dbgdev_address_watch(pmgr->dbgdev,
-							adw_info);
 }
 
+
+/* =========================================================================== */
+/*
+ * Handle abnormal process termination
+ * if we are in the midst of a debug session, we should kill all pending waves
+ * of the debugged process and unregister the process from the Debugger.
+ */
+long
+kfd_dbgmgr_abnormal_termination(struct kfd_dbgmgr *pmgr, struct kfd_process *process)
+{
+	long status = 0;
+	struct dbg_wave_control_info wac_info;
+
+	dev_info(NULL, "kfd: In func %s\n", __func__);
+
+	do {
+
+		if ((pmgr == NULL) || (pmgr->dev == NULL) || (pmgr->dbgdev == NULL)) {
+			/* Invalid Pointer */
+			dev_info(NULL, "Error! kfd: In func %s >> Illegal pointers\n", __func__);
+			status = -EINVAL;
+			break;
+		}
+		/* first, we kill all the wavefronts of this process */
+
+		wac_info.process = process;
+		wac_info.mode = HSA_DBG_WAVEMODE_BROADCAST_PROCESS;
+		wac_info.operand = HSA_DBG_WAVEOP_KILL;
+		wac_info.trapId  = 0x0; /* not used for the KILL */
+		wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value = 0; /* not used for kill */
+		wac_info.dbgWave_msg.MemoryVA = NULL; /* not used for kill  */
+
+		status = (long) pmgr->dbgdev->dbgdev_wave_control(pmgr->dbgdev, &wac_info);
+
+		if (status != 0) {
+			dev_info(NULL, "Error! kfd: In func %s: wave control failed, status is: %ld\n", __func__, status);
+			break;
+		}
+		if (pmgr->pasid == wac_info.process->pasid) {
+				/* if terminated process was registered for debug, then unregister it  */
+				status = kfd_dbgmgr_unregister(pmgr, process);
+				pmgr->pasid = 0;
+		}
+		if (status != 0)
+			dev_info(NULL,
+					"Error! kfd: In func %s: unregister failed, status is: %ld debugger can not be reused\n",
+					__func__, status);
+
+	} while (false);
+
+	return status;
+
+}
+
+
+/*///////////////////////////////////////////////////////////////////////////////////////// */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.h b/drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.h
index 257a745..2b6484e 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.h
@@ -26,252 +26,242 @@
 
 #include "kfd_priv.h"
 
-/* must align with hsakmttypes definition */
+/*
+ * SQ_IND_CMD_CMD enum
+ */
+
+
+/* must align with hsakmttypes definition. */
 #pragma pack(push, 4)
 
-enum HSA_DBG_WAVEOP {
-	HSA_DBG_WAVEOP_HALT = 1,	/* Halts a wavefront		*/
-	HSA_DBG_WAVEOP_RESUME = 2,	/* Resumes a wavefront		*/
-	HSA_DBG_WAVEOP_KILL = 3,	/* Kills a wavefront		*/
-	HSA_DBG_WAVEOP_DEBUG = 4,	/* Causes wavefront to enter
-						debug mode		*/
-	HSA_DBG_WAVEOP_TRAP = 5,	/* Causes wavefront to take
-						a trap			*/
+typedef enum _HSA_DBG_WAVEOP {
+	HSA_DBG_WAVEOP_HALT = 1,	/* Halts a wavefront  */
+	HSA_DBG_WAVEOP_RESUME = 2,	/* Resumes a wavefront  */
+	HSA_DBG_WAVEOP_KILL = 3,	/* Kills a wavefront  */
+	HSA_DBG_WAVEOP_DEBUG = 4,	/* Causes wavefront to enter debug mode  */
+	HSA_DBG_WAVEOP_TRAP = 5,	/* Causes wavefront to take a trap  */
 	HSA_DBG_NUM_WAVEOP = 5,
 	HSA_DBG_MAX_WAVEOP = 0xFFFFFFFF
-};
+} HSA_DBG_WAVEOP;
 
-enum HSA_DBG_WAVEMODE {
-	/* send command to a single wave */
-	HSA_DBG_WAVEMODE_SINGLE = 0,
-	/*
-	 * Broadcast to all wavefronts of all processes is not
-	 * supported for HSA user mode
-	 */
-
-	/* send to waves within current process */
-	HSA_DBG_WAVEMODE_BROADCAST_PROCESS = 2,
-	/* send to waves within current process on CU  */
-	HSA_DBG_WAVEMODE_BROADCAST_PROCESS_CU = 3,
+typedef enum _HSA_DBG_WAVEMODE {
+	HSA_DBG_WAVEMODE_SINGLE = 0,	/* send command to a single wave  */
+	/* Broadcast to all wavefronts of all processes is not supported for HSA user mode */
+	HSA_DBG_WAVEMODE_BROADCAST_PROCESS = 2,	/* send to waves within current process  */
+	HSA_DBG_WAVEMODE_BROADCAST_PROCESS_CU = 3,	/* send to waves within current process on CU  */
 	HSA_DBG_NUM_WAVEMODE = 3,
 	HSA_DBG_MAX_WAVEMODE = 0xFFFFFFFF
-};
+} HSA_DBG_WAVEMODE;
 
-enum HSA_DBG_WAVEMSG_TYPE {
+typedef enum _HSA_DBG_WAVEMSG_TYPE {
 	HSA_DBG_WAVEMSG_AUTO = 0,
 	HSA_DBG_WAVEMSG_USER = 1,
 	HSA_DBG_WAVEMSG_ERROR = 2,
 	HSA_DBG_NUM_WAVEMSG,
 	HSA_DBG_MAX_WAVEMSG = 0xFFFFFFFF
-};
+} HSA_DBG_WAVEMSG_TYPE;
 
-enum HSA_DBG_WATCH_MODE {
-	HSA_DBG_WATCH_READ = 0,		/* Read operations only */
-	HSA_DBG_WATCH_NONREAD = 1,	/* Write or Atomic operations only */
-	HSA_DBG_WATCH_ATOMIC = 2,	/* Atomic Operations only */
-	HSA_DBG_WATCH_ALL = 3,		/* Read, Write or Atomic operations */
+typedef enum _HSA_DBG_WATCH_MODE {
+	HSA_DBG_WATCH_READ = 0,	/* Read operations only  */
+	HSA_DBG_WATCH_NONREAD = 1,	/* Write or Atomic operations only  */
+	HSA_DBG_WATCH_ATOMIC = 2,	/* Atomic Operations only  */
+	HSA_DBG_WATCH_ALL = 3,	/* Read, Write or Atomic operations  */
 	HSA_DBG_WATCH_NUM,
 	HSA_DBG_WATCH_SIZE = 0xFFFFFFFF
-};
+} HSA_DBG_WATCH_MODE;
 
 /* This structure is hardware specific and may change in the future */
-struct HsaDbgWaveMsgAMDGen2 {
+typedef struct _HsaDbgWaveMsgAMDGen2 {
 	union {
-		struct ui32 {
-			uint32_t UserData:8;	/* user data */
-			uint32_t ShaderArray:1;	/* Shader array */
-			uint32_t Priv:1;	/* Privileged */
-			uint32_t Reserved0:4;	/* This field is reserved,
-						   should be 0 */
-			uint32_t WaveId:4;	/* wave id */
-			uint32_t SIMD:2;	/* SIMD id */
-			uint32_t HSACU:4;	/* Compute unit */
-			uint32_t ShaderEngine:2;/* Shader engine */
-			uint32_t MessageType:2;	/* see HSA_DBG_WAVEMSG_TYPE */
-			uint32_t Reserved1:4;	/* This field is reserved,
-						   should be 0 */
+		struct {
+			uint32_t UserData:8;	/*  user data  */
+			uint32_t ShaderArray:1;	/*  Shader array  */
+			uint32_t Priv:1;	/*  Privileged  */
+			uint32_t Reserved0:4;	/*  This field is reserved, should be 0  */
+			uint32_t WaveId:4;	/*  wave id  */
+			uint32_t SIMD:2;	/*  SIMD id  */
+			uint32_t HSACU:4;	/*  Compute unit  */
+			uint32_t ShaderEngine:2;	/*  Shader engine  */
+			uint32_t MessageType:2;	/*  see HSA_DBG_WAVEMSG_TYPE  */
+			uint32_t Reserved1:4;	/*  This field is reserved, should be 0  */
 		} ui32;
 		uint32_t Value;
 	};
-	uint32_t Reserved2;
-};
 
-union HsaDbgWaveMessageAMD {
-	struct HsaDbgWaveMsgAMDGen2 WaveMsgInfoGen2;
-	/* for future HsaDbgWaveMsgAMDGen3; */
-};
-
-struct HsaDbgWaveMessage {
-	void *MemoryVA;		/* ptr to associated host-accessible data */
-	union HsaDbgWaveMessageAMD DbgWaveMsg;
-};
+	uint32_t Reserved2;
 
-/*
- * TODO: This definitions to be MOVED to kfd_event, once it is implemented.
- *
- * HSA sync primitive, Event and HW Exception notification API definitions.
- * The API functions allow the runtime to define a so-called sync-primitive,
- * a SW object combining a user-mode provided "syncvar" and a scheduler event
- * that can be signaled through a defined GPU interrupt. A syncvar is
- * a process virtual memory location of a certain size that can be accessed
- * by CPU and GPU shader code within the process to set and query the content
- * within that memory. The definition of the content is determined by the HSA
- * runtime and potentially GPU shader code interfacing with the HSA runtime.
- * The syncvar values may be commonly written through an PM4 WRITE_DATA packet
- * in the user mode instruction stream. The OS scheduler event is typically
- * associated and signaled by an interrupt issued by the GPU, but other HSA
- * system interrupt conditions from other HW (e.g. IOMMUv2) may be surfaced
- * by the KFD by this mechanism, too. */
-
-/* these are the new definitions for events */
-enum HSA_EVENTTYPE {
-	HSA_EVENTTYPE_SIGNAL = 0,	/* user-mode generated GPU signal */
-	HSA_EVENTTYPE_NODECHANGE = 1,	/* HSA node change (attach/detach) */
-	HSA_EVENTTYPE_DEVICESTATECHANGE = 2,	/* HSA device state change
-						   (start/stop) */
-	HSA_EVENTTYPE_HW_EXCEPTION = 3,	/* GPU shader exception event */
-	HSA_EVENTTYPE_SYSTEM_EVENT = 4,	/* GPU SYSCALL with parameter info */
-	HSA_EVENTTYPE_DEBUG_EVENT = 5,	/* GPU signal for debugging */
-	HSA_EVENTTYPE_PROFILE_EVENT = 6,/* GPU signal for profiling */
-	HSA_EVENTTYPE_QUEUE_EVENT = 7,	/* GPU signal queue idle state
-					   (EOP pm4) */
+} HsaDbgWaveMsgAMDGen2;
+
+typedef union _HsaDbgWaveMessageAMD {
+	HsaDbgWaveMsgAMDGen2 WaveMsgInfoGen2;
+	/* for future HsaDbgWaveMsgAMDGen3;  */
+} HsaDbgWaveMessageAMD;
+
+typedef struct _HsaDbgWaveMessage {
+	void *MemoryVA;		/*  ptr to associated host-accessible data  */
+	HsaDbgWaveMessageAMD DbgWaveMsg;
+} HsaDbgWaveMessage;
+
+/* TODO: This definitions to be MOVED to kfd_event, once it is implemented.
+
+ HSA sync primitive, Event and HW Exception notification API definitions
+ The API functions allow the runtime to define a so-called sync-primitive, a SW object
+ combining a user-mode provided "syncvar" and a scheduler event that can be signaled
+ through a defined GPU interrupt. A syncvar is a process virtual memory location of
+ a certain size that can be accessed by CPU and GPU shader code within the process to set
+ and query the content within that memory. The definition of the content is determined by
+ the HSA runtime and potentially GPU shader code interfacing with the HSA runtime.
+ The syncvar values may be commonly written through an PM4 WRITE_DATA packet in the
+ user mode instruction stream. The OS scheduler event is typically associated and
+ signaled by an interrupt issued by the GPU, but other HSA system interrupt conditions
+ from other HW (e.g. IOMMUv2) may besurfaced by the KFD by this mechanism, too.  */
+
+/*  these are the new definitions for events  */
+
+typedef enum _HSA_EVENTTYPE {
+	HSA_EVENTTYPE_SIGNAL = 0,	/* /user-mode generated GPU signal  */
+	HSA_EVENTTYPE_NODECHANGE = 1,	/* HSA node change (attach/detach)  */
+	HSA_EVENTTYPE_DEVICESTATECHANGE = 2,	/* HSA device state change( start/stop )  */
+	HSA_EVENTTYPE_HW_EXCEPTION = 3,	/* GPU shader exception event  */
+	HSA_EVENTTYPE_SYSTEM_EVENT = 4,	/* GPU SYSCALL with parameter info  */
+	HSA_EVENTTYPE_DEBUG_EVENT = 5,	/* GPU signal for debugging  */
+	HSA_EVENTTYPE_PROFILE_EVENT = 6,	/* GPU signal for profiling  */
+	HSA_EVENTTYPE_QUEUE_EVENT = 7,	/* GPU signal queue idle state (EOP pm4)  */
 	/* ...  */
 	HSA_EVENTTYPE_MAXID,
 	HSA_EVENTTYPE_TYPE_SIZE = 0xFFFFFFFF
-};
+} HSA_EVENTTYPE;
+
+typedef uint32_t HSA_EVENTID;
 
-/* Sub-definitions for various event types: Syncvar */
-struct HsaSyncVar {
-	union SyncVar {
-		void *UserData;	/* pointer to user mode data */
-		uint64_t UserDataPtrValue; /* 64bit compatibility of value */
+/*  Subdefinitions for various event types: Syncvar  */
+
+typedef struct _HsaSyncVar {
+	union {
+		void *UserData;	/* pointer to user mode data  */
+		uint64_t UserDataPtrValue;	/* 64bit compatibility of value  */
 	} SyncVar;
 	uint64_t SyncVarSize;
-};
+} HsaSyncVar;
 
-/* Sub-definitions for various event types: NodeChange */
+/*
+ Subdefinitions for various event types: NodeChange
+*/
 
-enum HSA_EVENTTYPE_NODECHANGE_FLAGS {
+typedef enum _HSA_EVENTTYPE_NODECHANGE_FLAGS {
 	HSA_EVENTTYPE_NODECHANGE_ADD = 0,
 	HSA_EVENTTYPE_NODECHANGE_REMOVE = 1,
 	HSA_EVENTTYPE_NODECHANGE_SIZE = 0xFFFFFFFF
-};
+} HSA_EVENTTYPE_NODECHANGE_FLAGS;
 
-struct HsaNodeChange {
-	/* HSA node added/removed on the platform */
-	enum HSA_EVENTTYPE_NODECHANGE_FLAGS Flags;
-};
+typedef struct _HsaNodeChange {
+	HSA_EVENTTYPE_NODECHANGE_FLAGS Flags;	/*  HSA node added/removed on the platform  */
+} HsaNodeChange;
+
+/*
+ Sub-definitions for various event types: DeviceStateChange
+*/
 
-/* Sub-definitions for various event types: DeviceStateChange */
-enum HSA_EVENTTYPE_DEVICESTATECHANGE_FLAGS {
-	/* device started (and available) */
-	HSA_EVENTTYPE_DEVICESTATUSCHANGE_START = 0,
-	/* device stopped (i.e. unavailable) */
-	HSA_EVENTTYPE_DEVICESTATUSCHANGE_STOP = 1,
+typedef enum _HSA_EVENTTYPE_DEVICESTATECHANGE_FLAGS {
+	HSA_EVENTTYPE_DEVICESTATUSCHANGE_START = 0,	/* device started (and available)  */
+	HSA_EVENTTYPE_DEVICESTATUSCHANGE_STOP = 1,	/* device stopped (i.e. unavailable)  */
 	HSA_EVENTTYPE_DEVICESTATUSCHANGE_SIZE = 0xFFFFFFFF
-};
+} HSA_EVENTTYPE_DEVICESTATECHANGE_FLAGS;
 
-enum HSA_DEVICE {
+typedef enum _HSA_DEVICE {
 	HSA_DEVICE_CPU = 0,
 	HSA_DEVICE_GPU = 1,
 	MAX_HSA_DEVICE = 2
-};
+} HSA_DEVICE;
 
-struct HsaDeviceStateChange {
+typedef struct _HsaDeviceStateChange {
 	uint32_t NodeId;	/* F-NUMA node that contains the device */
-	enum HSA_DEVICE Device;	/* device type: GPU or CPU */
-	enum HSA_EVENTTYPE_DEVICESTATECHANGE_FLAGS Flags; /* event flags */
-};
+	HSA_DEVICE Device;	/* device type: GPU or CPU */
+	HSA_EVENTTYPE_DEVICESTATECHANGE_FLAGS Flags;	/* event flags   */
+} HsaDeviceStateChange;
 
-struct HsaEventData {
-	enum HSA_EVENTTYPE EventType; /* event type */
-	union EventData {
-		/*
-		 * return data associated with HSA_EVENTTYPE_SIGNAL
-		 * and other events
-		 */
-		struct HsaSyncVar SyncVar;
+typedef struct _HsaEventData {
+	HSA_EVENTTYPE EventType;	/* event type */
+	union {
+		/* return data associated with HSA_EVENTTYPE_SIGNAL and other events */
+		HsaSyncVar SyncVar;
 
 		/* data associated with HSA_EVENTTYPE_NODE_CHANGE */
-		struct HsaNodeChange NodeChangeState;
+		HsaNodeChange NodeChangeState;
 
 		/* data associated with HSA_EVENTTYPE_DEVICE_STATE_CHANGE */
-		struct HsaDeviceStateChange DeviceState;
+		HsaDeviceStateChange DeviceState;
 	} EventData;
 
-	/* the following data entries are internal to the KFD & thunk itself */
+	/* the following data entries are internal to the KFD & thunk itself. */
 
-	/* internal thunk store for Event data (OsEventHandle) */
-	uint64_t HWData1;
-	/* internal thunk store for Event data (HWAddress) */
-	uint64_t HWData2;
-	/* internal thunk store for Event data (HWData) */
-	uint32_t HWData3;
-};
+	uint64_t HWData1;	/* internal thunk store for Event data  (OsEventHandle) */
+	uint64_t HWData2;	/* internal thunk store for Event data  (HWAddress) */
+	uint32_t HWData3;	/* internal thunk store for Event data  (HWData) */
+} HsaEventData;
 
-struct HsaEventDescriptor {
-	/* event type to allocate */
-	enum HSA_EVENTTYPE EventType;
-	/* H-NUMA node containing GPU device that is event source */
-	uint32_t NodeId;
-	/* pointer to user mode syncvar data, syncvar->UserDataPtrValue
-	 * may be NULL
-	 */
-	struct HsaSyncVar SyncVar;
-};
+typedef struct _HsaEventDescriptor {
+	HSA_EVENTTYPE EventType;	/* event type to allocate */
+	uint32_t NodeId;	/* H-NUMA node containing GPU device that is event source */
+	HsaSyncVar SyncVar;	/* pointer to user mode syncvar data, syncvar->UserDataPtrValue may be NULL */
+} HsaEventDescriptor;
+
+typedef struct _HsaEvent {
+	HSA_EVENTID EventId;
+	HsaEventData EventData;
+} HsaEvent;
 
-struct HsaEvent {
-	uint32_t EventId;
-	struct HsaEventData EventData;
-};
 
 #pragma pack(pop)
 
-enum DBGDEV_TYPE {
+typedef enum _DBGDEV_TYPE {
 	DBGDEV_TYPE_ILLEGAL = 0,
 	DBGDEV_TYPE_NODIQ = 1,
 	DBGDEV_TYPE_DIQ = 2,
 	DBGDEV_TYPE_TEST = 3
-};
+} DBGDEV_TYPE;
 
 struct dbg_address_watch_info {
 	struct kfd_process *process;
-	enum HSA_DBG_WATCH_MODE *watch_mode;
+	HSA_DBG_WATCH_MODE *watch_mode;
 	uint64_t *watch_address;
 	uint64_t *watch_mask;
-	struct HsaEvent *watch_event;
+	HsaEvent *watch_event;
 	uint32_t num_watch_points;
 };
 
 struct dbg_wave_control_info {
 	struct kfd_process *process;
 	uint32_t trapId;
-	enum HSA_DBG_WAVEOP operand;
-	enum HSA_DBG_WAVEMODE mode;
-	struct HsaDbgWaveMessage dbgWave_msg;
+	HSA_DBG_WAVEOP operand;
+	HSA_DBG_WAVEMODE mode;
+	HsaDbgWaveMessage dbgWave_msg;
 };
 
 struct kfd_dbgdev {
 
 	/* The device that owns this data. */
+
 	struct kfd_dev *dev;
 
 	/* kernel queue for DIQ */
+
 	struct kernel_queue *kq;
 
 	/* a pointer to the pqm of the calling process */
+
 	struct process_queue_manager *pqm;
 
 	/* type of debug device ( DIQ, non DIQ, etc. ) */
-	enum DBGDEV_TYPE type;
+
+	DBGDEV_TYPE type;
 
 	/* virtualized function pointers to device dbg */
+
 	int (*dbgdev_register)(struct kfd_dbgdev *dbgdev);
 	int (*dbgdev_unregister)(struct kfd_dbgdev *dbgdev);
-	int (*dbgdev_address_watch)(struct kfd_dbgdev *dbgdev,
-				struct dbg_address_watch_info *adw_info);
-	int (*dbgdev_wave_control)(struct kfd_dbgdev *dbgdev,
-				struct dbg_wave_control_info *wac_info);
+	int (*dbgdev_address_watch)(struct kfd_dbgdev *dbgdev, struct dbg_address_watch_info *adw_info);
+	int (*dbgdev_wave_control)(struct kfd_dbgdev *dbgdev, struct dbg_wave_control_info *wac_info);
 
 };
 
@@ -282,13 +272,12 @@ struct kfd_dbgmgr {
 };
 
 /* prototypes for debug manager functions */
-struct mutex *kfd_get_dbgmgr_mutex(void);
+struct mutex *get_dbgmgr_mutex(void);
 void kfd_dbgmgr_destroy(struct kfd_dbgmgr *pmgr);
 bool kfd_dbgmgr_create(struct kfd_dbgmgr **ppmgr, struct kfd_dev *pdev);
 long kfd_dbgmgr_register(struct kfd_dbgmgr *pmgr, struct kfd_process *p);
 long kfd_dbgmgr_unregister(struct kfd_dbgmgr *pmgr, struct kfd_process *p);
-long kfd_dbgmgr_wave_control(struct kfd_dbgmgr *pmgr,
-				struct dbg_wave_control_info *wac_info);
-long kfd_dbgmgr_address_watch(struct kfd_dbgmgr *pmgr,
-			struct dbg_address_watch_info *adw_info);
-#endif /* KFD_DBGMGR_H_ */
+long kfd_dbgmgr_wave_control(struct kfd_dbgmgr *pmgr, struct dbg_wave_control_info *wac_info);
+long kfd_dbgmgr_address_watch(struct kfd_dbgmgr *pmgr, struct dbg_address_watch_info *adw_info);
+long kfd_dbgmgr_abnormal_termination(struct kfd_dbgmgr *pmgr, struct kfd_process *process);
+#endif				/* KFD_DBGMGR_H_ */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
index 3f95f7c..20592ba 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
@@ -24,9 +24,11 @@
 #include <linux/bsearch.h>
 #include <linux/pci.h>
 #include <linux/slab.h>
+#include <linux/highmem.h>
 #include "kfd_priv.h"
 #include "kfd_device_queue_manager.h"
 #include "kfd_pm4_headers.h"
+#include "cwsr_trap_handler_carrizo.h"
 
 #define MQD_SIZE_ALIGNED 768
 
@@ -38,7 +40,8 @@ static const struct kfd_device_info kaveri_device_info = {
 	.ih_ring_entry_size = 4 * sizeof(uint32_t),
 	.event_interrupt_class = &event_interrupt_class_cik,
 	.num_of_watch_points = 4,
-	.mqd_size_aligned = MQD_SIZE_ALIGNED
+	.mqd_size_aligned = MQD_SIZE_ALIGNED,
+	.is_need_iommu_device = true
 };
 
 static const struct kfd_device_info carrizo_device_info = {
@@ -49,14 +52,50 @@ static const struct kfd_device_info carrizo_device_info = {
 	.ih_ring_entry_size = 4 * sizeof(uint32_t),
 	.event_interrupt_class = &event_interrupt_class_cik,
 	.num_of_watch_points = 4,
-	.mqd_size_aligned = MQD_SIZE_ALIGNED
+	.mqd_size_aligned = MQD_SIZE_ALIGNED,
+	.is_need_iommu_device = true
 };
 
+static const struct kfd_device_info tonga_device_info = {
+	.asic_family = CHIP_TONGA,
+	.max_pasid_bits = 16,
+	.max_no_of_hqd  = 24,
+	.ih_ring_entry_size = 4 * sizeof(uint32_t),
+	.event_interrupt_class = &event_interrupt_class_cik,
+	.num_of_watch_points = 4,
+	.mqd_size_aligned = MQD_SIZE_ALIGNED,
+	.is_need_iommu_device = false
+};
+
+static const struct kfd_device_info fiji_device_info = {
+	.asic_family = CHIP_FIJI,
+	.max_pasid_bits = 16,
+	.max_no_of_hqd  = 24,
+	.ih_ring_entry_size = 4 * sizeof(uint32_t),
+	.event_interrupt_class = &event_interrupt_class_cik,
+	.num_of_watch_points = 4,
+	.mqd_size_aligned = MQD_SIZE_ALIGNED,
+	.is_need_iommu_device = false
+}
+;
 struct kfd_deviceid {
 	unsigned short did;
 	const struct kfd_device_info *device_info;
 };
 
+/*
+ * //
+// TONGA/AMETHYST device IDs (performance segment)
+//
+#define DEVICE_ID_VI_TONGA_P_6920               0x6920  // unfused
+#define DEVICE_ID_VI_TONGA_P_6921               0x6921  // Amethyst XT
+#define DEVICE_ID_VI_TONGA_P_6928               0x6928  // Tonga GL XT
+#define DEVICE_ID_VI_TONGA_P_692B               0x692B  // Tonga GL PRO
+#define DEVICE_ID_VI_TONGA_P_692F               0x692F  // Tonga GL PRO VF
+#define DEVICE_ID_VI_TONGA_P_6938               0x6938  // Tonga XT
+#define DEVICE_ID_VI_TONGA_P_6939               0x6939  // Tonga PRO
+ *
+ */
 /* Please keep this sorted by increasing device id. */
 static const struct kfd_deviceid supported_devices[] = {
 	{ 0x1304, &kaveri_device_info },	/* Kaveri */
@@ -85,13 +124,23 @@ static const struct kfd_deviceid supported_devices[] = {
 	{ 0x9874, &carrizo_device_info },	/* Carrizo */
 	{ 0x9875, &carrizo_device_info },	/* Carrizo */
 	{ 0x9876, &carrizo_device_info },	/* Carrizo */
-	{ 0x9877, &carrizo_device_info }	/* Carrizo */
+	{ 0x9877, &carrizo_device_info },	/* Carrizo */
+	{ 0x6920, &tonga_device_info   },	/* Tonga */
+	{ 0x6921, &tonga_device_info   },	/* Tonga */
+	{ 0x6928, &tonga_device_info   },	/* Tonga */
+	{ 0x692B, &tonga_device_info   },	/* Tonga */
+	{ 0x692F, &tonga_device_info   },	/* Tonga */
+	{ 0x6938, &tonga_device_info   },	/* Tonga */
+	{ 0x6939, &tonga_device_info   },	/* Tonga */
+	{ 0x7300, &fiji_device_info    }	/* Fiji */
 };
 
 static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size,
 				unsigned int chunk_size);
 static void kfd_gtt_sa_fini(struct kfd_dev *kfd);
 
+static int kfd_resume(struct kfd_dev *kfd);
+
 static const struct kfd_device_info *lookup_device_info(unsigned short did)
 {
 	size_t i;
@@ -117,6 +166,8 @@ struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd,
 	if (!device_info)
 		return NULL;
 
+	BUG_ON(!f2g);
+
 	kfd = kzalloc(sizeof(*kfd), GFP_KERNEL);
 	if (!kfd)
 		return NULL;
@@ -170,15 +221,8 @@ static bool device_iommu_pasid_init(struct kfd_dev *kfd)
 				pasid_limit,
 				kfd->doorbell_process_limit - 1);
 
-	err = amd_iommu_init_device(kfd->pdev, pasid_limit);
-	if (err < 0) {
-		dev_err(kfd_device, "error initializing iommu device\n");
-		return false;
-	}
-
 	if (!kfd_set_pasid_limit(pasid_limit)) {
 		dev_err(kfd_device, "error setting pasid limit\n");
-		amd_iommu_free_device(kfd->pdev);
 		return false;
 	}
 
@@ -219,13 +263,81 @@ static int iommu_invalid_ppr_cb(struct pci_dev *pdev, int pasid,
 	return AMD_IOMMU_INV_PRI_RSP_INVALID;
 }
 
+static int kfd_cwsr_init(struct kfd_dev *kfd)
+{
+	/*
+	 * Initialize the CWSR required memory for TBA and TMA
+	 * only support CWSR on VI and up with FW version >=625.
+	 */
+	if (cwsr_enable &&
+		(kfd->mec_fw_version >= KFD_CWSR_CZ_FW_VER)) {
+		void *cwsr_addr = NULL;
+		unsigned int size = sizeof(cwsr_trap_carrizo_hex);
+
+		if (size > PAGE_SIZE) {
+			pr_err("amdkfd: wrong CWSR ISA size.\n");
+			return -EINVAL;
+		}
+		kfd->cwsr_size =
+			ALIGN(size, PAGE_SIZE) + PAGE_SIZE;
+		kfd->cwsr_pages = alloc_pages(GFP_KERNEL | __GFP_HIGHMEM,
+					get_order(kfd->cwsr_size));
+		if (!kfd->cwsr_pages) {
+			pr_err("amdkfd: error alloc CWSR isa memory.\n");
+			return -ENOMEM;
+		}
+		/*Only first page used for cwsr ISA code */
+		cwsr_addr = kmap(kfd->cwsr_pages);
+		memset(cwsr_addr, 0, PAGE_SIZE);
+		memcpy(cwsr_addr, cwsr_trap_carrizo_hex, size);
+		kunmap(kfd->cwsr_pages);
+		kfd->tma_offset = ALIGN(size, PAGE_SIZE);
+		kfd->cwsr_enabled = true;
+		dev_info(kfd_device,
+			"Reserved %d pages for cwsr.\n",
+			(kfd->cwsr_size >> PAGE_SHIFT));
+	}
+
+	return 0;
+}
+
+static void kfd_cwsr_fini(struct kfd_dev *kfd)
+{
+	if (kfd->cwsr_pages)
+		__free_pages(kfd->cwsr_pages, get_order(kfd->cwsr_size));
+}
+
 bool kgd2kfd_device_init(struct kfd_dev *kfd,
 			 const struct kgd2kfd_shared_resources *gpu_resources)
 {
 	unsigned int size;
+	unsigned int vmid_bitmap_kfd, vmid_num_kfd;
+
+	kfd->mec_fw_version = kfd->kfd2kgd->get_fw_version(kfd->kgd,
+			KGD_ENGINE_MEC1);
 
 	kfd->shared_resources = *gpu_resources;
 
+	vmid_bitmap_kfd = kfd->shared_resources.compute_vmid_bitmap;
+	kfd->vm_info.first_vmid_kfd = ffs(vmid_bitmap_kfd) - 1;
+	kfd->vm_info.last_vmid_kfd = fls(vmid_bitmap_kfd) - 1;
+	vmid_num_kfd = kfd->vm_info.last_vmid_kfd
+			- kfd->vm_info.first_vmid_kfd + 1;
+	kfd->vm_info.vmid_num_kfd = vmid_num_kfd;
+
+	/* If MEC firmware is too old, turn off hws multiple process mapping */
+	if (kfd->mec_fw_version	< KFD_MULTI_PROC_MAPPING_HWS_SUPPORT)
+		kfd->max_proc_per_quantum = 0;
+	/* Verify module parameters regarding mapped process number*/
+	else if ((hws_max_conc_proc < 0)
+			|| (hws_max_conc_proc > vmid_num_kfd)) {
+		dev_err(kfd_device,
+			"hws_max_conc_proc (%d) must be between 0 and %d, use %d instead\n",
+			hws_max_conc_proc, vmid_num_kfd, vmid_num_kfd);
+		kfd->max_proc_per_quantum = vmid_num_kfd;
+	} else
+		kfd->max_proc_per_quantum = hws_max_conc_proc;
+
 	/* calculate max size of mqds needed for queues */
 	size = max_num_of_queues_per_device *
 			kfd->device_info->mqd_size_aligned;
@@ -280,16 +392,6 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
 		goto kfd_interrupt_error;
 	}
 
-	if (!device_iommu_pasid_init(kfd)) {
-		dev_err(kfd_device,
-			"Error initializing iommuv2 for device (%x:%x)\n",
-			kfd->pdev->vendor, kfd->pdev->device);
-		goto device_iommu_pasid_error;
-	}
-	amd_iommu_set_invalidate_ctx_cb(kfd->pdev,
-						iommu_pasid_shutdown_callback);
-	amd_iommu_set_invalid_ppr_cb(kfd->pdev, iommu_invalid_ppr_cb);
-
 	kfd->dqm = device_queue_manager_init(kfd);
 	if (!kfd->dqm) {
 		dev_err(kfd_device,
@@ -298,13 +400,21 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
 		goto device_queue_manager_error;
 	}
 
-	if (kfd->dqm->ops.start(kfd->dqm) != 0) {
-		dev_err(kfd_device,
-			"Error starting queuen manager for device (%x:%x)\n",
-			kfd->pdev->vendor, kfd->pdev->device);
-		goto dqm_start_error;
+	if (kfd->device_info->is_need_iommu_device) {
+		if (!device_iommu_pasid_init(kfd)) {
+			dev_err(kfd_device,
+				"Error initializing iommuv2 for device (%x:%x)\n",
+				kfd->pdev->vendor, kfd->pdev->device);
+			goto device_iommu_pasid_error;
+		}
 	}
 
+	if (kfd_cwsr_init(kfd))
+		goto device_iommu_pasid_error;
+
+	if (kfd_resume(kfd))
+		goto kfd_resume_error;
+
 	kfd->dbgmgr = NULL;
 
 	kfd->init_complete = true;
@@ -316,11 +426,11 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
 
 	goto out;
 
-dqm_start_error:
+kfd_resume_error:
+	kfd_cwsr_fini(kfd);
+device_iommu_pasid_error:
 	device_queue_manager_uninit(kfd->dqm);
 device_queue_manager_error:
-	amd_iommu_free_device(kfd->pdev);
-device_iommu_pasid_error:
 	kfd_interrupt_exit(kfd);
 kfd_interrupt_error:
 	kfd_topology_remove_device(kfd);
@@ -338,8 +448,9 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
 void kgd2kfd_device_exit(struct kfd_dev *kfd)
 {
 	if (kfd->init_complete) {
+		kgd2kfd_suspend(kfd);
+		kfd_cwsr_fini(kfd);
 		device_queue_manager_uninit(kfd->dqm);
-		amd_iommu_free_device(kfd->pdev);
 		kfd_interrupt_exit(kfd);
 		kfd_topology_remove_device(kfd);
 		kfd_gtt_sa_fini(kfd);
@@ -355,32 +466,68 @@ void kgd2kfd_suspend(struct kfd_dev *kfd)
 
 	if (kfd->init_complete) {
 		kfd->dqm->ops.stop(kfd->dqm);
-		amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL);
-		amd_iommu_set_invalid_ppr_cb(kfd->pdev, NULL);
-		amd_iommu_free_device(kfd->pdev);
+		if (kfd->device_info->is_need_iommu_device) {
+			amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL);
+			amd_iommu_set_invalid_ppr_cb(kfd->pdev, NULL);
+			amd_iommu_free_device(kfd->pdev);
+		}
 	}
 }
 
-int kgd2kfd_resume(struct kfd_dev *kfd)
+int kgd2kfd_evict_bo(struct kfd_dev *dev, void *mem)
 {
-	unsigned int pasid_limit;
-	int err;
+	return evict_bo(dev, mem);
+}
 
+int kgd2kfd_restore(struct kfd_dev *kfd)
+{
+	return restore(kfd);
+}
+
+int kgd2kfd_resume(struct kfd_dev *kfd)
+{
 	BUG_ON(kfd == NULL);
 
-	pasid_limit = kfd_get_pasid_limit();
+	if (!kfd->init_complete)
+		return 0;
+
+	return kfd_resume(kfd);
+
+}
+
+static int kfd_resume(struct kfd_dev *kfd)
+{
+	int err = 0;
+
+	if (kfd->device_info->is_need_iommu_device) {
+		unsigned int pasid_limit = kfd_get_pasid_limit();
 
-	if (kfd->init_complete) {
 		err = amd_iommu_init_device(kfd->pdev, pasid_limit);
-		if (err < 0)
+		if (err)
 			return -ENXIO;
 		amd_iommu_set_invalidate_ctx_cb(kfd->pdev,
-						iommu_pasid_shutdown_callback);
-		amd_iommu_set_invalid_ppr_cb(kfd->pdev, iommu_invalid_ppr_cb);
-		kfd->dqm->ops.start(kfd->dqm);
+				iommu_pasid_shutdown_callback);
+		amd_iommu_set_invalid_ppr_cb(kfd->pdev,
+				iommu_invalid_ppr_cb);
 	}
 
-	return 0;
+	err = kfd->dqm->ops.start(kfd->dqm);
+	if (err) {
+		dev_err(kfd_device,
+			"Error starting queue manager for device (%x:%x)\n",
+			kfd->pdev->vendor, kfd->pdev->device);
+		goto dqm_start_error;
+	}
+
+	kfd->kfd2kgd->write_config_static_mem(kfd->kgd, true, 1, 3, 0);
+
+	return err;
+
+dqm_start_error:
+	if (kfd->device_info->is_need_iommu_device)
+		amd_iommu_free_device(kfd->pdev);
+
+	return err;
 }
 
 /* This is called directly from KGD at ISR. */
@@ -399,6 +546,58 @@ void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry)
 	spin_unlock(&kfd->interrupt_lock);
 }
 
+int kgd2kfd_quiesce_mm(struct kfd_dev *kfd, struct mm_struct *mm)
+{
+	struct kfd_process *p;
+	struct kfd_process_device *pdd;
+	int r;
+
+	BUG_ON(kfd == NULL);
+	if (!kfd->init_complete)
+		return 0;
+
+	/* Because we are called from arbitrary context (workqueue) as opposed
+	 * to process context, kfd_process could attempt to exit while we are
+	 * running so the lookup function returns a read-locked process. */
+	p = kfd_lookup_process_by_mm(mm);
+	if (!p)
+		return -ENODEV;
+
+	r = -ENODEV;
+	pdd = kfd_get_process_device_data(kfd, p);
+	if (pdd)
+		r = process_evict_queues(kfd->dqm, &pdd->qpd);
+
+	up_read(&p->lock);
+	return r;
+}
+
+int kgd2kfd_resume_mm(struct kfd_dev *kfd, struct mm_struct *mm)
+{
+	struct kfd_process *p;
+	struct kfd_process_device *pdd;
+	int r;
+
+	BUG_ON(kfd == NULL);
+	if (!kfd->init_complete)
+		return 0;
+
+	/* Because we are called from arbitrary context (workqueue) as opposed
+	 * to process context, kfd_process could attempt to exit while we are
+	 * running so the lookup function returns a read-locked process. */
+	p = kfd_lookup_process_by_mm(mm);
+	if (!p)
+		return -ENODEV;
+
+	r = -ENODEV;
+	pdd = kfd_get_process_device_data(kfd, p);
+	if (pdd)
+		r = process_restore_queues(kfd->dqm, &pdd->qpd);
+
+	up_read(&p->lock);
+	return r;
+}
+
 static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size,
 				unsigned int chunk_size)
 {
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index 42de22b..e123390 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -44,9 +44,10 @@ static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
 					struct queue *q,
 					struct qcm_process_device *qpd);
 
-static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock);
-static int destroy_queues_cpsch(struct device_queue_manager *dqm,
-				bool preempt_static_queues, bool lock);
+static int execute_queues_cpsch(struct device_queue_manager *dqm);
+static int unmap_queues_cpsch(struct device_queue_manager *dqm,
+		enum kfd_unmap_queues_filter filter,
+		uint32_t filter_param, bool reset);
 
 static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
 					struct queue *q,
@@ -116,11 +117,11 @@ static int allocate_vmid(struct device_queue_manager *dqm,
 	if (dqm->vmid_bitmap == 0)
 		return -ENOMEM;
 
-	bit = find_first_bit((unsigned long *)&dqm->vmid_bitmap, CIK_VMID_NUM);
+	bit = find_first_bit((unsigned long *)&dqm->vmid_bitmap,
+				dqm->dev->vm_info.vmid_num_kfd);
 	clear_bit(bit, (unsigned long *)&dqm->vmid_bitmap);
 
-	/* Kaveri kfd vmid's starts from vmid 8 */
-	allocated_vmid = bit + KFD_VMID_START_OFFSET;
+	allocated_vmid = bit + dqm->dev->vm_info.first_vmid_kfd;
 	pr_debug("kfd: vmid allocation %d\n", allocated_vmid);
 	qpd->vmid = allocated_vmid;
 	q->properties.vmid = allocated_vmid;
@@ -128,6 +129,11 @@ static int allocate_vmid(struct device_queue_manager *dqm,
 	set_pasid_vmid_mapping(dqm, q->process->pasid, q->properties.vmid);
 	program_sh_mem_settings(dqm, qpd);
 
+	dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->kgd,
+			allocated_vmid,
+			qpd->page_table_base);
+	/*invalidate the VM context after pasid and vmid mapping is set up*/
+	radeon_flush_tlb(dqm->dev, qpd->pqm->process->pasid);
 	return 0;
 }
 
@@ -135,7 +141,7 @@ static void deallocate_vmid(struct device_queue_manager *dqm,
 				struct qcm_process_device *qpd,
 				struct queue *q)
 {
-	int bit = qpd->vmid - KFD_VMID_START_OFFSET;
+	int bit = qpd->vmid - dqm->dev->vm_info.first_vmid_kfd;
 
 	/* Release the vmid mapping */
 	set_pasid_vmid_mapping(dqm, 0, qpd->vmid);
@@ -175,6 +181,14 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
 	}
 	*allocated_vmid = qpd->vmid;
 	q->properties.vmid = qpd->vmid;
+	/*
+	 * Eviction state logic: we only mark active queues as evicted
+	 * to avoid the overhead of restoring inactive queues later
+	 */
+	if (qpd->evicted)
+		q->properties.is_evicted = (q->properties.queue_size > 0 &&
+					    q->properties.queue_percent > 0 &&
+					    q->properties.queue_address != 0);
 
 	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
 		retval = create_compute_queue_nocpsch(dqm, q, qpd);
@@ -281,8 +295,12 @@ static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
 			q->pipe,
 			q->queue);
 
+	dqm->dev->kfd2kgd->alloc_memory_of_scratch(
+			dqm->dev->kgd, qpd->sh_hidden_private_base, qpd->vmid);
+
 	retval = mqd->load_mqd(mqd, q->mqd, q->pipe,
-			q->queue, (uint32_t __user *) q->properties.write_ptr);
+			q->queue, (uint32_t __user *) q->properties.write_ptr,
+			qpd->page_table_base);
 	if (retval != 0) {
 		deallocate_hqd(dqm, q);
 		mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
@@ -362,34 +380,56 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q)
 {
 	int retval;
 	struct mqd_manager *mqd;
+	struct kfd_process_device *pdd;
+
 	bool prev_active = false;
 
 	BUG_ON(!dqm || !q || !q->mqd);
 
 	mutex_lock(&dqm->lock);
+
+	pdd = kfd_get_process_device_data(q->device, q->process);
+	if (!pdd) {
+		mutex_unlock(&dqm->lock);
+		return -ENODEV;
+	}
 	mqd = dqm->ops.get_mqd_manager(dqm,
 			get_mqd_type_from_queue_type(q->properties.type));
 	if (mqd == NULL) {
 		mutex_unlock(&dqm->lock);
 		return -ENOMEM;
 	}
+	/*
+	 * Eviction state logic: we only mark active queues as evicted
+	 * to avoid the overhead of restoring inactive queues later
+	 */
+	if (pdd->qpd.evicted > 0)
+		q->properties.is_evicted = (q->properties.queue_size > 0 &&
+					    q->properties.queue_percent > 0 &&
+					    q->properties.queue_address != 0);
 
+	/* save previous activity state for counters */
 	if (q->properties.is_active)
 		prev_active = true;
 
-	/*
-	 *
-	 * check active state vs. the previous state
-	 * and modify counter accordingly
-	 */
+
 	retval = mqd->update_mqd(mqd, q->mqd, &q->properties);
+	if (sched_policy == KFD_SCHED_POLICY_NO_HWS &&
+		q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
+		retval = mqd->load_mqd(mqd, q->mqd, q->pipe,
+			q->queue,
+			(uint32_t __user *)q->properties.write_ptr, 0);
+	/*
+		 * check active state vs. the previous state
+		 * and modify counter accordingly
+	*/
 	if ((q->properties.is_active) && (!prev_active))
 		dqm->queue_count++;
 	else if ((!q->properties.is_active) && (prev_active))
 		dqm->queue_count--;
 
 	if (sched_policy != KFD_SCHED_POLICY_NO_HWS)
-		retval = execute_queues_cpsch(dqm, false);
+		retval = execute_queues_cpsch(dqm);
 
 	mutex_unlock(&dqm->lock);
 	return retval;
@@ -415,15 +455,115 @@ static struct mqd_manager *get_mqd_manager_nocpsch(
 	return mqd;
 }
 
+int process_evict_queues(struct device_queue_manager *dqm,
+		struct qcm_process_device *qpd)
+{
+	struct queue *q, *next;
+	struct mqd_manager *mqd;
+	int retval = 0;
+
+	BUG_ON(!dqm || !qpd);
+
+	mutex_lock(&dqm->lock);
+	if (qpd->evicted++ > 0) { /* already evicted, do nothing */
+		mutex_unlock(&dqm->lock);
+		return 0;
+	}
+	/* unactivate all active queues on the qpd */
+	list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
+		mqd = dqm->ops.get_mqd_manager(dqm,
+			get_mqd_type_from_queue_type(q->properties.type));
+		if (!mqd) { /* should not be here */
+			BUG();
+			continue;
+		}
+		/* if the queue is not active anyway, it is not evicted */
+		if (q->properties.is_active == true)
+			q->properties.is_evicted = true;
+
+		retval = mqd->update_mqd(mqd, q->mqd, &q->properties);
+		if (sched_policy == KFD_SCHED_POLICY_NO_HWS &&
+				q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
+			retval = mqd->load_mqd(mqd, q->mqd, q->pipe,
+				q->queue,
+				(uint32_t __user *)q->properties.write_ptr, 0);
+		if (q->properties.is_evicted)
+			dqm->queue_count--;
+	}
+	if (sched_policy != KFD_SCHED_POLICY_NO_HWS)
+		retval = execute_queues_cpsch(dqm);
+
+	mutex_unlock(&dqm->lock);
+	return retval;
+
+}
+
+int process_restore_queues(struct device_queue_manager *dqm,
+		struct qcm_process_device *qpd)
+{
+	struct queue *q, *next;
+	struct mqd_manager *mqd;
+	int retval = 0;
+
+
+	BUG_ON(!dqm || !qpd);
+
+	mutex_lock(&dqm->lock);
+	if (qpd->evicted == 0) { /* already restored, do nothing */
+		mutex_unlock(&dqm->lock);
+		return 0;
+	}
+
+	if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
+		qpd->evicted--;
+		mutex_unlock(&dqm->lock);
+		return 0;
+	}
+
+	/* activate all active queues on the qpd */
+	list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
+		mqd = dqm->ops.get_mqd_manager(dqm,
+			get_mqd_type_from_queue_type(q->properties.type));
+		if (!mqd) { /* should not be here */
+			BUG();
+			continue;
+		}
+		if (q->properties.is_evicted) {
+			q->properties.is_evicted = false;
+			retval = mqd->update_mqd(mqd, q->mqd, &q->properties);
+			if (sched_policy == KFD_SCHED_POLICY_NO_HWS &&
+				q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
+				retval =
+						mqd->load_mqd(
+								mqd,
+								q->mqd,
+								q->pipe,
+								q->queue,
+				(uint32_t __user *)q->properties.write_ptr,
+								0);
+			dqm->queue_count++;
+		}
+	}
+	if (sched_policy != KFD_SCHED_POLICY_NO_HWS)
+		retval = execute_queues_cpsch(dqm);
+
+	if (retval == 0)
+		qpd->evicted = 0;
+	mutex_unlock(&dqm->lock);
+	return retval;
+
+}
+
 static int register_process_nocpsch(struct device_queue_manager *dqm,
 					struct qcm_process_device *qpd)
 {
+	struct kfd_process_device *pdd;
 	struct device_process_node *n;
 	int retval;
 
 	BUG_ON(!dqm || !qpd);
 
-	pr_debug("kfd: In func %s\n", __func__);
+	pr_debug("In func %s\n", __func__);
 
 	n = kzalloc(sizeof(struct device_process_node), GFP_KERNEL);
 	if (!n)
@@ -434,6 +574,11 @@ static int register_process_nocpsch(struct device_queue_manager *dqm,
 	mutex_lock(&dqm->lock);
 	list_add(&n->list, &dqm->queues);
 
+	pdd = qpd_to_pdd(qpd);
+	qpd->page_table_base =
+		dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm);
+	pr_debug("Retrieved PD address == 0x%08u\n", qpd->page_table_base);
+
 	retval = dqm->ops_asic_specific.register_process(dqm, qpd);
 
 	dqm->processes_count++;
@@ -499,7 +644,6 @@ static void init_interrupts(struct device_queue_manager *dqm)
 		if (is_pipe_enabled(dqm, 0, i))
 			dqm->dev->kfd2kgd->init_interrupts(dqm->dev->kgd, i);
 }
-
 static int init_scheduler(struct device_queue_manager *dqm)
 {
 	int retval = 0;
@@ -534,7 +678,7 @@ static int initialize_nocpsch(struct device_queue_manager *dqm)
 	for (i = 0; i < get_pipes_per_mec(dqm); i++)
 		dqm->allocated_queues[i] = (1 << get_queues_per_pipe(dqm)) - 1;
 
-	dqm->vmid_bitmap = (1 << VMID_PER_DEVICE) - 1;
+	dqm->vmid_bitmap = (1 << dqm->dev->vm_info.vmid_num_kfd) - 1;
 	dqm->sdma_bitmap = (1 << CIK_SDMA_QUEUES) - 1;
 
 	init_scheduler(dqm);
@@ -607,8 +751,8 @@ static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
 	if (retval != 0)
 		return retval;
 
-	q->properties.sdma_queue_id = q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;
-	q->properties.sdma_engine_id = q->sdma_id / CIK_SDMA_ENGINE_NUM;
+	q->properties.sdma_queue_id = q->sdma_id / CIK_SDMA_QUEUES_PER_ENGINE;
+	q->properties.sdma_engine_id = q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;
 
 	pr_debug("kfd: sdma id is:    %d\n", q->sdma_id);
 	pr_debug("     sdma queue id: %d\n", q->properties.sdma_queue_id);
@@ -623,7 +767,7 @@ static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
 	}
 
 	retval = mqd->load_mqd(mqd, q->mqd, 0,
-				0, NULL);
+				0, NULL, 0);
 	if (retval != 0) {
 		deallocate_sdma_queue(dqm, q->sdma_id);
 		mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
@@ -646,8 +790,7 @@ static int set_sched_resources(struct device_queue_manager *dqm)
 
 	pr_debug("kfd: In func %s\n", __func__);
 
-	res.vmid_mask = (1 << VMID_PER_DEVICE) - 1;
-	res.vmid_mask <<= KFD_VMID_START_OFFSET;
+	res.vmid_mask = dqm->dev->shared_resources.compute_vmid_bitmap;
 
 	res.queue_mask = 0;
 	for (i = 0; i < KGD_MAX_QUEUES; ++i) {
@@ -696,6 +839,7 @@ static int initialize_cpsch(struct device_queue_manager *dqm)
 	dqm->queue_count = dqm->processes_count = 0;
 	dqm->sdma_queue_count = 0;
 	dqm->active_runlist = false;
+	dqm->sdma_bitmap = (1 << CIK_SDMA_QUEUES) - 1;
 	retval = dqm->ops_asic_specific.initialize(dqm);
 	if (retval != 0)
 		goto fail_init_pipelines;
@@ -716,7 +860,7 @@ static int start_cpsch(struct device_queue_manager *dqm)
 
 	retval = 0;
 
-	retval = pm_init(&dqm->packets, dqm);
+	retval = pm_init(&dqm->packets, dqm, dqm->dev->mec_fw_version);
 	if (retval != 0)
 		goto fail_packet_manager_init;
 
@@ -743,7 +887,9 @@ static int start_cpsch(struct device_queue_manager *dqm)
 			kfd_bind_process_to_device(dqm->dev,
 						node->qpd->pqm->process);
 
-	execute_queues_cpsch(dqm, true);
+	mutex_lock(&dqm->lock);
+	execute_queues_cpsch(dqm);
+	mutex_unlock(&dqm->lock);
 
 	return 0;
 fail_allocate_vidmem:
@@ -760,7 +906,11 @@ static int stop_cpsch(struct device_queue_manager *dqm)
 
 	BUG_ON(!dqm);
 
-	destroy_queues_cpsch(dqm, true, true);
+	mutex_lock(&dqm->lock);
+
+	unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, false);
+
+	mutex_unlock(&dqm->lock);
 
 	list_for_each_entry(node, &dqm->queues, list) {
 		pdd = qpd_to_pdd(node->qpd);
@@ -799,7 +949,7 @@ static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
 	list_add(&kq->list, &qpd->priv_queue_list);
 	dqm->queue_count++;
 	qpd->is_debug = true;
-	execute_queues_cpsch(dqm, false);
+	execute_queues_cpsch(dqm);
 	mutex_unlock(&dqm->lock);
 
 	return 0;
@@ -815,11 +965,11 @@ static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
 
 	mutex_lock(&dqm->lock);
 	/* here we actually preempt the DIQ */
-	destroy_queues_cpsch(dqm, true, false);
+	unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, false);
 	list_del(&kq->list);
 	dqm->queue_count--;
 	qpd->is_debug = false;
-	execute_queues_cpsch(dqm, false);
+	execute_queues_cpsch(dqm);
 	/*
 	 * Unconditionally decrement this counter, regardless of the queue's
 	 * type.
@@ -830,14 +980,6 @@ static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
 	mutex_unlock(&dqm->lock);
 }
 
-static void select_sdma_engine_id(struct queue *q)
-{
-	static int sdma_id;
-
-	q->sdma_id = sdma_id;
-	sdma_id = (sdma_id + 1) % 2;
-}
-
 static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 			struct qcm_process_device *qpd, int *allocate_vmid)
 {
@@ -860,9 +1002,15 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 		goto out;
 	}
 
-	if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
-		select_sdma_engine_id(q);
-
+	if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
+		retval = allocate_sdma_queue(dqm, &q->sdma_id);
+		if (retval != 0)
+			goto out;
+		q->properties.sdma_queue_id =
+			q->sdma_id / CIK_SDMA_QUEUES_PER_ENGINE;
+		q->properties.sdma_engine_id =
+			q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;
+	}
 	mqd = dqm->ops.get_mqd_manager(dqm,
 			get_mqd_type_from_queue_type(q->properties.type));
 
@@ -870,8 +1018,19 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 		mutex_unlock(&dqm->lock);
 		return -ENOMEM;
 	}
+	/*
+	 * Eviction state logic: we only mark active queues as evicted
+	 * to avoid the overhead of restoring inactive queues later
+	 */
+	if (qpd->evicted)
+		q->properties.is_evicted = (q->properties.queue_size > 0 &&
+					    q->properties.queue_percent > 0 &&
+					    q->properties.queue_address != 0);
 
 	dqm->ops_asic_specific.init_sdma_vm(dqm, q, qpd);
+
+	q->properties.tba_addr = qpd->tba_addr;
+	q->properties.tma_addr = qpd->tma_addr;
 	retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
 				&q->gart_mqd_addr, &q->properties);
 	if (retval != 0)
@@ -880,7 +1039,7 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 	list_add(&q->list, &qpd->queues_list);
 	if (q->properties.is_active) {
 		dqm->queue_count++;
-		retval = execute_queues_cpsch(dqm, false);
+		retval = execute_queues_cpsch(dqm);
 	}
 
 	if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
@@ -917,20 +1076,20 @@ int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
 	return 0;
 }
 
-static int destroy_sdma_queues(struct device_queue_manager *dqm,
+static int unmap_sdma_queues(struct device_queue_manager *dqm,
 				unsigned int sdma_engine)
 {
 	return pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_SDMA,
-			KFD_PREEMPT_TYPE_FILTER_DYNAMIC_QUEUES, 0, false,
+			KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, false,
 			sdma_engine);
 }
 
-static int destroy_queues_cpsch(struct device_queue_manager *dqm,
-				bool preempt_static_queues, bool lock)
+/* dqm->lock mutex has to be locked before calling this function */
+static int unmap_queues_cpsch(struct device_queue_manager *dqm,
+		enum kfd_unmap_queues_filter filter,
+		uint32_t filter_param, bool reset)
 {
 	int retval;
-	enum kfd_preempt_type_filter preempt_type;
-	struct kfd_process_device *pdd;
 
 	BUG_ON(!dqm);
 
@@ -940,23 +1099,21 @@ static int destroy_queues_cpsch(struct device_queue_manager *dqm,
 		mutex_lock(&dqm->lock);
 	if (!dqm->active_runlist)
 		goto out;
+	if (dqm->active_runlist == false)
+		return retval;
 
 	pr_debug("kfd: Before destroying queues, sdma queue count is : %u\n",
 		dqm->sdma_queue_count);
 
 	if (dqm->sdma_queue_count > 0) {
-		destroy_sdma_queues(dqm, 0);
-		destroy_sdma_queues(dqm, 1);
+		unmap_sdma_queues(dqm, 0);
+		unmap_sdma_queues(dqm, 1);
 	}
 
-	preempt_type = preempt_static_queues ?
-			KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES :
-			KFD_PREEMPT_TYPE_FILTER_DYNAMIC_QUEUES;
-
 	retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE,
-			preempt_type, 0, false, 0);
+			filter, filter_param, reset, 0);
 	if (retval != 0)
-		goto out;
+		return retval;
 
 	*dqm->fence_addr = KFD_FENCE_INIT;
 	pm_send_query_status(&dqm->packets, dqm->fence_gpu_addr,
@@ -965,55 +1122,47 @@ static int destroy_queues_cpsch(struct device_queue_manager *dqm,
 	retval = amdkfd_fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED,
 				QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS);
 	if (retval != 0) {
-		pdd = kfd_get_process_device_data(dqm->dev,
-				kfd_get_process(current));
-		pdd->reset_wavefronts = true;
-		goto out;
+		pr_err("kfd: unmapping queues failed.");
+		return retval;
 	}
+
 	pm_release_ib(&dqm->packets);
 	dqm->active_runlist = false;
 
-out:
-	if (lock)
-		mutex_unlock(&dqm->lock);
 	return retval;
 }
 
-static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock)
+/* dqm->lock mutex has to be locked before calling this function */
+static int execute_queues_cpsch(struct device_queue_manager *dqm)
 {
 	int retval;
 
 	BUG_ON(!dqm);
 
-	if (lock)
-		mutex_lock(&dqm->lock);
-
-	retval = destroy_queues_cpsch(dqm, false, false);
+	retval = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES,
+			0, false);
 	if (retval != 0) {
 		pr_err("kfd: the cp might be in an unrecoverable state due to an unsuccessful queues preemption");
-		goto out;
+		return retval;
 	}
 
 	if (dqm->queue_count <= 0 || dqm->processes_count <= 0) {
 		retval = 0;
-		goto out;
+		return retval;
 	}
 
 	if (dqm->active_runlist) {
 		retval = 0;
-		goto out;
+		return retval;
 	}
 
 	retval = pm_send_runlist(&dqm->packets, &dqm->queues);
 	if (retval != 0) {
 		pr_err("kfd: failed to execute runlist");
-		goto out;
+		return retval;
 	}
 	dqm->active_runlist = true;
 
-out:
-	if (lock)
-		mutex_unlock(&dqm->lock);
 	return retval;
 }
 
@@ -1051,14 +1200,16 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,
 		goto failed;
 	}
 
-	if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
+	if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
 		dqm->sdma_queue_count--;
+		deallocate_sdma_queue(dqm, q->sdma_id);
+	}
 
 	list_del(&q->list);
 	if (q->properties.is_active)
 		dqm->queue_count--;
 
-	execute_queues_cpsch(dqm, false);
+	retval = execute_queues_cpsch(dqm);
 
 	mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
 
@@ -1072,7 +1223,7 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,
 
 	mutex_unlock(&dqm->lock);
 
-	return 0;
+	return retval;
 
 failed:
 failed_try_destroy_debugged_queue:
@@ -1156,6 +1307,172 @@ static bool set_cache_memory_policy(struct device_queue_manager *dqm,
 	return false;
 }
 
+static int set_trap_handler(struct device_queue_manager *dqm,
+				struct qcm_process_device *qpd,
+				uint64_t tba_addr,
+				uint64_t tma_addr)
+{
+	uint64_t *tma;
+
+	tma = (uint64_t *)(qpd->cwsr_kaddr + dqm->dev->tma_offset);
+	tma[0] = tba_addr;
+	tma[1] = tma_addr;
+	return 0;
+}
+
+
+static int set_page_directory_base(struct device_queue_manager *dqm,
+					struct qcm_process_device *qpd)
+{
+	struct kfd_process_device *pdd;
+	uint32_t pd_base;
+	int retval = 0;
+
+	BUG_ON(!dqm || !qpd);
+
+	mutex_lock(&dqm->lock);
+
+	pdd = qpd_to_pdd(qpd);
+
+	/* Retrieve PD base */
+	pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm);
+
+	/* If it has not changed, just get out */
+	if (qpd->page_table_base == pd_base)
+		goto out;
+
+	/* Update PD Base in QPD */
+	qpd->page_table_base = pd_base;
+	pr_debug("Updated PD address == 0x%08u\n", pd_base);
+
+	/*
+	 * Preempt queues, destroy runlist and create new runlist. Queues
+	 * will have the update PD base address
+	 */
+	if (sched_policy != KFD_SCHED_POLICY_NO_HWS)
+		retval = execute_queues_cpsch(dqm);
+
+out:
+	mutex_unlock(&dqm->lock);
+
+	return retval;
+}
+
+static int process_termination_nocpsch(struct device_queue_manager *dqm,
+		struct qcm_process_device *qpd)
+{
+	struct queue *q, *next;
+	struct mqd_manager *mqd;
+	struct device_process_node *cur, *next_dpn;
+
+	mutex_lock(&dqm->lock);
+
+	/* Clear all user mode queues */
+	list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
+		mqd = dqm->ops.get_mqd_manager(dqm,
+			get_mqd_type_from_queue_type(q->properties.type));
+		if (!mqd) {
+			mutex_unlock(&dqm->lock);
+			return -ENOMEM;
+		}
+
+		if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
+			dqm->sdma_queue_count--;
+			deallocate_sdma_queue(dqm, q->sdma_id);
+		}
+
+		list_del(&q->list);
+		if (q->properties.is_active)
+			dqm->queue_count--;
+
+		dqm->total_queue_count--;
+		mqd->destroy_mqd(mqd, q->mqd,
+				KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
+				QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS,
+				q->pipe, q->queue);
+		mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
+		if (list_empty(&qpd->queues_list))
+			deallocate_vmid(dqm, qpd, q);
+	}
+
+	/* Unregister process */
+	list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
+		if (qpd == cur->qpd) {
+			list_del(&cur->list);
+			kfree(cur);
+			dqm->processes_count--;
+			break;
+		}
+	}
+
+	mutex_unlock(&dqm->lock);
+
+	return 0;
+}
+
+
+static int process_termination_cpsch(struct device_queue_manager *dqm,
+		struct qcm_process_device *qpd)
+{
+	int retval;
+	struct queue *q, *next;
+	struct kernel_queue *kq, *kq_next;
+	struct mqd_manager *mqd;
+	struct device_process_node *cur, *next_dpn;
+
+	retval = 0;
+
+	mutex_lock(&dqm->lock);
+
+	/* Clean all kernel queues */
+	list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) {
+		list_del(&kq->list);
+		dqm->queue_count--;
+		qpd->is_debug = false;
+		dqm->total_queue_count--;
+	}
+
+	/* Clear all user mode queues */
+	list_for_each_entry(q, &qpd->queues_list, list) {
+		if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
+			dqm->sdma_queue_count--;
+			deallocate_sdma_queue(dqm, q->sdma_id);
+		}
+
+		if (q->properties.is_active)
+			dqm->queue_count--;
+
+		dqm->total_queue_count--;
+	}
+
+	/* Unregister process */
+	list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
+		if (qpd == cur->qpd) {
+			list_del(&cur->list);
+			kfree(cur);
+			dqm->processes_count--;
+			break;
+		}
+	}
+
+	retval = execute_queues_cpsch(dqm);
+
+	/* lastly, free mqd resources */
+	list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
+		mqd = dqm->ops.get_mqd_manager(dqm,
+			get_mqd_type_from_queue_type(q->properties.type));
+		if (!mqd) {
+			mutex_unlock(&dqm->lock);
+			return -ENOMEM;
+		}
+		list_del(&q->list);
+		mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
+	}
+
+	mutex_unlock(&dqm->lock);
+	return retval;
+}
+
 struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
 {
 	struct device_queue_manager *dqm;
@@ -1186,6 +1503,9 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
 		dqm->ops.create_kernel_queue = create_kernel_queue_cpsch;
 		dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch;
 		dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
+		dqm->ops.set_trap_handler = set_trap_handler;
+		dqm->ops.set_page_directory_base = set_page_directory_base;
+		dqm->ops.process_termination = process_termination_cpsch;
 		break;
 	case KFD_SCHED_POLICY_NO_HWS:
 		/* initialize dqm for no cp scheduling */
@@ -1200,6 +1520,9 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
 		dqm->ops.initialize = initialize_nocpsch;
 		dqm->ops.uninitialize = uninitialize_nocpsch;
 		dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
+		dqm->ops.set_trap_handler = set_trap_handler;
+		dqm->ops.set_page_directory_base = set_page_directory_base;
+		dqm->ops.process_termination = process_termination_nocpsch;
 		break;
 	default:
 		BUG();
@@ -1214,6 +1537,11 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
 	case CHIP_KAVERI:
 		device_queue_manager_init_cik(&dqm->ops_asic_specific);
 		break;
+
+	case CHIP_TONGA:
+	case CHIP_FIJI:
+		device_queue_manager_init_vi_tonga(&dqm->ops_asic_specific);
+		break;
 	}
 
 	if (dqm->ops.initialize(dqm) != 0) {
@@ -1231,3 +1559,20 @@ void device_queue_manager_uninit(struct device_queue_manager *dqm)
 	dqm->ops.uninitialize(dqm);
 	kfree(dqm);
 }
+
+int kfd_process_vm_fault(struct device_queue_manager *dqm,
+				unsigned int pasid)
+{
+	struct kfd_process_device *pdd;
+	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
+	int ret = 0;
+
+	if (!p)
+		return -EINVAL;
+	pdd = kfd_get_process_device_data(dqm->dev, p);
+	if (pdd)
+		ret = process_evict_queues(dqm, &pdd->qpd);
+	up_read(&p->lock);
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
index faf820a..d6af017 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
@@ -29,10 +29,7 @@
 #include "kfd_priv.h"
 #include "kfd_mqd_manager.h"
 
-#define QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS	(500)
-#define CIK_VMID_NUM				(8)
-#define KFD_VMID_START_OFFSET			(8)
-#define VMID_PER_DEVICE				CIK_VMID_NUM
+#define QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS	(9000)
 #define KFD_DQM_FIRST_PIPE			(0)
 #define CIK_SDMA_QUEUES				(4)
 #define CIK_SDMA_QUEUES_PER_ENGINE		(2)
@@ -79,6 +76,12 @@ struct device_process_node {
  * @set_cache_memory_policy: Sets memory policy (cached/ non cached) for the
  * memory apertures.
  *
+ * @set_page_directory_base: Sets the PD base address (GPU local memory)
+ * in all the queues of the relevant process running on the specified device.
+ * It preempts the queues, updates the value and execute the runlist again.
+ *
+ * @process_termination: Clears all process queues belongs to that device.
+ *
  */
 
 struct device_queue_manager_ops {
@@ -122,6 +125,16 @@ struct device_queue_manager_ops {
 					   enum cache_policy alternate_policy,
 					   void __user *alternate_aperture_base,
 					   uint64_t alternate_aperture_size);
+
+	int	(*set_trap_handler)(struct device_queue_manager *dqm,
+				    struct qcm_process_device *qpd,
+				    uint64_t tba_addr,
+				    uint64_t tma_addr);
+
+	int	(*set_page_directory_base)(struct device_queue_manager *dqm,
+					struct qcm_process_device *qpd);
+	int (*process_termination)(struct device_queue_manager *dqm,
+			struct qcm_process_device *qpd);
 };
 
 struct device_queue_manager_asic_ops {
@@ -178,12 +191,20 @@ struct device_queue_manager {
 
 void device_queue_manager_init_cik(struct device_queue_manager_asic_ops *ops);
 void device_queue_manager_init_vi(struct device_queue_manager_asic_ops *ops);
+void device_queue_manager_init_vi_tonga(
+		struct device_queue_manager_asic_ops *ops);
 void program_sh_mem_settings(struct device_queue_manager *dqm,
 					struct qcm_process_device *qpd);
 unsigned int get_queues_num(struct device_queue_manager *dqm);
 unsigned int get_queues_per_pipe(struct device_queue_manager *dqm);
 unsigned int get_pipes_per_mec(struct device_queue_manager *dqm);
 
+int process_evict_queues(struct device_queue_manager *dqm,
+		struct qcm_process_device *qpd);
+int process_restore_queues(struct device_queue_manager *dqm,
+		struct qcm_process_device *qpd);
+
+
 static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd)
 {
 	return (pdd->lds_base >> 16) & 0xFF;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c
index 48dc056..da55e39c 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c
@@ -24,6 +24,7 @@
 #include "kfd_device_queue_manager.h"
 #include "cik_regs.h"
 #include "oss/oss_2_4_sh_mask.h"
+#include "gca/gfx_7_2_sh_mask.h"
 
 static bool set_cache_memory_policy_cik(struct device_queue_manager *dqm,
 				   struct qcm_process_device *qpd,
@@ -125,6 +126,7 @@ static int register_process_cik(struct device_queue_manager *dqm,
 	} else {
 		temp = get_sh_mem_bases_nybble_64(pdd);
 		qpd->sh_mem_bases = compute_sh_mem_bases_64bit(temp);
+		qpd->sh_mem_config |= 1  << SH_MEM_CONFIG__PRIVATE_ATC__SHIFT;
 	}
 
 	pr_debug("kfd: is32bit process: %d sh_mem_bases nybble: 0x%X and register 0x%X\n",
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c
index 7e9cae9..c023e50 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c
@@ -39,6 +39,31 @@ static int initialize_cpsch_vi(struct device_queue_manager *dqm);
 static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q,
 				struct qcm_process_device *qpd);
 
+/*
+ * Tonga device queue manager functions
+ */
+static bool set_cache_memory_policy_vi_tonga(struct device_queue_manager *dqm,
+			struct qcm_process_device *qpd,
+			enum cache_policy default_policy,
+			enum cache_policy alternate_policy,
+			void __user *alternate_aperture_base,
+			uint64_t alternate_aperture_size);
+static int register_process_vi_tonga(struct device_queue_manager *dqm,
+			struct qcm_process_device *qpd);
+static void init_sdma_vm_tonga(struct device_queue_manager *dqm,
+			struct queue *q,
+			struct qcm_process_device *qpd);
+
+void device_queue_manager_init_vi_tonga(
+		struct device_queue_manager_asic_ops *ops)
+{
+	ops->set_cache_memory_policy = set_cache_memory_policy_vi_tonga;
+	ops->register_process = register_process_vi_tonga;
+	ops->initialize = initialize_cpsch_vi;
+	ops->init_sdma_vm = init_sdma_vm_tonga;
+}
+
+
 void device_queue_manager_init_vi(struct device_queue_manager_asic_ops *ops)
 {
 	ops->set_cache_memory_policy = set_cache_memory_policy_vi;
@@ -104,6 +129,33 @@ static bool set_cache_memory_policy_vi(struct device_queue_manager *dqm,
 	return true;
 }
 
+static bool set_cache_memory_policy_vi_tonga(struct device_queue_manager *dqm,
+		struct qcm_process_device *qpd,
+		enum cache_policy default_policy,
+		enum cache_policy alternate_policy,
+		void __user *alternate_aperture_base,
+		uint64_t alternate_aperture_size)
+{
+	uint32_t default_mtype;
+	uint32_t ape1_mtype;
+
+	default_mtype = (default_policy == cache_policy_coherent) ?
+			MTYPE_UC :
+			MTYPE_NC_NV;
+
+	ape1_mtype = (alternate_policy == cache_policy_coherent) ?
+			MTYPE_UC :
+			MTYPE_NC_NV;
+
+	qpd->sh_mem_config =
+			SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
+				   SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT |
+			default_mtype << SH_MEM_CONFIG__DEFAULT_MTYPE__SHIFT |
+			ape1_mtype << SH_MEM_CONFIG__APE1_MTYPE__SHIFT;
+
+	return true;
+}
+
 static int register_process_vi(struct device_queue_manager *dqm,
 					struct qcm_process_device *qpd)
 {
@@ -137,6 +189,8 @@ static int register_process_vi(struct device_queue_manager *dqm,
 		qpd->sh_mem_bases = compute_sh_mem_bases_64bit(temp);
 		qpd->sh_mem_config |= SH_MEM_ADDRESS_MODE_HSA64 <<
 			SH_MEM_CONFIG__ADDRESS_MODE__SHIFT;
+		qpd->sh_mem_config |= 1  <<
+			SH_MEM_CONFIG__PRIVATE_ATC__SHIFT;
 	}
 
 	pr_debug("kfd: is32bit process: %d sh_mem_bases nybble: 0x%X and register 0x%X\n",
@@ -145,6 +199,41 @@ static int register_process_vi(struct device_queue_manager *dqm,
 	return 0;
 }
 
+static int register_process_vi_tonga(struct device_queue_manager *dqm,
+			struct qcm_process_device *qpd)
+{
+	struct kfd_process_device *pdd;
+	unsigned int temp;
+
+	BUG_ON(!dqm || !qpd);
+
+	pdd = qpd_to_pdd(qpd);
+
+	/* check if sh_mem_config register already configured */
+	if (qpd->sh_mem_config == 0) {
+		qpd->sh_mem_config =
+				SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
+					SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT |
+				MTYPE_UC <<
+					SH_MEM_CONFIG__DEFAULT_MTYPE__SHIFT |
+				MTYPE_UC <<
+					SH_MEM_CONFIG__APE1_MTYPE__SHIFT;
+
+		qpd->sh_mem_ape1_limit = 0;
+		qpd->sh_mem_ape1_base = 0;
+	}
+
+	/* On dGPU we're always in GPUVM64 addressing mode with 64-bit
+	 * aperture addresses. */
+	temp = get_sh_mem_bases_nybble_64(pdd);
+	qpd->sh_mem_bases = compute_sh_mem_bases_64bit(temp);
+
+	pr_debug("kfd: sh_mem_bases nybble: 0x%X and register 0x%X\n",
+		temp, qpd->sh_mem_bases);
+
+	return 0;
+}
+
 static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q,
 				struct qcm_process_device *qpd)
 {
@@ -161,6 +250,23 @@ static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q,
 	q->properties.sdma_vm_addr = value;
 }
 
+static void init_sdma_vm_tonga(struct device_queue_manager *dqm,
+			struct queue *q,
+			struct qcm_process_device *qpd)
+{
+	uint32_t value = 0;
+
+	if (q->process->is_32bit_user_mode)
+		value |= (1 << SDMA0_RLC0_VIRTUAL_ADDR__PTR32__SHIFT) |
+				get_sh_mem_bases_32(qpd_to_pdd(qpd));
+	else
+		value |= ((get_sh_mem_bases_nybble_64(qpd_to_pdd(qpd))) <<
+				SDMA0_RLC0_VIRTUAL_ADDR__SHARED_BASE__SHIFT) &
+				SDMA0_RLC0_VIRTUAL_ADDR__SHARED_BASE_MASK;
+	q->properties.sdma_vm_addr = value;
+}
+
+
 static int initialize_cpsch_vi(struct device_queue_manager *dqm)
 {
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
index 453c5d6..d6a7e2a 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
@@ -142,12 +142,11 @@ int kfd_doorbell_mmap(struct kfd_process *process, struct vm_area_struct *vma)
 
 	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 
-	pr_debug("kfd: mapping doorbell page in %s\n"
+	pr_debug("kfd: mapping doorbell page in kfd_doorbell_mmap\n"
 		 "     target user address == 0x%08llX\n"
 		 "     physical address    == 0x%08llX\n"
 		 "     vm_flags            == 0x%04lX\n"
 		 "     size                == 0x%04lX\n",
-		 __func__,
 		 (unsigned long long) vma->vm_start, address, vma->vm_flags,
 		 doorbell_process_allocation());
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
index d1ce83d..23b5936 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
@@ -32,11 +32,10 @@
 #include "kfd_events.h"
 #include <linux/device.h>
 
-/*
- * A task can only be on a single wait_queue at a time, but we need to support
+/* A task can only be on a single wait_queue at a time, but we need to support
  * waiting on multiple events (any/all).
- * Instead of each event simply having a wait_queue with sleeping tasks, it
- * has a singly-linked list of tasks.
+ * Instead of each event simply having a wait_queue with sleeping tasks, it has a
+ * singly-linked list of tasks.
  * A thread that wants to sleep creates an array of these, one for each event
  * and adds one to each event's waiter chain.
  */
@@ -52,12 +51,11 @@ struct kfd_event_waiter {
 	uint32_t input_index;
 };
 
-/*
- * Over-complicated pooled allocator for event notification slots.
+/* Over-complicated pooled allocator for event notification slots.
  *
- * Each signal event needs a 64-bit signal slot where the signaler will write
- * a 1 before sending an interrupt.l (This is needed because some interrupts
- * do not contain enough spare data bits to identify an event.)
+ * Each signal event needs a 64-bit signal slot where the signaler will write a 1
+ * before sending an interrupt.l (This is needed because some interrupts do not
+ * contain enough spare data bits to identify an event.)
  * We get whole pages from vmalloc and map them to the process VA.
  * Individual signal events are then allocated a slot in a page.
  */
@@ -65,6 +63,7 @@ struct kfd_event_waiter {
 struct signal_page {
 	struct list_head event_pages;	/* kfd_process.signal_event_pages */
 	uint64_t *kernel_address;
+	uint64_t handle;
 	uint64_t __user *user_address;
 	uint32_t page_index;		/* Index into the mmap aperture. */
 	unsigned int free_slots;
@@ -74,8 +73,7 @@ struct signal_page {
 #define SLOTS_PER_PAGE KFD_SIGNAL_EVENT_LIMIT
 #define SLOT_BITMAP_SIZE BITS_TO_LONGS(SLOTS_PER_PAGE)
 #define BITS_PER_PAGE (ilog2(SLOTS_PER_PAGE)+1)
-#define SIGNAL_PAGE_SIZE (sizeof(struct signal_page) + \
-				SLOT_BITMAP_SIZE * sizeof(long))
+#define SIGNAL_PAGE_SIZE (sizeof(struct signal_page) + SLOT_BITMAP_SIZE * sizeof(long))
 
 /*
  * For signal events, the event ID is used as the interrupt user data.
@@ -85,23 +83,27 @@ struct signal_page {
 #define INTERRUPT_DATA_BITS 8
 #define SIGNAL_EVENT_ID_SLOT_SHIFT 0
 
+/* We can only create 8 debug events */
+
+#define KFD_DEBUG_EVENT_LIMIT 8
+#define KFD_DEBUG_EVENT_MASK 0x1F
+#define KFD_DEBUG_EVENT_SHIFT 5
+
 static uint64_t *page_slots(struct signal_page *page)
 {
 	return page->kernel_address;
 }
 
-static bool allocate_free_slot(struct kfd_process *process,
-				struct signal_page **out_page,
-				unsigned int *out_slot_index)
+static bool
+allocate_free_slot(struct kfd_process *process,
+		   struct signal_page **out_page,
+		   unsigned int *out_slot_index)
 {
 	struct signal_page *page;
 
 	list_for_each_entry(page, &process->signal_event_pages, event_pages) {
 		if (page->free_slots > 0) {
-			unsigned int slot =
-				find_first_zero_bit(page->used_slot_bitmap,
-							SLOTS_PER_PAGE);
-
+			unsigned int slot = find_first_zero_bit(page->used_slot_bitmap, SLOTS_PER_PAGE);
 			__set_bit(slot, page->used_slot_bitmap);
 			page->free_slots--;
 
@@ -130,6 +132,8 @@ static bool allocate_signal_page(struct file *devkfd, struct kfd_process *p)
 {
 	void *backing_store;
 	struct signal_page *page;
+	unsigned int slot;
+	int i;
 
 	page = kzalloc(SIGNAL_PAGE_SIZE, GFP_KERNEL);
 	if (!page)
@@ -137,17 +141,23 @@ static bool allocate_signal_page(struct file *devkfd, struct kfd_process *p)
 
 	page->free_slots = SLOTS_PER_PAGE;
 
-	backing_store = (void *) __get_free_pages(GFP_KERNEL | __GFP_ZERO,
+	backing_store = (void *) __get_free_pages(GFP_KERNEL | __GFP_ZERO, \
 					get_order(KFD_SIGNAL_EVENT_LIMIT * 8));
 	if (!backing_store)
 		goto fail_alloc_signal_store;
 
 	/* prevent user-mode info leaks */
-	memset(backing_store, (uint8_t) UNSIGNALED_EVENT_SLOT,
-		KFD_SIGNAL_EVENT_LIMIT * 8);
-
+	memset(backing_store, (uint8_t) UNSIGNALED_EVENT_SLOT, KFD_SIGNAL_EVENT_LIMIT * 8);
 	page->kernel_address = backing_store;
 
+	/* Set bits of debug events to prevent allocation */
+	for (i = 0 ; i < KFD_DEBUG_EVENT_LIMIT ; i++) {
+		slot = (i << KFD_DEBUG_EVENT_SHIFT) |
+				KFD_DEBUG_EVENT_MASK;
+		__set_bit(slot, page->used_slot_bitmap);
+		page->free_slots--;
+	}
+
 	if (list_empty(&p->signal_event_pages))
 		page->page_index = 0;
 	else
@@ -169,10 +179,10 @@ static bool allocate_signal_page(struct file *devkfd, struct kfd_process *p)
 	return false;
 }
 
-static bool allocate_event_notification_slot(struct file *devkfd,
-					struct kfd_process *p,
-					struct signal_page **page,
-					unsigned int *signal_slot_index)
+static bool
+allocate_event_notification_slot(struct file *devkfd, struct kfd_process *p,
+				 struct signal_page **page,
+				 unsigned int *signal_slot_index)
 {
 	bool ret;
 
@@ -186,6 +196,88 @@ static bool allocate_event_notification_slot(struct file *devkfd,
 	return ret;
 }
 
+static bool
+allocate_signal_page_dgpu(struct kfd_process *p,
+		uint64_t *kernel_address, uint64_t handle)
+{
+	struct signal_page *my_page;
+
+	my_page = kzalloc(SIGNAL_PAGE_SIZE, GFP_KERNEL);
+	if (!my_page)
+		return false;
+
+	/* prevent user-mode info leaks */
+	memset(kernel_address, (uint8_t) UNSIGNALED_EVENT_SLOT,
+			KFD_SIGNAL_EVENT_LIMIT * 8);
+
+	my_page->kernel_address = kernel_address;
+	my_page->handle = handle;
+	my_page->user_address = NULL;
+	my_page->free_slots = SLOTS_PER_PAGE;
+	if (list_empty(&p->signal_event_pages))
+			my_page->page_index = 0;
+	else
+		my_page->page_index = list_tail_entry(&p->signal_event_pages,
+		   struct signal_page,
+		   event_pages)->page_index + 1;
+
+	pr_debug("allocated new event signal page at %p, for process %p\n",
+			my_page, p);
+	pr_debug("page index is %d\n", my_page->page_index);
+
+	list_add(&my_page->event_pages, &p->signal_event_pages);
+
+	return true;
+}
+
+void kfd_free_signal_page_dgpu(struct kfd_process *p, uint64_t handle)
+{
+	struct signal_page *page, *tmp;
+
+	list_for_each_entry_safe(page, tmp, &p->signal_event_pages,
+				event_pages) {
+		if (page->handle == handle) {
+			list_del(&page->event_pages);
+			kfree(page);
+			break;
+		}
+	}
+}
+
+static bool
+allocate_debug_event_notification_slot(struct file *devkfd,
+				struct kfd_process *p,
+				struct signal_page **out_page,
+				unsigned int *out_slot_index)
+{
+	struct signal_page *page;
+	unsigned int slot;
+	bool ret;
+
+	if (list_empty(&p->signal_event_pages)) {
+		ret = allocate_signal_page(devkfd, p);
+		if (ret == false)
+			return ret;
+	}
+
+	page = list_entry((&p->signal_event_pages)->next, struct signal_page,
+				event_pages);
+	slot = (p->debug_event_count << KFD_DEBUG_EVENT_SHIFT) |
+			KFD_DEBUG_EVENT_MASK;
+
+	pr_debug("page == %p\n", page);
+	pr_debug("slot == %d\n", slot);
+
+	page_slots(page)[slot] = UNSIGNALED_EVENT_SLOT;
+	*out_page = page;
+	*out_slot_index = slot;
+
+	pr_debug("allocated debug event signal slot in page %p, slot %d\n",
+			page, slot);
+
+	return true;
+}
+
 /* Assumes that the process's event_mutex is locked. */
 static void release_event_notification_slot(struct signal_page *page,
 						size_t slot_index)
@@ -202,10 +294,7 @@ static struct signal_page *lookup_signal_page_by_index(struct kfd_process *p,
 {
 	struct signal_page *page;
 
-	/*
-	 * This is safe because we don't delete signal pages until the
-	 * process exits.
-	 */
+	/* This is safe because we don't delete signal pages until the process exits. */
 	list_for_each_entry(page, &p->signal_event_pages, event_pages)
 		if (page->page_index == page_index)
 			return page;
@@ -213,10 +302,7 @@ static struct signal_page *lookup_signal_page_by_index(struct kfd_process *p,
 	return NULL;
 }
 
-/*
- * Assumes that p->event_mutex is held and of course that p is not going
- * away (current or locked).
- */
+/* Assumes that p->event_mutex is held and of course that p is not going away (current or locked). */
 static struct kfd_event *lookup_event_by_id(struct kfd_process *p, uint32_t id)
 {
 	struct kfd_event *ev;
@@ -231,32 +317,27 @@ static struct kfd_event *lookup_event_by_id(struct kfd_process *p, uint32_t id)
 static u32 make_signal_event_id(struct signal_page *page,
 					 unsigned int signal_slot_index)
 {
-	return page->page_index |
-			(signal_slot_index << SIGNAL_EVENT_ID_SLOT_SHIFT);
+	return page->page_index | (signal_slot_index << SIGNAL_EVENT_ID_SLOT_SHIFT);
 }
 
-/*
- * Produce a kfd event id for a nonsignal event.
- * These are arbitrary numbers, so we do a sequential search through
- * the hash table for an unused number.
+/* Produce a kfd event id for a nonsignal event.
+ * These are arbitrary numbers, so we do a sequential search through the hash table
+ * for an unused number.
  */
 static u32 make_nonsignal_event_id(struct kfd_process *p)
 {
 	u32 id;
 
 	for (id = p->next_nonsignal_event_id;
-		id < KFD_LAST_NONSIGNAL_EVENT_ID &&
-		lookup_event_by_id(p, id) != NULL;
-		id++)
+	     id < KFD_LAST_NONSIGNAL_EVENT_ID && lookup_event_by_id(p, id) != NULL;
+	     id++)
 		;
 
 	if (id < KFD_LAST_NONSIGNAL_EVENT_ID) {
 
-		/*
-		 * What if id == LAST_NONSIGNAL_EVENT_ID - 1?
-		 * Then next_nonsignal_event_id = LAST_NONSIGNAL_EVENT_ID so
-		 * the first loop fails immediately and we proceed with the
-		 * wraparound loop below.
+		/* What if id == LAST_NONSIGNAL_EVENT_ID - 1?
+		 * Then next_nonsignal_event_id = LAST_NONSIGNAL_EVENT_ID so the first loop
+		 * fails immediately and we proceed with the wraparound loop below.
 		 */
 		p->next_nonsignal_event_id = id + 1;
 
@@ -264,54 +345,68 @@ static u32 make_nonsignal_event_id(struct kfd_process *p)
 	}
 
 	for (id = KFD_FIRST_NONSIGNAL_EVENT_ID;
-		id < KFD_LAST_NONSIGNAL_EVENT_ID &&
-		lookup_event_by_id(p, id) != NULL;
-		id++)
+	     id < KFD_LAST_NONSIGNAL_EVENT_ID && lookup_event_by_id(p, id) != NULL;
+	     id++)
 		;
 
 
 	if (id < KFD_LAST_NONSIGNAL_EVENT_ID) {
 		p->next_nonsignal_event_id = id + 1;
 		return id;
+	} else {
+		p->next_nonsignal_event_id = KFD_FIRST_NONSIGNAL_EVENT_ID;
+		return 0;
 	}
-
-	p->next_nonsignal_event_id = KFD_FIRST_NONSIGNAL_EVENT_ID;
-	return 0;
 }
 
-static struct kfd_event *lookup_event_by_page_slot(struct kfd_process *p,
-						struct signal_page *page,
-						unsigned int signal_slot)
+static struct kfd_event *
+lookup_event_by_page_slot(struct kfd_process *p,
+			  struct signal_page *page, unsigned int signal_slot)
 {
 	return lookup_event_by_id(p, make_signal_event_id(page, signal_slot));
 }
 
-static int create_signal_event(struct file *devkfd,
-				struct kfd_process *p,
-				struct kfd_event *ev)
+static int
+create_signal_event(struct file *devkfd, struct kfd_process *p, struct kfd_event *ev)
 {
-	if (p->signal_event_count == KFD_SIGNAL_EVENT_LIMIT) {
+	if ((ev->type == KFD_EVENT_TYPE_SIGNAL) &&
+			(p->signal_event_count == KFD_SIGNAL_EVENT_LIMIT)) {
 		pr_warn("amdkfd: Signal event wasn't created because limit was reached\n");
 		return -ENOMEM;
+	} else if ((ev->type == KFD_EVENT_TYPE_DEBUG) &&
+			(p->debug_event_count == KFD_DEBUG_EVENT_LIMIT)) {
+		pr_warn("amdkfd: Debug event wasn't created because limit was reached\n");
+		return -ENOMEM;
 	}
 
-	if (!allocate_event_notification_slot(devkfd, p, &ev->signal_page,
+	if (ev->type == KFD_EVENT_TYPE_SIGNAL) {
+		if (!allocate_event_notification_slot(devkfd, p,
+						&ev->signal_page,
 						&ev->signal_slot_index)) {
-		pr_warn("amdkfd: Signal event wasn't created because out of kernel memory\n");
-		return -ENOMEM;
-	}
+			pr_warn("amdkfd: Signal event wasn't created because out of kernel memory\n");
+			return -ENOMEM;
+		}
 
-	p->signal_event_count++;
+		p->signal_event_count++;
 
-	ev->user_signal_address =
-			&ev->signal_page->user_address[ev->signal_slot_index];
+		if ((p->signal_event_count & KFD_DEBUG_EVENT_MASK) ==
+				KFD_DEBUG_EVENT_MASK)
+			p->signal_event_count++;
 
-	ev->event_id = make_signal_event_id(ev->signal_page,
-						ev->signal_slot_index);
+	} else if (ev->type == KFD_EVENT_TYPE_DEBUG) {
+		if (!allocate_debug_event_notification_slot(devkfd, p,
+						&ev->signal_page,
+						&ev->signal_slot_index)) {
+			pr_warn("amdkfd: Debug event wasn't created because out of kernel memory\n");
+			return -ENOMEM;
+		}
 
-	pr_debug("signal event number %zu created with id %d, address %p\n",
-			p->signal_event_count, ev->event_id,
-			ev->user_signal_address);
+		p->debug_event_count++;
+	}
+
+	ev->user_signal_address = &ev->signal_page->user_address[ev->signal_slot_index];
+
+	ev->event_id = make_signal_event_id(ev->signal_page, ev->signal_slot_index);
 
 	pr_debug("signal event number %zu created with id %d, address %p\n",
 			p->signal_event_count, ev->event_id,
@@ -320,12 +415,10 @@ static int create_signal_event(struct file *devkfd,
 	return 0;
 }
 
-/*
- * No non-signal events are supported yet.
- * We create them as events that never signal.
- * Set event calls from user-mode are failed.
- */
-static int create_other_event(struct kfd_process *p, struct kfd_event *ev)
+/* No non-signal events are supported yet.
+ * We create them as events that never signal. Set event calls from user-mode are failed. */
+static int
+create_other_event(struct kfd_process *p, struct kfd_event *ev)
 {
 	ev->event_id = make_nonsignal_event_id(p);
 	if (ev->event_id == 0)
@@ -341,20 +434,25 @@ void kfd_event_init_process(struct kfd_process *p)
 	INIT_LIST_HEAD(&p->signal_event_pages);
 	p->next_nonsignal_event_id = KFD_FIRST_NONSIGNAL_EVENT_ID;
 	p->signal_event_count = 0;
+	p->debug_event_count = 0;
 }
 
 static void destroy_event(struct kfd_process *p, struct kfd_event *ev)
 {
 	if (ev->signal_page != NULL) {
-		release_event_notification_slot(ev->signal_page,
-						ev->signal_slot_index);
-		p->signal_event_count--;
+		if (ev->type == KFD_EVENT_TYPE_SIGNAL) {
+			release_event_notification_slot(ev->signal_page,
+							ev->signal_slot_index);
+			p->signal_event_count--;
+			if ((p->signal_event_count & KFD_DEBUG_EVENT_MASK) ==
+					KFD_DEBUG_EVENT_MASK)
+				p->signal_event_count--;
+		} else if (ev->type == KFD_EVENT_TYPE_DEBUG) {
+			p->debug_event_count--;
+		}
 	}
 
-	/*
-	 * Abandon the list of waiters. Individual waiting threads will
-	 * clean up their own data.
-	 */
+	/* Abandon the list of waiters. Individual waiting threads will clean up their own data.*/
 	list_del(&ev->waiters);
 
 	hash_del(&ev->events);
@@ -371,18 +469,17 @@ static void destroy_events(struct kfd_process *p)
 		destroy_event(p, ev);
 }
 
-/*
- * We assume that the process is being destroyed and there is no need to
- * unmap the pages or keep bookkeeping data in order.
- */
+/* We assume that the process is being destroyed and there is no need to unmap the pages
+ * or keep bookkeeping data in order. */
 static void shutdown_signal_pages(struct kfd_process *p)
 {
 	struct signal_page *page, *tmp;
 
-	list_for_each_entry_safe(page, tmp, &p->signal_event_pages,
-					event_pages) {
-		free_pages((unsigned long)page->kernel_address,
-				get_order(KFD_SIGNAL_EVENT_LIMIT * 8));
+	list_for_each_entry_safe(page, tmp, &p->signal_event_pages, event_pages) {
+		if (page->user_address) {
+			free_pages((unsigned long)page->kernel_address,
+					get_order(KFD_SIGNAL_EVENT_LIMIT * 8));
+		}
 		kfree(page);
 	}
 }
@@ -395,8 +492,7 @@ void kfd_event_free_process(struct kfd_process *p)
 
 static bool event_can_be_gpu_signaled(const struct kfd_event *ev)
 {
-	return ev->type == KFD_EVENT_TYPE_SIGNAL ||
-					ev->type == KFD_EVENT_TYPE_DEBUG;
+	return ev->type == KFD_EVENT_TYPE_SIGNAL || ev->type == KFD_EVENT_TYPE_DEBUG;
 }
 
 static bool event_can_be_cpu_signaled(const struct kfd_event *ev)
@@ -407,11 +503,12 @@ static bool event_can_be_cpu_signaled(const struct kfd_event *ev)
 int kfd_event_create(struct file *devkfd, struct kfd_process *p,
 		     uint32_t event_type, bool auto_reset, uint32_t node_id,
 		     uint32_t *event_id, uint32_t *event_trigger_data,
-		     uint64_t *event_page_offset, uint32_t *event_slot_index)
+		     uint64_t *event_page_offset, uint32_t *event_slot_index,
+		     void *kern_addr)
 {
 	int ret = 0;
-	struct kfd_event *ev = kzalloc(sizeof(*ev), GFP_KERNEL);
 
+	struct kfd_event *ev = kzalloc(sizeof(*ev), GFP_KERNEL);
 	if (!ev)
 		return -ENOMEM;
 
@@ -421,17 +518,20 @@ int kfd_event_create(struct file *devkfd, struct kfd_process *p,
 
 	INIT_LIST_HEAD(&ev->waiters);
 
-	*event_page_offset = 0;
-
 	mutex_lock(&p->event_mutex);
 
+	if (kern_addr && list_empty(&p->signal_event_pages))
+		allocate_signal_page_dgpu(p, kern_addr, *event_page_offset);
+
+	*event_page_offset = 0;
+
 	switch (event_type) {
 	case KFD_EVENT_TYPE_SIGNAL:
 	case KFD_EVENT_TYPE_DEBUG:
 		ret = create_signal_event(devkfd, p, ev);
 		if (!ret) {
 			*event_page_offset = (ev->signal_page->page_index |
-					KFD_MMAP_EVENTS_MASK);
+					KFD_MMAP_TYPE_EVENTS);
 			*event_page_offset <<= PAGE_SHIFT;
 			*event_slot_index = ev->signal_slot_index;
 		}
@@ -538,8 +638,7 @@ int kfd_reset_event(struct kfd_process *p, uint32_t event_id)
 
 static void acknowledge_signal(struct kfd_process *p, struct kfd_event *ev)
 {
-	page_slots(ev->signal_page)[ev->signal_slot_index] =
-						UNSIGNALED_EVENT_SLOT;
+	page_slots(ev->signal_page)[ev->signal_slot_index] = UNSIGNALED_EVENT_SLOT;
 }
 
 static bool is_slot_signaled(struct signal_page *page, unsigned int index)
@@ -547,8 +646,7 @@ static bool is_slot_signaled(struct signal_page *page, unsigned int index)
 	return page_slots(page)[index] != UNSIGNALED_EVENT_SLOT;
 }
 
-static void set_event_from_interrupt(struct kfd_process *p,
-					struct kfd_event *ev)
+static void set_event_from_interrupt(struct kfd_process *p, struct kfd_event *ev)
 {
 	if (ev && event_can_be_gpu_signaled(ev)) {
 		acknowledge_signal(p, ev);
@@ -561,42 +659,39 @@ void kfd_signal_event_interrupt(unsigned int pasid, uint32_t partial_id,
 {
 	struct kfd_event *ev;
 
-	/*
-	 * Because we are called from arbitrary context (workqueue) as opposed
+	/* Because we are called from arbitrary context (workqueue) as opposed
 	 * to process context, kfd_process could attempt to exit while we are
-	 * running so the lookup function returns a locked process.
-	 */
+	 * running so the lookup function returns a read-locked process. */
 	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
-
 	if (!p)
 		return; /* Presumably process exited. */
 
 	mutex_lock(&p->event_mutex);
 
-	if (valid_id_bits >= INTERRUPT_DATA_BITS) {
+	if ((valid_id_bits >= INTERRUPT_DATA_BITS) &&
+			((partial_id & KFD_DEBUG_EVENT_MASK) ==
+					KFD_DEBUG_EVENT_MASK)) {
 		/* Partial ID is a full ID. */
 		ev = lookup_event_by_id(p, partial_id);
 		set_event_from_interrupt(p, ev);
 	} else {
-		/*
-		 * Partial ID is in fact partial. For now we completely
-		 * ignore it, but we could use any bits we did receive to
-		 * search faster.
-		 */
+		/* Partial ID is in fact partial. For now we completely ignore it,
+		 * but we could use any bits we did receive to search faster. */
 		struct signal_page *page;
 		unsigned i;
 
-		list_for_each_entry(page, &p->signal_event_pages, event_pages)
-			for (i = 0; i < SLOTS_PER_PAGE; i++)
+		list_for_each_entry(page, &p->signal_event_pages, event_pages) {
+			for (i = 0; i < SLOTS_PER_PAGE; i++) {
 				if (is_slot_signaled(page, i)) {
-					ev = lookup_event_by_page_slot(p,
-								page, i);
+					ev = lookup_event_by_page_slot(p, page, i);
 					set_event_from_interrupt(p, ev);
 				}
+			}
+		}
 	}
 
 	mutex_unlock(&p->event_mutex);
-	mutex_unlock(&p->mutex);
+	up_read(&p->lock);
 }
 
 static struct kfd_event_waiter *alloc_event_waiters(uint32_t num_events)
@@ -604,20 +699,20 @@ static struct kfd_event_waiter *alloc_event_waiters(uint32_t num_events)
 	struct kfd_event_waiter *event_waiters;
 	uint32_t i;
 
-	event_waiters = kmalloc_array(num_events,
-					sizeof(struct kfd_event_waiter),
-					GFP_KERNEL);
+	event_waiters = kmalloc(num_events * sizeof(struct kfd_event_waiter), GFP_KERNEL);
 
-	for (i = 0; (event_waiters) && (i < num_events) ; i++) {
-		INIT_LIST_HEAD(&event_waiters[i].waiters);
-		event_waiters[i].sleeping_task = current;
-		event_waiters[i].activated = false;
+	if (event_waiters) {
+		for (i = 0; i < num_events; i++) {
+			INIT_LIST_HEAD(&event_waiters[i].waiters);
+			event_waiters[i].sleeping_task = current;
+			event_waiters[i].activated = false;
+		}
 	}
 
 	return event_waiters;
 }
 
-static int init_event_waiter(struct kfd_process *p,
+static int init_event_waiter_get_status(struct kfd_process *p,
 		struct kfd_event_waiter *waiter,
 		uint32_t event_id,
 		uint32_t input_index)
@@ -632,13 +727,21 @@ static int init_event_waiter(struct kfd_process *p,
 	waiter->activated = ev->signaled;
 	ev->signaled = ev->signaled && !ev->auto_reset;
 
-	list_add(&waiter->waiters, &ev->waiters);
-
 	return 0;
 }
 
+static void init_event_waiter_add_to_waitlist(struct kfd_event_waiter *waiter)
+{
+	struct kfd_event *ev = waiter->event;
+
+	/* Only add to the wait list if we actually need to
+	 * wait on this event. */
+	if (!waiter->activated)
+		list_add(&waiter->waiters, &ev->waiters);
+}
+
 static bool test_event_condition(bool all, uint32_t num_events,
-				struct kfd_event_waiter *event_waiters)
+		struct kfd_event_waiter *event_waiters)
 {
 	uint32_t i;
 	uint32_t activated_count = 0;
@@ -663,23 +766,15 @@ static bool copy_signaled_event_data(uint32_t num_events,
 		struct kfd_event_waiter *event_waiters,
 		struct kfd_event_data __user *data)
 {
-	struct kfd_hsa_memory_exception_data *src;
-	struct kfd_hsa_memory_exception_data __user *dst;
-	struct kfd_event_waiter *waiter;
-	struct kfd_event *event;
 	uint32_t i;
 
-	for (i = 0; i < num_events; i++) {
-		waiter = &event_waiters[i];
-		event = waiter->event;
-		if (waiter->activated && event->type == KFD_EVENT_TYPE_MEMORY) {
-			dst = &data[waiter->input_index].memory_exception_data;
-			src = &event->memory_exception_data;
-			if (copy_to_user(dst, src,
-				sizeof(struct kfd_hsa_memory_exception_data)))
+	for (i = 0; i < num_events; i++)
+		if (event_waiters[i].activated &&
+			event_waiters[i].event->type == KFD_EVENT_TYPE_MEMORY)
+			if (copy_to_user(&data[event_waiters[i].input_index].memory_exception_data,
+					&event_waiters[i].event->memory_exception_data,
+					sizeof(struct kfd_hsa_memory_exception_data)))
 				return false;
-		}
-	}
 
 	return true;
 
@@ -695,11 +790,9 @@ static long user_timeout_to_jiffies(uint32_t user_timeout_ms)
 	if (user_timeout_ms == KFD_EVENT_TIMEOUT_INFINITE)
 		return MAX_SCHEDULE_TIMEOUT;
 
-	/*
-	 * msecs_to_jiffies interprets all values above 2^31-1 as infinite,
+	/* msecs_to_jiffies interprets all values above 2^31-1 as infinite,
 	 * but we consider them finite.
-	 * This hack is wrong, but nobody is likely to notice.
-	 */
+	 * This hack is wrong, but nobody is likely to notice. */
 	user_timeout_ms = min_t(uint32_t, user_timeout_ms, 0x7FFFFFFF);
 
 	return msecs_to_jiffies(user_timeout_ms) + 1;
@@ -724,11 +817,16 @@ int kfd_wait_on_events(struct kfd_process *p,
 			(struct kfd_event_data __user *) data;
 	uint32_t i;
 	int ret = 0;
+
 	struct kfd_event_waiter *event_waiters = NULL;
 	long timeout = user_timeout_to_jiffies(user_timeout_ms);
 
 	mutex_lock(&p->event_mutex);
 
+	/* Set to something unreasonable - this is really
+	 * just a bool for now. */
+	*wait_result = KFD_WAIT_TIMEOUT;
+
 	event_waiters = alloc_event_waiters(num_events);
 	if (!event_waiters) {
 		ret = -ENOMEM;
@@ -744,14 +842,34 @@ int kfd_wait_on_events(struct kfd_process *p,
 			goto fail;
 		}
 
-		ret = init_event_waiter(p, &event_waiters[i],
+		ret = init_event_waiter_get_status(p, &event_waiters[i],
 				event_data.event_id, i);
 		if (ret)
 			goto fail;
 	}
 
+	/* Check condition once. */
+	if (test_event_condition(all, num_events, event_waiters)) {
+		if (copy_signaled_event_data(num_events,
+				event_waiters, events))
+			*wait_result = KFD_WAIT_COMPLETE;
+		else
+			*wait_result = KFD_WAIT_ERROR;
+		free_waiters(num_events, event_waiters);
+	} else {
+		/* Add to wait lists if we need to wait. */
+		for (i = 0; i < num_events; i++)
+			init_event_waiter_add_to_waitlist(&event_waiters[i]);
+	}
+
 	mutex_unlock(&p->event_mutex);
 
+	/* Return if all waits were already satisfied. */
+	if (*wait_result != KFD_WAIT_TIMEOUT) {
+		__set_current_state(TASK_RUNNING);
+		return ret;
+	}
+
 	while (true) {
 		if (fatal_signal_pending(current)) {
 			ret = -EINTR;
@@ -760,17 +878,17 @@ int kfd_wait_on_events(struct kfd_process *p,
 
 		if (signal_pending(current)) {
 			/*
-			 * This is wrong when a nonzero, non-infinite timeout
-			 * is specified. We need to use
-			 * ERESTARTSYS_RESTARTBLOCK, but struct restart_block
-			 * contains a union with data for each user and it's
-			 * in generic kernel code that I don't want to
-			 * touch yet.
+			 * This is wrong when a nonzero, non-infinite timeout is specified.
+			 * We need to use ERESTARTSYS_RESTARTBLOCK, but struct restart_block
+			 * contains a union with data for each user and it's in generic
+			 * kernel code that I don't want to touch yet.
 			 */
 			ret = -ERESTARTSYS;
 			break;
 		}
 
+		set_current_state(TASK_INTERRUPTIBLE);
+
 		if (test_event_condition(all, num_events, event_waiters)) {
 			if (copy_signaled_event_data(num_events,
 					event_waiters, events))
@@ -785,7 +903,7 @@ int kfd_wait_on_events(struct kfd_process *p,
 			break;
 		}
 
-		timeout = schedule_timeout_interruptible(timeout);
+		timeout = schedule_timeout(timeout);
 	}
 	__set_current_state(TASK_RUNNING);
 
@@ -825,8 +943,7 @@ int kfd_event_mmap(struct kfd_process *p, struct vm_area_struct *vma)
 	page = lookup_signal_page_by_index(p, page_index);
 	if (!page) {
 		/* Probably KFD bug, but mmap is user-accessible. */
-		pr_debug("signal page could not be found for page_index %u\n",
-				page_index);
+		pr_debug("signal page could not be found for page_index %u\n", page_index);
 		return -EINVAL;
 	}
 
@@ -858,23 +975,29 @@ int kfd_event_mmap(struct kfd_process *p, struct vm_area_struct *vma)
 static void lookup_events_by_type_and_signal(struct kfd_process *p,
 		int type, void *event_data)
 {
-	struct kfd_hsa_memory_exception_data *ev_data;
 	struct kfd_event *ev;
 	int bkt;
 	bool send_signal = true;
 
-	ev_data = (struct kfd_hsa_memory_exception_data *) event_data;
-
-	hash_for_each(p->events, bkt, ev, events)
+	hash_for_each(p->events, bkt, ev, events) {
 		if (ev->type == type) {
 			send_signal = false;
 			dev_dbg(kfd_device,
 					"Event found: id %X type %d",
 					ev->event_id, ev->type);
 			set_event(ev);
-			if (ev->type == KFD_EVENT_TYPE_MEMORY && ev_data)
-				ev->memory_exception_data = *ev_data;
+			if (ev->type == KFD_EVENT_TYPE_MEMORY && event_data)
+				ev->memory_exception_data =
+						*(struct kfd_hsa_memory_exception_data *)event_data;
 		}
+	}
+
+	if (type == KFD_EVENT_TYPE_MEMORY) {
+		dev_warn(kfd_device,
+			"Sending SIGSEGV to HSA Process with PID %d ",
+				p->lead_thread->pid);
+		send_sig(SIGSEGV, p->lead_thread, 0);
+	}
 
 	/* Send SIGTERM no event of type "type" has been found*/
 	if (send_signal) {
@@ -901,7 +1024,7 @@ void kfd_signal_iommu_event(struct kfd_dev *dev, unsigned int pasid,
 	/*
 	 * Because we are called from arbitrary context (workqueue) as opposed
 	 * to process context, kfd_process could attempt to exit while we are
-	 * running so the lookup function returns a locked process.
+	 * running so the lookup function returns a read-locked process.
 	 */
 	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
 
@@ -916,24 +1039,24 @@ void kfd_signal_iommu_event(struct kfd_dev *dev, unsigned int pasid,
 	memory_exception_data.gpu_id = dev->id;
 	memory_exception_data.va = address;
 	/* Set failure reason */
-	memory_exception_data.failure.NotPresent = 1;
-	memory_exception_data.failure.NoExecute = 0;
-	memory_exception_data.failure.ReadOnly = 0;
+	memory_exception_data.failure.NotPresent = true;
+	memory_exception_data.failure.NoExecute = false;
+	memory_exception_data.failure.ReadOnly = false;
 	if (vma) {
 		if (vma->vm_start > address) {
-			memory_exception_data.failure.NotPresent = 1;
-			memory_exception_data.failure.NoExecute = 0;
-			memory_exception_data.failure.ReadOnly = 0;
+			memory_exception_data.failure.NotPresent = true;
+			memory_exception_data.failure.NoExecute = false;
+			memory_exception_data.failure.ReadOnly = false;
 		} else {
-			memory_exception_data.failure.NotPresent = 0;
+			memory_exception_data.failure.NotPresent = false;
 			if (is_write_requested && !(vma->vm_flags & VM_WRITE))
-				memory_exception_data.failure.ReadOnly = 1;
+				memory_exception_data.failure.ReadOnly = true;
 			else
-				memory_exception_data.failure.ReadOnly = 0;
+				memory_exception_data.failure.ReadOnly = false;
 			if (is_execute_requested && !(vma->vm_flags & VM_EXEC))
-				memory_exception_data.failure.NoExecute = 1;
+				memory_exception_data.failure.NoExecute = true;
 			else
-				memory_exception_data.failure.NoExecute = 0;
+				memory_exception_data.failure.NoExecute = false;
 		}
 	}
 
@@ -946,7 +1069,7 @@ void kfd_signal_iommu_event(struct kfd_dev *dev, unsigned int pasid,
 			&memory_exception_data);
 
 	mutex_unlock(&p->event_mutex);
-	mutex_unlock(&p->mutex);
+	up_read(&p->lock);
 }
 
 void kfd_signal_hw_exception_event(unsigned int pasid)
@@ -954,7 +1077,7 @@ void kfd_signal_hw_exception_event(unsigned int pasid)
 	/*
 	 * Because we are called from arbitrary context (workqueue) as opposed
 	 * to process context, kfd_process could attempt to exit while we are
-	 * running so the lookup function returns a locked process.
+	 * running so the lookup function returns a read-locked process.
 	 */
 	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
 
@@ -967,5 +1090,42 @@ void kfd_signal_hw_exception_event(unsigned int pasid)
 	lookup_events_by_type_and_signal(p, KFD_EVENT_TYPE_HW_EXCEPTION, NULL);
 
 	mutex_unlock(&p->event_mutex);
-	mutex_unlock(&p->mutex);
+	up_read(&p->lock);
+}
+
+void kfd_signal_vm_fault_event(struct kfd_dev *dev, unsigned int pasid,
+				struct kfd_vm_fault_info *info)
+{
+	struct kfd_event *ev;
+	int bkt;
+	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
+	struct kfd_hsa_memory_exception_data memory_exception_data;
+
+	if (!p)
+		return; /* Presumably process exited. */
+	memset(&memory_exception_data, 0, sizeof(memory_exception_data));
+	memory_exception_data.gpu_id = dev->id;
+	/* Set failure reason */
+	if (info) {
+		memory_exception_data.va = (info->page_addr) << PAGE_SHIFT;
+		memory_exception_data.failure.NotPresent =
+			info->prot_valid ? true : false;
+		memory_exception_data.failure.NoExecute =
+			info->prot_exec ? true : false;
+		memory_exception_data.failure.ReadOnly =
+			info->prot_write ? true : false;
+	}
+	mutex_lock(&p->event_mutex);
+
+	hash_for_each(p->events, bkt, ev, events) {
+		if (ev->type == KFD_EVENT_TYPE_MEMORY) {
+			ev->memory_exception_data = memory_exception_data;
+			set_event(ev);
+		}
+	}
+
+	mutex_unlock(&p->event_mutex);
+	up_read(&p->lock);
+
 }
+
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.h b/drivers/gpu/drm/amd/amdkfd/kfd_events.h
index 28f6838..d7987eb 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_events.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.h
@@ -34,8 +34,7 @@
 #define KFD_FIRST_NONSIGNAL_EVENT_ID KFD_EVENT_ID_NONSIGNAL_MASK
 #define KFD_LAST_NONSIGNAL_EVENT_ID UINT_MAX
 
-/*
- * Written into kfd_signal_slot_t to indicate that the event is not signaled.
+/* Written into kfd_signal_slot_t to indicate that the event is not signaled.
  * Since the event protocol may need to write the event ID into memory, this
  * must not be a valid event ID.
  * For the sake of easy memset-ing, this must be a byte pattern.
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c b/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
index 2b65510..587f847 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
@@ -33,7 +33,7 @@
 #include <linux/time.h>
 #include "kfd_priv.h"
 #include <linux/mm.h>
-#include <linux/mman.h>
+#include <uapi/asm-generic/mman-common.h>
 #include <asm/processor.h>
 
 /*
@@ -278,21 +278,36 @@
 #define MAKE_GPUVM_APP_BASE(gpu_num) \
 	(((uint64_t)(gpu_num) << 61) + 0x1000000000000L)
 
-#define MAKE_GPUVM_APP_LIMIT(base) \
-	(((uint64_t)(base) & \
-		0xFFFFFF0000000000UL) | 0xFFFFFFFFFFL)
+#define MAKE_GPUVM_APP_LIMIT(base, size) \
+	(((uint64_t)(base) & 0xFFFFFF0000000000UL) + (size) - 1)
 
-#define MAKE_SCRATCH_APP_BASE(gpu_num) \
-	(((uint64_t)(gpu_num) << 61) + 0x100000000L)
+#define MAKE_SCRATCH_APP_BASE() \
+	(((uint64_t)(0x1UL) << 61) + 0x100000000L)
 
 #define MAKE_SCRATCH_APP_LIMIT(base) \
 	(((uint64_t)base & 0xFFFFFFFF00000000UL) | 0xFFFFFFFF)
 
-#define MAKE_LDS_APP_BASE(gpu_num) \
-	(((uint64_t)(gpu_num) << 61) + 0x0)
+#define MAKE_LDS_APP_BASE() \
+	(((uint64_t)(0x1UL) << 61) + 0x0)
+
 #define MAKE_LDS_APP_LIMIT(base) \
 	(((uint64_t)(base) & 0xFFFFFFFF00000000UL) | 0xFFFFFFFF)
 
+
+#define DGPU_VM_BASE_DEFAULT 0x100000
+
+int kfd_set_process_dgpu_aperture(struct kfd_process_device *pdd,
+					uint64_t base, uint64_t limit)
+{
+	if (base < (pdd->qpd.cwsr_base + pdd->dev->cwsr_size)) {
+		pr_err("Set dgpu vm base 0x%llx failed.\n", base);
+		return -EINVAL;
+	}
+	pdd->dgpu_base = base;
+	pdd->dgpu_limit = limit;
+	return 0;
+}
+
 int kfd_init_apertures(struct kfd_process *process)
 {
 	uint8_t id  = 0;
@@ -300,13 +315,16 @@ int kfd_init_apertures(struct kfd_process *process)
 	struct kfd_process_device *pdd;
 
 	/*Iterating over all devices*/
-	while ((dev = kfd_topology_enum_kfd_devices(id)) != NULL &&
-		id < NUM_OF_SUPPORTED_GPUS) {
+	while (kfd_topology_enum_kfd_devices(id, &dev) == 0) {
+		if (!dev) {
+			id++; /* Skip non GPU devices */
+			continue;
+		}
 
 		pdd = kfd_create_process_device_data(dev, process);
 		if (pdd == NULL) {
 			pr_err("Failed to create process device data\n");
-			return -1;
+			goto err;
 		}
 		/*
 		 * For 64 bit process aperture will be statically reserved in
@@ -322,19 +340,24 @@ int kfd_init_apertures(struct kfd_process *process)
 			 * node id couldn't be 0 - the three MSB bits of
 			 * aperture shoudn't be 0
 			 */
-			pdd->lds_base = MAKE_LDS_APP_BASE(id + 1);
+			pdd->lds_base = MAKE_LDS_APP_BASE();
 
 			pdd->lds_limit = MAKE_LDS_APP_LIMIT(pdd->lds_base);
 
 			pdd->gpuvm_base = MAKE_GPUVM_APP_BASE(id + 1);
 
-			pdd->gpuvm_limit =
-					MAKE_GPUVM_APP_LIMIT(pdd->gpuvm_base);
+			pdd->gpuvm_limit = MAKE_GPUVM_APP_LIMIT(
+				pdd->gpuvm_base,
+				dev->shared_resources.gpuvm_size);
 
-			pdd->scratch_base = MAKE_SCRATCH_APP_BASE(id + 1);
+			pdd->scratch_base = MAKE_SCRATCH_APP_BASE();
 
 			pdd->scratch_limit =
 				MAKE_SCRATCH_APP_LIMIT(pdd->scratch_base);
+
+			if (KFD_IS_DGPU(dev->device_info->asic_family))
+				pdd->qpd.cwsr_base = DGPU_VM_BASE_DEFAULT;
+
 		}
 
 		dev_dbg(kfd_device, "node id %u\n", id);
@@ -350,6 +373,32 @@ int kfd_init_apertures(struct kfd_process *process)
 	}
 
 	return 0;
+
+err:
+	return -1;
 }
 
+void radeon_flush_tlb(struct kfd_dev *dev, uint32_t pasid)
+{
+	uint8_t vmid;
+	int first_vmid_to_scan = 8;
+	int last_vmid_to_scan = 15;
 
+	const struct kfd2kgd_calls *f2g = dev->kfd2kgd;
+	/* Scan all registers in the range ATC_VMID8_PASID_MAPPING .. ATC_VMID15_PASID_MAPPING
+	 * to check which VMID the current process is mapped to
+	 * and flush TLB for this VMID if found*/
+	for (vmid = first_vmid_to_scan; vmid <= last_vmid_to_scan; vmid++) {
+		if (f2g->get_atc_vmid_pasid_mapping_valid(
+			dev->kgd, vmid)) {
+			if (f2g->get_atc_vmid_pasid_mapping_pasid(
+				dev->kgd, vmid) == pasid) {
+				dev_dbg(kfd_device,
+					"TLB of vmid %u", vmid);
+				f2g->write_vmid_invalidate_request(
+					dev->kgd, vmid);
+				break;
+			}
+		}
+	}
+}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c b/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c
index 7f134aa..a8cdbc8 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c
@@ -172,8 +172,7 @@ static void interrupt_wq(struct work_struct *work)
 				sizeof(uint32_t))];
 
 	while (dequeue_ih_ring_entry(dev, ih_ring_entry))
-		dev->device_info->event_interrupt_class->interrupt_wq(dev,
-								ih_ring_entry);
+		dev->device_info->event_interrupt_class->interrupt_wq(dev, ih_ring_entry);
 }
 
 bool interrupt_is_wanted(struct kfd_dev *dev, const uint32_t *ih_ring_entry)
@@ -181,8 +180,7 @@ bool interrupt_is_wanted(struct kfd_dev *dev, const uint32_t *ih_ring_entry)
 	/* integer and bitwise OR so there is no boolean short-circuiting */
 	unsigned wanted = 0;
 
-	wanted |= dev->device_info->event_interrupt_class->interrupt_isr(dev,
-								ih_ring_entry);
+	wanted |= dev->device_info->event_interrupt_class->interrupt_isr(dev, ih_ring_entry);
 
 	return wanted != 0;
 }
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
index d135cd0..513cfe6 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
@@ -143,7 +143,7 @@ static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev,
 		kq->queue->pipe = KFD_CIK_HIQ_PIPE;
 		kq->queue->queue = KFD_CIK_HIQ_QUEUE;
 		kq->mqd->load_mqd(kq->mqd, kq->queue->mqd, kq->queue->pipe,
-					kq->queue->queue, NULL);
+					kq->queue->queue, NULL, 0);
 	} else {
 		/* allocate fence for DIQ */
 
@@ -213,20 +213,23 @@ static int acquire_packet_buffer(struct kernel_queue *kq,
 
 	BUG_ON(!kq || !buffer_ptr);
 
+	/* When rptr == wptr, the buffer is empty.
+	 * When rptr == wptr + 1, the buffer is full.
+	 * It is always rptr that advances to the position of wptr, rather than
+	 * the opposite. So we can only use up to queue_size_dwords - 1 dwords.
+	 */
 	rptr = *kq->rptr_kernel;
 	wptr = *kq->wptr_kernel;
 	queue_address = (unsigned int *)kq->pq_kernel_addr;
 	queue_size_dwords = kq->queue->properties.queue_size / sizeof(uint32_t);
 
-	pr_debug("rptr: %d\n", rptr);
-	pr_debug("wptr: %d\n", wptr);
-	pr_debug("queue_address 0x%p\n", queue_address);
+	pr_debug("amdkfd: In func %s\n rptr: %d\n wptr: %d\n queue_address 0x%p\n",
+			__func__, rptr, wptr, queue_address);
 
-	available_size = (rptr - 1 - wptr + queue_size_dwords) %
+	available_size = (rptr + queue_size_dwords - 1 - wptr) %
 							queue_size_dwords;
 
-	if (packet_size_in_dwords >= queue_size_dwords ||
-			packet_size_in_dwords >= available_size) {
+	if (packet_size_in_dwords > available_size) {
 		/*
 		 * make sure calling functions know
 		 * acquire_packet_buffer() failed
@@ -236,6 +239,13 @@ static int acquire_packet_buffer(struct kernel_queue *kq,
 	}
 
 	if (wptr + packet_size_in_dwords >= queue_size_dwords) {
+		/* make sure after rolling back to position 0, there is
+		 * still enough space. */
+		if (packet_size_in_dwords >= rptr) {
+			*buffer_ptr = NULL;
+			return -ENOMEM;
+		}
+		/* fill nops, roll back and start at position 0 */
 		while (wptr > 0) {
 			queue_address[wptr] = kq->nop_packet;
 			wptr = (wptr + 1) % queue_size_dwords;
@@ -295,6 +305,8 @@ struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
 
 	switch (dev->device_info->asic_family) {
 	case CHIP_CARRIZO:
+	case CHIP_TONGA:
+	case CHIP_FIJI:
 		kernel_queue_init_vi(&kq->ops_asic_specific);
 		break;
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_module.c b/drivers/gpu/drm/amd/amdkfd/kfd_module.c
index 850a562..e9b886d 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_module.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_module.c
@@ -29,10 +29,11 @@
 #define KFD_DRIVER_AUTHOR	"AMD Inc. and others"
 
 #define KFD_DRIVER_DESC		"Standalone HSA driver for AMD's GPUs"
-#define KFD_DRIVER_DATE		"20150421"
-#define KFD_DRIVER_MAJOR	0
-#define KFD_DRIVER_MINOR	7
-#define KFD_DRIVER_PATCHLEVEL	2
+#define KFD_DRIVER_DATE		"20160129"
+#define KFD_DRIVER_MAJOR	1
+#define KFD_DRIVER_MINOR	8
+#define KFD_DRIVER_PATCHLEVEL	1
+#define KFD_DRIVER_RC_LEVEL	""
 
 static const struct kgd2kfd_calls kgd2kfd = {
 	.exit		= kgd2kfd_exit,
@@ -42,6 +43,10 @@ static const struct kgd2kfd_calls kgd2kfd = {
 	.interrupt	= kgd2kfd_interrupt,
 	.suspend	= kgd2kfd_suspend,
 	.resume		= kgd2kfd_resume,
+	.evict_bo	= kgd2kfd_evict_bo,
+	.restore	= kgd2kfd_restore,
+	.quiesce_mm	= kgd2kfd_quiesce_mm,
+	.resume_mm	= kgd2kfd_resume_mm,
 };
 
 int sched_policy = KFD_SCHED_POLICY_HWS;
@@ -49,6 +54,15 @@ module_param(sched_policy, int, 0444);
 MODULE_PARM_DESC(sched_policy,
 	"Scheduling policy (0 = HWS (Default), 1 = HWS without over-subscription, 2 = Non-HWS (Used for debugging only)");
 
+int hws_max_conc_proc = 0;
+module_param(hws_max_conc_proc, int, 0444);
+MODULE_PARM_DESC(hws_max_conc_proc,
+	"Max # processes HWS can execute concurrently when sched_policy=0 (0 = no concurrency (Default), #VMIDs for KFD = Maximum)");
+
+int cwsr_enable = 1;
+module_param(cwsr_enable, int, 0444);
+MODULE_PARM_DESC(cwsr_enable, "CWSR enable (0 = Off, 1 = On (Default))");
+
 int max_num_of_queues_per_device = KFD_MAX_NUM_OF_QUEUES_PER_DEVICE_DEFAULT;
 module_param(max_num_of_queues_per_device, int, 0444);
 MODULE_PARM_DESC(max_num_of_queues_per_device,
@@ -61,6 +75,11 @@ MODULE_PARM_DESC(send_sigterm,
 
 static int amdkfd_init_completed;
 
+int debug_largebar = 0;
+module_param(debug_largebar, int, 0444);
+MODULE_PARM_DESC(debug_largebar,
+	"Debug large-bar flag used to simulate large-bar capability on non-large bar machine (0 = disable, 1 = enable)");
+
 int kgd2kfd_init(unsigned interface_version, const struct kgd2kfd_calls **g2f)
 {
 	if (!amdkfd_init_completed)
@@ -149,4 +168,5 @@ MODULE_DESCRIPTION(KFD_DRIVER_DESC);
 MODULE_LICENSE("GPL and additional rights");
 MODULE_VERSION(__stringify(KFD_DRIVER_MAJOR) "."
 	       __stringify(KFD_DRIVER_MINOR) "."
-	       __stringify(KFD_DRIVER_PATCHLEVEL));
+	       __stringify(KFD_DRIVER_PATCHLEVEL)
+	       KFD_DRIVER_RC_LEVEL);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
index b1ef136..ef1dc9b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
@@ -31,6 +31,9 @@ struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type,
 		return mqd_manager_init_cik(type, dev);
 	case CHIP_CARRIZO:
 		return mqd_manager_init_vi(type, dev);
+	case CHIP_TONGA:
+	case CHIP_FIJI:
+		return mqd_manager_init_vi_tonga(type, dev);
 	}
 
 	return NULL;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
index 213a71e..eb60192 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
@@ -67,7 +67,8 @@ struct mqd_manager {
 
 	int	(*load_mqd)(struct mqd_manager *mm, void *mqd,
 				uint32_t pipe_id, uint32_t queue_id,
-				uint32_t __user *wptr);
+				uint32_t __user *wptr,
+				uint32_t page_table_base);
 
 	int	(*update_mqd)(struct mqd_manager *mm, void *mqd,
 				struct queue_properties *q);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
index e492692..62dbdca 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
@@ -31,11 +31,71 @@
 #include "cik_structs.h"
 #include "oss/oss_2_4_sh_mask.h"
 
+#define AQL_ENABLE 1
+
 static inline struct cik_mqd *get_mqd(void *mqd)
 {
 	return (struct cik_mqd *)mqd;
 }
 
+static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
+{
+	return (struct cik_sdma_rlc_registers *)mqd;
+}
+
+static void update_cu_mask(struct mqd_manager *mm, void *mqd,
+			struct queue_properties *q)
+{
+	struct cik_mqd *m;
+	struct kfd_cu_info cu_info;
+	uint32_t mgmt_se_mask;
+	uint32_t cu_sh_mask, cu_sh_shift;
+	uint32_t cu_mask;
+	int se, sh;
+
+	if (q->cu_mask == 0)
+		return;
+
+	m = get_mqd(mqd);
+	m->compute_static_thread_mgmt_se0 = 0;
+	m->compute_static_thread_mgmt_se1 = 0;
+	m->compute_static_thread_mgmt_se2 = 0;
+	m->compute_static_thread_mgmt_se3 = 0;
+
+	mm->dev->kfd2kgd->get_cu_info(mm->dev->kgd, &cu_info);
+	cu_mask = q->cu_mask;
+	for (se = 0; se < cu_info.num_shader_engines && cu_mask; se++) {
+		mgmt_se_mask = 0;
+		for (sh = 0; sh < 2 && cu_mask; sh++) {
+			cu_sh_shift = hweight32(cu_info.cu_bitmap[se][sh]);
+			cu_sh_mask = (1 << cu_sh_shift) - 1;
+			mgmt_se_mask |= (cu_mask & cu_sh_mask) << (sh * 16);
+			cu_mask >>= cu_sh_shift;
+		}
+		switch (se) {
+		case 0:
+			m->compute_static_thread_mgmt_se0 = mgmt_se_mask;
+			break;
+		case 1:
+			m->compute_static_thread_mgmt_se1 = mgmt_se_mask;
+			break;
+		case 2:
+			m->compute_static_thread_mgmt_se2 = mgmt_se_mask;
+			break;
+		case 3:
+			m->compute_static_thread_mgmt_se3 = mgmt_se_mask;
+			break;
+		default:
+			break;
+		}
+	}
+	pr_debug("kfd: update cu mask to %#x %#x %#x %#x\n",
+		m->compute_static_thread_mgmt_se0,
+		m->compute_static_thread_mgmt_se1,
+		m->compute_static_thread_mgmt_se2,
+		m->compute_static_thread_mgmt_se3);
+}
+
 static int init_mqd(struct mqd_manager *mm, void **mqd,
 		struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
 		struct queue_properties *q)
@@ -152,15 +212,16 @@ static void uninit_mqd_sdma(struct mqd_manager *mm, void *mqd,
 }
 
 static int load_mqd(struct mqd_manager *mm, void *mqd, uint32_t pipe_id,
-			uint32_t queue_id, uint32_t __user *wptr)
+			uint32_t queue_id, uint32_t __user *wptr,
+			uint32_t page_table_base)
 {
 	return mm->dev->kfd2kgd->hqd_load
-		(mm->dev->kgd, mqd, pipe_id, queue_id, wptr);
+		(mm->dev->kgd, mqd, pipe_id, queue_id, wptr, page_table_base);
 }
 
 static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
 			uint32_t pipe_id, uint32_t queue_id,
-			uint32_t __user *wptr)
+			uint32_t __user *wptr, uint32_t page_table_base)
 {
 	return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd);
 }
@@ -197,11 +258,14 @@ static int update_mqd(struct mqd_manager *mm, void *mqd,
 		m->cp_hqd_pq_control |= NO_UPDATE_RPTR;
 	}
 
+	update_cu_mask(mm, mqd, q);
+
 	m->cp_hqd_active = 0;
 	q->is_active = false;
 	if (q->queue_size > 0 &&
 			q->queue_address != 0 &&
-			q->queue_percent > 0) {
+			q->queue_percent > 0 &&
+			!q->is_evicted) {
 		m->cp_hqd_active = 1;
 		q->is_active = true;
 	}
@@ -239,7 +303,8 @@ static int update_mqd_sdma(struct mqd_manager *mm, void *mqd,
 	q->is_active = false;
 	if (q->queue_size > 0 &&
 			q->queue_address != 0 &&
-			q->queue_percent > 0) {
+			q->queue_percent > 0 &&
+			!q->is_evicted) {
 		m->sdma_rlc_rb_cntl |=
 				1 << SDMA0_RLC0_RB_CNTL__RB_ENABLE__SHIFT;
 
@@ -388,7 +453,8 @@ static int update_mqd_hiq(struct mqd_manager *mm, void *mqd,
 	q->is_active = false;
 	if (q->queue_size > 0 &&
 			q->queue_address != 0 &&
-			q->queue_percent > 0) {
+			q->queue_percent > 0 &&
+			!q->is_evicted) {
 		m->cp_hqd_active = 1;
 		q->is_active = true;
 	}
@@ -396,16 +462,6 @@ static int update_mqd_hiq(struct mqd_manager *mm, void *mqd,
 	return 0;
 }
 
-struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
-{
-	struct cik_sdma_rlc_registers *m;
-
-	BUG_ON(!mqd);
-
-	m = (struct cik_sdma_rlc_registers *)mqd;
-
-	return m;
-}
 
 struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
 		struct kfd_dev *dev)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
index a9b9882..4260c2f 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
@@ -28,9 +28,9 @@
 #include "kfd_priv.h"
 #include "kfd_mqd_manager.h"
 #include "vi_structs.h"
-#include "gca/gfx_8_0_sh_mask.h"
-#include "gca/gfx_8_0_enum.h"
-
+#include "asic_reg/gca/gfx_8_0_sh_mask.h"
+#include "asic_reg/gca/gfx_8_0_enum.h"
+#include "oss/oss_3_0_sh_mask.h"
 #define CP_MQD_CONTROL__PRIV_STATE__SHIFT 0x8
 
 static inline struct vi_mqd *get_mqd(void *mqd)
@@ -38,6 +38,64 @@ static inline struct vi_mqd *get_mqd(void *mqd)
 	return (struct vi_mqd *)mqd;
 }
 
+static inline struct vi_sdma_mqd *get_sdma_mqd(void *mqd)
+{
+	return (struct vi_sdma_mqd *)mqd;
+}
+
+static void update_cu_mask(struct mqd_manager *mm, void *mqd,
+			struct queue_properties *q)
+{
+	struct vi_mqd *m;
+	struct kfd_cu_info cu_info;
+	uint32_t mgmt_se_mask;
+	uint32_t cu_sh_mask, cu_sh_shift;
+	uint32_t cu_mask;
+	int se, sh;
+
+	if (q->cu_mask == 0)
+		return;
+
+	m = get_mqd(mqd);
+	m->compute_static_thread_mgmt_se0 = 0;
+	m->compute_static_thread_mgmt_se1 = 0;
+	m->compute_static_thread_mgmt_se2 = 0;
+	m->compute_static_thread_mgmt_se3 = 0;
+
+	mm->dev->kfd2kgd->get_cu_info(mm->dev->kgd, &cu_info);
+	cu_mask = q->cu_mask;
+	for (se = 0; se < cu_info.num_shader_engines && cu_mask; se++) {
+		mgmt_se_mask = 0;
+		for (sh = 0; sh < 2 && cu_mask; sh++) {
+			cu_sh_shift = hweight32(cu_info.cu_bitmap[se][sh]);
+			cu_sh_mask = (1 << cu_sh_shift) - 1;
+			mgmt_se_mask |= (cu_mask & cu_sh_mask) << (sh * 16);
+			cu_mask >>= cu_sh_shift;
+		}
+		switch (se) {
+		case 0:
+			m->compute_static_thread_mgmt_se0 = mgmt_se_mask;
+			break;
+		case 1:
+			m->compute_static_thread_mgmt_se1 = mgmt_se_mask;
+			break;
+		case 2:
+			m->compute_static_thread_mgmt_se2 = mgmt_se_mask;
+			break;
+		case 3:
+			m->compute_static_thread_mgmt_se3 = mgmt_se_mask;
+			break;
+		default:
+			break;
+		}
+	}
+	pr_debug("kfd: update cu mask to %#x %#x %#x %#x\n",
+		m->compute_static_thread_mgmt_se0,
+		m->compute_static_thread_mgmt_se1,
+		m->compute_static_thread_mgmt_se2,
+		m->compute_static_thread_mgmt_se3);
+}
+
 static int init_mqd(struct mqd_manager *mm, void **mqd,
 			struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
 			struct queue_properties *q)
@@ -84,6 +142,25 @@ static int init_mqd(struct mqd_manager *mm, void **mqd,
 	if (q->format == KFD_QUEUE_FORMAT_AQL)
 		m->cp_hqd_iq_rptr = 1;
 
+	if (q->tba_addr) {
+		m->cp_hqd_persistent_state |=
+			(1 << CP_HQD_PERSISTENT_STATE__QSWITCH_MODE__SHIFT);
+		m->compute_pgm_rsrc2 |=
+			(1 << COMPUTE_PGM_RSRC2__TRAP_PRESENT__SHIFT);
+		m->cp_hqd_ctx_save_base_addr_lo =
+			lower_32_bits(q->ctx_save_restore_area_address);
+		m->cp_hqd_ctx_save_base_addr_hi =
+			upper_32_bits(q->ctx_save_restore_area_address);
+		m->cp_hqd_ctx_save_size = q->ctx_save_restore_area_size;
+		m->cp_hqd_cntl_stack_size = q->ctl_stack_size;
+		m->cp_hqd_cntl_stack_offset = q->ctl_stack_size;
+		m->cp_hqd_wg_state_offset = q->ctl_stack_size;
+		m->compute_tba_lo = lower_32_bits(q->tba_addr >> 8);
+		m->compute_tba_hi = upper_32_bits(q->tba_addr >> 8);
+		m->compute_tma_lo = lower_32_bits(q->tma_addr >> 8);
+		m->compute_tma_hi = upper_32_bits(q->tma_addr >> 8);
+	}
+
 	*mqd = m;
 	if (gart_addr != NULL)
 		*gart_addr = addr;
@@ -94,10 +171,10 @@ static int init_mqd(struct mqd_manager *mm, void **mqd,
 
 static int load_mqd(struct mqd_manager *mm, void *mqd,
 			uint32_t pipe_id, uint32_t queue_id,
-			uint32_t __user *wptr)
+			uint32_t __user *wptr, uint32_t page_table_base)
 {
 	return mm->dev->kfd2kgd->hqd_load
-		(mm->dev->kgd, mqd, pipe_id, queue_id, wptr);
+		(mm->dev->kgd, mqd, pipe_id, queue_id, wptr, page_table_base);
 }
 
 static int __update_mqd(struct mqd_manager *mm, void *mqd,
@@ -155,12 +232,19 @@ static int __update_mqd(struct mqd_manager *mm, void *mqd,
 		m->cp_hqd_pq_control |= CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK |
 				2 << CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT;
 	}
+	if (q->tba_addr)
+		m->cp_hqd_ctx_save_control =
+			atc_bit << CP_HQD_CTX_SAVE_CONTROL__ATC__SHIFT |
+			mtype << CP_HQD_CTX_SAVE_CONTROL__MTYPE__SHIFT;
+
+	update_cu_mask(mm, mqd, q);
 
 	m->cp_hqd_active = 0;
 	q->is_active = false;
 	if (q->queue_size > 0 &&
 			q->queue_address != 0 &&
-			q->queue_percent > 0) {
+			q->queue_percent > 0 &&
+			!q->is_evicted) {
 		m->cp_hqd_active = 1;
 		q->is_active = true;
 	}
@@ -175,6 +259,12 @@ static int update_mqd(struct mqd_manager *mm, void *mqd,
 	return __update_mqd(mm, mqd, q, MTYPE_CC, 1);
 }
 
+static int update_mqd_tonga(struct mqd_manager *mm, void *mqd,
+			struct queue_properties *q)
+{
+	return __update_mqd(mm, mqd, q, MTYPE_UC, 0);
+}
+
 static int destroy_mqd(struct mqd_manager *mm, void *mqd,
 			enum kfd_preempt_type type,
 			unsigned int timeout, uint32_t pipe_id,
@@ -233,6 +323,111 @@ static int update_mqd_hiq(struct mqd_manager *mm, void *mqd,
 	return retval;
 }
 
+static int init_mqd_sdma(struct mqd_manager *mm, void **mqd,
+		struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
+		struct queue_properties *q)
+{
+	int retval;
+	struct vi_sdma_mqd *m;
+
+
+	BUG_ON(!mm || !mqd || !mqd_mem_obj);
+
+	retval = kfd_gtt_sa_allocate(mm->dev,
+			sizeof(struct vi_sdma_mqd),
+			mqd_mem_obj);
+
+	if (retval != 0)
+		return -ENOMEM;
+
+	m = (struct vi_sdma_mqd *) (*mqd_mem_obj)->cpu_ptr;
+
+	memset(m, 0, sizeof(struct vi_sdma_mqd));
+
+	*mqd = m;
+	if (gart_addr != NULL)
+		*gart_addr = (*mqd_mem_obj)->gpu_addr;
+
+	retval = mm->update_mqd(mm, m, q);
+
+	return retval;
+}
+
+static void uninit_mqd_sdma(struct mqd_manager *mm, void *mqd,
+		struct kfd_mem_obj *mqd_mem_obj)
+{
+	BUG_ON(!mm || !mqd);
+	kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
+}
+
+static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
+		uint32_t pipe_id, uint32_t queue_id,
+		uint32_t __user *wptr, uint32_t page_table_base)
+{
+	return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd);
+}
+
+static int update_mqd_sdma(struct mqd_manager *mm, void *mqd,
+		struct queue_properties *q)
+{
+	struct vi_sdma_mqd *m;
+	BUG_ON(!mm || !mqd || !q);
+
+	m = get_sdma_mqd(mqd);
+	m->sdmax_rlcx_rb_cntl = (ffs(q->queue_size / sizeof(unsigned int)) - 1)
+		<< SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT |
+		q->vmid << SDMA0_RLC0_RB_CNTL__RB_VMID__SHIFT |
+		1 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
+		6 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT;
+
+	m->sdmax_rlcx_rb_base = lower_32_bits(q->queue_address >> 8);
+	m->sdmax_rlcx_rb_base_hi = upper_32_bits(q->queue_address >> 8);
+	m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
+	m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
+	m->sdmax_rlcx_doorbell = q->doorbell_off <<
+		SDMA0_RLC0_DOORBELL__OFFSET__SHIFT |
+		1 << SDMA0_RLC0_DOORBELL__ENABLE__SHIFT;
+
+	m->sdmax_rlcx_virtual_addr = q->sdma_vm_addr;
+
+	m->sdma_engine_id = q->sdma_engine_id;
+	m->sdma_queue_id = q->sdma_queue_id;
+
+	q->is_active = false;
+	if (q->queue_size > 0 &&
+			q->queue_address != 0 &&
+			q->queue_percent > 0 &&
+			!q->is_evicted) {
+		m->sdmax_rlcx_rb_cntl |=
+			1 << SDMA0_RLC0_RB_CNTL__RB_ENABLE__SHIFT;
+
+		q->is_active = true;
+	}
+
+	return 0;
+}
+
+/*
+ *  * preempt type here is ignored because there is only one way
+ *  * to preempt sdma queue
+ */
+static int destroy_mqd_sdma(struct mqd_manager *mm, void *mqd,
+		enum kfd_preempt_type type,
+		unsigned int timeout, uint32_t pipe_id,
+		uint32_t queue_id)
+{
+	return mm->dev->kfd2kgd->hqd_sdma_destroy(mm->dev->kgd, mqd, timeout);
+}
+
+static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
+		uint64_t queue_address, uint32_t pipe_id,
+		uint32_t queue_id)
+{
+	return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd);
+}
+
+
+
 struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
 		struct kfd_dev *dev)
 {
@@ -268,6 +463,12 @@ struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
 		mqd->is_occupied = is_occupied;
 		break;
 	case KFD_MQD_TYPE_SDMA:
+		mqd->init_mqd = init_mqd_sdma;
+		mqd->uninit_mqd = uninit_mqd_sdma;
+		mqd->load_mqd = load_mqd_sdma;
+		mqd->update_mqd = update_mqd_sdma;
+		mqd->destroy_mqd = destroy_mqd_sdma;
+		mqd->is_occupied = is_occupied_sdma;
 		break;
 	default:
 		kfree(mqd);
@@ -276,3 +477,17 @@ struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
 
 	return mqd;
 }
+
+struct mqd_manager *mqd_manager_init_vi_tonga(enum KFD_MQD_TYPE type,
+			struct kfd_dev *dev)
+{
+	struct mqd_manager *mqd;
+
+	mqd = mqd_manager_init_vi(type, dev);
+	if (!mqd)
+		return NULL;
+	if ((type == KFD_MQD_TYPE_CP) || (type == KFD_MQD_TYPE_COMPUTE))
+		mqd->update_mqd = update_mqd_tonga;
+	return mqd;
+}
+
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
index a470019..1a443e5 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
@@ -57,25 +57,37 @@ static void pm_calc_rlib_size(struct packet_manager *pm,
 {
 	unsigned int process_count, queue_count;
 	unsigned int map_queue_size;
+	unsigned int max_proc_per_quantum = 1;
 
-	BUG_ON(!pm || !rlib_size || !over_subscription);
+	struct kfd_dev	*dev = pm->dqm->dev;
+
+	BUG_ON(!pm || !rlib_size || !over_subscription || !dev);
 
 	process_count = pm->dqm->processes_count;
 	queue_count = pm->dqm->queue_count;
 
-	/* check if there is over subscription*/
+	/* check if there is over subscription
+	 * Note: the arbitration between the number of VMIDs and
+	 * hws_max_conc_proc has been done in
+	 * kgd2kfd_device_init().
+	 */
+
 	*over_subscription = false;
-	if ((process_count > 1) || queue_count > get_queues_num(pm->dqm)) {
+
+	if (dev->max_proc_per_quantum > 1)
+		max_proc_per_quantum = dev->max_proc_per_quantum;
+
+	if ((process_count > max_proc_per_quantum) ||
+		queue_count > get_queues_num(pm->dqm)) {
 		*over_subscription = true;
 		pr_debug("kfd: over subscribed runlist\n");
 	}
 
-	map_queue_size =
-		(pm->dqm->dev->device_info->asic_family == CHIP_CARRIZO) ?
+	map_queue_size = KFD_IS_VI(pm->dqm->dev->device_info->asic_family) ?
 		sizeof(struct pm4_mes_map_queues) :
 		sizeof(struct pm4_map_queues);
 	/* calculate run list ib allocation size */
-	*rlib_size = process_count * sizeof(struct pm4_map_process) +
+	*rlib_size = process_count * pm->pmf->get_map_process_packet_size() +
 		     queue_count * map_queue_size;
 
 	/*
@@ -102,11 +114,14 @@ static int pm_allocate_runlist_ib(struct packet_manager *pm,
 
 	pm_calc_rlib_size(pm, rl_buffer_size, is_over_subscription);
 
+	mutex_lock(&pm->lock);
+
 	retval = kfd_gtt_sa_allocate(pm->dqm->dev, *rl_buffer_size,
 					&pm->ib_buffer_obj);
 
 	if (retval != 0) {
 		pr_err("kfd: failed to allocate runlist IB\n");
+		mutex_unlock(&pm->lock);
 		return retval;
 	}
 
@@ -115,6 +130,8 @@ static int pm_allocate_runlist_ib(struct packet_manager *pm,
 
 	memset(*rl_buffer, 0, *rl_buffer_size);
 	pm->allocated = true;
+
+	mutex_unlock(&pm->lock);
 	return retval;
 }
 
@@ -122,9 +139,24 @@ static int pm_create_runlist(struct packet_manager *pm, uint32_t *buffer,
 			uint64_t ib, size_t ib_size_in_dwords, bool chain)
 {
 	struct pm4_runlist *packet;
+	int concurrent_proc_cnt = 0;
+	struct kfd_dev *kfd = pm->dqm->dev;
 
 	BUG_ON(!pm || !buffer || !ib);
 
+	/* Determine the number of processes to map together to HW:
+	 * it can not exceed the number of VMIDs available to the
+	 * scheduler, and it is determined by the smaller of the number
+	 * of processes in the runlist and kfd module parameter
+	 * hws_max_conc_proc.
+	 * Note: the arbitration between the number of VMIDs and
+	 * hws_max_conc_proc has been done in
+	 * kgd2kfd_device_init().
+	 */
+	concurrent_proc_cnt = min(pm->dqm->processes_count,
+			kfd->max_proc_per_quantum);
+
+
 	packet = (struct pm4_runlist *)buffer;
 
 	memset(buffer, 0, sizeof(struct pm4_runlist));
@@ -135,6 +167,7 @@ static int pm_create_runlist(struct packet_manager *pm, uint32_t *buffer,
 	packet->bitfields4.chain = chain ? 1 : 0;
 	packet->bitfields4.offload_polling = 0;
 	packet->bitfields4.valid = 1;
+	packet->bitfields4.process_cnt = concurrent_proc_cnt;
 	packet->ordinal2 = lower_32_bits(ib);
 	packet->bitfields3.ib_base_hi = upper_32_bits(ib);
 
@@ -183,6 +216,90 @@ static int pm_create_map_process(struct packet_manager *pm, uint32_t *buffer,
 	return 0;
 }
 
+static int pm_create_map_process_scratch_kv(struct packet_manager *pm,
+		uint32_t *buffer, struct qcm_process_device *qpd)
+{
+	struct pm4_map_process_scratch_kv *packet;
+	struct queue *cur;
+	uint32_t num_queues;
+
+	BUG_ON(!pm || !buffer || !qpd);
+
+	packet = (struct pm4_map_process_scratch_kv *)buffer;
+
+	pr_debug("kfd: In func %s\n", __func__);
+
+	memset(buffer, 0, sizeof(struct pm4_map_process_scratch_kv));
+
+	packet->header.u32all = build_pm4_header(IT_MAP_PROCESS,
+				sizeof(struct pm4_map_process_scratch_kv));
+	packet->bitfields2.diq_enable = (qpd->is_debug) ? 1 : 0;
+	packet->bitfields2.process_quantum = 1;
+	packet->bitfields2.pasid = qpd->pqm->process->pasid;
+	packet->bitfields3.page_table_base = qpd->page_table_base;
+	packet->bitfields14.gds_size = qpd->gds_size;
+	packet->bitfields14.num_gws = qpd->num_gws;
+	packet->bitfields14.num_oac = qpd->num_oac;
+	num_queues = 0;
+	list_for_each_entry(cur, &qpd->queues_list, list)
+		num_queues++;
+	packet->bitfields14.num_queues = (qpd->is_debug) ? 0 : num_queues;
+
+	packet->sh_mem_config = qpd->sh_mem_config;
+	packet->sh_mem_bases = qpd->sh_mem_bases;
+	packet->sh_mem_ape1_base = qpd->sh_mem_ape1_base;
+	packet->sh_mem_ape1_limit = qpd->sh_mem_ape1_limit;
+
+	packet->sh_hidden_private_base_vmid = qpd->sh_hidden_private_base;
+
+	packet->gds_addr_lo = lower_32_bits(qpd->gds_context_area);
+	packet->gds_addr_hi = upper_32_bits(qpd->gds_context_area);
+
+	return 0;
+}
+
+static int pm_create_map_process_scratch(struct packet_manager *pm,
+		uint32_t *buffer, struct qcm_process_device *qpd)
+{
+	struct pm4_map_process_scratch *packet;
+	struct queue *cur;
+	uint32_t num_queues;
+
+	BUG_ON(!pm || !buffer || !qpd);
+
+	packet = (struct pm4_map_process_scratch *)buffer;
+
+	pr_debug("kfd: In func %s\n", __func__);
+
+	memset(buffer, 0, sizeof(struct pm4_map_process_scratch));
+
+	packet->header.u32all = build_pm4_header(IT_MAP_PROCESS,
+					sizeof(struct pm4_map_process_scratch));
+	packet->bitfields2.diq_enable = (qpd->is_debug) ? 1 : 0;
+	packet->bitfields2.process_quantum = 1;
+	packet->bitfields2.pasid = qpd->pqm->process->pasid;
+	packet->bitfields3.page_table_base = qpd->page_table_base;
+	packet->bitfields10.gds_size = qpd->gds_size;
+	packet->bitfields10.num_gws = qpd->num_gws;
+	packet->bitfields10.num_oac = qpd->num_oac;
+	num_queues = 0;
+	list_for_each_entry(cur, &qpd->queues_list, list)
+		num_queues++;
+	packet->bitfields10.num_queues = (qpd->is_debug) ? 0 : num_queues;
+
+	packet->sh_mem_config = qpd->sh_mem_config;
+	packet->sh_mem_bases = qpd->sh_mem_bases;
+	packet->sh_mem_ape1_base = qpd->sh_mem_ape1_base;
+	packet->sh_mem_ape1_limit = qpd->sh_mem_ape1_limit;
+
+	packet->sh_hidden_private_base_vmid = qpd->sh_hidden_private_base;
+
+	packet->gds_addr_lo = lower_32_bits(qpd->gds_context_area);
+	packet->gds_addr_hi = upper_32_bits(qpd->gds_context_area);
+
+	return 0;
+}
+
 static int pm_create_map_queue_vi(struct packet_manager *pm, uint32_t *buffer,
 		struct queue *q, bool is_static)
 {
@@ -220,7 +337,7 @@ static int pm_create_map_queue_vi(struct packet_manager *pm, uint32_t *buffer,
 			queue_type__mes_map_queues__debug_interface_queue_vi;
 		break;
 	case KFD_QUEUE_TYPE_SDMA:
-		packet->bitfields2.engine_sel =
+		packet->bitfields2.engine_sel = q->properties.sdma_engine_id +
 				engine_sel__mes_map_queues__sdma0_vi;
 		use_static = false; /* no static queues under SDMA */
 		break;
@@ -280,7 +397,7 @@ static int pm_create_map_queue(struct packet_manager *pm, uint32_t *buffer,
 				engine_sel__mes_map_queues__compute;
 		break;
 	case KFD_QUEUE_TYPE_SDMA:
-		packet->bitfields2.engine_sel =
+		packet->bitfields2.engine_sel = q->properties.sdma_engine_id +
 				engine_sel__mes_map_queues__sdma0;
 		use_static = false; /* no static queues under SDMA */
 		break;
@@ -349,12 +466,12 @@ static int pm_create_runlist_ib(struct packet_manager *pm,
 			return -ENOMEM;
 		}
 
-		retval = pm_create_map_process(pm, &rl_buffer[rl_wptr], qpd);
+		retval = pm->pmf->map_process(pm, &rl_buffer[rl_wptr], qpd);
 		if (retval != 0)
 			return retval;
 
 		proccesses_mapped++;
-		inc_wptr(&rl_wptr, sizeof(struct pm4_map_process),
+		inc_wptr(&rl_wptr, pm->pmf->get_map_process_packet_size(),
 				alloc_size_bytes);
 
 		list_for_each_entry(kq, &qpd->priv_queue_list, list) {
@@ -364,8 +481,7 @@ static int pm_create_runlist_ib(struct packet_manager *pm,
 			pr_debug("kfd: static_queue, mapping kernel q %d, is debug status %d\n",
 				kq->queue->queue, qpd->is_debug);
 
-			if (pm->dqm->dev->device_info->asic_family ==
-					CHIP_CARRIZO)
+			if (KFD_IS_VI(pm->dqm->dev->device_info->asic_family))
 				retval = pm_create_map_queue_vi(pm,
 						&rl_buffer[rl_wptr],
 						kq->queue,
@@ -390,8 +506,7 @@ static int pm_create_runlist_ib(struct packet_manager *pm,
 			pr_debug("kfd: static_queue, mapping user queue %d, is debug status %d\n",
 				q->queue, qpd->is_debug);
 
-			if (pm->dqm->dev->device_info->asic_family ==
-					CHIP_CARRIZO)
+			if (KFD_IS_VI(pm->dqm->dev->device_info->asic_family))
 				retval = pm_create_map_queue_vi(pm,
 						&rl_buffer[rl_wptr],
 						q,
@@ -424,7 +539,23 @@ static int pm_create_runlist_ib(struct packet_manager *pm,
 	return 0;
 }
 
-int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm)
+static int get_map_process_packet_size(void)
+{
+	return sizeof(struct pm4_map_process);
+}
+
+static int get_map_process_packet_size_scratch_kv(void)
+{
+	return sizeof(struct pm4_map_process_scratch_kv);
+}
+
+static int get_map_process_packet_size_scratch(void)
+{
+	return sizeof(struct pm4_map_process_scratch);
+}
+
+int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm,
+		uint16_t fw_ver)
 {
 	BUG_ON(!dqm);
 
@@ -435,8 +566,37 @@ int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm)
 		mutex_destroy(&pm->lock);
 		return -ENOMEM;
 	}
+	pm->pmf = kzalloc(sizeof(struct packet_manager_firmware), GFP_KERNEL);
 	pm->allocated = false;
 
+	switch (pm->dqm->dev->device_info->asic_family) {
+	case CHIP_KAVERI:
+		if (fw_ver >= KFD_SCRATCH_KV_FW_VER) {
+			pm->pmf->map_process = pm_create_map_process_scratch_kv;
+			pm->pmf->get_map_process_packet_size =
+					get_map_process_packet_size_scratch_kv;
+		} else {
+			pm->pmf->map_process = pm_create_map_process;
+			pm->pmf->get_map_process_packet_size =
+						get_map_process_packet_size;
+		}
+		break;
+	case CHIP_CARRIZO:
+	case CHIP_TONGA:
+	case CHIP_FIJI:
+		if (fw_ver >= KFD_SCRATCH_CZ_FW_VER) {
+			pm->pmf->map_process = pm_create_map_process_scratch;
+			pm->pmf->get_map_process_packet_size =
+					get_map_process_packet_size_scratch;
+		} else {
+			pm->pmf->map_process = pm_create_map_process;
+			pm->pmf->get_map_process_packet_size =
+						get_map_process_packet_size;
+		}
+		break;
+
+	}
+
 	return 0;
 }
 
@@ -446,6 +606,7 @@ void pm_uninit(struct packet_manager *pm)
 
 	mutex_destroy(&pm->lock);
 	kernel_queue_uninit(pm->priv_queue);
+	kfree(pm->pmf);
 }
 
 int pm_send_set_resources(struct packet_manager *pm,
@@ -578,7 +739,7 @@ int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address,
 }
 
 int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,
-			enum kfd_preempt_type_filter mode,
+			enum kfd_unmap_queues_filter filter,
 			uint32_t filter_param, bool reset,
 			unsigned int sdma_engine)
 {
@@ -598,8 +759,8 @@ int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,
 
 	packet = (struct pm4_unmap_queues *)buffer;
 	memset(buffer, 0, sizeof(struct pm4_unmap_queues));
-	pr_debug("kfd: static_queue: unmapping queues: mode is %d , reset is %d , type is %d\n",
-		mode, reset, type);
+	pr_debug("kfd: static_queue: unmapping queues: filter is %d , reset is %d , type is %d\n",
+		filter, reset, type);
 	packet->header.u32all = build_pm4_header(IT_UNMAP_QUEUES,
 					sizeof(struct pm4_unmap_queues));
 	switch (type) {
@@ -624,26 +785,26 @@ int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,
 		packet->bitfields2.action =
 				action__mes_unmap_queues__preempt_queues;
 
-	switch (mode) {
-	case KFD_PREEMPT_TYPE_FILTER_SINGLE_QUEUE:
+	switch (filter) {
+	case KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE:
 		packet->bitfields2.queue_sel =
 				queue_sel__mes_unmap_queues__perform_request_on_specified_queues;
 		packet->bitfields2.num_queues = 1;
 		packet->bitfields3b.doorbell_offset0 = filter_param;
 		break;
-	case KFD_PREEMPT_TYPE_FILTER_BY_PASID:
+	case KFD_UNMAP_QUEUES_FILTER_BY_PASID:
 		packet->bitfields2.queue_sel =
 				queue_sel__mes_unmap_queues__perform_request_on_pasid_queues;
 		packet->bitfields3a.pasid = filter_param;
 		break;
-	case KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES:
+	case KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES:
 		packet->bitfields2.queue_sel =
 				queue_sel__mes_unmap_queues__perform_request_on_all_active_queues;
 		break;
-	case KFD_PREEMPT_TYPE_FILTER_DYNAMIC_QUEUES:
+	case KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES:
 		/* in this case, we do not preempt static queues */
-		packet->bitfields2.queue_sel =
-				queue_sel__mes_unmap_queues__perform_request_on_dynamic_queues_only;
+		packet->bitfields2.queue_sel = 
+			queue_sel__mes_unmap_queues__perform_request_on_dynamic_queues_only;
 		break;
 	default:
 		BUG();
@@ -671,3 +832,4 @@ void pm_release_ib(struct packet_manager *pm)
 	}
 	mutex_unlock(&pm->lock);
 }
+
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers.h b/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers.h
index 5b393f3..e7570cc 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers.h
@@ -127,7 +127,8 @@ struct pm4_runlist {
 			uint32_t offload_polling:1;
 			uint32_t reserved3:1;
 			uint32_t valid:1;
-			uint32_t reserved4:8;
+			uint32_t process_cnt:4;
+			uint32_t reserved4:4;
 		} bitfields4;
 		uint32_t ordinal4;
 	};
@@ -186,6 +187,123 @@ struct pm4_map_process {
 };
 #endif
 
+/*--------------------MES_MAP_PROCESS_SCRATCH-------------------- */
+
+#ifndef PM4_MES_MAP_PROCESS_SCRATCH_DEFINED
+#define PM4_MES_MAP_PROCESS_SCRATCH_DEFINED
+
+struct pm4_map_process_scratch {
+	union {
+		union PM4_MES_TYPE_3_HEADER header;	/* header */
+		uint32_t ordinal1;
+	};
+
+	union {
+		struct {
+			uint32_t pasid:16;
+			uint32_t reserved1:8;
+			uint32_t diq_enable:1;
+			uint32_t process_quantum:7;
+		} bitfields2;
+		uint32_t ordinal2;
+	};
+
+	union {
+		struct {
+			uint32_t page_table_base:28;
+			uint32_t reserved3:4;
+		} bitfields3;
+		uint32_t ordinal3;
+	};
+
+	uint32_t reserved;
+
+	uint32_t sh_mem_bases;
+	uint32_t sh_mem_config;
+	uint32_t sh_mem_ape1_base;
+	uint32_t sh_mem_ape1_limit;
+
+	uint32_t sh_hidden_private_base_vmid;
+
+	uint32_t reserved2;
+	uint32_t reserved3;
+
+	uint32_t gds_addr_lo;
+	uint32_t gds_addr_hi;
+
+	union {
+		struct {
+			uint32_t num_gws:6;
+			uint32_t reserved4:2;
+			uint32_t num_oac:4;
+			uint32_t reserved5:4;
+			uint32_t gds_size:6;
+			uint32_t num_queues:10;
+		} bitfields10;
+		uint32_t ordinal10;
+	};
+
+	uint32_t completion_signal_lo;
+	uint32_t completion_signal_hi;
+
+};
+#endif
+
+#ifndef PM4_MES_MAP_PROCESS_DEFINED_KV_SCRATCH
+#define PM4_MES_MAP_PROCESS_DEFINED_KV_SCRATCH
+
+struct pm4_map_process_scratch_kv {
+	union {
+		union PM4_MES_TYPE_3_HEADER   header; /* header */
+		uint32_t            ordinal1;
+	};
+
+	union {
+		struct {
+			uint32_t pasid:16;
+			uint32_t reserved1:8;
+			uint32_t diq_enable:1;
+			uint32_t process_quantum:7;
+		} bitfields2;
+		uint32_t ordinal2;
+	};
+
+	union {
+		struct {
+			uint32_t page_table_base:28;
+			uint32_t reserved2:4;
+		} bitfields3;
+		uint32_t ordinal3;
+	};
+
+	uint32_t reserved3;
+	uint32_t sh_mem_bases;
+	uint32_t sh_mem_config;
+	uint32_t sh_mem_ape1_base;
+	uint32_t sh_mem_ape1_limit;
+	uint32_t sh_hidden_private_base_vmid;
+	uint32_t reserved4;
+	uint32_t reserved5;
+	uint32_t gds_addr_lo;
+	uint32_t gds_addr_hi;
+
+	union {
+		struct {
+			uint32_t num_gws:6;
+			uint32_t reserved6:2;
+			uint32_t num_oac:4;
+			uint32_t reserved7:4;
+			uint32_t gds_size:6;
+			uint32_t num_queues:10;
+		} bitfields14;
+		uint32_t ordinal14;
+	};
+
+	uint32_t completion_signal_lo32;
+uint32_t completion_signal_hi32;
+};
+#endif
+
 /*--------------------MES_MAP_QUEUES--------------------*/
 
 #ifndef PM4_MES_MAP_QUEUES_DEFINED
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 4750cab..ae636ae 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -30,13 +30,45 @@
 #include <linux/atomic.h>
 #include <linux/workqueue.h>
 #include <linux/spinlock.h>
+#include <linux/idr.h>
 #include <linux/kfd_ioctl.h>
+#include <linux/pid.h>
+#include <linux/interval_tree.h>
 #include <kgd_kfd_interface.h>
 
+#include "amd_rdma.h"
+
 #define KFD_SYSFS_FILE_MODE 0444
 
-#define KFD_MMAP_DOORBELL_MASK 0x8000000000000
-#define KFD_MMAP_EVENTS_MASK 0x4000000000000
+/* GPU ID hash width in bits */
+#define KFD_GPU_ID_HASH_WIDTH 16
+
+/* Use upper bits of mmap offset to store KFD driver specific information.
+ * BITS[63:62] - Encode MMAP type
+ * BITS[61:46] - Encode gpu_id. To identify to which GPU the offset belongs to
+ * BITS[45:40] - Reserved. Not Used.
+ * BITS[39:0]  - MMAP offset value. Used by TTM.
+ *
+ * NOTE: struct vm_area_struct.vm_pgoff uses offset in pages. Hence, these
+ *  defines are w.r.t to PAGE_SIZE
+ */
+#define KFD_MMAP_TYPE_SHIFT	(62 - PAGE_SHIFT)
+#define KFD_MMAP_TYPE_MASK	(0x3ULL << KFD_MMAP_TYPE_SHIFT)
+#define KFD_MMAP_TYPE_DOORBELL	(0x3ULL << KFD_MMAP_TYPE_SHIFT)
+#define KFD_MMAP_TYPE_EVENTS	(0x2ULL << KFD_MMAP_TYPE_SHIFT)
+#define KFD_MMAP_TYPE_MAP_BO	(0x1ULL << KFD_MMAP_TYPE_SHIFT)
+#define KFD_MMAP_TYPE_RESERVED_MEM	(0x0ULL << KFD_MMAP_TYPE_SHIFT)
+
+#define KFD_MMAP_GPU_ID_SHIFT (46 - PAGE_SHIFT)
+#define KFD_MMAP_GPU_ID_MASK (((1ULL << KFD_GPU_ID_HASH_WIDTH) - 1) \
+				<< KFD_MMAP_GPU_ID_SHIFT)
+#define KFD_MMAP_GPU_ID(gpu_id) ((((uint64_t)gpu_id) << KFD_MMAP_GPU_ID_SHIFT)\
+				& KFD_MMAP_GPU_ID_MASK)
+#define KFD_MMAP_GPU_ID_GET(offset)    ((offset & KFD_MMAP_GPU_ID_MASK) \
+				>> KFD_MMAP_GPU_ID_SHIFT)
+
+#define KFD_MMAP_OFFSET_VALUE_MASK	(0xFFFFFFFFFFULL >> PAGE_SHIFT)
+#define KFD_MMAP_OFFSET_VALUE_GET(offset) (offset & KFD_MMAP_OFFSET_VALUE_MASK)
 
 /*
  * When working with cp scheduler we should assign the HIQ manually or via
@@ -48,8 +80,6 @@
 #define KFD_CIK_HIQ_PIPE 4
 #define KFD_CIK_HIQ_QUEUE 0
 
-/* GPU ID hash width in bits */
-#define KFD_GPU_ID_HASH_WIDTH 16
 
 /* Macro for allocating structures */
 #define kfd_alloc_struct(ptr_to_struct)	\
@@ -74,12 +104,26 @@ extern int max_num_of_queues_per_device;
 /* Kernel module parameter to specify the scheduling policy */
 extern int sched_policy;
 
+extern int cwsr_enable;
+
+/*
+ * Kernel module parameter to specify the maximum process
+ * number per HW scheduler
+ */
+extern int hws_max_conc_proc;
+
 /*
  * Kernel module parameter to specify whether to send sigterm to HSA process on
  * unhandled exception
  */
 extern int send_sigterm;
 
+/*
+ * This kernel module is used to simulate large bar machine on non-large bar
+ * enabled machines.
+ */
+extern int debug_largebar;
+
 /**
  * enum kfd_sched_policy
  *
@@ -114,14 +158,17 @@ enum cache_policy {
 
 enum asic_family_type {
 	CHIP_KAVERI = 0,
-	CHIP_CARRIZO
+	CHIP_CARRIZO,
+	CHIP_TONGA,
+	CHIP_FIJI
 };
 
+#define KFD_IS_VI(chip) ((chip) >= CHIP_CARRIZO && (chip) <= CHIP_FIJI)
+#define KFD_IS_DGPU(chip) ((chip) >= CHIP_TONGA && (chip) <= CHIP_FIJI)
+
 struct kfd_event_interrupt_class {
-	bool (*interrupt_isr)(struct kfd_dev *dev,
-				const uint32_t *ih_ring_entry);
-	void (*interrupt_wq)(struct kfd_dev *dev,
-				const uint32_t *ih_ring_entry);
+	bool (*interrupt_isr)(struct kfd_dev *dev, const uint32_t *ih_ring_entry);
+	void (*interrupt_wq)(struct kfd_dev *dev, const uint32_t *ih_ring_entry);
 };
 
 struct kfd_device_info {
@@ -132,6 +179,7 @@ struct kfd_device_info {
 	size_t ih_ring_entry_size;
 	uint8_t num_of_watch_points;
 	uint16_t mqd_size_aligned;
+	bool is_need_iommu_device;
 };
 
 struct kfd_mem_obj {
@@ -141,6 +189,12 @@ struct kfd_mem_obj {
 	uint32_t *cpu_ptr;
 };
 
+struct kfd_vmid_info {
+	uint32_t first_vmid_kfd;
+	uint32_t last_vmid_kfd;
+	uint32_t vmid_num_kfd;
+};
+
 struct kfd_dev {
 	struct kgd_dev *kgd;
 
@@ -165,11 +219,12 @@ struct kfd_dev {
 					   */
 
 	struct kgd2kfd_shared_resources shared_resources;
+	struct kfd_vmid_info vm_info;
 
 	const struct kfd2kgd_calls *kfd2kgd;
 	struct mutex doorbell_mutex;
-	DECLARE_BITMAP(doorbell_available_index,
-			KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
+	unsigned long doorbell_available_index[DIV_ROUND_UP(
+		KFD_MAX_NUM_OF_QUEUES_PER_PROCESS, BITS_PER_LONG)];
 
 	void *gtt_mem;
 	uint64_t gtt_start_gpu_addr;
@@ -179,6 +234,11 @@ struct kfd_dev {
 	unsigned int gtt_sa_chunk_size;
 	unsigned int gtt_sa_num_of_chunks;
 
+	/* QCM Device instance */
+	struct device_queue_manager *dqm;
+
+	bool init_complete;
+
 	/* Interrupts */
 	void *interrupt_ring;
 	size_t interrupt_ring_size;
@@ -187,10 +247,6 @@ struct kfd_dev {
 	struct work_struct interrupt_work;
 	spinlock_t interrupt_lock;
 
-	/* QCM Device instance */
-	struct device_queue_manager *dqm;
-
-	bool init_complete;
 	/*
 	 * Interrupts of interest to KFD are copied
 	 * from the HW ring into a SW ring.
@@ -198,7 +254,26 @@ struct kfd_dev {
 	bool interrupts_active;
 
 	/* Debug manager */
-	struct kfd_dbgmgr           *dbgmgr;
+	struct kfd_dbgmgr *dbgmgr;
+
+	/* MEC firmware version*/
+	uint16_t mec_fw_version;
+
+	/* Maximum process number mapped to HW scheduler */
+	unsigned int max_proc_per_quantum;
+
+	/* cwsr */
+	bool cwsr_enabled;
+	struct page *cwsr_pages;
+	uint32_t cwsr_size;
+	uint32_t tma_offset;  /*Offset for TMA from the  start of cwsr_mem*/
+};
+
+struct kfd_bo {
+	void *mem;
+	struct interval_tree_node it;
+	struct kfd_dev *dev;
+	struct list_head cb_data_head;
 };
 
 /* KGD2KFD callbacks */
@@ -221,22 +296,22 @@ void kfd_chardev_exit(void);
 struct device *kfd_chardev(void);
 
 /**
- * enum kfd_preempt_type_filter
+ * enum kfd_unmap_queues_filter
  *
- * @KFD_PREEMPT_TYPE_FILTER_SINGLE_QUEUE: Preempts single queue.
+ * @KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE: Preempts single queue.
  *
- * @KFD_PRERMPT_TYPE_FILTER_ALL_QUEUES: Preempts all queues in the
+ * @KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES: Preempts all queues in the
  *						running queues list.
  *
- * @KFD_PRERMPT_TYPE_FILTER_BY_PASID: Preempts queues that belongs to
+ * @KFD_UNMAP_QUEUES_FILTER_BY_PASID: Preempts queues that belongs to
  *						specific process.
  *
  */
-enum kfd_preempt_type_filter {
-	KFD_PREEMPT_TYPE_FILTER_SINGLE_QUEUE,
-	KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES,
-	KFD_PREEMPT_TYPE_FILTER_DYNAMIC_QUEUES,
-	KFD_PREEMPT_TYPE_FILTER_BY_PASID
+enum kfd_unmap_queues_filter {
+	KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE,
+	KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES,
+	KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES,
+	KFD_UNMAP_QUEUES_FILTER_BY_PASID
 };
 
 enum kfd_preempt_type {
@@ -324,6 +399,7 @@ struct queue_properties {
 	uint32_t __iomem *doorbell_ptr;
 	uint32_t doorbell_off;
 	bool is_interop;
+	bool is_evicted; /* true -> queue is evicted */
 	bool is_active;
 	/* Not relevant for user mode queues in cp scheduling */
 	unsigned int vmid;
@@ -336,6 +412,11 @@ struct queue_properties {
 	uint32_t eop_ring_buffer_size;
 	uint64_t ctx_save_restore_area_address;
 	uint32_t ctx_save_restore_area_size;
+	uint32_t ctl_stack_size;
+	uint64_t tba_addr;
+	uint64_t tma_addr;
+	/* Relevant for CU */
+	uint32_t cu_mask;
 };
 
 /**
@@ -424,6 +505,7 @@ struct qcm_process_device {
 	unsigned int queue_count;
 	unsigned int vmid;
 	bool is_debug;
+	unsigned evicted; /* eviction counter, 0=active */
 	/*
 	 * All the memory management data should be here too
 	 */
@@ -436,8 +518,22 @@ struct qcm_process_device {
 	uint32_t gds_size;
 	uint32_t num_gws;
 	uint32_t num_oac;
+	uint32_t sh_hidden_private_base;
+
+	/*cwsr memory*/
+	int  cwsr_mem_handle;
+	uint64_t cwsr_base;
+	uint64_t tba_addr;
+	uint64_t tma_addr;
+	void *cwsr_kaddr;
 };
 
+/*8 byte handle containing GPU ID in the most significant 4 bytes and
+ * idr_handle in the least significant 4 bytes*/
+#define MAKE_HANDLE(gpu_id, idr_handle) (((uint64_t)(gpu_id) << 32) + idr_handle)
+#define GET_GPU_ID(handle) (handle >> 32)
+#define GET_IDR_HANDLE(handle) (handle & 0xFFFFFFFF)
+
 /* Data that is per-process-per device. */
 struct kfd_process_device {
 	/*
@@ -449,6 +545,8 @@ struct kfd_process_device {
 	/* The device that owns this data. */
 	struct kfd_dev *dev;
 
+	/* The process that owns this kfd_process_device. */
+	struct kfd_process *process;
 
 	/* per-process-per device QCM data structure */
 	struct qcm_process_device qpd;
@@ -460,10 +558,23 @@ struct kfd_process_device {
 	uint64_t gpuvm_limit;
 	uint64_t scratch_base;
 	uint64_t scratch_limit;
+	uint64_t dgpu_base;
+	uint64_t dgpu_limit;
+	uint64_t mapped_size;
+	uint64_t last_eviction;
+	bool	 evicted;
+
+	uint64_t sh_hidden_private_base_vmid;
 
 	/* Is this process/pasid bound to this device? (amd_iommu_bind_pasid) */
 	bool bound;
 
+	/* VM context for GPUVM allocations */
+	void *vm;
+
+	/* GPUVM allocations storage */
+	struct idr alloc_idr;
+
 	/* This flag tells if we should reset all
 	 * wavefronts on process termination
 	 */
@@ -482,7 +593,7 @@ struct kfd_process {
 
 	struct mm_struct *mm;
 
-	struct mutex mutex;
+	struct rw_semaphore lock;
 
 	/*
 	 * In any process, the thread that started main() is the lead
@@ -513,6 +624,8 @@ struct kfd_process {
 	/* Size is queue_array_size, up to MAX_PROCESS_QUEUES. */
 	struct kfd_queue **queues;
 
+	unsigned long allocated_queue_bitmap[DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS, BITS_PER_LONG)];
+
 	/*Is the user space process 32 bit?*/
 	bool is_32bit_user_mode;
 
@@ -520,10 +633,12 @@ struct kfd_process {
 	struct mutex event_mutex;
 	/* All events in process hashed by ID, linked on kfd_event.events. */
 	DECLARE_HASHTABLE(events, 4);
-	struct list_head signal_event_pages;	/* struct slot_page_header.
-								event_pages */
+	struct list_head signal_event_pages;	/* struct slot_page_header.event_pages */
 	u32 next_nonsignal_event_id;
 	size_t signal_event_count;
+	size_t debug_event_count;
+
+	struct rb_root bo_interval_tree;
 };
 
 /**
@@ -546,9 +661,10 @@ struct amdkfd_ioctl_desc {
 
 void kfd_process_create_wq(void);
 void kfd_process_destroy_wq(void);
-struct kfd_process *kfd_create_process(const struct task_struct *);
+struct kfd_process *kfd_create_process(struct file *filep);
 struct kfd_process *kfd_get_process(const struct task_struct *);
 struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid);
+struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm);
 
 struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
 							struct kfd_process *p);
@@ -558,6 +674,29 @@ struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
 struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
 							struct kfd_process *p);
 
+int kfd_reserved_mem_mmap(struct kfd_process *process, struct vm_area_struct *vma);
+
+/* KFD process API for creating and translating handles */
+int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
+					void *mem, uint64_t start,
+					uint64_t length);
+void *kfd_process_device_translate_handle(struct kfd_process_device *p,
+					int handle);
+struct kfd_bo *kfd_process_device_find_bo(struct kfd_process_device *pdd,
+					int handle);
+void *kfd_process_find_bo_from_interval(struct kfd_process *p,
+					uint64_t start_addr,
+					uint64_t last_addr);
+void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,
+					int handle);
+
+void run_rdma_free_callback(struct kfd_bo *buf_obj);
+struct kfd_process *kfd_lookup_process_by_pid(struct pid *pid);
+
+/* kfd dgpu memory */
+int kfd_map_memory_to_gpu(struct kfd_dev *dev, void *mem,
+		struct kfd_process *p, struct kfd_process_device *pdd);
+
 /* Process device data iterator */
 struct kfd_process_device *kfd_get_first_process_device_data(struct kfd_process *p);
 struct kfd_process_device *kfd_get_next_process_device_data(struct kfd_process *p,
@@ -600,7 +739,11 @@ int kfd_topology_add_device(struct kfd_dev *gpu);
 int kfd_topology_remove_device(struct kfd_dev *gpu);
 struct kfd_dev *kfd_device_by_id(uint32_t gpu_id);
 struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev);
-struct kfd_dev *kfd_topology_enum_kfd_devices(uint8_t idx);
+struct kfd_dev *kfd_device_by_kgd(const struct kgd_dev *kgd);
+uint32_t kfd_get_gpu_id(struct kfd_dev *dev);
+int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_dev **kdev);
+int kfd_numa_node_to_apic_id(int numa_node_id);
+int kfd_get_proximity_domain(const struct pci_bus *bus);
 
 /* Interrupts */
 int kfd_interrupt_init(struct kfd_dev *dev);
@@ -615,9 +758,12 @@ int kgd2kfd_resume(struct kfd_dev *kfd);
 
 /* amdkfd Apertures */
 int kfd_init_apertures(struct kfd_process *process);
+int kfd_set_process_dgpu_aperture(struct kfd_process_device *pdd,
+				uint64_t base, uint64_t limit);
 
 /* Queue Context Management */
-struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd);
+inline uint32_t lower_32(uint64_t x);
+inline uint32_t upper_32(uint64_t x);
 
 int init_queue(struct queue **q, const struct queue_properties *properties);
 void uninit_queue(struct queue *q);
@@ -630,11 +776,15 @@ struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
 		struct kfd_dev *dev);
 struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
 		struct kfd_dev *dev);
+struct mqd_manager *mqd_manager_init_vi_tonga(enum KFD_MQD_TYPE type,
+		struct kfd_dev *dev);
 struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev);
 void device_queue_manager_uninit(struct device_queue_manager *dqm);
 struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
 					enum kfd_queue_type type);
 void kernel_queue_uninit(struct kernel_queue *kq);
+int kfd_process_vm_fault(struct device_queue_manager *dqm,
+				unsigned int pasid);
 
 /* Process Queue Manager */
 struct process_queue_node {
@@ -649,18 +799,16 @@ int pqm_create_queue(struct process_queue_manager *pqm,
 			    struct kfd_dev *dev,
 			    struct file *f,
 			    struct queue_properties *properties,
-			    unsigned int flags,
-			    enum kfd_queue_type type,
 			    unsigned int *qid);
 int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid);
 int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid,
 			struct queue_properties *p);
+int pqm_set_cu_mask(struct process_queue_manager *pqm, unsigned int qid,
+			struct queue_properties *p);
 struct kernel_queue *pqm_get_kernel_queue(struct process_queue_manager *pqm,
 						unsigned int qid);
-
-int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
-				unsigned int fence_value,
-				unsigned long timeout);
+int kgd2kfd_quiesce_mm(struct kfd_dev *kfd, struct mm_struct *mm);
+int kgd2kfd_resume_mm(struct kfd_dev *kfd, struct mm_struct *mm);
 
 /* Packet Manager */
 
@@ -668,7 +816,9 @@ int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
 
 #define KFD_FENCE_COMPLETED (100)
 #define KFD_FENCE_INIT   (10)
-#define KFD_UNMAP_LATENCY (150)
+#define KFD_UNMAP_LATENCY (40)
+
+struct packet_manager_firmware;
 
 struct packet_manager {
 	struct device_queue_manager *dqm;
@@ -676,9 +826,19 @@ struct packet_manager {
 	struct mutex lock;
 	bool allocated;
 	struct kfd_mem_obj *ib_buffer_obj;
+
+	struct packet_manager_firmware *pmf;
 };
 
-int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm);
+struct packet_manager_firmware {
+	/* Support different firmware versions for map process packet */
+	int (*map_process)(struct packet_manager *pm, uint32_t *buffer,
+				struct qcm_process_device *qpd);
+	int (*get_map_process_packet_size)(void);
+};
+
+int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm,
+		uint16_t fw_ver);
 void pm_uninit(struct packet_manager *pm);
 int pm_send_set_resources(struct packet_manager *pm,
 				struct scheduling_resources *res);
@@ -687,7 +847,7 @@ int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address,
 				uint32_t fence_value);
 
 int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,
-			enum kfd_preempt_type_filter mode,
+			enum kfd_unmap_queues_filter mode,
 			uint32_t filter_param, bool reset,
 			unsigned int sdma_engine);
 
@@ -696,6 +856,9 @@ void pm_release_ib(struct packet_manager *pm);
 uint64_t kfd_get_number_elems(struct kfd_dev *kfd);
 phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev,
 					struct kfd_process *process);
+int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
+				unsigned int fence_value,
+				unsigned long timeout);
 
 /* Events */
 extern const struct kfd_event_interrupt_class event_interrupt_class_cik;
@@ -714,8 +877,7 @@ int kfd_wait_on_events(struct kfd_process *p,
 		       uint32_t num_events, void __user *data,
 		       bool all, uint32_t user_timeout_ms,
 		       enum kfd_event_wait_result *wait_result);
-void kfd_signal_event_interrupt(unsigned int pasid, uint32_t partial_id,
-				uint32_t valid_id_bits);
+void kfd_signal_event_interrupt(unsigned int pasid, uint32_t partial_id, uint32_t valid_id_bits);
 void kfd_signal_iommu_event(struct kfd_dev *dev,
 		unsigned int pasid, unsigned long address,
 		bool is_write_requested, bool is_execute_requested);
@@ -723,11 +885,28 @@ void kfd_signal_hw_exception_event(unsigned int pasid);
 int kfd_set_event(struct kfd_process *p, uint32_t event_id);
 int kfd_reset_event(struct kfd_process *p, uint32_t event_id);
 int kfd_event_create(struct file *devkfd, struct kfd_process *p,
-		     uint32_t event_type, bool auto_reset, uint32_t node_id,
-		     uint32_t *event_id, uint32_t *event_trigger_data,
-		     uint64_t *event_page_offset, uint32_t *event_slot_index);
+	     uint32_t event_type, bool auto_reset, uint32_t node_id,
+	     uint32_t *event_id, uint32_t *event_trigger_data,
+	     uint64_t *event_page_offset, uint32_t *event_slot_index,
+	     void *kern_addr);
 int kfd_event_destroy(struct kfd_process *p, uint32_t event_id);
+void kfd_free_signal_page_dgpu(struct kfd_process *p, uint64_t handle);
+
+void kfd_signal_vm_fault_event(struct kfd_dev *dev, unsigned int pasid,
+				struct kfd_vm_fault_info *info);
+
+void radeon_flush_tlb(struct kfd_dev *dev, uint32_t pasid);
 
 int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p);
+int kgd2kfd_evict_bo(struct kfd_dev *dev, void *mem);
+int kgd2kfd_restore(struct kfd_dev *kfd);
+int evict_size(struct kfd_process *p, int size, int type);
+int evict_bo(struct kfd_dev *dev, void *mem);
+int restore(struct kfd_dev *kfd);
+
+#define KFD_SCRATCH_CZ_FW_VER 600
+#define KFD_SCRATCH_KV_FW_VER 413
+#define KFD_MULTI_PROC_MAPPING_HWS_SUPPORT 600
+#define KFD_CWSR_CZ_FW_VER 625
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 035bbc9..a069c3d 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -28,6 +28,10 @@
 #include <linux/amd-iommu.h>
 #include <linux/notifier.h>
 #include <linux/compat.h>
+#include <linux/mm.h>
+#include <asm/tlb.h>
+#include <linux/highmem.h>
+#include <uapi/asm-generic/mman-common.h>
 
 struct mm_struct;
 
@@ -41,6 +45,7 @@ struct mm_struct;
  */
 #define INITIAL_QUEUE_ARRAY_SIZE 16
 
+static int evict_pdd(struct kfd_process_device *pdd);
 /*
  * List of struct kfd_process (field kfd_process).
  * Unique/indexed by mm_struct*
@@ -58,8 +63,14 @@ struct kfd_process_release_work {
 	struct kfd_process *p;
 };
 
-static struct kfd_process *find_process(const struct task_struct *thread);
+#define MIN_IDR_ID 1
+#define MAX_IDR_ID 0 /*0 - for unlimited*/
+
+static struct kfd_process *find_process(const struct task_struct *thread,
+		bool lock);
 static struct kfd_process *create_process(const struct task_struct *thread);
+static int kfd_process_init_cwsr(struct kfd_process *p, struct file *filep);
+
 
 void kfd_process_create_wq(void)
 {
@@ -75,10 +86,12 @@ void kfd_process_destroy_wq(void)
 	}
 }
 
-struct kfd_process *kfd_create_process(const struct task_struct *thread)
+struct kfd_process *kfd_create_process(struct file *filep)
 {
 	struct kfd_process *process;
 
+	struct task_struct *thread = current;
+
 	BUG_ON(!kfd_process_wq);
 
 	if (thread->mm == NULL)
@@ -99,7 +112,7 @@ struct kfd_process *kfd_create_process(const struct task_struct *thread)
 	mutex_lock(&kfd_processes_mutex);
 
 	/* A prior open of /dev/kfd could have already created the process. */
-	process = find_process(thread);
+	process = find_process(thread, false);
 	if (process)
 		pr_debug("kfd: process already found\n");
 
@@ -110,6 +123,8 @@ struct kfd_process *kfd_create_process(const struct task_struct *thread)
 
 	up_write(&thread->mm->mmap_sem);
 
+	kfd_process_init_cwsr(process, filep);
+
 	return process;
 }
 
@@ -124,7 +139,7 @@ struct kfd_process *kfd_get_process(const struct task_struct *thread)
 	if (thread->group_leader->mm != thread->mm)
 		return ERR_PTR(-EINVAL);
 
-	process = find_process(thread);
+	process = find_process(thread, false);
 
 	return process;
 }
@@ -141,23 +156,164 @@ static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
 	return NULL;
 }
 
-static struct kfd_process *find_process(const struct task_struct *thread)
+static struct kfd_process *find_process(const struct task_struct *thread,
+		bool lock)
 {
 	struct kfd_process *p;
 	int idx;
 
 	idx = srcu_read_lock(&kfd_processes_srcu);
 	p = find_process_by_mm(thread->mm);
+	if (p && lock)
+		down_read(&p->lock);
 	srcu_read_unlock(&kfd_processes_srcu, idx);
 
 	return p;
 }
 
+/* This returns with process->lock read-locked. */
+struct kfd_process *kfd_lookup_process_by_pid(struct pid *pid)
+{
+	struct task_struct *task = NULL;
+	struct kfd_process *p    = NULL;
+
+	if (!pid)
+		task = current;
+	else
+		task = get_pid_task(pid, PIDTYPE_PID);
+
+	if (task)
+		p = find_process(task, true);
+
+	return p;
+}
+
+int evict_size(struct kfd_process *process, int size, int type)
+{
+	struct kfd_process_device *pdd, *temp_pdd = NULL;
+	struct kfd_process *p = process;
+	int temp = 0;
+
+	down_write(&p->lock);
+
+	if (type == EVICT_FIRST_PDD) {
+
+		list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
+			pr_debug("Releasing pdd (topology id %d) for process (pasid %d) in workqueue\n",
+					pdd->dev->id, p->pasid);
+			if (pdd->mapped_size >= size) {
+				evict_pdd(pdd);
+				return 0;
+			}
+
+		}
+	} else if (type == EVICT_BIGGEST_PDD) {
+
+		list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
+			pr_debug("Releasing pdd (topology id %d) for process (pasid %d) in workqueue\n",
+					pdd->dev->id, p->pasid);
+			if (pdd->mapped_size >= temp) {
+				temp = pdd->mapped_size;
+				temp_pdd = pdd;
+			}
+
+		}
+		if (temp_pdd->mapped_size > size) {
+			evict_pdd(temp_pdd);
+			return 0;
+		}
+
+	}
+	up_write(&p->lock);
+	return 0;
+
+}
+
+int evict_bo(struct kfd_dev *dev, void *mem)
+{
+	struct kfd_process_device *pdd;
+
+	pdd =  dev->kfd2kgd->get_pdd_from_buffer_object(dev->kgd,
+			((struct kgd_mem *)mem));
+
+	if (pdd)
+		evict_pdd(pdd);
+
+	return 0;
+}
+
+static int evict_pdd(struct kfd_process_device *pdd)
+{
+	void *mem;
+	int id;
+
+	/*process_evict_queues(struct device_queue_manager *dqm, pdd->qpd)*/
+	/*
+	* Remove all handles from idr and release appropriate
+	* local memory object
+	*/
+	idr_for_each_entry(&pdd->alloc_idr, mem, id) {
+		pdd->dev->kfd2kgd->unmap_memory_to_gpu(
+			pdd->dev->kgd, mem, pdd->vm);
+	}
+	pdd->last_eviction = jiffies;
+	pdd->mapped_size = 0;
+	pdd->evicted = true;
+
+	/*flush_tlb_all();*/
+
+	return 0;
+}
+
+int restore(struct kfd_dev *kfd)
+{
+	struct kfd_process *p = NULL;
+	/*  TODO still working on how to get the process */
+	struct kfd_process_device *pdd = kfd_get_process_device_data(kfd, p);
+	void *mem;
+	int id;
+
+	/* need to run on all processes*/
+	down_write(&p->lock);
+
+	list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
+		pr_debug("Releasing pdd (topology id %d) for process (pasid %d) in workqueue\n",
+				pdd->dev->id, p->pasid);
+
+		/*
+		 * Remove all handles from idr and release appropriate
+		 * local memory object
+		 */
+		if (pdd->evicted) {
+			idr_for_each_entry(&pdd->alloc_idr, mem, id) {
+				pdd->dev->kfd2kgd->map_memory_to_gpu(
+						pdd->dev->kgd,
+						mem, pdd->vm);
+				pdd->last_eviction = 0;
+				pdd->mapped_size = 0;
+			}
+
+			/*process_restore_queues
+			 * (struct device_queue_manager *dqm, pdd->qpd)*/
+		} else {
+			pdd->evicted = false;
+		}
+	}
+	up_write(&p->lock);
+	return 0;
+}
+
+/* No process locking is needed in this function, because the process
+ * is not findable any more. We must assume that no other thread is
+ * using it any more, otherwise we couldn't safely free the process
+ * stucture in the end. */
 static void kfd_process_wq_release(struct work_struct *work)
 {
 	struct kfd_process_release_work *my_work;
-	struct kfd_process_device *pdd, *temp;
+	struct kfd_process_device *pdd, *temp, *peer_pdd;
 	struct kfd_process *p;
+	struct kfd_bo *buf_obj;
+	int id;
 
 	my_work = (struct kfd_process_release_work *) work;
 
@@ -166,19 +322,40 @@ static void kfd_process_wq_release(struct work_struct *work)
 	pr_debug("Releasing process (pasid %d) in workqueue\n",
 			p->pasid);
 
-	mutex_lock(&p->mutex);
-
-	list_for_each_entry_safe(pdd, temp, &p->per_device_data,
-							per_device_list) {
+	list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
 		pr_debug("Releasing pdd (topology id %d) for process (pasid %d) in workqueue\n",
 				pdd->dev->id, p->pasid);
 
-		if (pdd->reset_wavefronts)
-			dbgdev_wave_reset_wavefronts(pdd->dev, p);
+		if (pdd->dev->device_info->is_need_iommu_device)
+			amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid);
+
+		/*
+		 * Remove all handles from idr and release appropriate
+		 * local memory object
+		 */
+		idr_for_each_entry(&pdd->alloc_idr, buf_obj, id) {
+			list_for_each_entry(peer_pdd,
+				&p->per_device_data, per_device_list) {
+					pdd->dev->kfd2kgd->unmap_memory_to_gpu(
+						peer_pdd->dev->kgd,
+						buf_obj->mem, peer_pdd->vm);
+			}
+
+			run_rdma_free_callback(buf_obj);
+			pdd->dev->kfd2kgd->free_memory_of_gpu(
+					pdd->dev->kgd, buf_obj->mem);
+			kfd_process_device_remove_obj_handle(pdd, id);
+		}
+	}
 
-		amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid);
+	list_for_each_entry_safe(pdd, temp, &p->per_device_data,
+				 per_device_list) {
+		radeon_flush_tlb(pdd->dev, p->pasid);
+		/* Destroy the GPUVM VM context */
+		if (pdd->vm)
+			pdd->dev->kfd2kgd->destroy_process_vm(
+				pdd->dev->kgd, pdd->vm);
 		list_del(&pdd->per_device_list);
-
 		kfree(pdd);
 	}
 
@@ -186,15 +363,11 @@ static void kfd_process_wq_release(struct work_struct *work)
 
 	kfd_pasid_free(p->pasid);
 
-	mutex_unlock(&p->mutex);
-
-	mutex_destroy(&p->mutex);
-
 	kfree(p->queues);
 
 	kfree(p);
 
-	kfree(work);
+	kfree((void *)work);
 }
 
 static void kfd_process_destroy_delayed(struct rcu_head *rcu)
@@ -223,6 +396,8 @@ static void kfd_process_notifier_release(struct mmu_notifier *mn,
 {
 	struct kfd_process *p;
 	struct kfd_process_device *pdd = NULL;
+	struct kfd_dev *dev = NULL;
+	long status = -EFAULT;
 
 	/*
 	 * The kfd_process structure can not be free because the
@@ -236,9 +411,31 @@ static void kfd_process_notifier_release(struct mmu_notifier *mn,
 	mutex_unlock(&kfd_processes_mutex);
 	synchronize_srcu(&kfd_processes_srcu);
 
-	mutex_lock(&p->mutex);
+	down_write(&p->lock);
+
+	/* Iterate over all process device data structures and if the pdd is in
+	 * debug mode,we should first force unregistration, then we will be
+	 * able to destroy the queues   */
+	list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
+		dev = pdd->dev;
+		mutex_lock(get_dbgmgr_mutex());
+
+		if ((dev != NULL) &&
+			(dev->dbgmgr) &&
+			(dev->dbgmgr->pasid == p->pasid)) {
+
+			status = kfd_dbgmgr_unregister(dev->dbgmgr, p);
+			if (status == 0) {
+				kfd_dbgmgr_destroy(dev->dbgmgr);
+				dev->dbgmgr = NULL;
+			}
+		}
+		mutex_unlock(get_dbgmgr_mutex());
+	}
+
+
+	/* now we can uninit the pqm: */
 
-	/* In case our notifier is called before IOMMU notifier */
 	pqm_uninit(&p->pqm);
 
 	/* Iterate over all process device data structure and check
@@ -256,7 +453,7 @@ static void kfd_process_notifier_release(struct mmu_notifier *mn,
 		}
 	}
 
-	mutex_unlock(&p->mutex);
+	up_write(&p->lock);
 
 	/*
 	 * Because we drop mm_count inside kfd_process_destroy_delayed
@@ -272,6 +469,94 @@ static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
 	.release = kfd_process_notifier_release,
 };
 
+static int kfd_process_init_cwsr(struct kfd_process *p, struct file *filep)
+{
+	int err;
+	unsigned long  offset;
+	struct kfd_process_device *temp, *pdd = NULL;
+	void *mem = NULL;
+	struct kfd_dev *dev = NULL;
+	struct qcm_process_device *qpd = NULL;
+
+	down_write(&p->lock);
+	list_for_each_entry_safe(pdd, temp, &p->per_device_data,
+				per_device_list) {
+		dev = pdd->dev;
+		qpd = &pdd->qpd;
+		if (!dev->cwsr_enabled || qpd->tba_addr)
+			continue;
+		if (qpd->cwsr_base) {
+			/* cwsr_base is only set for DGPU */
+
+			/* can't hold the process lock while
+			 * allocating from KGD */
+			up_write(&p->lock);
+
+			err = dev->kfd2kgd->alloc_memory_of_gpu(
+				dev->kgd, qpd->cwsr_base, dev->cwsr_size,
+				pdd->vm, (struct kgd_mem **)&mem,
+				NULL, &qpd->cwsr_kaddr, pdd,
+				ALLOC_MEM_FLAGS_GTT |
+				ALLOC_MEM_FLAGS_NONPAGED |
+				ALLOC_MEM_FLAGS_EXECUTE_ACCESS |
+				ALLOC_MEM_FLAGS_NO_SUBSTITUTE);
+			if (err)
+				goto err_alloc_tba;
+			err = kfd_map_memory_to_gpu(dev, mem, p, pdd);
+			if (err)
+				goto err_map_tba;
+
+			down_write(&p->lock);
+			/* Check if someone else allocated the memory
+			 * while we weren't looking */
+			if (qpd->tba_addr) {
+				up_write(&p->lock);
+				dev->kfd2kgd->unmap_memory_to_gpu(dev->kgd,
+					(struct kgd_mem *)mem, pdd->vm);
+				dev->kfd2kgd->free_memory_of_gpu(dev->kgd, mem);
+				down_write(&p->lock);
+			} else {
+				qpd->cwsr_mem_handle =
+					kfd_process_device_create_obj_handle(
+						pdd, mem, qpd->cwsr_base,
+						dev->cwsr_size);
+				if (qpd->cwsr_mem_handle < 0)
+					goto err_create_handle;
+
+				memcpy(qpd->cwsr_kaddr, kmap(dev->cwsr_pages),
+				       PAGE_SIZE);
+				kunmap(dev->cwsr_pages);
+				qpd->tba_addr = qpd->cwsr_base;
+			}
+		} else {
+			offset = (kfd_get_gpu_id(dev) |
+				KFD_MMAP_TYPE_RESERVED_MEM) << PAGE_SHIFT;
+			qpd->tba_addr = (uint64_t)vm_mmap(filep, 0,
+				dev->cwsr_size,	PROT_READ | PROT_EXEC,
+				MAP_SHARED, offset);
+			qpd->cwsr_kaddr = (void *)qpd->tba_addr;
+		}
+		if (IS_ERR_VALUE(qpd->tba_addr)) {
+			pr_err("Failure to set tba address. error -%d.\n",
+				(int)qpd->tba_addr);
+			qpd->tba_addr = 0;
+			qpd->cwsr_kaddr = NULL;
+		} else
+			qpd->tma_addr = qpd->tba_addr + dev->tma_offset;
+			pr_debug("set tba :0x%llx, tma:0x%llx for pqm.\n",
+				qpd->tba_addr, qpd->tma_addr);
+	}
+
+err_create_handle:
+	up_write(&p->lock);
+	return err;
+
+err_map_tba:
+	dev->kfd2kgd->free_memory_of_gpu(dev->kgd, mem);
+err_alloc_tba:
+	return err;
+}
+
 static struct kfd_process *create_process(const struct task_struct *thread)
 {
 	struct kfd_process *process;
@@ -282,6 +567,8 @@ static struct kfd_process *create_process(const struct task_struct *thread)
 	if (!process)
 		goto err_alloc_process;
 
+	process->bo_interval_tree = RB_ROOT;
+
 	process->queues = kmalloc_array(INITIAL_QUEUE_ARRAY_SIZE,
 					sizeof(process->queues[0]), GFP_KERNEL);
 	if (!process->queues)
@@ -291,7 +578,7 @@ static struct kfd_process *create_process(const struct task_struct *thread)
 	if (process->pasid == 0)
 		goto err_alloc_pasid;
 
-	mutex_init(&process->mutex);
+	init_rwsem(&process->lock);
 
 	process->mm = thread->mm;
 
@@ -364,8 +651,22 @@ struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
 		INIT_LIST_HEAD(&pdd->qpd.queues_list);
 		INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
 		pdd->qpd.dqm = dev->dqm;
+		pdd->qpd.pqm = &p->pqm;
+		pdd->qpd.evicted = 0;
 		pdd->reset_wavefronts = false;
+		pdd->process = p;
 		list_add(&pdd->per_device_list, &p->per_device_data);
+
+		/* Init idr used for memory handle translation */
+		idr_init(&pdd->alloc_idr);
+
+		/* Create the GPUVM context for this specific device */
+		if (dev->kfd2kgd->create_process_vm(dev->kgd, &pdd->vm)) {
+			pr_err("Failed to create process VM object\n");
+			list_del(&pdd->per_device_list);
+			kfree(pdd);
+			pdd = NULL;
+		}
 	}
 
 	return pdd;
@@ -393,9 +694,11 @@ struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
 	if (pdd->bound)
 		return pdd;
 
-	err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread);
-	if (err < 0)
-		return ERR_PTR(err);
+	if (dev->device_info->is_need_iommu_device) {
+		err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread);
+		if (err < 0)
+			return ERR_PTR(err);
+	}
 
 	pdd->bound = true;
 
@@ -420,18 +723,21 @@ void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid)
 
 	pr_debug("Unbinding process %d from IOMMU\n", pasid);
 
-	if ((dev->dbgmgr) && (dev->dbgmgr->pasid == p->pasid))
-		kfd_dbgmgr_destroy(dev->dbgmgr);
-
-	pqm_uninit(&p->pqm);
+	mutex_lock(get_dbgmgr_mutex());
 
-	pdd = kfd_get_process_device_data(dev, p);
+	if ((dev->dbgmgr) && (dev->dbgmgr->pasid == p->pasid)) {
 
-	if (!pdd) {
-		mutex_unlock(&p->mutex);
-		return;
+		if (kfd_dbgmgr_unregister(dev->dbgmgr, p) == 0) {
+			kfd_dbgmgr_destroy(dev->dbgmgr);
+			dev->dbgmgr = NULL;
+		}
 	}
 
+	mutex_unlock(get_dbgmgr_mutex());
+
+	pqm_uninit(&p->pqm);
+
+	pdd = kfd_get_process_device_data(dev, p);
 	if (pdd->reset_wavefronts) {
 		dbgdev_wave_reset_wavefronts(pdd->dev, p);
 		pdd->reset_wavefronts = false;
@@ -444,9 +750,10 @@ void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid)
 	 * We don't call amd_iommu_unbind_pasid() here
 	 * because the IOMMU called us.
 	 */
-	pdd->bound = false;
+	if (pdd)
+		pdd->bound = false;
 
-	mutex_unlock(&p->mutex);
+	up_write(&p->lock);
 }
 
 struct kfd_process_device *kfd_get_first_process_device_data(struct kfd_process *p)
@@ -469,7 +776,116 @@ bool kfd_has_process_device_data(struct kfd_process *p)
 	return !(list_empty(&p->per_device_data));
 }
 
-/* This returns with process->mutex locked. */
+/* Create specific handle mapped to mem from process local memory idr
+ * Assumes that the process lock is held. */
+int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
+					void *mem, uint64_t start,
+					uint64_t length)
+{
+	int handle;
+	struct kfd_bo *buf_obj;
+	struct kfd_process *p;
+
+	BUG_ON(pdd == NULL);
+	BUG_ON(mem == NULL);
+
+	p = pdd->process;
+
+	buf_obj = kmalloc(sizeof(*buf_obj), GFP_KERNEL);
+
+	if (!buf_obj)
+		return -ENOMEM;
+
+	buf_obj->it.start = start;
+	buf_obj->it.last = start + length - 1;
+	interval_tree_insert(&buf_obj->it, &p->bo_interval_tree);
+
+	buf_obj->mem = mem;
+	buf_obj->dev = pdd->dev;
+
+	INIT_LIST_HEAD(&buf_obj->cb_data_head);
+
+	idr_preload(GFP_KERNEL);
+
+	handle = idr_alloc(&pdd->alloc_idr, buf_obj, MIN_IDR_ID, MAX_IDR_ID,
+			GFP_NOWAIT);
+
+	idr_preload_end();
+
+	return handle;
+}
+
+struct kfd_bo *kfd_process_device_find_bo(struct kfd_process_device *pdd,
+					int handle)
+{
+	BUG_ON(pdd == NULL);
+
+	if (handle < 0)
+		return NULL;
+
+	return (struct kfd_bo *)idr_find(&pdd->alloc_idr, handle);
+}
+
+/* Translate specific handle from process local memory idr
+ * Assumes that the process lock is held. */
+void *kfd_process_device_translate_handle(struct kfd_process_device *pdd,
+					int handle)
+{
+	struct kfd_bo *buf_obj;
+
+	buf_obj = kfd_process_device_find_bo(pdd, handle);
+
+	return buf_obj->mem;
+}
+
+void *kfd_process_find_bo_from_interval(struct kfd_process *p,
+					uint64_t start_addr,
+					uint64_t last_addr)
+{
+	struct interval_tree_node *it_node;
+	struct kfd_bo *buf_obj;
+
+	it_node = interval_tree_iter_first(&p->bo_interval_tree,
+			start_addr, last_addr);
+	if (!it_node) {
+		pr_err("%llu - %llu does not relate to an existing buffer\n",
+				start_addr, last_addr);
+		return NULL;
+	}
+
+	BUG_ON(NULL != interval_tree_iter_next(it_node,
+			start_addr, last_addr));
+
+	buf_obj = container_of(it_node, struct kfd_bo, it);
+
+	return buf_obj;
+}
+
+/* Remove specific handle from process local memory idr
+ * Assumes that the process lock is held. */
+void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,
+					int handle)
+{
+	struct kfd_bo *buf_obj;
+	struct kfd_process *p;
+
+	BUG_ON(pdd == NULL);
+
+	p = pdd->process;
+
+	if (handle < 0)
+		return;
+
+	buf_obj = kfd_process_device_find_bo(pdd, handle);
+
+	idr_remove(&pdd->alloc_idr, handle);
+
+	interval_tree_remove(&buf_obj->it, &p->bo_interval_tree);
+
+	kfree(buf_obj);
+}
+
+/* This returns with process->lock read-locked. */
 struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid)
 {
 	struct kfd_process *p;
@@ -479,7 +895,7 @@ struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid)
 
 	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
 		if (p->pasid == pasid) {
-			mutex_lock(&p->mutex);
+			down_read(&p->lock);
 			break;
 		}
 	}
@@ -488,3 +904,53 @@ struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid)
 
 	return p;
 }
+
+/* This returns with process->lock read-locked. */
+struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm)
+{
+	struct kfd_process *p;
+
+	int idx = srcu_read_lock(&kfd_processes_srcu);
+
+	p = find_process_by_mm(mm);
+	if (p != NULL)
+		down_read(&p->lock);
+
+	srcu_read_unlock(&kfd_processes_srcu, idx);
+
+	return p;
+}
+
+int kfd_reserved_mem_mmap(struct kfd_process *process, struct vm_area_struct *vma)
+{
+	unsigned long pfn, i;
+	int ret = 0;
+	struct kfd_dev *dev = kfd_device_by_id(vma->vm_pgoff);
+
+	if (dev == NULL)
+		return -EINVAL;
+	if ((vma->vm_start & (PAGE_SIZE - 1)) ||
+		(vma->vm_end & (PAGE_SIZE - 1))) {
+		pr_err("KFD only support page aligned memory map.\n");
+		return -EINVAL;
+	}
+
+	pr_debug("kfd reserved mem mmap been called.\n");
+	/* We supported  two reserved memory mmap in the future .
+	    1. Trap handler code and parameter (TBA and TMA , 2 pages total)
+	    2. Relaunch stack (control  block, 1 page for Carrizo)
+	 */
+
+	for (i = 0; i < ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT); ++i) {
+		pfn = page_to_pfn(&dev->cwsr_pages[i]);
+		vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND
+			| VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP;
+		/* mapping the page to user process */
+		ret = remap_pfn_range(vma, vma->vm_start + (i << PAGE_SHIFT),
+				pfn, PAGE_SIZE, vma->vm_page_prot);
+		if (ret)
+			break;
+	}
+	return ret;
+}
+
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
index 97f7f7e..e79cd42 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
@@ -89,23 +89,36 @@ void pqm_uninit(struct process_queue_manager *pqm)
 {
 	int retval;
 	struct process_queue_node *pqn, *next;
+	struct kfd_process_device *pdd;
+	struct kfd_dev *dev = NULL;
 
 	BUG_ON(!pqm);
 
 	pr_debug("In func %s\n", __func__);
 
 	list_for_each_entry_safe(pqn, next, &pqm->queues, process_queue_list) {
-		retval = pqm_destroy_queue(
-				pqm,
-				(pqn->q != NULL) ?
-					pqn->q->properties.queue_id :
-					pqn->kq->queue->properties.queue_id);
-
-		if (retval != 0) {
-			pr_err("kfd: failed to destroy queue\n");
-			return;
+		if (pqn->q)
+			dev = pqn->q->device;
+		else if (pqn->kq)
+			dev = pqn->kq->dev;
+		else
+			BUG();
+
+		pdd = kfd_get_process_device_data(dev, pqm->process);
+		if (pdd) {
+			retval = dev->dqm->ops.process_termination
+				(dev->dqm, &pdd->qpd);
+			if (retval != 0)
+				pdd->reset_wavefronts = true;
 		}
 	}
+
+	list_for_each_entry_safe(pqn, next, &pqm->queues, process_queue_list) {
+		uninit_queue(pqn->q);
+		list_del(&pqn->process_queue_list);
+		kfree(pqn);
+	}
+
 	kfree(pqm->queue_slot_bitmap);
 	pqm->queue_slot_bitmap = NULL;
 }
@@ -148,23 +161,19 @@ int pqm_create_queue(struct process_queue_manager *pqm,
 			    struct kfd_dev *dev,
 			    struct file *f,
 			    struct queue_properties *properties,
-			    unsigned int flags,
-			    enum kfd_queue_type type,
 			    unsigned int *qid)
 {
 	int retval;
 	struct kfd_process_device *pdd;
-	struct queue_properties q_properties;
 	struct queue *q;
 	struct process_queue_node *pqn;
 	struct kernel_queue *kq;
 	int num_queues = 0;
 	struct queue *cur;
+	enum kfd_queue_type type = properties->type;
 
 	BUG_ON(!pqm || !dev || !properties || !qid);
 
-	memset(&q_properties, 0, sizeof(struct queue_properties));
-	memcpy(&q_properties, properties, sizeof(struct queue_properties));
 	q = NULL;
 	kq = NULL;
 
@@ -192,10 +201,9 @@ int pqm_create_queue(struct process_queue_manager *pqm,
 	if (retval != 0)
 		return retval;
 
-	if (list_empty(&pqm->queues)) {
-		pdd->qpd.pqm = pqm;
+	if (list_empty(&pdd->qpd.queues_list) &&
+			list_empty(&pdd->qpd.priv_queue_list))
 		dev->dqm->ops.register_process(dev->dqm, &pdd->qpd);
-	}
 
 	pqn = kzalloc(sizeof(struct process_queue_node), GFP_KERNEL);
 	if (!pqn) {
@@ -205,35 +213,34 @@ int pqm_create_queue(struct process_queue_manager *pqm,
 
 	switch (type) {
 	case KFD_QUEUE_TYPE_SDMA:
-		if (dev->dqm->queue_count >=
-			CIK_SDMA_QUEUES_PER_ENGINE * CIK_SDMA_ENGINE_NUM) {
-			pr_err("Over-subscription is not allowed for SDMA.\n");
-			retval = -EPERM;
-			goto err_create_queue;
-		}
-
-		retval = create_cp_queue(pqm, dev, &q, properties, f, *qid);
-		if (retval != 0)
-			goto err_create_queue;
-		pqn->q = q;
-		pqn->kq = NULL;
-		retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd,
-						&q->properties.vmid);
-		pr_debug("DQM returned %d for create_queue\n", retval);
-		print_queue(q);
-		break;
+                if (dev->dqm->sdma_queue_count >= CIK_SDMA_QUEUES) {
+                        pr_err("kfd: over-subscription is not allowed for SDMA.\n");
+                        retval = -EPERM;
+                        goto err_create_queue;
+                }
+
+                retval = create_cp_queue(pqm, dev, &q, properties, f, *qid);
+                if (retval != 0)
+                        goto err_create_queue;
+                pqn->q = q;
+                pqn->kq = NULL;
+                retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd,
+                                                &q->properties.vmid);
+                pr_debug("DQM returned %d for create_queue\n", retval);
+                print_queue(q);
+                break;
 
 	case KFD_QUEUE_TYPE_COMPUTE:
 		/* check if there is over subscription */
 		if ((sched_policy == KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION) &&
-		((dev->dqm->processes_count >= VMID_PER_DEVICE) ||
+		((dev->dqm->processes_count >= dev->vm_info.vmid_num_kfd) ||
 		(dev->dqm->queue_count >= get_queues_num(dev->dqm)))) {
 			pr_err("kfd: over-subscription is not allowed in radeon_kfd.sched_policy == 1\n");
 			retval = -EPERM;
 			goto err_create_queue;
 		}
 
-		retval = create_cp_queue(pqm, dev, &q, &q_properties, f, *qid);
+		retval = create_cp_queue(pqm, dev, &q, properties, f, *qid);
 		if (retval != 0)
 			goto err_create_queue;
 		pqn->q = q;
@@ -270,9 +277,8 @@ int pqm_create_queue(struct process_queue_manager *pqm,
 	list_add(&pqn->process_queue_list, &pqm->queues);
 
 	if (q) {
-		*properties = q->properties;
 		pr_debug("kfd: PQM done creating queue\n");
-		print_queue_properties(properties);
+		print_queue_properties(&q->properties);
 	}
 
 	return retval;
@@ -282,7 +288,8 @@ int pqm_create_queue(struct process_queue_manager *pqm,
 err_allocate_pqn:
 	/* check if queues list is empty unregister process from device */
 	clear_bit(*qid, pqm->queue_slot_bitmap);
-	if (list_empty(&pqm->queues))
+	if (list_empty(&pdd->qpd.queues_list) &&
+			list_empty(&pdd->qpd.priv_queue_list))
 		dev->dqm->ops.unregister_process(dev->dqm, &pdd->qpd);
 	return retval;
 }
@@ -331,9 +338,11 @@ int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid)
 	if (pqn->q) {
 		dqm = pqn->q->device->dqm;
 		retval = dqm->ops.destroy_queue(dqm, &pdd->qpd, pqn->q);
-		if (retval != 0)
+		if (retval != 0) {
+			if (retval == -ETIME)
+				pdd->reset_wavefronts = true;
 			return retval;
-
+		}
 		uninit_queue(pqn->q);
 	}
 
@@ -341,7 +350,8 @@ int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid)
 	kfree(pqn);
 	clear_bit(qid, pqm->queue_slot_bitmap);
 
-	if (list_empty(&pqm->queues))
+	if (list_empty(&pdd->qpd.queues_list) &&
+			list_empty(&pdd->qpd.priv_queue_list))
 		dqm->ops.unregister_process(dqm, &pdd->qpd);
 
 	return retval;
@@ -375,6 +385,31 @@ int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid,
 	return 0;
 }
 
+int pqm_set_cu_mask(struct process_queue_manager *pqm, unsigned int qid,
+			struct queue_properties *p)
+{
+	int retval;
+	struct process_queue_node *pqn;
+
+	BUG_ON(!pqm);
+
+	pqn = get_queue_by_qid(pqm, qid);
+	if (!pqn) {
+		pr_debug("amdkfd: No queue %d exists for update operation\n",
+				qid);
+		return -EFAULT;
+	}
+
+	pqn->q->properties.cu_mask = p->cu_mask;
+
+	retval = pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,
+							pqn->q);
+	if (retval != 0)
+		return retval;
+
+	return 0;
+}
+
 struct kernel_queue *pqm_get_kernel_queue(
 					struct process_queue_manager *pqm,
 					unsigned int qid)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_rdma.c b/drivers/gpu/drm/amd/amdkfd/kfd_rdma.c
new file mode 100644
index 0000000..69bdaf1
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_rdma.c
@@ -0,0 +1,296 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/device.h>
+#include <linux/export.h>
+#include <linux/pid.h>
+#include <linux/err.h>
+#include <linux/slab.h>
+#include "amd_rdma.h"
+#include "kfd_priv.h"
+
+
+struct rdma_cb {
+	struct list_head node;
+	struct amd_p2p_info amd_p2p_data;
+	void  (*free_callback)(void *client_priv);
+	void  *client_priv;
+};
+
+/**
+ * This function makes the pages underlying a range of GPU virtual memory
+ * accessible for DMA operations from another PCIe device
+ *
+ * \param   address       - The start address in the Unified Virtual Address
+ *			    space in the specified process
+ * \param   length        - The length of requested mapping
+ * \param   pid           - Pointer to structure pid to which address belongs.
+ *			    Could be NULL for current process address space.
+ * \param   p2p_data    - On return: Pointer to structure describing
+ *			    underlying pages/locations
+ * \param   free_callback - Pointer to callback which will be called when access
+ *			    to such memory must be stopped immediately: Memory
+ *			    was freed, GECC events, etc.
+ *			    Client should  immediately stop any transfer
+ *			    operations and returned as soon as possible.
+ *			    After return all resources associated with address
+ *			    will be release and no access will be allowed.
+ * \param   client_priv   - Pointer to be passed as parameter on
+ *			    'free_callback;
+ *
+ * \return  0 if operation was successful
+ */
+static int get_pages(uint64_t address, uint64_t length, struct pid *pid,
+		struct amd_p2p_info **amd_p2p_data,
+		void  (*free_callback)(void *client_priv),
+		void  *client_priv)
+{
+	struct kfd_bo *buf_obj;
+	struct kgd_mem *mem;
+	struct sg_table *sg_table_tmp;
+	struct kfd_dev *dev;
+	uint64_t last = address + length - 1;
+	uint64_t offset;
+	struct kfd_process *p;
+	struct rdma_cb *rdma_cb_data;
+	int ret = 0;
+
+	p = kfd_lookup_process_by_pid(pid);
+	if (!p) {
+		pr_err("could not find the process in %s.\n",
+				__func__);
+		return -EINVAL;
+	}
+
+	buf_obj = kfd_process_find_bo_from_interval(p, address, last);
+	if (!buf_obj) {
+		pr_err("can not find a kfd_bo for the range\n");
+		ret = -EINVAL;
+		goto out;
+	}
+
+	rdma_cb_data = kmalloc(sizeof(*rdma_cb_data), GFP_KERNEL);
+	if (!rdma_cb_data) {
+		*amd_p2p_data = NULL;
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	mem = buf_obj->mem;
+	dev = buf_obj->dev;
+	offset = address - buf_obj->it.start;
+
+	ret = dev->kfd2kgd->pin_get_sg_table_bo(dev->kgd, mem,
+			offset, length, &sg_table_tmp);
+
+	if (ret) {
+		pr_err("pin_get_sg_table_bo failed.\n");
+		*amd_p2p_data = NULL;
+		goto free_mem;
+	}
+
+	rdma_cb_data->amd_p2p_data.va = address;
+	rdma_cb_data->amd_p2p_data.size = length;
+	rdma_cb_data->amd_p2p_data.pid = pid;
+	rdma_cb_data->amd_p2p_data.priv = buf_obj;
+	rdma_cb_data->amd_p2p_data.pages = sg_table_tmp;
+
+	rdma_cb_data->free_callback = free_callback;
+	rdma_cb_data->client_priv = client_priv;
+
+	list_add(&rdma_cb_data->node, &buf_obj->cb_data_head);
+
+	*amd_p2p_data = &rdma_cb_data->amd_p2p_data;
+
+	goto out;
+
+free_mem:
+	kfree(rdma_cb_data);
+out:
+	up_read(&p->lock);
+
+	return ret;
+}
+
+static int put_pages_helper(struct amd_p2p_info *p2p_data)
+{
+	struct kfd_bo *buf_obj;
+	struct kfd_dev *dev;
+	struct sg_table *sg_table_tmp;
+	struct rdma_cb *rdma_cb_data;
+
+	if (!p2p_data) {
+		pr_err("amd_p2p_info pointer is invalid.\n");
+		return -EINVAL;
+	}
+
+	rdma_cb_data = container_of(p2p_data, struct rdma_cb, amd_p2p_data);
+
+	buf_obj = p2p_data->priv;
+	dev = buf_obj->dev;
+	sg_table_tmp = p2p_data->pages;
+
+	list_del(&rdma_cb_data->node);
+	kfree(rdma_cb_data);
+
+	dev->kfd2kgd->unpin_put_sg_table_bo(buf_obj->mem, sg_table_tmp);
+
+
+	return 0;
+}
+
+void run_rdma_free_callback(struct kfd_bo *buf_obj)
+{
+	struct rdma_cb *tmp, *rdma_cb_data;
+
+	list_for_each_entry_safe(rdma_cb_data, tmp,
+			&buf_obj->cb_data_head, node) {
+		if (rdma_cb_data->free_callback)
+			rdma_cb_data->free_callback(
+					rdma_cb_data->client_priv);
+
+		put_pages_helper(&rdma_cb_data->amd_p2p_data);
+	}
+}
+
+/**
+ *
+ * This function release resources previously allocated by get_pages() call.
+ *
+ * \param   p_p2p_data - A pointer to pointer to amd_p2p_info entries
+ * 			allocated by get_pages() call.
+ *
+ * \return  0 if operation was successful
+ */
+static int put_pages(struct amd_p2p_info **p_p2p_data)
+{
+	struct kfd_process *p = NULL;
+	int ret = 0;
+
+	if (!(*p_p2p_data)) {
+		pr_err("amd_p2p_info pointer is invalid.\n");
+		return -EINVAL;
+	}
+
+	p = kfd_lookup_process_by_pid((*p_p2p_data)->pid);
+	if (!p) {
+		pr_err("could not find the process in %s\n",
+				__func__);
+		return -EINVAL;
+	}
+
+	ret = put_pages_helper(*p_p2p_data);
+
+	if (!ret)
+		*p_p2p_data = NULL;
+
+	up_read(&p->lock);
+
+	return ret;
+}
+
+/**
+ * Check if given address belongs to GPU address space.
+ *
+ * \param   address - Address to check
+ * \param   pid     - Process to which given address belongs.
+ *		      Could be NULL if current one.
+ *
+ * \return  0  - This is not GPU address managed by AMD driver
+ *	    1  - This is GPU address managed by AMD driver
+ */
+static int is_gpu_address(uint64_t address, struct pid *pid)
+{
+	struct kfd_bo *buf_obj;
+	struct kfd_process *p;
+
+	p = kfd_lookup_process_by_pid(pid);
+	if (!p) {
+		pr_err("could not find the process in %s.\n",
+				__func__);
+		return 0;
+	}
+
+	buf_obj = kfd_process_find_bo_from_interval(p, address, address);
+
+	up_read(&p->lock);
+	if (!buf_obj)
+		return 0;
+	else
+		return 1;
+}
+
+/**
+ * Return the single page size to be used when building scatter/gather table
+ * for given range.
+ *
+ * \param   address   - Address
+ * \param   length    - Range length
+ * \param   pid       - Process id structure. Could be NULL if current one.
+ * \param   page_size - On return: Page size
+ *
+ * \return  0 if operation was successful
+ */
+static int get_page_size(uint64_t address, uint64_t length, struct pid *pid,
+			unsigned long *page_size)
+{
+	/*
+	 * As local memory is always consecutive, we can assume the local
+	 * memory page size to be arbitrary.
+	 * Currently we assume the local memory page size to be the same
+	 * as system memory, which is 4KB.
+	 */
+	*page_size = PAGE_SIZE;
+
+	return 0;
+}
+
+
+/**
+ * Singleton object: rdma interface function pointers
+ */
+static const struct amd_rdma_interface  rdma_ops = {
+	.get_pages = get_pages,
+	.put_pages = put_pages,
+	.is_gpu_address = is_gpu_address,
+	.get_page_size = get_page_size,
+};
+
+/**
+ * amdkfd_query_rdma_interface - Return interface (function pointers table) for
+ *				 rdma interface
+ *
+ *
+ * \param interace     - OUT: Pointer to interface
+ *
+ * \return 0 if operation was successful.
+ */
+int amdkfd_query_rdma_interface(const struct amd_rdma_interface **ops)
+{
+	*ops  = &rdma_ops;
+
+	return 0;
+}
+EXPORT_SYMBOL(amdkfd_query_rdma_interface);
+
+
+
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index 8c6e47c..f4f91a8 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -28,16 +28,19 @@
 #include <linux/hash.h>
 #include <linux/cpufreq.h>
 #include <linux/log2.h>
+#include <linux/dmi.h>
+#include <linux/atomic.h>
 
 #include "kfd_priv.h"
 #include "kfd_crat.h"
 #include "kfd_topology.h"
 
-static struct list_head topology_device_list;
-static int topology_crat_parsed;
+/* topology_device_list - Master list of all topology devices */
+struct list_head topology_device_list;
 static struct kfd_system_properties sys_props;
 
 static DECLARE_RWSEM(topology_lock);
+static atomic_t topology_crat_proximity_domain;
 
 struct kfd_dev *kfd_device_by_id(uint32_t gpu_id)
 {
@@ -57,311 +60,61 @@ struct kfd_dev *kfd_device_by_id(uint32_t gpu_id)
 	return device;
 }
 
-struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev)
+uint32_t kfd_get_gpu_id(struct kfd_dev *dev)
 {
 	struct kfd_topology_device *top_dev;
-	struct kfd_dev *device = NULL;
+	uint32_t gpu_id = 0;
 
 	down_read(&topology_lock);
 
 	list_for_each_entry(top_dev, &topology_device_list, list)
-		if (top_dev->gpu->pdev == pdev) {
-			device = top_dev->gpu;
+		if (top_dev->gpu == dev) {
+			gpu_id = top_dev->gpu_id;
 			break;
 		}
 
 	up_read(&topology_lock);
 
-	return device;
-}
-
-static int kfd_topology_get_crat_acpi(void *crat_image, size_t *size)
-{
-	struct acpi_table_header *crat_table;
-	acpi_status status;
-
-	if (!size)
-		return -EINVAL;
-
-	/*
-	 * Fetch the CRAT table from ACPI
-	 */
-	status = acpi_get_table(CRAT_SIGNATURE, 0, &crat_table);
-	if (status == AE_NOT_FOUND) {
-		pr_warn("CRAT table not found\n");
-		return -ENODATA;
-	} else if (ACPI_FAILURE(status)) {
-		const char *err = acpi_format_exception(status);
-
-		pr_err("CRAT table error: %s\n", err);
-		return -EINVAL;
-	}
-
-	if (*size >= crat_table->length && crat_image != NULL)
-		memcpy(crat_image, crat_table, crat_table->length);
-
-	*size = crat_table->length;
-
-	return 0;
+	return gpu_id;
 }
 
-static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev,
-		struct crat_subtype_computeunit *cu)
-{
-	BUG_ON(!dev);
-	BUG_ON(!cu);
-
-	dev->node_props.cpu_cores_count = cu->num_cpu_cores;
-	dev->node_props.cpu_core_id_base = cu->processor_id_low;
-	if (cu->hsa_capability & CRAT_CU_FLAGS_IOMMU_PRESENT)
-		dev->node_props.capability |= HSA_CAP_ATS_PRESENT;
-
-	pr_info("CU CPU: cores=%d id_base=%d\n", cu->num_cpu_cores,
-			cu->processor_id_low);
-}
-
-static void kfd_populated_cu_info_gpu(struct kfd_topology_device *dev,
-		struct crat_subtype_computeunit *cu)
-{
-	BUG_ON(!dev);
-	BUG_ON(!cu);
-
-	dev->node_props.simd_id_base = cu->processor_id_low;
-	dev->node_props.simd_count = cu->num_simd_cores;
-	dev->node_props.lds_size_in_kb = cu->lds_size_in_kb;
-	dev->node_props.max_waves_per_simd = cu->max_waves_simd;
-	dev->node_props.wave_front_size = cu->wave_front_size;
-	dev->node_props.mem_banks_count = cu->num_banks;
-	dev->node_props.array_count = cu->num_arrays;
-	dev->node_props.cu_per_simd_array = cu->num_cu_per_array;
-	dev->node_props.simd_per_cu = cu->num_simd_per_cu;
-	dev->node_props.max_slots_scratch_cu = cu->max_slots_scatch_cu;
-	if (cu->hsa_capability & CRAT_CU_FLAGS_HOT_PLUGGABLE)
-		dev->node_props.capability |= HSA_CAP_HOT_PLUGGABLE;
-	pr_info("CU GPU: simds=%d id_base=%d\n", cu->num_simd_cores,
-				cu->processor_id_low);
-}
-
-/* kfd_parse_subtype_cu is called when the topology mutex is already acquired */
-static int kfd_parse_subtype_cu(struct crat_subtype_computeunit *cu)
-{
-	struct kfd_topology_device *dev;
-	int i = 0;
-
-	BUG_ON(!cu);
-
-	pr_info("Found CU entry in CRAT table with proximity_domain=%d caps=%x\n",
-			cu->proximity_domain, cu->hsa_capability);
-	list_for_each_entry(dev, &topology_device_list, list) {
-		if (cu->proximity_domain == i) {
-			if (cu->flags & CRAT_CU_FLAGS_CPU_PRESENT)
-				kfd_populated_cu_info_cpu(dev, cu);
-
-			if (cu->flags & CRAT_CU_FLAGS_GPU_PRESENT)
-				kfd_populated_cu_info_gpu(dev, cu);
-			break;
-		}
-		i++;
-	}
-
-	return 0;
-}
-
-/*
- * kfd_parse_subtype_mem is called when the topology mutex is
- * already acquired
- */
-static int kfd_parse_subtype_mem(struct crat_subtype_memory *mem)
+struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev)
 {
-	struct kfd_mem_properties *props;
-	struct kfd_topology_device *dev;
-	int i = 0;
-
-	BUG_ON(!mem);
-
-	pr_info("Found memory entry in CRAT table with proximity_domain=%d\n",
-			mem->promixity_domain);
-	list_for_each_entry(dev, &topology_device_list, list) {
-		if (mem->promixity_domain == i) {
-			props = kfd_alloc_struct(props);
-			if (props == NULL)
-				return -ENOMEM;
-
-			if (dev->node_props.cpu_cores_count == 0)
-				props->heap_type = HSA_MEM_HEAP_TYPE_FB_PRIVATE;
-			else
-				props->heap_type = HSA_MEM_HEAP_TYPE_SYSTEM;
-
-			if (mem->flags & CRAT_MEM_FLAGS_HOT_PLUGGABLE)
-				props->flags |= HSA_MEM_FLAGS_HOT_PLUGGABLE;
-			if (mem->flags & CRAT_MEM_FLAGS_NON_VOLATILE)
-				props->flags |= HSA_MEM_FLAGS_NON_VOLATILE;
-
-			props->size_in_bytes =
-				((uint64_t)mem->length_high << 32) +
-							mem->length_low;
-			props->width = mem->width;
+	struct kfd_topology_device *top_dev;
+	struct kfd_dev *device = NULL;
 
-			dev->mem_bank_count++;
-			list_add_tail(&props->list, &dev->mem_props);
+	down_read(&topology_lock);
 
+	list_for_each_entry(top_dev, &topology_device_list, list)
+		if (top_dev->gpu && top_dev->gpu->pdev == pdev) {
+			device = top_dev->gpu;
 			break;
 		}
-		i++;
-	}
-
-	return 0;
-}
-
-/*
- * kfd_parse_subtype_cache is called when the topology mutex
- * is already acquired
- */
-static int kfd_parse_subtype_cache(struct crat_subtype_cache *cache)
-{
-	struct kfd_cache_properties *props;
-	struct kfd_topology_device *dev;
-	uint32_t id;
-
-	BUG_ON(!cache);
-
-	id = cache->processor_id_low;
 
-	pr_info("Found cache entry in CRAT table with processor_id=%d\n", id);
-	list_for_each_entry(dev, &topology_device_list, list)
-		if (id == dev->node_props.cpu_core_id_base ||
-		    id == dev->node_props.simd_id_base) {
-			props = kfd_alloc_struct(props);
-			if (props == NULL)
-				return -ENOMEM;
-
-			props->processor_id_low = id;
-			props->cache_level = cache->cache_level;
-			props->cache_size = cache->cache_size;
-			props->cacheline_size = cache->cache_line_size;
-			props->cachelines_per_tag = cache->lines_per_tag;
-			props->cache_assoc = cache->associativity;
-			props->cache_latency = cache->cache_latency;
-
-			if (cache->flags & CRAT_CACHE_FLAGS_DATA_CACHE)
-				props->cache_type |= HSA_CACHE_TYPE_DATA;
-			if (cache->flags & CRAT_CACHE_FLAGS_INST_CACHE)
-				props->cache_type |= HSA_CACHE_TYPE_INSTRUCTION;
-			if (cache->flags & CRAT_CACHE_FLAGS_CPU_CACHE)
-				props->cache_type |= HSA_CACHE_TYPE_CPU;
-			if (cache->flags & CRAT_CACHE_FLAGS_SIMD_CACHE)
-				props->cache_type |= HSA_CACHE_TYPE_HSACU;
-
-			dev->cache_count++;
-			dev->node_props.caches_count++;
-			list_add_tail(&props->list, &dev->cache_props);
-
-			break;
-		}
+	up_read(&topology_lock);
 
-	return 0;
+	return device;
 }
 
-/*
- * kfd_parse_subtype_iolink is called when the topology mutex
- * is already acquired
- */
-static int kfd_parse_subtype_iolink(struct crat_subtype_iolink *iolink)
+struct kfd_dev *kfd_device_by_kgd(const struct kgd_dev *kgd)
 {
-	struct kfd_iolink_properties *props;
-	struct kfd_topology_device *dev;
-	uint32_t i = 0;
-	uint32_t id_from;
-	uint32_t id_to;
-
-	BUG_ON(!iolink);
-
-	id_from = iolink->proximity_domain_from;
-	id_to = iolink->proximity_domain_to;
+	struct kfd_topology_device *top_dev;
+	struct kfd_dev *device = NULL;
 
-	pr_info("Found IO link entry in CRAT table with id_from=%d\n", id_from);
-	list_for_each_entry(dev, &topology_device_list, list) {
-		if (id_from == i) {
-			props = kfd_alloc_struct(props);
-			if (props == NULL)
-				return -ENOMEM;
-
-			props->node_from = id_from;
-			props->node_to = id_to;
-			props->ver_maj = iolink->version_major;
-			props->ver_min = iolink->version_minor;
-
-			/*
-			 * weight factor (derived from CDIR), currently always 1
-			 */
-			props->weight = 1;
-
-			props->min_latency = iolink->minimum_latency;
-			props->max_latency = iolink->maximum_latency;
-			props->min_bandwidth = iolink->minimum_bandwidth_mbs;
-			props->max_bandwidth = iolink->maximum_bandwidth_mbs;
-			props->rec_transfer_size =
-					iolink->recommended_transfer_size;
-
-			dev->io_link_count++;
-			dev->node_props.io_links_count++;
-			list_add_tail(&props->list, &dev->io_link_props);
+	down_read(&topology_lock);
 
+	list_for_each_entry(top_dev, &topology_device_list, list)
+		if (top_dev->gpu && top_dev->gpu->kgd == kgd) {
+			device = top_dev->gpu;
 			break;
 		}
-		i++;
-	}
 
-	return 0;
-}
-
-static int kfd_parse_subtype(struct crat_subtype_generic *sub_type_hdr)
-{
-	struct crat_subtype_computeunit *cu;
-	struct crat_subtype_memory *mem;
-	struct crat_subtype_cache *cache;
-	struct crat_subtype_iolink *iolink;
-	int ret = 0;
-
-	BUG_ON(!sub_type_hdr);
-
-	switch (sub_type_hdr->type) {
-	case CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY:
-		cu = (struct crat_subtype_computeunit *)sub_type_hdr;
-		ret = kfd_parse_subtype_cu(cu);
-		break;
-	case CRAT_SUBTYPE_MEMORY_AFFINITY:
-		mem = (struct crat_subtype_memory *)sub_type_hdr;
-		ret = kfd_parse_subtype_mem(mem);
-		break;
-	case CRAT_SUBTYPE_CACHE_AFFINITY:
-		cache = (struct crat_subtype_cache *)sub_type_hdr;
-		ret = kfd_parse_subtype_cache(cache);
-		break;
-	case CRAT_SUBTYPE_TLB_AFFINITY:
-		/*
-		 * For now, nothing to do here
-		 */
-		pr_info("Found TLB entry in CRAT table (not processing)\n");
-		break;
-	case CRAT_SUBTYPE_CCOMPUTE_AFFINITY:
-		/*
-		 * For now, nothing to do here
-		 */
-		pr_info("Found CCOMPUTE entry in CRAT table (not processing)\n");
-		break;
-	case CRAT_SUBTYPE_IOLINK_AFFINITY:
-		iolink = (struct crat_subtype_iolink *)sub_type_hdr;
-		ret = kfd_parse_subtype_iolink(iolink);
-		break;
-	default:
-		pr_warn("Unknown subtype (%d) in CRAT\n",
-				sub_type_hdr->type);
-	}
+	up_read(&topology_lock);
 
-	return ret;
+	return device;
 }
 
+/* Called with write topology_lock acquired */
 static void kfd_release_topology_device(struct kfd_topology_device *dev)
 {
 	struct kfd_mem_properties *mem;
@@ -398,20 +151,22 @@ static void kfd_release_topology_device(struct kfd_topology_device *dev)
 	sys_props.num_devices--;
 }
 
-static void kfd_release_live_view(void)
+void kfd_release_live_view(void)
 {
 	struct kfd_topology_device *dev;
 
+	down_write(&topology_lock);
 	while (topology_device_list.next != &topology_device_list) {
 		dev = container_of(topology_device_list.next,
 				 struct kfd_topology_device, list);
 		kfd_release_topology_device(dev);
-}
-
+	}
+	up_write(&topology_lock);
 	memset(&sys_props, 0, sizeof(sys_props));
 }
 
-static struct kfd_topology_device *kfd_create_topology_device(void)
+struct kfd_topology_device *kfd_create_topology_device(
+				struct list_head *device_list)
 {
 	struct kfd_topology_device *dev;
 
@@ -425,65 +180,12 @@ static struct kfd_topology_device *kfd_create_topology_device(void)
 	INIT_LIST_HEAD(&dev->cache_props);
 	INIT_LIST_HEAD(&dev->io_link_props);
 
-	list_add_tail(&dev->list, &topology_device_list);
+	list_add_tail(&dev->list, device_list);
 	sys_props.num_devices++;
 
 	return dev;
 }
 
-static int kfd_parse_crat_table(void *crat_image)
-{
-	struct kfd_topology_device *top_dev;
-	struct crat_subtype_generic *sub_type_hdr;
-	uint16_t node_id;
-	int ret;
-	struct crat_header *crat_table = (struct crat_header *)crat_image;
-	uint16_t num_nodes;
-	uint32_t image_len;
-
-	if (!crat_image)
-		return -EINVAL;
-
-	num_nodes = crat_table->num_domains;
-	image_len = crat_table->length;
-
-	pr_info("Parsing CRAT table with %d nodes\n", num_nodes);
-
-	for (node_id = 0; node_id < num_nodes; node_id++) {
-		top_dev = kfd_create_topology_device();
-		if (!top_dev) {
-			kfd_release_live_view();
-			return -ENOMEM;
-		}
-	}
-
-	sys_props.platform_id =
-		(*((uint64_t *)crat_table->oem_id)) & CRAT_OEMID_64BIT_MASK;
-	sys_props.platform_oem = *((uint64_t *)crat_table->oem_table_id);
-	sys_props.platform_rev = crat_table->revision;
-
-	sub_type_hdr = (struct crat_subtype_generic *)(crat_table+1);
-	while ((char *)sub_type_hdr + sizeof(struct crat_subtype_generic) <
-			((char *)crat_image) + image_len) {
-		if (sub_type_hdr->flags & CRAT_SUBTYPE_FLAGS_ENABLED) {
-			ret = kfd_parse_subtype(sub_type_hdr);
-			if (ret != 0) {
-				kfd_release_live_view();
-				return ret;
-			}
-		}
-
-		sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
-				sub_type_hdr->length);
-	}
-
-	sys_props.generation_count++;
-	topology_crat_parsed = 1;
-
-	return 0;
-}
-
-
 #define sysfs_show_gen_prop(buffer, fmt, ...) \
 		snprintf(buffer, PAGE_SIZE, "%s"fmt, buffer, __VA_ARGS__)
 #define sysfs_show_32bit_prop(buffer, name, value) \
@@ -601,7 +303,7 @@ static ssize_t kfd_cache_show(struct kobject *kobj, struct attribute *attr,
 		char *buffer)
 {
 	ssize_t ret;
-	uint32_t i;
+	uint32_t i, j;
 	struct kfd_cache_properties *cache;
 
 	/* Making sure that the buffer is an empty string */
@@ -619,12 +321,18 @@ static ssize_t kfd_cache_show(struct kobject *kobj, struct attribute *attr,
 	sysfs_show_32bit_prop(buffer, "latency", cache->cache_latency);
 	sysfs_show_32bit_prop(buffer, "type", cache->cache_type);
 	snprintf(buffer, PAGE_SIZE, "%ssibling_map ", buffer);
-	for (i = 0; i < KFD_TOPOLOGY_CPU_SIBLINGS; i++)
-		ret = snprintf(buffer, PAGE_SIZE, "%s%d%s",
-				buffer, cache->sibling_map[i],
-				(i == KFD_TOPOLOGY_CPU_SIBLINGS-1) ?
-						"\n" : ",");
-
+	for (i = 0; i < CRAT_SIBLINGMAP_SIZE; i++)
+		for (j = 0; j < sizeof(cache->sibling_map[0])*8; j++) {
+			/* Check each bit */
+			if (cache->sibling_map[i] & (1 << j))
+				ret = snprintf(buffer, PAGE_SIZE,
+					 "%s%d%s", buffer, 1, ",");
+			else
+				ret = snprintf(buffer, PAGE_SIZE,
+					 "%s%d%s", buffer, 0, ",");
+		}
+	/* Replace the last "," with end of line */
+	*(buffer + strlen(buffer) - 1) = 0xA;
 	return ret;
 }
 
@@ -644,6 +352,7 @@ static ssize_t node_show(struct kobject *kobj, struct attribute *attr,
 	char public_name[KFD_TOPOLOGY_PUBLIC_NAME_SIZE];
 	uint32_t i;
 	uint32_t log_max_watch_addr;
+	struct kfd_local_mem_info local_mem_info;
 
 	/* Making sure that the buffer is an empty string */
 	buffer[0] = 0;
@@ -683,7 +392,6 @@ static ssize_t node_show(struct kobject *kobj, struct attribute *attr,
 	} else {
 		sysfs_show_32bit_prop(buffer, "mem_banks_count",
 				dev->node_props.mem_banks_count);
-	}
 
 	sysfs_show_32bit_prop(buffer, "caches_count",
 			dev->node_props.caches_count);
@@ -732,17 +440,30 @@ static ssize_t node_show(struct kobject *kobj, struct attribute *attr,
 				HSA_CAP_WATCH_POINTS_TOTALBITS_MASK);
 		}
 
+		if (dev->gpu->device_info->asic_family == CHIP_TONGA)
+			dev->node_props.capability |=
+					HSA_CAP_AQL_QUEUE_DOUBLE_MAP;
+
 		sysfs_show_32bit_prop(buffer, "max_engine_clk_fcompute",
-			dev->gpu->kfd2kgd->get_max_engine_clock_in_mhz(
-					dev->gpu->kgd));
+			dev->node_props.max_engine_clk_fcompute);
 
-		sysfs_show_64bit_prop(buffer, "local_mem_size",
-				(unsigned long long int) 0);
+		/*
+		 * If the ASIC is CZ, set local memory size to 0 to disable
+		 * local memory support
+		 */
+		if (dev->gpu->device_info->asic_family != CHIP_CARRIZO) {
+			dev->gpu->kfd2kgd->get_local_mem_info(dev->gpu->kgd,
+				&local_mem_info);
+			sysfs_show_64bit_prop(buffer, "local_mem_size",
+					local_mem_info.local_mem_size_private +
+					local_mem_info.local_mem_size_public);
+		}
+		else
+			sysfs_show_64bit_prop(buffer, "local_mem_size",
+					(unsigned long long int) 0);
 
 		sysfs_show_32bit_prop(buffer, "fw_version",
-			dev->gpu->kfd2kgd->get_fw_version(
-						dev->gpu->kgd,
-						KGD_ENGINE_MEC1));
+				dev->gpu->mec_fw_version);
 		sysfs_show_32bit_prop(buffer, "capability",
 				dev->node_props.capability);
 	}
@@ -938,6 +659,7 @@ static int kfd_build_sysfs_node_entry(struct kfd_topology_device *dev,
 	return 0;
 }
 
+/* Called with write topology lock acquired */
 static int kfd_build_sysfs_node_tree(void)
 {
 	struct kfd_topology_device *dev;
@@ -954,6 +676,7 @@ static int kfd_build_sysfs_node_tree(void)
 	return 0;
 }
 
+/* Called with write topology lock acquired */
 static void kfd_remove_sysfs_node_tree(void)
 {
 	struct kfd_topology_device *dev;
@@ -1025,88 +748,200 @@ static void kfd_topology_release_sysfs(void)
 	}
 }
 
+/* Called with write topology_lock acquired */
+static int kfd_topology_update_device_list(struct list_head *temp_list,
+					struct list_head *master_list)
+{
+	int num = 0;
+
+	while (!list_empty(temp_list)) {
+		list_move_tail(temp_list->next, master_list);
+		num++;
+	}
+	return num;
+}
+
+static void kfd_debug_print_topology(void)
+{
+	struct kfd_topology_device *dev;
+
+	down_read(&topology_lock);
+
+	dev = list_last_entry(&topology_device_list, struct kfd_topology_device, list);
+	if (dev) {
+		if (dev->node_props.cpu_cores_count && dev->node_props.simd_count) {
+			pr_info("Topology: Add APU node [0x%0x:0x%0x]\n",
+				dev->node_props.device_id, dev->node_props.vendor_id);
+		}
+		else if (dev->node_props.cpu_cores_count)
+			pr_info("Topology: Add CPU node\n");
+		else if (dev->node_props.simd_count)
+			pr_info("Topology: Add dGPU node [0x%0x:0x%0x]\n",
+				dev->node_props.device_id, dev->node_props.vendor_id);
+	}
+	up_read(&topology_lock);
+}
+
+/* Helper function for intializing platform_xx members of kfd_system_properties
+ */
+static void kfd_update_system_properties(void)
+{
+	struct kfd_topology_device *dev;
+
+	down_read(&topology_lock);
+	dev = list_last_entry(&topology_device_list, struct kfd_topology_device, list);
+	if (dev) {
+		sys_props.platform_id =
+			(*((uint64_t *)dev->oem_id)) & CRAT_OEMID_64BIT_MASK;
+		sys_props.platform_oem = *((uint64_t *)dev->oem_table_id);
+		sys_props.platform_rev = dev->oem_revision;
+	}
+	up_read(&topology_lock);
+}
+
+static void find_system_memory(const struct dmi_header *dm,
+	void *private)
+{
+	struct kfd_mem_properties *mem;
+	u16 mem_width, mem_clock;
+	struct kfd_topology_device *kdev =
+		(struct kfd_topology_device *)private;
+	const u8 *dmi_data = (const u8 *)(dm + 1);
+
+	if (dm->type == DMI_ENTRY_MEM_DEVICE && dm->length >= 0x15) {
+		mem_width = (u16)(*(const u16 *)(dmi_data + 0x6));
+		mem_clock = (u16)(*(const u16 *)(dmi_data + 0x11));
+		list_for_each_entry(mem, &kdev->mem_props, list) {
+			if (mem_width != 0xFFFF && mem_width != 0)
+				mem->width = mem_width;
+			if (mem_clock != 0)
+				mem->mem_clk_max = mem_clock;
+		}
+	}
+}
+/* kfd_add_non_crat_information - Add information that is not currently
+ *	defined in CRAT but is necessary for KFD topology
+ * @dev - topology device to which addition info is added
+ */
+static void kfd_add_non_crat_information(struct kfd_topology_device *kdev)
+{
+	/* Check if CPU only node. */
+	if (kdev->gpu == NULL) {
+		/* Add system memory information */
+		dmi_walk(find_system_memory, kdev);
+	}
+	/* TODO: For GPU node, rearrange code from kfd_topology_add_device */
+}
+
 int kfd_topology_init(void)
 {
 	void *crat_image = NULL;
 	size_t image_size = 0;
 	int ret;
+	struct list_head temp_topology_device_list;
+	int cpu_only_node = 0;
+	struct kfd_topology_device *kdev;
+	int proximity_domain;
+	int num_nodes;
+
+	/* topology_device_list - Master list of all topology devices
+	 * temp_topology_device_list - temporary list created while parsing CRAT
+	 * or VCRAT. Once parsing is complete the contents of list is moved to
+	 * topology_device_list
+	 */
 
-	/*
-	 * Initialize the head for the topology device list
+	 /* Initialize the head for the both the lists
 	 */
 	INIT_LIST_HEAD(&topology_device_list);
+	INIT_LIST_HEAD(&temp_topology_device_list);
 	init_rwsem(&topology_lock);
-	topology_crat_parsed = 0;
 
 	memset(&sys_props, 0, sizeof(sys_props));
 
+	/* Proximity domains in ACPI CRAT tables start counting at
+	 * 0. The same should be true for virtual CRAT tables created
+	 * at this stage. GPUs added later in kfd_topology_add_device
+	 * use a counter. */
+	proximity_domain = 0;
+
 	/*
-	 * Get the CRAT image from the ACPI
+	 * Get the CRAT image from the ACPI. If ACPI doesn't have one
+	 * create a virtual CRAT.
+	 * NOTE: The current implementation expects all AMD APUs to have
+	 *	CRAT. If no CRAT is available, it is assumed to be a CPU
 	 */
-	ret = kfd_topology_get_crat_acpi(crat_image, &image_size);
-	if (ret == 0 && image_size > 0) {
-		pr_info("Found CRAT image with size=%zd\n", image_size);
-		crat_image = kmalloc(image_size, GFP_KERNEL);
-		if (!crat_image) {
-			ret = -ENOMEM;
-			pr_err("No memory for allocating CRAT image\n");
-			goto err;
-		}
-		ret = kfd_topology_get_crat_acpi(crat_image, &image_size);
-
-		if (ret == 0) {
-			down_write(&topology_lock);
-			ret = kfd_parse_crat_table(crat_image);
-			if (ret == 0)
-				ret = kfd_topology_update_sysfs();
-			up_write(&topology_lock);
-		} else {
-			pr_err("Couldn't get CRAT table size from ACPI\n");
-		}
-		kfree(crat_image);
-	} else if (ret == -ENODATA) {
-		ret = 0;
-	} else {
-		pr_err("Couldn't get CRAT table size from ACPI\n");
+	ret = kfd_create_crat_image_acpi(&crat_image, &image_size);
+	if (ret != 0) {
+		ret = kfd_create_crat_image_virtual(&crat_image, &image_size,
+				COMPUTE_UNIT_CPU, NULL,
+				proximity_domain);
+		cpu_only_node = 1;
+	}
+
+	if (ret == 0)
+		ret = kfd_parse_crat_table(crat_image,
+				&temp_topology_device_list,
+				proximity_domain);
+	else {
+		pr_err("Error getting/creating CRAT table\n");
+		goto err;
+	}
+
+	down_write(&topology_lock);
+	num_nodes = kfd_topology_update_device_list(&temp_topology_device_list,
+						    &topology_device_list);
+	atomic_set(&topology_crat_proximity_domain, num_nodes-1);
+	ret = kfd_topology_update_sysfs();
+	up_write(&topology_lock);
+
+	if (ret == 0) {
+		sys_props.generation_count++;
+		kfd_update_system_properties();
+		kfd_debug_print_topology();
+		pr_info("Finished initializing topology\n");
+	}
+	else
+		pr_err("Failed to update topology in sysfs ret=%d\n", ret);
+
+	/* For nodes with GPU, this information gets added
+	 * when GPU is detected (kfd_topology_add_device). */
+	if (cpu_only_node) {
+		/* Add additional information to CPU only node created above */
+		down_write(&topology_lock);
+		kdev = list_first_entry(&topology_device_list,
+				struct kfd_topology_device, list);
+		up_write(&topology_lock);
+		kfd_add_non_crat_information(kdev);
 	}
 
 err:
-	pr_info("Finished initializing topology ret=%d\n", ret);
+	kfd_destroy_crat_image(crat_image);
 	return ret;
 }
 
 void kfd_topology_shutdown(void)
 {
+	down_write(&topology_lock);
 	kfd_topology_release_sysfs();
+	up_write(&topology_lock);
 	kfd_release_live_view();
 }
 
-static void kfd_debug_print_topology(void)
-{
-	struct kfd_topology_device *dev;
-	uint32_t i = 0;
-
-	pr_info("DEBUG PRINT OF TOPOLOGY:");
-	list_for_each_entry(dev, &topology_device_list, list) {
-		pr_info("Node: %d\n", i);
-		pr_info("\tGPU assigned: %s\n", (dev->gpu ? "yes" : "no"));
-		pr_info("\tCPU count: %d\n", dev->node_props.cpu_cores_count);
-		pr_info("\tSIMD count: %d", dev->node_props.simd_count);
-		i++;
-	}
-}
-
 static uint32_t kfd_generate_gpu_id(struct kfd_dev *gpu)
 {
 	uint32_t hashout;
 	uint32_t buf[7];
 	uint64_t local_mem_size;
 	int i;
+	struct kfd_local_mem_info local_mem_info;
 
 	if (!gpu)
 		return 0;
 
-	local_mem_size = gpu->kfd2kgd->get_vmem_size(gpu->kgd);
+	gpu->kfd2kgd->get_local_mem_info(gpu->kgd, &local_mem_info);
+
+	local_mem_size = local_mem_info.local_mem_size_private +
+			local_mem_info.local_mem_size_public;
 
 	buf[0] = gpu->pdev->devfn;
 	buf[1] = gpu->pdev->subsystem_vendor;
@@ -1121,7 +956,13 @@ static uint32_t kfd_generate_gpu_id(struct kfd_dev *gpu)
 
 	return hashout;
 }
-
+/* kfd_assign_gpu - Attach @gpu to the correct kfd topology device. If
+ *		the GPU device is not already present in the topology device list
+ *		then return NULL. This means a new topology device has to be
+ *		created for this GPU.
+ * TODO: Rather than assiging @gpu to first topology device withtout
+ *		gpu attached, it will better to have more stringent check.
+ */
 static struct kfd_topology_device *kfd_assign_gpu(struct kfd_dev *gpu)
 {
 	struct kfd_topology_device *dev;
@@ -1129,13 +970,14 @@ static struct kfd_topology_device *kfd_assign_gpu(struct kfd_dev *gpu)
 
 	BUG_ON(!gpu);
 
+	down_write(&topology_lock);
 	list_for_each_entry(dev, &topology_device_list, list)
 		if (dev->gpu == NULL && dev->node_props.simd_count > 0) {
 			dev->gpu = gpu;
 			out_dev = dev;
 			break;
 		}
-
+	up_write(&topology_lock);
 	return out_dev;
 }
 
@@ -1147,70 +989,146 @@ static void kfd_notify_gpu_change(uint32_t gpu_id, int arrival)
 	 */
 }
 
+/* kfd_fill_mem_clk_max_info - Since CRAT doesn't have memory clock info,
+ *		patch this after CRAT parsing.
+ */
+static void kfd_fill_mem_clk_max_info(struct kfd_topology_device *dev)
+{
+	struct kfd_mem_properties *mem;
+	struct kfd_local_mem_info local_mem_info;
+
+	if (dev == NULL)
+		return;
+
+	/* Currently, amdgpu driver (amdgpu_mc) deals only with GPUs with
+	 * single bank of VRAM local memory.
+	 * for dGPUs - VCRAT reports only one bank of Local Memory
+	 * for APUs - If CRAT from ACPI reports more than one bank, then
+	 *	all the banks will report the same mem_clk_max information
+	 */
+	dev->gpu->kfd2kgd->get_local_mem_info(dev->gpu->kgd,
+		&local_mem_info);
+
+	list_for_each_entry(mem, &dev->mem_props, list)
+		mem->mem_clk_max = local_mem_info.mem_clk_max;
+}
+
 int kfd_topology_add_device(struct kfd_dev *gpu)
 {
 	uint32_t gpu_id;
 	struct kfd_topology_device *dev;
-	int res;
+	struct kfd_cu_info cu_info;
+	int res = 0;
+	struct list_head temp_topology_device_list;
+	void *crat_image = NULL;
+	size_t image_size = 0;
+	int proximity_domain;
 
 	BUG_ON(!gpu);
 
+	INIT_LIST_HEAD(&temp_topology_device_list);
+
 	gpu_id = kfd_generate_gpu_id(gpu);
 
 	pr_debug("kfd: Adding new GPU (ID: 0x%x) to topology\n", gpu_id);
 
-	down_write(&topology_lock);
-	/*
-	 * Try to assign the GPU to existing topology device (generated from
-	 * CRAT table
+	proximity_domain = atomic_inc_return(&
+				topology_crat_proximity_domain);
+
+	/* Check to see if this gpu device exists in the topology_device_list.
+	 * If so, assign the gpu to that device,
+	 * else create a Virtual CRAT for this gpu device and then parse that CRAT
+	 * to create a new topology device. Once created assign the gpu to that
+	 * topology device
 	 */
 	dev = kfd_assign_gpu(gpu);
 	if (!dev) {
-		pr_info("GPU was not found in the current topology. Extending.\n");
-		kfd_debug_print_topology();
-		dev = kfd_create_topology_device();
-		if (!dev) {
-			res = -ENOMEM;
+		res = kfd_create_crat_image_virtual(&crat_image, &image_size,
+				COMPUTE_UNIT_GPU,
+				gpu, proximity_domain);
+		if (res == 0)
+			res = kfd_parse_crat_table(crat_image,
+				&temp_topology_device_list, proximity_domain);
+		else {
+			pr_err("Error in VCRAT for GPU (ID: 0x%x)\n", gpu_id);
 			goto err;
 		}
-		dev->gpu = gpu;
 
-		/*
-		 * TODO: Make a call to retrieve topology information from the
-		 * GPU vBIOS
-		 */
+		down_write(&topology_lock);
+		kfd_topology_update_device_list(&temp_topology_device_list,
+			&topology_device_list);
 
 		/*
 		 * Update the SYSFS tree, since we added another topology device
 		 */
-		if (kfd_topology_update_sysfs() < 0)
-			kfd_topology_release_sysfs();
-
+		res = kfd_topology_update_sysfs();
+		up_write(&topology_lock);
+
+		if (res == 0)
+			sys_props.generation_count++;
+		else
+			pr_err("Failed to update GPU (ID: 0x%x) to sysfs topology. res=%d\n",
+						gpu_id, res);
+		dev = kfd_assign_gpu(gpu);
+		BUG_ON(!dev);
 	}
 
 	dev->gpu_id = gpu_id;
 	gpu->id = gpu_id;
+
+	/* TODO: Move the following lines to function
+	 *	kfd_add_non_crat_information */
+
+	/* Fill-in additional information that is not available in CRAT but
+	 * needed for the topology */
+
+	dev->gpu->kfd2kgd->get_cu_info(dev->gpu->kgd, &cu_info);
+	dev->node_props.simd_arrays_per_engine = cu_info.num_shader_arrays_per_engine;
+
 	dev->node_props.vendor_id = gpu->pdev->vendor;
 	dev->node_props.device_id = gpu->pdev->device;
-	dev->node_props.location_id = (gpu->pdev->bus->number << 24) +
-			(gpu->pdev->devfn & 0xffffff);
-	/*
-	 * TODO: Retrieve max engine clock values from KGD
-	 */
-
-	if (dev->gpu->device_info->asic_family == CHIP_CARRIZO) {
-		dev->node_props.capability |= HSA_CAP_DOORBELL_PACKET_TYPE;
-		pr_info("amdkfd: adding doorbell packet type capability\n");
+	dev->node_props.location_id = PCI_DEVID(gpu->pdev->bus->number,
+		gpu->pdev->devfn);
+	dev->node_props.max_engine_clk_fcompute =
+		dev->gpu->kfd2kgd->get_max_engine_clock_in_mhz(dev->gpu->kgd);
+	dev->node_props.max_engine_clk_ccompute =
+		cpufreq_quick_get_max(0) / 1000;
+
+	kfd_fill_mem_clk_max_info(dev);
+
+	switch (dev->gpu->device_info->asic_family) {
+	case CHIP_KAVERI:
+		dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_PRE_1_0 <<
+			HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) &
+			HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK);
+		break;
+	case CHIP_CARRIZO:
+	case CHIP_TONGA:
+	case CHIP_FIJI:
+		pr_debug("amdkfd: adding doorbell packet type capability\n");
+		dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_1_0 <<
+			HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) &
+			HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK);
+		break;
 	}
 
-	res = 0;
+	/* Fix errors in CZ CRAT.
+	 * simd_count: Carrizo CRAT reports wrong simd_count, probably because it
+	 *		doesn't consider masked out CUs
+	 * capability flag: Carrizo CRAT doesn't report IOMMU flags.
+	 */
+	if (dev->gpu->device_info->asic_family == CHIP_CARRIZO) {
+		dev->node_props.simd_count =
+			cu_info.simd_per_cu * cu_info.cu_active_number;
+		dev->node_props.capability |= HSA_CAP_ATS_PRESENT;
+        }
 
+	kfd_debug_print_topology();
 err:
-	up_write(&topology_lock);
-
 	if (res == 0)
 		kfd_notify_gpu_change(gpu_id, 1);
 
+	kfd_destroy_crat_image(crat_image);
 	return res;
 }
 
@@ -1243,22 +1161,26 @@ int kfd_topology_remove_device(struct kfd_dev *gpu)
 	return res;
 }
 
-/*
- * When idx is out of bounds, the function will return NULL
+/* kfd_topology_enum_kfd_devices - Enumerate through all devices in KFD
+ *	topology. If GPU device is found @idx, then valid kfd_dev pointer is
+ *	returned through @kdev
+ * Return -	0: On success (@kdev will be NULL for non GPU nodes)
+ *		-1: If end of list
  */
-struct kfd_dev *kfd_topology_enum_kfd_devices(uint8_t idx)
+int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_dev **kdev)
 {
 
 	struct kfd_topology_device *top_dev;
-	struct kfd_dev *device = NULL;
 	uint8_t device_idx = 0;
 
+	*kdev = NULL;
 	down_read(&topology_lock);
 
 	list_for_each_entry(top_dev, &topology_device_list, list) {
 		if (device_idx == idx) {
-			device = top_dev->gpu;
-			break;
+			*kdev = top_dev->gpu;
+			up_read(&topology_lock);
+			return 0;
 		}
 
 		device_idx++;
@@ -1266,6 +1188,57 @@ struct kfd_dev *kfd_topology_enum_kfd_devices(uint8_t idx)
 
 	up_read(&topology_lock);
 
-	return device;
+	return -1;
+
+}
+
+static int kfd_cpumask_to_apic_id(const struct cpumask *cpumask)
+{
+	const struct cpuinfo_x86 *cpuinfo;
+	int first_cpu_of_nuna_node;
+
+	if (cpumask == NULL || cpumask == cpu_none_mask)
+		return -1;
+	first_cpu_of_nuna_node = cpumask_first(cpumask);
+	cpuinfo = &cpu_data(first_cpu_of_nuna_node);
+
+	return cpuinfo->apicid;
+}
+
+/* kfd_numa_node_to_apic_id - Returns the APIC ID of the first logical processor
+ *	of the given NUMA node (numa_node_id)
+ * Return -1 on failure
+ */
+int kfd_numa_node_to_apic_id(int numa_node_id)
+{
+	if (numa_node_id == -1) {
+		pr_warn("Invalid NUMA Node. Use online CPU mask\n");
+		return kfd_cpumask_to_apic_id(cpu_online_mask);
+	}
+	return kfd_cpumask_to_apic_id(cpumask_of_node(numa_node_id));
+}
+
+/* kfd_get_proximity_domain - Find proximity_domain (node id) to which
+ *  given PCI bus belongs to. CRAT table contains only the APIC ID
+ *  of the parent NUMA node. So use that as the search parameter.
+ * Return -1 on failure
+ */
+int kfd_get_proximity_domain(const struct pci_bus *bus)
+{
+	struct kfd_topology_device *dev;
+	int proximity_domain = -1;
+
+	down_read(&topology_lock);
+
+	list_for_each_entry(dev, &topology_device_list, list)
+		if (dev->node_props.cpu_cores_count &&
+			dev->node_props.cpu_core_id_base ==
+			kfd_cpumask_to_apic_id(cpumask_of_pcibus(bus))) {
+				proximity_domain = dev->proximity_domain;
+			break;
+		}
+
+	up_read(&topology_lock);
 
+	return proximity_domain;
 }
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
index c3ddb9b..ab28188 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
@@ -39,8 +39,16 @@
 #define HSA_CAP_WATCH_POINTS_SUPPORTED		0x00000080
 #define HSA_CAP_WATCH_POINTS_TOTALBITS_MASK	0x00000f00
 #define HSA_CAP_WATCH_POINTS_TOTALBITS_SHIFT	8
-#define HSA_CAP_RESERVED			0xfffff000
+#define HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK	0x00003000
+#define HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT	12
+#define HSA_CAP_RESERVED			0xffffc000
+
+#define HSA_CAP_DOORBELL_TYPE_PRE_1_0		0x0
+#define HSA_CAP_DOORBELL_TYPE_1_0		0x1
+#define HSA_CAP_WATCH_POINTS_TOTALBITS_MASK	0x00000f00
+#define HSA_CAP_WATCH_POINTS_TOTALBITS_SHIFT	8
 #define HSA_CAP_DOORBELL_PACKET_TYPE		0x00001000
+#define HSA_CAP_AQL_QUEUE_DOUBLE_MAP		0x00004000
 
 struct kfd_node_properties {
 	uint32_t cpu_cores_count;
@@ -91,8 +99,6 @@ struct kfd_mem_properties {
 	struct attribute	attr;
 };
 
-#define KFD_TOPOLOGY_CPU_SIBLINGS 256
-
 #define HSA_CACHE_TYPE_DATA		0x00000001
 #define HSA_CACHE_TYPE_INSTRUCTION	0x00000002
 #define HSA_CACHE_TYPE_CPU		0x00000004
@@ -109,7 +115,7 @@ struct kfd_cache_properties {
 	uint32_t		cache_assoc;
 	uint32_t		cache_latency;
 	uint32_t		cache_type;
-	uint8_t			sibling_map[KFD_TOPOLOGY_CPU_SIBLINGS];
+	uint8_t			sibling_map[CRAT_SIBLINGMAP_SIZE];
 	struct kobject		*kobj;
 	struct attribute	attr;
 };
@@ -135,8 +141,8 @@ struct kfd_iolink_properties {
 struct kfd_topology_device {
 	struct list_head		list;
 	uint32_t			gpu_id;
+	uint32_t			proximity_domain;
 	struct kfd_node_properties	node_props;
-	uint32_t			mem_bank_count;
 	struct list_head		mem_props;
 	uint32_t			cache_count;
 	struct list_head		cache_props;
@@ -150,6 +156,9 @@ struct kfd_topology_device {
 	struct attribute		attr_gpuid;
 	struct attribute		attr_name;
 	struct attribute		attr_props;
+	uint8_t		oem_id[CRAT_OEMID_LENGTH];
+	uint8_t		oem_table_id[CRAT_OEMTABLEID_LENGTH];
+	uint32_t	oem_revision;
 };
 
 struct kfd_system_properties {
@@ -164,6 +173,8 @@ struct kfd_system_properties {
 	struct attribute	attr_props;
 };
 
-
+struct kfd_topology_device *kfd_create_topology_device(
+		struct list_head *device_list);
+void kfd_release_live_view(void);
 
 #endif /* __KFD_TOPOLOGY_H__ */
-- 
2.7.4