aboutsummaryrefslogtreecommitdiffstats
path: root/common/recipes-kernel/linux/linux-yocto-4.14.71/1368-drm-amdkfd-revert-kfd-part-to-a-previous-state.patch
diff options
context:
space:
mode:
Diffstat (limited to 'common/recipes-kernel/linux/linux-yocto-4.14.71/1368-drm-amdkfd-revert-kfd-part-to-a-previous-state.patch')
-rw-r--r--common/recipes-kernel/linux/linux-yocto-4.14.71/1368-drm-amdkfd-revert-kfd-part-to-a-previous-state.patch31349
1 files changed, 31349 insertions, 0 deletions
diff --git a/common/recipes-kernel/linux/linux-yocto-4.14.71/1368-drm-amdkfd-revert-kfd-part-to-a-previous-state.patch b/common/recipes-kernel/linux/linux-yocto-4.14.71/1368-drm-amdkfd-revert-kfd-part-to-a-previous-state.patch
new file mode 100644
index 00000000..f6937453
--- /dev/null
+++ b/common/recipes-kernel/linux/linux-yocto-4.14.71/1368-drm-amdkfd-revert-kfd-part-to-a-previous-state.patch
@@ -0,0 +1,31349 @@
+From 3538bdf4c8b2d8f1f93fd806656bad0c82c6e60f Mon Sep 17 00:00:00 2001
+From: Chaudhary Amit Kumar <chaudharyamit.kumar@amd.com>
+Date: Thu, 18 Oct 2018 18:06:09 +0530
+Subject: [PATCH 1368/4131] drm/amdkfd: revert kfd part to a previous state
+
+Revert following files to "2ba6b00 drm/amd/powerplay: add profile mode for vega10.":
+
+ - drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd*
+ - drivers/gpu/drm/amd/amdkfd/*
+ - drivers/gpu/drm/amd/include/kgd_kfd_interface.h
+ - include/uapi/linux/kfd_ioctl.h
+ - drivers/gpu/drm/radeon/radeon_kfd*
+
+Due to upstream, porting kfd patches to 4.13 all-open has many conflicts.
+It's hard to elegantly fix these conflicts. So we revert the kfd part to a
+previous commit, where we began to first port dkms patches in 4.12 hybrid.
+Then sequentially port all kfd patches.
+
+Change-Id: I75eda45f41ced2f4c444ded126e2b80b53d15f2a
+Signed-off-by: Le.Ma <Le.Ma@amd.com>
+Acked-by: Junwei Zhang <Jerry.Zhang@amd.com>
+Signed-off-by: kalyan.alle <kalyan.alle@amd.com>
+Signed-off-by: Chaudhary Amit Kumar <chaudharyamit.kumar@amd.com>
+---
+ drivers/gpu/drm/amd/amdgpu/Makefile | 4 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu.h | 97 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 354 +--
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 184 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c | 196 --
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c | 542 +---
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c | 642 +----
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c | 1227 ----------
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 2578 --------------------
+ drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 246 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 50 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h | 1 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 13 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 82 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c | 133 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 106 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_object.h | 22 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c | 5 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 1 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c | 21 +-
+ drivers/gpu/drm/amd/amdgpu/vid.h | 6 -
+ drivers/gpu/drm/amd/amdkfd/Kconfig | 3 +-
+ drivers/gpu/drm/amd/amdkfd/Makefile | 23 +-
+ drivers/gpu/drm/amd/amdkfd/backport/Makefile | 7 -
+ drivers/gpu/drm/amd/amdkfd/backport/backport.h | 6 -
+ drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c | 73 +-
+ drivers/gpu/drm/amd/amdkfd/cik_int.h | 24 +-
+ drivers/gpu/drm/amd/amdkfd/cik_regs.h | 3 +-
+ .../gpu/drm/amd/amdkfd/cwsr_trap_handler_carrizo.h | 1384 -----------
+ .../gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx9.asm | 1388 -----------
+ drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 1361 +----------
+ drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 1304 ----------
+ drivers/gpu/drm/amd/amdkfd/kfd_crat.h | 42 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c | 219 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.h | 32 -
+ drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.c | 24 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.h | 27 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c | 75 -
+ drivers/gpu/drm/amd/amdkfd/kfd_device.c | 890 +------
+ .../gpu/drm/amd/amdkfd/kfd_device_queue_manager.c | 1222 +++-------
+ .../gpu/drm/amd/amdkfd/kfd_device_queue_manager.h | 56 +-
+ .../drm/amd/amdkfd/kfd_device_queue_manager_cik.c | 80 +-
+ .../drm/amd/amdkfd/kfd_device_queue_manager_v9.c | 90 -
+ .../drm/amd/amdkfd/kfd_device_queue_manager_vi.c | 123 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c | 106 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_events.c | 253 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c | 124 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c | 133 -
+ drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c | 91 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_ipc.c | 275 ---
+ drivers/gpu/drm/amd/amdkfd/kfd_ipc.h | 51 -
+ drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c | 149 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.h | 17 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_cik.c | 128 -
+ drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_v9.c | 377 ---
+ drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_vi.c | 361 ---
+ drivers/gpu/drm/amd/amdkfd/kfd_module.c | 61 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c | 54 -
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h | 18 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c | 240 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c | 528 ----
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c | 329 +--
+ drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c | 556 +++--
+ drivers/gpu/drm/amd/amdkfd/kfd_pasid.c | 7 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_peerdirect.c | 513 ----
+ drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers.h | 330 ++-
+ drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_ai.h | 583 -----
+ drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_diq.h | 97 +
+ drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_vi.h | 140 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 546 +----
+ drivers/gpu/drm/amd/amdkfd/kfd_process.c | 919 +------
+ .../gpu/drm/amd/amdkfd/kfd_process_queue_manager.c | 290 +--
+ drivers/gpu/drm/amd/amdkfd/kfd_queue.c | 12 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_rdma.c | 294 ---
+ drivers/gpu/drm/amd/amdkfd/kfd_topology.c | 1177 ++++-----
+ drivers/gpu/drm/amd/amdkfd/kfd_topology.h | 41 +-
+ drivers/gpu/drm/amd/amdkfd/soc15_int.h | 84 -
+ drivers/gpu/drm/amd/include/kgd_kfd_interface.h | 231 +-
+ drivers/gpu/drm/drm_pci.c | 1 +
+ drivers/gpu/drm/radeon/radeon_kfd.c | 19 +-
+ include/drm/drm_drv.h | 2 +
+ include/drm/drm_pci.h | 7 +
+ 82 files changed, 3407 insertions(+), 20703 deletions(-)
+ delete mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c
+ mode change 100755 => 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
+ mode change 100755 => 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
+ delete mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
+ delete mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+ mode change 100755 => 100644 drivers/gpu/drm/amd/amdkfd/Makefile
+ delete mode 100644 drivers/gpu/drm/amd/amdkfd/backport/Makefile
+ delete mode 100644 drivers/gpu/drm/amd/amdkfd/backport/backport.h
+ delete mode 100644 drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_carrizo.h
+ delete mode 100644 drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx9.asm
+ delete mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_crat.c
+ delete mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c
+ delete mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c
+ delete mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
+ delete mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_ipc.c
+ delete mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_ipc.h
+ delete mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_v9.c
+ delete mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
+ delete mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_peerdirect.c
+ delete mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_ai.h
+ mode change 100755 => 100644 drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+ delete mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_rdma.c
+ delete mode 100644 drivers/gpu/drm/amd/amdkfd/soc15_int.h
+ mode change 100755 => 100644 drivers/gpu/drm/amd/include/kgd_kfd_interface.h
+
+diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
+index 6b373d0..bc6f49e 100755
+--- a/drivers/gpu/drm/amd/amdgpu/Makefile
++++ b/drivers/gpu/drm/amd/amdgpu/Makefile
+@@ -32,7 +32,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
+ amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
+ amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
+ amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o amdgpu_atomfirmware.o \
+- amdgpu_queue_mgr.o amdgpu_vf_error.o amdgpu_sem.o amdgpu_amdkfd_fence.o
++ amdgpu_queue_mgr.o amdgpu_vf_error.o amdgpu_sem.o
+
+ # add asic specific block
+ amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
+@@ -110,8 +110,6 @@ amdgpu-y += \
+ amdgpu_amdkfd.o \
+ amdgpu_amdkfd_gfx_v7.o \
+ amdgpu_amdkfd_gfx_v8.o \
+- amdgpu_amdkfd_gfx_v9.o \
+- amdgpu_amdkfd_gpuvm.o
+
+ # add cgs
+ amdgpu-y += amdgpu_cgs.o
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+index bcf95e7..b07c90e 100755
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+@@ -67,7 +67,6 @@
+ #include "amdgpu_vce.h"
+ #include "amdgpu_vcn.h"
+ #include "amdgpu_dm.h"
+-#include "amdgpu_mn.h"
+
+ #include "gpu_scheduler.h"
+ #include "amdgpu_virt.h"
+@@ -125,7 +124,6 @@ extern int amdgpu_cntl_sb_buf_per_se;
+ extern int amdgpu_param_buf_per_se;
+ extern int amdgpu_job_hang_limit;
+ extern int amdgpu_lbpw;
+-extern int amdgpu_compute_multipipe;
+
+ #ifdef CONFIG_DRM_AMDGPU_SI
+ extern int amdgpu_si_support;
+@@ -184,8 +182,7 @@ struct amdgpu_cs_parser;
+ struct amdgpu_job;
+ struct amdgpu_irq_src;
+ struct amdgpu_fpriv;
+-struct kfd_vm_fault_info;
+-struct amdgpu_bo_va_mapping;
++struct kfd_process_device;
+
+ enum amdgpu_cp_irq {
+ AMDGPU_CP_IRQ_GFX_EOP = 0,
+@@ -300,25 +297,14 @@ struct amdgpu_buffer_funcs {
+
+ /* provided by hw blocks that can write ptes, e.g., sdma */
+ struct amdgpu_vm_pte_funcs {
+- /* number of dw to reserve per operation */
+- unsigned copy_pte_num_dw;
+-
+ /* copy pte entries from GART */
+ void (*copy_pte)(struct amdgpu_ib *ib,
+ uint64_t pe, uint64_t src,
+ unsigned count);
+-
+ /* write pte one entry at a time with addr mapping */
+ void (*write_pte)(struct amdgpu_ib *ib, uint64_t pe,
+ uint64_t value, unsigned count,
+ uint32_t incr);
+-
+- /* maximum nums of PTEs/PDEs in a single operation */
+- uint32_t set_max_nums_pte_pde;
+-
+- /* number of dw to reserve per operation */
+- unsigned set_pte_pde_num_dw;
+-
+ /* for linear pte/pde updates without addr mapping */
+ void (*set_pte_pde)(struct amdgpu_ib *ib,
+ uint64_t pe,
+@@ -397,15 +383,7 @@ struct amdgpu_clock {
+ */
+
+ #define AMDGPU_GEM_DOMAIN_MAX 0x3
+-
+-struct amdgpu_gem_object {
+- struct drm_gem_object base;
+- struct list_head list;
+- struct amdgpu_bo *bo;
+-};
+-
+-struct kgd_mem;
+-#define gem_to_amdgpu_bo(gobj) container_of((gobj), struct amdgpu_gem_object, base)->bo
++#define gem_to_amdgpu_bo(gobj) container_of((gobj), struct amdgpu_bo, gem_base)
+
+ void amdgpu_gem_object_free(struct drm_gem_object *obj);
+ int amdgpu_gem_object_open(struct drm_gem_object *obj,
+@@ -421,8 +399,6 @@ amdgpu_gem_prime_import_sg_table(struct drm_device *dev,
+ struct dma_buf *amdgpu_gem_prime_export(struct drm_device *dev,
+ struct drm_gem_object *gobj,
+ int flags);
+-struct drm_gem_object *
+-amdgpu_gem_prime_foreign_bo(struct amdgpu_device *adev, struct amdgpu_bo *bo);
+ int amdgpu_gem_prime_pin(struct drm_gem_object *obj);
+ void amdgpu_gem_prime_unpin(struct drm_gem_object *obj);
+ struct reservation_object *amdgpu_gem_prime_res_obj(struct drm_gem_object *);
+@@ -484,10 +460,9 @@ struct amdgpu_sa_bo {
+ */
+ void amdgpu_gem_force_release(struct amdgpu_device *adev);
+ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
+- int alignment, u32 initial_domain,
+- u64 flags, bool kernel,
+- struct reservation_object *resv,
+- struct drm_gem_object **obj);
++ int alignment, u32 initial_domain,
++ u64 flags, bool kernel,
++ struct drm_gem_object **obj);
+
+ int amdgpu_mode_dumb_create(struct drm_file *file_priv,
+ struct drm_device *dev,
+@@ -545,9 +520,6 @@ struct amdgpu_mc {
+ u64 private_aperture_end;
+ /* protects concurrent invalidation */
+ spinlock_t invalidate_lock;
+-
+- struct kfd_vm_fault_info *vm_fault_info;
+- atomic_t vm_fault_info_updated;
+ };
+
+ /*
+@@ -730,7 +702,7 @@ int amdgpu_queue_mgr_fini(struct amdgpu_device *adev,
+ struct amdgpu_queue_mgr *mgr);
+ int amdgpu_queue_mgr_map(struct amdgpu_device *adev,
+ struct amdgpu_queue_mgr *mgr,
+- u32 hw_ip, u32 instance, u32 ring,
++ int hw_ip, int instance, int ring,
+ struct amdgpu_ring **out_ring);
+
+ /*
+@@ -966,7 +938,6 @@ struct amdgpu_gfx_config {
+ };
+
+ struct amdgpu_cu_info {
+- uint32_t simd_per_cu;
+ uint32_t max_waves_per_simd;
+ uint32_t wave_front_size;
+ uint32_t max_scratch_slots_per_cu;
+@@ -1094,7 +1065,6 @@ struct amdgpu_cs_parser {
+ /* buffer objects */
+ struct ww_acquire_ctx ticket;
+ struct amdgpu_bo_list *bo_list;
+- struct amdgpu_mn *mn;
+ struct amdgpu_bo_list_entry vm_pd;
+ struct list_head validated;
+ struct dma_fence *fence;
+@@ -1236,6 +1206,20 @@ void amdgpu_benchmark(struct amdgpu_device *adev, int test_number);
+ void amdgpu_test_moves(struct amdgpu_device *adev);
+
+ /*
++ * MMU Notifier
++ */
++#if defined(CONFIG_MMU_NOTIFIER)
++int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr);
++void amdgpu_mn_unregister(struct amdgpu_bo *bo);
++#else
++static inline int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
++{
++ return -ENODEV;
++}
++static inline void amdgpu_mn_unregister(struct amdgpu_bo *bo) {}
++#endif
++
++/*
+ * Debugfs
+ */
+ struct amdgpu_debugfs {
+@@ -1435,7 +1419,10 @@ struct amdgpu_direct_gma {
+ };
+
+ #if defined(CONFIG_ZONE_DEVICE) && \
+- (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0) || defined(OS_NAME_RHEL_7_3) || defined(OS_NAME_SLE))
++ (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0) || \
++ defined(OS_NAME_RHEL_7_3) || \
++ defined(OS_NAME_RHEL_7_4) || \
++ defined(OS_NAME_SLE))
+ #define CONFIG_ENABLE_SSG
+ #endif
+
+@@ -1603,14 +1590,18 @@ struct amdgpu_device {
+ /* sdma */
+ struct amdgpu_sdma sdma;
+
+- /* uvd */
+- struct amdgpu_uvd uvd;
++ union {
++ struct {
++ /* uvd */
++ struct amdgpu_uvd uvd;
+
+- /* vce */
+- struct amdgpu_vce vce;
++ /* vce */
++ struct amdgpu_vce vce;
++ };
+
+- /* vcn */
+- struct amdgpu_vcn vcn;
++ /* vcn */
++ struct amdgpu_vcn vcn;
++ };
+
+ /* firmwares */
+ struct amdgpu_firmware firmware;
+@@ -1655,7 +1646,6 @@ struct amdgpu_device {
+ /* record hw reset is performed */
+ bool has_hw_reset;
+ u8 reset_magic[AMDGPU_RESET_MAGIC_NUM];
+- spinlock_t tlb_invalidation_lock;
+
+ /* record last mm index being written through WREG32*/
+ unsigned long last_mm_index;
+@@ -1861,6 +1851,18 @@ void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes,
+ u64 num_vis_bytes);
+ void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain);
+ bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo);
++int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages);
++int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
++ uint32_t flags);
++bool amdgpu_ttm_tt_has_userptr(struct ttm_tt *ttm);
++struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm);
++bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
++ unsigned long end);
++bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
++ int *last_invalidated);
++bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm);
++uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
++ struct ttm_mem_reg *mem);
+ void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64 base);
+ void amdgpu_gart_location(struct amdgpu_device *adev, struct amdgpu_mc *mc);
+ void amdgpu_ttm_set_active_vram_size(struct amdgpu_device *adev, u64 size);
+@@ -1943,9 +1945,10 @@ static inline int amdgpu_acpi_init(struct amdgpu_device *adev) { return 0; }
+ static inline void amdgpu_acpi_fini(struct amdgpu_device *adev) { }
+ #endif
+
+-int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
+- uint64_t addr, struct amdgpu_bo **bo,
+- struct amdgpu_bo_va_mapping **mapping);
++struct amdgpu_bo_va_mapping *
++amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
++ uint64_t addr, struct amdgpu_bo **bo);
++int amdgpu_cs_sysvm_access_required(struct amdgpu_cs_parser *parser);
+
+ #if defined(CONFIG_DRM_AMD_DC)
+ int amdgpu_dm_display_resume(struct amdgpu_device *adev );
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+index ec8141f..ef56352 100755
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+@@ -20,29 +20,23 @@
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+-#undef pr_fmt
+-#define pr_fmt(fmt) "kfd2kgd: " fmt
+-
+ #include "amdgpu_amdkfd.h"
+-#include <linux/dma-buf.h>
++#include "amd_shared.h"
+ #include <drm/drmP.h>
+ #include "amdgpu.h"
+ #include "amdgpu_gfx.h"
+ #include <linux/module.h>
+
+-#define AMDKFD_SKIP_UNCOMPILED_CODE 1
+-
++const struct kfd2kgd_calls *kfd2kgd;
+ const struct kgd2kfd_calls *kgd2kfd;
+-bool (*kgd2kfd_init_p)(unsigned int, const struct kgd2kfd_calls**);
+-
+-unsigned int global_compute_vmid_bitmap = 0xFF00;
++bool (*kgd2kfd_init_p)(unsigned, const struct kgd2kfd_calls**);
+
+ int amdgpu_amdkfd_init(void)
+ {
+ int ret;
+
+ #if defined(CONFIG_HSA_AMD_MODULE)
+- int (*kgd2kfd_init_p)(unsigned int, const struct kgd2kfd_calls**);
++ int (*kgd2kfd_init_p)(unsigned, const struct kgd2kfd_calls**);
+
+ kgd2kfd_init_p = symbol_request(kgd2kfd_init);
+
+@@ -63,68 +57,56 @@ int amdgpu_amdkfd_init(void)
+ #else
+ ret = -ENOENT;
+ #endif
+- amdgpu_amdkfd_gpuvm_init_mem_limits();
+- return ret;
+-}
+
+-void amdgpu_amdkfd_fini(void)
+-{
+- if (kgd2kfd) {
+- kgd2kfd->exit();
+- symbol_put(kgd2kfd_init);
+- }
++ return ret;
+ }
+
+-void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev)
++bool amdgpu_amdkfd_load_interface(struct amdgpu_device *adev)
+ {
+- const struct kfd2kgd_calls *kfd2kgd;
+-
+- if (!kgd2kfd)
+- return;
+-
+ switch (adev->asic_type) {
+ #ifdef CONFIG_DRM_AMDGPU_CIK
+ case CHIP_KAVERI:
+- case CHIP_HAWAII:
+ kfd2kgd = amdgpu_amdkfd_gfx_7_get_functions();
+ break;
+ #endif
+ case CHIP_CARRIZO:
+- case CHIP_TONGA:
+- case CHIP_FIJI:
+- case CHIP_POLARIS10:
+- case CHIP_POLARIS11:
+ kfd2kgd = amdgpu_amdkfd_gfx_8_0_get_functions();
+ break;
+- case CHIP_VEGA10:
+- case CHIP_RAVEN:
+- kfd2kgd = amdgpu_amdkfd_gfx_9_0_get_functions();
+- break;
+ default:
+- dev_info(adev->dev, "kfd not supported on this ASIC\n");
+- return;
++ return false;
++ }
++
++ return true;
++}
++
++void amdgpu_amdkfd_fini(void)
++{
++ if (kgd2kfd) {
++ kgd2kfd->exit();
++ symbol_put(kgd2kfd_init);
+ }
++}
+
+- adev->kfd = kgd2kfd->probe((struct kgd_dev *)adev,
+- adev->pdev, kfd2kgd);
++void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev)
++{
++ if (kgd2kfd)
++ adev->kfd = kgd2kfd->probe((struct kgd_dev *)adev,
++ adev->pdev, kfd2kgd);
+ }
+
+ void amdgpu_amdkfd_device_init(struct amdgpu_device *adev)
+ {
+ int i;
+ int last_valid_bit;
+-
+ if (adev->kfd) {
+ struct kgd2kfd_shared_resources gpu_resources = {
+- .compute_vmid_bitmap = global_compute_vmid_bitmap,
++ .compute_vmid_bitmap = 0xFF00,
+ .num_pipe_per_mec = adev->gfx.mec.num_pipe_per_mec,
+- .num_queue_per_pipe = adev->gfx.mec.num_queue_per_pipe,
+- .gpuvm_size = (uint64_t)amdgpu_vm_size << 30
++ .num_queue_per_pipe = adev->gfx.mec.num_queue_per_pipe
+ };
+
+ /* this is going to have a few of the MSBs set that we need to
+- * clear
+- */
++ * clear */
+ bitmap_complement(gpu_resources.queue_bitmap,
+ adev->gfx.mec.queue_bitmap,
+ KGD_MAX_QUEUES);
+@@ -138,8 +120,7 @@ void amdgpu_amdkfd_device_init(struct amdgpu_device *adev)
+ gpu_resources.queue_bitmap);
+
+ /* According to linux/bitmap.h we shouldn't use bitmap_clear if
+- * nbits is not compile time constant
+- */
++ * nbits is not compile time constant */
+ last_valid_bit = 1 /* only first MEC can have compute queues */
+ * adev->gfx.mec.num_pipe_per_mec
+ * adev->gfx.mec.num_queue_per_pipe;
+@@ -150,28 +131,6 @@ void amdgpu_amdkfd_device_init(struct amdgpu_device *adev)
+ &gpu_resources.doorbell_physical_address,
+ &gpu_resources.doorbell_aperture_size,
+ &gpu_resources.doorbell_start_offset);
+- if (adev->asic_type >= CHIP_VEGA10) {
+- /* On SOC15 the BIF is involved in routing
+- * doorbells using the low 12 bits of the
+- * address. Communicate the assignments to
+- * KFD. KFD uses two doorbell pages per
+- * process in case of 64-bit doorbells so we
+- * can use each doorbell assignment twice.
+- */
+- gpu_resources.sdma_doorbell[0][0] =
+- AMDGPU_DOORBELL64_sDMA_ENGINE0;
+- gpu_resources.sdma_doorbell[0][1] =
+- AMDGPU_DOORBELL64_sDMA_ENGINE0 + 0x200;
+- gpu_resources.sdma_doorbell[1][0] =
+- AMDGPU_DOORBELL64_sDMA_ENGINE1;
+- gpu_resources.sdma_doorbell[1][1] =
+- AMDGPU_DOORBELL64_sDMA_ENGINE1 + 0x200;
+- /* Doorbells 0x0f0-0ff and 0x2f0-2ff are reserved for
+- * SDMA, IH and VCN. So don't use them for the CP.
+- */
+- gpu_resources.reserved_doorbell_mask = 0x1f0;
+- gpu_resources.reserved_doorbell_val = 0x0f0;
+- }
+
+ kgd2kfd->device_init(adev->kfd, &gpu_resources);
+ }
+@@ -208,81 +167,24 @@ int amdgpu_amdkfd_resume(struct amdgpu_device *adev)
+ return r;
+ }
+
+-int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
+- uint32_t vmid, uint64_t gpu_addr,
+- uint32_t *ib_cmd, uint32_t ib_len)
+-{
+- struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+- struct amdgpu_job *job;
+- struct amdgpu_ib *ib;
+- struct amdgpu_ring *ring;
+- struct dma_fence *f = NULL;
+- int ret;
+-
+- switch (engine) {
+- case KGD_ENGINE_MEC1:
+- ring = &adev->gfx.compute_ring[0];
+- break;
+- case KGD_ENGINE_SDMA1:
+- ring = &adev->sdma.instance[0].ring;
+- break;
+- case KGD_ENGINE_SDMA2:
+- ring = &adev->sdma.instance[1].ring;
+- break;
+- default:
+- pr_err("Invalid engine in IB submission: %d\n", engine);
+- ret = -EINVAL;
+- goto err;
+- }
+-
+- ret = amdgpu_job_alloc(adev, 1, &job, NULL);
+- if (ret)
+- goto err;
+-
+- ib = &job->ibs[0];
+- memset(ib, 0, sizeof(struct amdgpu_ib));
+-
+- ib->gpu_addr = gpu_addr;
+- ib->ptr = ib_cmd;
+- ib->length_dw = ib_len;
+- /* This works for NO_HWS. TODO: need to handle without knowing VMID */
+- job->vm_id = vmid;
+-
+- ret = amdgpu_ib_schedule(ring, 1, ib, job, &f);
+- if (ret) {
+- DRM_ERROR("amdgpu: failed to schedule IB.\n");
+- goto err_ib_sched;
+- }
+-
+- ret = dma_fence_wait(f, false);
+-
+-err_ib_sched:
+- dma_fence_put(f);
+- amdgpu_job_free(job);
+-err:
+- return ret;
+-}
+-
+-u32 pool_to_domain(enum kgd_memory_pool p)
+-{
+- switch (p) {
+- case KGD_POOL_FRAMEBUFFER: return AMDGPU_GEM_DOMAIN_VRAM;
+- default: return AMDGPU_GEM_DOMAIN_GTT;
+- }
+-}
+-
+ int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
+ void **mem_obj, uint64_t *gpu_addr,
+ void **cpu_ptr)
+ {
+ struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+- struct amdgpu_bo *bo = NULL;
++ struct kgd_mem **mem = (struct kgd_mem **) mem_obj;
+ int r;
+- uint64_t gpu_addr_tmp = 0;
+- void *cpu_ptr_tmp = NULL;
++
++ BUG_ON(kgd == NULL);
++ BUG_ON(gpu_addr == NULL);
++ BUG_ON(cpu_ptr == NULL);
++
++ *mem = kmalloc(sizeof(struct kgd_mem), GFP_KERNEL);
++ if ((*mem) == NULL)
++ return -ENOMEM;
+
+ r = amdgpu_bo_create(adev, size, PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_GTT,
+- AMDGPU_GEM_CREATE_CPU_GTT_USWC, NULL, NULL, 0, &bo);
++ AMDGPU_GEM_CREATE_CPU_GTT_USWC, NULL, NULL, 0, &(*mem)->bo);
+ if (r) {
+ dev_err(adev->dev,
+ "failed to allocate BO for amdkfd (%d)\n", r);
+@@ -290,87 +192,64 @@ int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
+ }
+
+ /* map the buffer */
+- r = amdgpu_bo_reserve(bo, true);
++ r = amdgpu_bo_reserve((*mem)->bo, true);
+ if (r) {
+ dev_err(adev->dev, "(%d) failed to reserve bo for amdkfd\n", r);
+ goto allocate_mem_reserve_bo_failed;
+ }
+
+- r = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT,
+- &gpu_addr_tmp);
++ r = amdgpu_bo_pin((*mem)->bo, AMDGPU_GEM_DOMAIN_GTT,
++ &(*mem)->gpu_addr);
+ if (r) {
+ dev_err(adev->dev, "(%d) failed to pin bo for amdkfd\n", r);
+ goto allocate_mem_pin_bo_failed;
+ }
++ *gpu_addr = (*mem)->gpu_addr;
+
+- r = amdgpu_bo_kmap(bo, &cpu_ptr_tmp);
++ r = amdgpu_bo_kmap((*mem)->bo, &(*mem)->cpu_ptr);
+ if (r) {
+ dev_err(adev->dev,
+ "(%d) failed to map bo to kernel for amdkfd\n", r);
+ goto allocate_mem_kmap_bo_failed;
+ }
++ *cpu_ptr = (*mem)->cpu_ptr;
+
+- *mem_obj = bo;
+- *gpu_addr = gpu_addr_tmp;
+- *cpu_ptr = cpu_ptr_tmp;
+-
+- amdgpu_bo_unreserve(bo);
++ amdgpu_bo_unreserve((*mem)->bo);
+
+ return 0;
+
+ allocate_mem_kmap_bo_failed:
+- amdgpu_bo_unpin(bo);
++ amdgpu_bo_unpin((*mem)->bo);
+ allocate_mem_pin_bo_failed:
+- amdgpu_bo_unreserve(bo);
++ amdgpu_bo_unreserve((*mem)->bo);
+ allocate_mem_reserve_bo_failed:
+- amdgpu_bo_unref(&bo);
++ amdgpu_bo_unref(&(*mem)->bo);
+
+ return r;
+ }
+
+ void free_gtt_mem(struct kgd_dev *kgd, void *mem_obj)
+ {
+- struct amdgpu_bo *bo = (struct amdgpu_bo *) mem_obj;
++ struct kgd_mem *mem = (struct kgd_mem *) mem_obj;
+
+- amdgpu_bo_reserve(bo, true);
+- amdgpu_bo_kunmap(bo);
+- amdgpu_bo_unpin(bo);
+- amdgpu_bo_unreserve(bo);
+- amdgpu_bo_unref(&(bo));
++ BUG_ON(mem == NULL);
++
++ amdgpu_bo_reserve(mem->bo, true);
++ amdgpu_bo_kunmap(mem->bo);
++ amdgpu_bo_unpin(mem->bo);
++ amdgpu_bo_unreserve(mem->bo);
++ amdgpu_bo_unref(&(mem->bo));
++ kfree(mem);
+ }
+
+-void get_local_mem_info(struct kgd_dev *kgd,
+- struct kfd_local_mem_info *mem_info)
++uint64_t get_vmem_size(struct kgd_dev *kgd)
+ {
+- uint64_t address_mask;
+- resource_size_t aper_limit;
+- struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
++ struct amdgpu_device *adev =
++ (struct amdgpu_device *)kgd;
+
+- address_mask = adev->dev->dma_mask ? ~*adev->dev->dma_mask :
+- ~((1ULL << 32) - 1);
+- aper_limit = adev->mc.aper_base + adev->mc.aper_size;
+-
+- memset(mem_info, 0, sizeof(*mem_info));
+- if (!(adev->mc.aper_base & address_mask ||
+- aper_limit & address_mask)) {
+- mem_info->local_mem_size_public = adev->mc.visible_vram_size;
+- mem_info->local_mem_size_private = adev->mc.real_vram_size -
+- adev->mc.visible_vram_size;
+- } else {
+- mem_info->local_mem_size_public = 0;
+- mem_info->local_mem_size_private = adev->mc.real_vram_size;
+- }
+- mem_info->vram_width = adev->mc.vram_width;
++ BUG_ON(kgd == NULL);
+
+- pr_debug("Address base: 0x%llx limit 0x%llx public 0x%llx private 0x%llx\n",
+- adev->mc.aper_base, aper_limit,
+- mem_info->local_mem_size_public,
+- mem_info->local_mem_size_private);
+-
+- if (amdgpu_sriov_vf(adev))
+- mem_info->mem_clk_max = adev->clock.default_mclk / 100;
+- else
+- mem_info->mem_clk_max = amdgpu_dpm_get_mclk(adev, false) / 100;
++ return adev->mc.real_vram_size;
+ }
+
+ uint64_t get_gpu_clock_counter(struct kgd_dev *kgd)
+@@ -385,113 +264,6 @@ uint64_t get_gpu_clock_counter(struct kgd_dev *kgd)
+ uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd)
+ {
+ struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+-
+- /* the sclk is in quantas of 10kHz */
+- if (amdgpu_sriov_vf(adev))
+- return adev->clock.default_sclk / 100;
+-
+- return amdgpu_dpm_get_sclk(adev, false) / 100;
+-}
+-
+-void get_cu_info(struct kgd_dev *kgd, struct kfd_cu_info *cu_info)
+-{
+- struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+- struct amdgpu_cu_info acu_info = adev->gfx.cu_info;
+-
+- memset(cu_info, 0, sizeof(*cu_info));
+- if (sizeof(cu_info->cu_bitmap) != sizeof(acu_info.bitmap))
+- return;
+-
+- cu_info->cu_active_number = acu_info.number;
+- cu_info->cu_ao_mask = acu_info.ao_cu_mask;
+- memcpy(&cu_info->cu_bitmap[0], &acu_info.bitmap[0],
+- sizeof(acu_info.bitmap));
+- cu_info->num_shader_engines = adev->gfx.config.max_shader_engines;
+- cu_info->num_shader_arrays_per_engine = adev->gfx.config.max_sh_per_se;
+- cu_info->num_cu_per_sh = adev->gfx.config.max_cu_per_sh;
+- cu_info->simd_per_cu = acu_info.simd_per_cu;
+- cu_info->max_waves_per_simd = acu_info.max_waves_per_simd;
+- cu_info->wave_front_size = acu_info.wave_front_size;
+- cu_info->max_scratch_slots_per_cu = acu_info.max_scratch_slots_per_cu;
+- cu_info->lds_size = acu_info.lds_size;
+-}
+-
+-int amdgpu_amdkfd_get_dmabuf_info(struct kgd_dev *kgd, int dma_buf_fd,
+- struct kgd_dev **dma_buf_kgd,
+- uint64_t *bo_size, void *metadata_buffer,
+- size_t buffer_size, uint32_t *metadata_size,
+- uint32_t *flags)
+-{
+- struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+- struct dma_buf *dma_buf;
+- struct drm_gem_object *obj;
+- struct amdgpu_bo *bo;
+- uint64_t metadata_flags;
+- int r = -EINVAL;
+-
+- dma_buf = dma_buf_get(dma_buf_fd);
+- if (IS_ERR(dma_buf))
+- return PTR_ERR(dma_buf);
+-
+- if (dma_buf->ops != &drm_gem_prime_dmabuf_ops)
+- /* Can't handle non-graphics buffers */
+- goto out_put;
+-
+- obj = dma_buf->priv;
+- if (obj->dev->driver != adev->ddev->driver)
+- /* Can't handle buffers from different drivers */
+- goto out_put;
+-
+- adev = obj->dev->dev_private;
+- bo = gem_to_amdgpu_bo(obj);
+- if (!(bo->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM |
+- AMDGPU_GEM_DOMAIN_GTT |
+- AMDGPU_GEM_DOMAIN_DGMA)))
+- /* Only VRAM, GTT and DGMA BOs are supported */
+- goto out_put;
+-
+- r = 0;
+- if (dma_buf_kgd)
+- *dma_buf_kgd = (struct kgd_dev *)adev;
+- if (bo_size)
+- *bo_size = amdgpu_bo_size(bo);
+- if (metadata_size)
+- *metadata_size = bo->metadata_size;
+- if (metadata_buffer)
+- r = amdgpu_bo_get_metadata(bo, metadata_buffer, buffer_size,
+- metadata_size, &metadata_flags);
+- if (flags) {
+- /* If the preferred domain is DGMA, set flags to VRAM because
+- * KFD doesn't support allocating DGMA memory
+- */
+- *flags = (bo->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM |
+- AMDGPU_GEM_DOMAIN_DGMA)) ?
+- ALLOC_MEM_FLAGS_VRAM : ALLOC_MEM_FLAGS_GTT;
+-
+- if (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)
+- *flags |= ALLOC_MEM_FLAGS_PUBLIC;
+- }
+-
+-out_put:
+- dma_buf_put(dma_buf);
+- return r;
+-}
+-
+-uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd)
+-{
+- struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+- uint64_t usage =
+- amdgpu_vram_mgr_usage(&adev->mman.bdev.man[TTM_PL_VRAM]);
+- return usage;
+-}
+-
+-bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev,
+- u32 vmid)
+-{
+- if (adev->kfd) {
+- if ((1 << vmid) & global_compute_vmid_bitmap)
+- return true;
+- }
+-
+- return false;
++ /* The sclk is in quantas of 10kHz */
++ return adev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk / 100;
+ }
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+index b259ba7..8e8c10e 100755
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+@@ -27,109 +27,21 @@
+
+ #include <linux/types.h>
+ #include <linux/mm.h>
+-#include <linux/workqueue.h>
+-#include <linux/mmu_context.h>
+ #include <kgd_kfd_interface.h>
+-#include "amdgpu.h"
+-
+-extern const struct kgd2kfd_calls *kgd2kfd;
+
+ struct amdgpu_device;
+
+-struct kfd_bo_va_list {
+- struct list_head bo_list;
+- struct amdgpu_bo_va *bo_va;
+- void *kgd_dev;
+- bool is_mapped;
+- bool map_fail;
+- uint64_t va;
+- uint64_t pte_flags;
+-};
+-
+ struct kgd_mem {
+- struct mutex lock;
+ struct amdgpu_bo *bo;
+- struct list_head bo_va_list;
+- /* protected by amdkfd_process_info.lock */
+- struct ttm_validate_buffer validate_list;
+- struct ttm_validate_buffer resv_list;
+- uint32_t domain;
+- unsigned int mapped_to_gpu_memory;
+- void *kptr;
+- uint64_t va;
+-
+- uint32_t mapping_flags;
+-
+- atomic_t invalid;
+- struct amdkfd_process_info *process_info;
+- struct page **user_pages;
+-
+- struct amdgpu_sync sync;
+-
+- /* flags bitfield */
+- bool coherent : 1;
+- bool no_substitute : 1;
+- bool aql_queue : 1;
+-};
+-
+-/* KFD Memory Eviction */
+-struct amdgpu_amdkfd_fence {
+- struct dma_fence base;
+- void *mm;
+- spinlock_t lock;
+- char timeline_name[TASK_COMM_LEN];
+-};
+-
+-struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
+- void *mm);
+-bool amd_kfd_fence_check_mm(struct dma_fence *f, void *mm);
+-struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f);
+-
+-struct amdkfd_process_info {
+- /* List head of all VMs that belong to a KFD process */
+- struct list_head vm_list_head;
+- /* List head for all KFD BOs that belong to a KFD process. */
+- struct list_head kfd_bo_list;
+- /* List of userptr BOs that are valid or invalid */
+- struct list_head userptr_valid_list;
+- struct list_head userptr_inval_list;
+- /* Lock to protect kfd_bo_list */
+- struct mutex lock;
+-
+- /* Number of VMs */
+- unsigned int n_vms;
+- /* Eviction Fence */
+- struct amdgpu_amdkfd_fence *eviction_fence;
+-
+- /* MMU-notifier related fields */
+- atomic_t evicted_bos;
+- struct delayed_work work;
+- struct pid *pid;
+-};
+-
+-/* struct amdkfd_vm -
+- * For Memory Eviction KGD requires a mechanism to keep track of all KFD BOs
+- * belonging to a KFD process. All the VMs belonging to the same process point
+- * to the same amdkfd_process_info.
+- */
+-struct amdkfd_vm {
+- /* Keep base as the first parameter for pointer compatibility between
+- * amdkfd_vm and amdgpu_vm.
+- */
+- struct amdgpu_vm base;
+-
+- /* List node in amdkfd_process_info.vm_list_head*/
+- struct list_head vm_list_node;
+-
+- struct amdgpu_device *adev;
+- /* Points to the KFD process VM info*/
+- struct amdkfd_process_info *process_info;
++ uint64_t gpu_addr;
++ void *cpu_ptr;
+ };
+
+
+ int amdgpu_amdkfd_init(void);
+ void amdgpu_amdkfd_fini(void);
+
++bool amdgpu_amdkfd_load_interface(struct amdgpu_device *adev);
+
+ void amdgpu_amdkfd_suspend(struct amdgpu_device *adev);
+ int amdgpu_amdkfd_resume(struct amdgpu_device *adev);
+@@ -139,105 +51,17 @@ void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev);
+ void amdgpu_amdkfd_device_init(struct amdgpu_device *adev);
+ void amdgpu_amdkfd_device_fini(struct amdgpu_device *adev);
+
+-int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm);
+-int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
+- uint32_t vmid, uint64_t gpu_addr,
+- uint32_t *ib_cmd, uint32_t ib_len);
+-int amdgpu_amdkfd_gpuvm_restore_process_bos(void *process_info,
+- struct dma_fence **ef);
+ struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void);
+ struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void);
+-struct kfd2kgd_calls *amdgpu_amdkfd_gfx_9_0_get_functions(void);
+-int amdgpu_amdkfd_copy_mem_to_mem(struct kgd_dev *kgd, struct kgd_mem *src_mem,
+- uint64_t src_offset, struct kgd_mem *dst_mem,
+- uint64_t dest_offset, uint64_t size, struct dma_fence **f,
+- uint64_t *actual_size);
+-
+-bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev,
+- u32 vmid);
+
+ /* Shared API */
+-int map_bo(struct amdgpu_device *rdev, uint64_t va, void *vm,
+- struct amdgpu_bo *bo, struct amdgpu_bo_va **bo_va);
+ int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
+ void **mem_obj, uint64_t *gpu_addr,
+ void **cpu_ptr);
+ void free_gtt_mem(struct kgd_dev *kgd, void *mem_obj);
+-void get_local_mem_info(struct kgd_dev *kgd,
+- struct kfd_local_mem_info *mem_info);
++uint64_t get_vmem_size(struct kgd_dev *kgd);
+ uint64_t get_gpu_clock_counter(struct kgd_dev *kgd);
+
+ uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd);
+-void get_cu_info(struct kgd_dev *kgd, struct kfd_cu_info *cu_info);
+-int amdgpu_amdkfd_get_dmabuf_info(struct kgd_dev *kgd, int dma_buf_fd,
+- struct kgd_dev **dmabuf_kgd,
+- uint64_t *bo_size, void *metadata_buffer,
+- size_t buffer_size, uint32_t *metadata_size,
+- uint32_t *flags);
+-uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd);
+-
+-#define read_user_wptr(mmptr, wptr, dst) \
+- ({ \
+- bool valid = false; \
+- if ((mmptr) && (wptr)) { \
+- if ((mmptr) == current->mm) { \
+- valid = !get_user((dst), (wptr)); \
+- } else if (current->mm == NULL) { \
+- use_mm(mmptr); \
+- valid = !get_user((dst), (wptr)); \
+- unuse_mm(mmptr); \
+- } \
+- } \
+- valid; \
+- })
+-
+-/* GPUVM API */
+-int amdgpu_amdkfd_gpuvm_sync_memory(
+- struct kgd_dev *kgd, struct kgd_mem *mem, bool intr);
+-int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
+- struct kgd_dev *kgd, uint64_t va, uint64_t size,
+- void *vm, struct kgd_mem **mem,
+- uint64_t *offset, uint32_t flags);
+-int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
+- struct kgd_dev *kgd, struct kgd_mem *mem, void *vm);
+-int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
+- struct kgd_dev *kgd, struct kgd_mem *mem, void *vm);
+-int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
+- struct kgd_dev *kgd, struct kgd_mem *mem, void *vm);
+
+-int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, void **vm,
+- void **process_info,
+- struct dma_fence **ef);
+-void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev *kgd, void *vm);
+-
+-uint32_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm);
+-
+-int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd,
+- struct kfd_vm_fault_info *info);
+-
+-int amdgpu_amdkfd_gpuvm_mmap_bo(
+- struct kgd_dev *kgd, struct vm_area_struct *vma);
+-
+-int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd,
+- struct kgd_mem *mem, void **kptr);
+-
+-int amdgpu_amdkfd_gpuvm_pin_get_sg_table(struct kgd_dev *kgd,
+- struct kgd_mem *mem, uint64_t offset,
+- uint64_t size, struct sg_table **ret_sg);
+-void amdgpu_amdkfd_gpuvm_unpin_put_sg_table(
+- struct kgd_mem *mem, struct sg_table *sg);
+-int amdgpu_amdkfd_gpuvm_import_dmabuf(struct kgd_dev *kgd,
+- struct dma_buf *dmabuf,
+- uint64_t va, void *vm,
+- struct kgd_mem **mem, uint64_t *size,
+- uint64_t *mmap_offset);
+-int amdgpu_amdkfd_gpuvm_export_dmabuf(struct kgd_dev *kgd, void *vm,
+- struct kgd_mem *mem,
+- struct dma_buf **dmabuf);
+-int amdgpu_amdkfd_gpuvm_evict_mem(struct kgd_mem *mem, struct mm_struct *mm);
+-int amdgpu_amdkfd_gpuvm_restore_mem(struct kgd_mem *mem, struct mm_struct *mm);
+-
+-void amdgpu_amdkfd_gpuvm_init_mem_limits(void);
+-void amdgpu_amdkfd_unreserve_system_memory_limit(struct amdgpu_bo *bo);
+ #endif /* AMDGPU_AMDKFD_H_INCLUDED */
+-
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c
+deleted file mode 100644
+index 3961937..0000000
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c
++++ /dev/null
+@@ -1,196 +0,0 @@
+-/*
+- * Copyright 2016 Advanced Micro Devices, Inc.
+- *
+- * Permission is hereby granted, free of charge, to any person obtaining a
+- * copy of this software and associated documentation files (the "Software"),
+- * to deal in the Software without restriction, including without limitation
+- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+- * and/or sell copies of the Software, and to permit persons to whom the
+- * Software is furnished to do so, subject to the following conditions:
+- *
+- * The above copyright notice and this permission notice shall be included in
+- * all copies or substantial portions of the Software.
+- *
+- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+- * OTHER DEALINGS IN THE SOFTWARE.
+- */
+-
+-#include <linux/spinlock.h>
+-#include <linux/atomic.h>
+-#include <linux/stacktrace.h>
+-#include <linux/sched.h>
+-#include <linux/slab.h>
+-#include "amdgpu_amdkfd.h"
+-
+-const struct dma_fence_ops amd_kfd_fence_ops;
+-static atomic_t fence_seq = ATOMIC_INIT(0);
+-
+-static int amd_kfd_fence_signal(struct dma_fence *f);
+-
+-/* Eviction Fence
+- * Fence helper functions to deal with KFD memory eviction.
+- * Big Idea - Since KFD submissions are done by user queues, a BO cannot be
+- * evicted unless all the user queues for that process are evicted.
+- *
+- * All the BOs in a process share an eviction fence. When process X wants
+- * to map VRAM memory but TTM can't find enough space, TTM will attempt to
+- * evict BOs from its LRU list. TTM checks if the BO is valuable to evict
+- * by calling ttm_bo_driver->eviction_valuable().
+- *
+- * ttm_bo_driver->eviction_valuable() - will return false if the BO belongs
+- * to process X. Otherwise, it will return true to indicate BO can be
+- * evicted by TTM.
+- *
+- * If ttm_bo_driver->eviction_valuable returns true, then TTM will continue
+- * the evcition process for that BO by calling ttm_bo_evict --> amdgpu_bo_move
+- * --> amdgpu_copy_buffer(). This sets up job in GPU scheduler.
+- *
+- * GPU Scheduler (amd_sched_main) - sets up a cb (fence_add_callback) to
+- * nofity when the BO is free to move. fence_add_callback --> enable_signaling
+- * --> amdgpu_amdkfd_fence.enable_signaling
+- *
+- * amdgpu_amdkfd_fence.enable_signaling - Start a work item that will quiesce
+- * user queues and signal fence. The work item will also start another delayed
+- * work item to restore BOs
+- */
+-
+-struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
+- void *mm)
+-{
+- struct amdgpu_amdkfd_fence *fence = NULL;
+-
+- fence = kzalloc(sizeof(*fence), GFP_KERNEL);
+- if (fence == NULL)
+- return NULL;
+-
+- /* mm_struct mm is used as void pointer to identify the parent
+- * KFD process. Don't dereference it. Fence and any threads using
+- * mm is guranteed to be released before process termination.
+- */
+- fence->mm = mm;
+- get_task_comm(fence->timeline_name, current);
+- spin_lock_init(&fence->lock);
+-
+- dma_fence_init(&fence->base, &amd_kfd_fence_ops, &fence->lock,
+- context, atomic_inc_return(&fence_seq));
+-
+- return fence;
+-}
+-
+-struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f)
+-{
+- struct amdgpu_amdkfd_fence *fence;
+-
+- if (!f)
+- return NULL;
+-
+- fence = container_of(f, struct amdgpu_amdkfd_fence, base);
+- if (fence && f->ops == &amd_kfd_fence_ops)
+- return fence;
+-
+- return NULL;
+-}
+-
+-static const char *amd_kfd_fence_get_driver_name(struct dma_fence *f)
+-{
+- return "amdgpu_amdkfd_fence";
+-}
+-
+-static const char *amd_kfd_fence_get_timeline_name(struct dma_fence *f)
+-{
+- struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
+-
+- return fence->timeline_name;
+-}
+-
+-/**
+- * amd_kfd_fence_enable_signaling - This gets called when TTM wants to evict
+- * a KFD BO and schedules a job to move the BO.
+- * If fence is already signaled return true.
+- * If fence is not signaled schedule a evict KFD process work item.
+- */
+-static bool amd_kfd_fence_enable_signaling(struct dma_fence *f)
+-{
+- struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
+-
+- if (!fence)
+- return false;
+-
+- if (dma_fence_is_signaled(f))
+- return true;
+-
+- if (!kgd2kfd->schedule_evict_and_restore_process(
+- (struct mm_struct *)fence->mm, f))
+- return true;
+-
+- return false;
+-}
+-
+-static int amd_kfd_fence_signal(struct dma_fence *f)
+-{
+- unsigned long flags;
+- int ret;
+-
+- spin_lock_irqsave(f->lock, flags);
+- /* Set enabled bit so cb will called */
+- set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &f->flags);
+- ret = dma_fence_signal_locked(f);
+- spin_unlock_irqrestore(f->lock, flags);
+-
+- return ret;
+-}
+-
+-/**
+- * amd_kfd_fence_release - callback that fence can be freed
+- *
+- * @fence: fence
+- *
+- * This function is called when the reference count becomes zero.
+- * It just RCU schedules freeing up the fence.
+-*/
+-static void amd_kfd_fence_release(struct dma_fence *f)
+-{
+- struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
+- /* Unconditionally signal the fence. The process is getting
+- * terminated.
+- */
+- if (WARN_ON(!fence))
+- return; /* Not an amdgpu_amdkfd_fence */
+-
+- amd_kfd_fence_signal(f);
+- kfree_rcu(f, rcu);
+-}
+-
+-/**
+- * amd_kfd_fence_check_mm - Check if @mm is same as that of the fence @f
+- * if same return TRUE else return FALSE.
+- *
+- * @f: [IN] fence
+- * @mm: [IN] mm that needs to be verified
+-*/
+-bool amd_kfd_fence_check_mm(struct dma_fence *f, void *mm)
+-{
+- struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
+-
+- if (!fence)
+- return false;
+- else if (fence->mm == mm)
+- return true;
+-
+- return false;
+-}
+-
+-const struct dma_fence_ops amd_kfd_fence_ops = {
+- .get_driver_name = amd_kfd_fence_get_driver_name,
+- .get_timeline_name = amd_kfd_fence_get_timeline_name,
+- .enable_signaling = amd_kfd_fence_enable_signaling,
+- .signaled = NULL,
+- .wait = dma_fence_default_wait,
+- .release = amd_kfd_fence_release,
+-};
+-
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
+old mode 100755
+new mode 100644
+index 6964ece..f6acf48
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
+@@ -20,9 +20,6 @@
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+-#undef pr_fmt
+-#define pr_fmt(fmt) "kfd2kgd: " fmt
+-
+ #include <linux/fdtable.h>
+ #include <linux/uaccess.h>
+ #include <linux/firmware.h>
+@@ -42,14 +39,6 @@
+ #include "gmc/gmc_7_1_sh_mask.h"
+ #include "cik_structs.h"
+
+-#define AMDKFD_SKIP_UNCOMPILED_CODE 1
+-
+-enum hqd_dequeue_request_type {
+- NO_ACTION = 0,
+- DRAIN_PIPE,
+- RESET_WAVES
+-};
+-
+ enum {
+ MAX_TRAPID = 8, /* 3 bits in the bitfield. */
+ MAX_WATCH_ADDRESSES = 4
+@@ -66,8 +55,8 @@ enum {
+ enum {
+ ADDRESS_WATCH_REG_CNTL_ATC_BIT = 0x10000000UL,
+ ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK = 0x00FFFFFF,
+- ADDRESS_WATCH_REG_ADDLOW_MASK_EXTENTION = 0x03000000,
+- /* extend the mask to 26 bits in order to match the low address field */
++ ADDRESS_WATCH_REG_ADDLOW_MASK_EXTENSION = 0x03000000,
++ /* extend the mask to 26 bits to match the low address field */
+ ADDRESS_WATCH_REG_ADDLOW_SHIFT = 6,
+ ADDRESS_WATCH_REG_ADDHIGH_MASK = 0xFFFF
+ };
+@@ -92,42 +81,30 @@ union TCP_WATCH_CNTL_BITS {
+ float f32All;
+ };
+
+-static int open_graphic_handle(struct kgd_dev *kgd, uint64_t va, void *vm,
+- int fd, uint32_t handle, struct kgd_mem **mem);
+-
+-static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type);
+-
+ /*
+ * Register access functions
+ */
+
+ static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
+- uint32_t sh_mem_config, uint32_t sh_mem_ape1_base,
+- uint32_t sh_mem_ape1_limit, uint32_t sh_mem_bases);
++ uint32_t sh_mem_config, uint32_t sh_mem_ape1_base,
++ uint32_t sh_mem_ape1_limit, uint32_t sh_mem_bases);
++
+ static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
+- unsigned int vmid);
++ unsigned int vmid);
++
+ static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
+- uint32_t hpd_size, uint64_t hpd_gpu_addr);
++ uint32_t hpd_size, uint64_t hpd_gpu_addr);
+ static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id);
+ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
+- uint32_t queue_id, uint32_t __user *wptr,
+- uint32_t wptr_shift, uint32_t wptr_mask,
+- struct mm_struct *mm);
+-static int kgd_hqd_dump(struct kgd_dev *kgd,
+- uint32_t pipe_id, uint32_t queue_id,
+- uint32_t (**dump)[2], uint32_t *n_regs);
+-static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
+- uint32_t __user *wptr, struct mm_struct *mm);
+-static int kgd_hqd_sdma_dump(struct kgd_dev *kgd,
+- uint32_t engine_id, uint32_t queue_id,
+- uint32_t (**dump)[2], uint32_t *n_regs);
++ uint32_t queue_id, uint32_t __user *wptr);
++static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd);
+ static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
+- uint32_t pipe_id, uint32_t queue_id);
+-static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd);
+-static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
+- enum kfd_preempt_type reset_type,
++ uint32_t pipe_id, uint32_t queue_id);
++
++static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
+ unsigned int utimeout, uint32_t pipe_id,
+ uint32_t queue_id);
++static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd);
+ static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
+ unsigned int utimeout);
+ static int kgd_address_watch_disable(struct kgd_dev *kgd);
+@@ -147,60 +124,21 @@ static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd, uint8_t vmid);
+ static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd,
+ uint8_t vmid);
+ static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid);
+-static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid);
+-static void set_num_of_requests(struct kgd_dev *dev, uint8_t num_of_req);
+-static int alloc_memory_of_scratch(struct kgd_dev *kgd,
+- uint64_t va, uint32_t vmid);
+-static int write_config_static_mem(struct kgd_dev *kgd, bool swizzle_enable,
+- uint8_t element_size, uint8_t index_stride, uint8_t mtype);
+-static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
+- uint32_t page_table_base);
+-static uint32_t read_vmid_from_vmfault_reg(struct kgd_dev *kgd);
+-
+-/* Because of REG_GET_FIELD() being used, we put this function in the
+- * asic specific file.
+- */
+-static int amdgpu_amdkfd_get_tile_config(struct kgd_dev *kgd,
+- struct tile_config *config)
+-{
+- struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+-
+- config->gb_addr_config = adev->gfx.config.gb_addr_config;
+- config->num_banks = REG_GET_FIELD(adev->gfx.config.mc_arb_ramcfg,
+- MC_ARB_RAMCFG, NOOFBANK);
+- config->num_ranks = REG_GET_FIELD(adev->gfx.config.mc_arb_ramcfg,
+- MC_ARB_RAMCFG, NOOFRANKS);
+
+- config->tile_config_ptr = adev->gfx.config.tile_mode_array;
+- config->num_tile_configs =
+- ARRAY_SIZE(adev->gfx.config.tile_mode_array);
+- config->macro_tile_config_ptr =
+- adev->gfx.config.macrotile_mode_array;
+- config->num_macro_tile_configs =
+- ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);
+-
+-
+- return 0;
+-}
++static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type);
+
+ static const struct kfd2kgd_calls kfd2kgd = {
+ .init_gtt_mem_allocation = alloc_gtt_mem,
+ .free_gtt_mem = free_gtt_mem,
+- .get_local_mem_info = get_local_mem_info,
++ .get_vmem_size = get_vmem_size,
+ .get_gpu_clock_counter = get_gpu_clock_counter,
+ .get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz,
+- .create_process_vm = amdgpu_amdkfd_gpuvm_create_process_vm,
+- .destroy_process_vm = amdgpu_amdkfd_gpuvm_destroy_process_vm,
+- .get_process_page_dir = amdgpu_amdkfd_gpuvm_get_process_page_dir,
+- .open_graphic_handle = open_graphic_handle,
+ .program_sh_mem_settings = kgd_program_sh_mem_settings,
+ .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
+ .init_pipeline = kgd_init_pipeline,
+ .init_interrupts = kgd_init_interrupts,
+ .hqd_load = kgd_hqd_load,
+ .hqd_sdma_load = kgd_hqd_sdma_load,
+- .hqd_dump = kgd_hqd_dump,
+- .hqd_sdma_dump = kgd_hqd_sdma_dump,
+ .hqd_is_occupied = kgd_hqd_is_occupied,
+ .hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied,
+ .hqd_destroy = kgd_hqd_destroy,
+@@ -209,50 +147,17 @@ static const struct kfd2kgd_calls kfd2kgd = {
+ .address_watch_execute = kgd_address_watch_execute,
+ .wave_control_execute = kgd_wave_control_execute,
+ .address_watch_get_offset = kgd_address_watch_get_offset,
+- .get_atc_vmid_pasid_mapping_pasid =
+- get_atc_vmid_pasid_mapping_pasid,
+- .get_atc_vmid_pasid_mapping_valid =
+- get_atc_vmid_pasid_mapping_valid,
+- .read_vmid_from_vmfault_reg = read_vmid_from_vmfault_reg,
++ .get_atc_vmid_pasid_mapping_pasid = get_atc_vmid_pasid_mapping_pasid,
++ .get_atc_vmid_pasid_mapping_valid = get_atc_vmid_pasid_mapping_valid,
+ .write_vmid_invalidate_request = write_vmid_invalidate_request,
+- .invalidate_tlbs = invalidate_tlbs,
+- .sync_memory = amdgpu_amdkfd_gpuvm_sync_memory,
+- .alloc_memory_of_gpu = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu,
+- .free_memory_of_gpu = amdgpu_amdkfd_gpuvm_free_memory_of_gpu,
+- .map_memory_to_gpu = amdgpu_amdkfd_gpuvm_map_memory_to_gpu,
+- .unmap_memory_to_gpu = amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu,
+- .get_fw_version = get_fw_version,
+- .set_num_of_requests = set_num_of_requests,
+- .get_cu_info = get_cu_info,
+- .alloc_memory_of_scratch = alloc_memory_of_scratch,
+- .write_config_static_mem = write_config_static_mem,
+- .mmap_bo = amdgpu_amdkfd_gpuvm_mmap_bo,
+- .map_gtt_bo_to_kernel = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel,
+- .set_vm_context_page_table_base = set_vm_context_page_table_base,
+- .pin_get_sg_table_bo = amdgpu_amdkfd_gpuvm_pin_get_sg_table,
+- .unpin_put_sg_table_bo = amdgpu_amdkfd_gpuvm_unpin_put_sg_table,
+- .get_dmabuf_info = amdgpu_amdkfd_get_dmabuf_info,
+- .import_dmabuf = amdgpu_amdkfd_gpuvm_import_dmabuf,
+- .export_dmabuf = amdgpu_amdkfd_gpuvm_export_dmabuf,
+- .get_vm_fault_info = amdgpu_amdkfd_gpuvm_get_vm_fault_info,
+- .submit_ib = amdgpu_amdkfd_submit_ib,
+- .get_tile_config = amdgpu_amdkfd_get_tile_config,
+- .restore_process_bos = amdgpu_amdkfd_gpuvm_restore_process_bos,
+- .copy_mem_to_mem = amdgpu_amdkfd_copy_mem_to_mem,
+- .get_vram_usage = amdgpu_amdkfd_get_vram_usage
++ .get_fw_version = get_fw_version
+ };
+
+-struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions()
++struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void)
+ {
+ return (struct kfd2kgd_calls *)&kfd2kgd;
+ }
+
+-static int open_graphic_handle(struct kgd_dev *kgd, uint64_t va, void *vm,
+- int fd, uint32_t handle, struct kgd_mem **mem)
+-{
+- return 0;
+-}
+-
+ static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd)
+ {
+ return (struct amdgpu_device *)kgd;
+@@ -281,7 +186,7 @@ static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id,
+ {
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+
+- uint32_t mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
++ uint32_t mec = (++pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
+ uint32_t pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
+
+ lock_srbm(kgd, mec, pipe, queue_id, 0);
+@@ -317,12 +222,12 @@ static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
+
+ /*
+ * We have to assume that there is no outstanding mapping.
+- * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0 because a
+- * mapping is in progress or because a mapping finished and the SW
+- * cleared it. So the protocol is to always wait & clear.
++ * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0 because
++ * a mapping is in progress or because a mapping finished and the
++ * SW cleared it. So the protocol is to always wait & clear.
+ */
+- uint32_t pasid_mapping = (pasid == 0) ? 0 :
+- (uint32_t)pasid | ATC_VMID0_PASID_MAPPING__VALID_MASK;
++ uint32_t pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid |
++ ATC_VMID0_PASID_MAPPING__VALID_MASK;
+
+ WREG32(mmATC_VMID0_PASID_MAPPING + vmid, pasid_mapping);
+
+@@ -368,7 +273,8 @@ static inline uint32_t get_sdma_base_addr(struct cik_sdma_rlc_registers *m)
+
+ retval = m->sdma_engine_id * SDMA1_REGISTER_OFFSET +
+ m->sdma_queue_id * KFD_CIK_SDMA_QUEUE_OFFSET;
+- pr_debug("sdma base address: 0x%x\n", retval);
++
++ pr_debug("kfd: sdma base address: 0x%x\n", retval);
+
+ return retval;
+ }
+@@ -384,138 +290,42 @@ static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
+ }
+
+ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
+- uint32_t queue_id, uint32_t __user *wptr,
+- uint32_t wptr_shift, uint32_t wptr_mask,
+- struct mm_struct *mm)
++ uint32_t queue_id, uint32_t __user *wptr)
+ {
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
++ uint32_t wptr_shadow, is_wptr_shadow_valid;
+ struct cik_mqd *m;
+- uint32_t *mqd_hqd;
+- uint32_t reg, wptr_val, data;
+- bool valid_wptr = false;
+
+ m = get_mqd(mqd);
+
+- acquire_queue(kgd, pipe_id, queue_id);
+-
+- /* HQD registers extend from CP_MQD_BASE_ADDR to CP_MQD_CONTROL. */
+- mqd_hqd = &m->cp_mqd_base_addr_lo;
+-
+- for (reg = mmCP_MQD_BASE_ADDR; reg <= mmCP_MQD_CONTROL; reg++)
+- WREG32(reg, mqd_hqd[reg - mmCP_MQD_BASE_ADDR]);
+-
+- /* Copy userspace write pointer value to register.
+- * Activate doorbell logic to monitor subsequent changes.
+- */
+- data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control,
+- CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
+- WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, data);
+-
+- /* read_user_ptr may take the mm->mmap_sem.
+- * release srbm_mutex to avoid circular dependency between
+- * srbm_mutex->mm_sem->reservation_ww_class_mutex->srbm_mutex.
+- */
+- release_queue(kgd);
+- valid_wptr = read_user_wptr(mm, wptr, wptr_val);
++ is_wptr_shadow_valid = !get_user(wptr_shadow, wptr);
++ if (is_wptr_shadow_valid)
++ m->cp_hqd_pq_wptr = wptr_shadow;
+
+ acquire_queue(kgd, pipe_id, queue_id);
+- if (valid_wptr)
+- WREG32(mmCP_HQD_PQ_WPTR, (wptr_val << wptr_shift) & wptr_mask);
+-
+- data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1);
+- WREG32(mmCP_HQD_ACTIVE, data);
+-
+
++ gfx_v7_0_mqd_commit(adev, m);
+ release_queue(kgd);
+
+ return 0;
+ }
+
+-static int kgd_hqd_dump(struct kgd_dev *kgd,
+- uint32_t pipe_id, uint32_t queue_id,
+- uint32_t (**dump)[2], uint32_t *n_regs)
+-{
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+- uint32_t i = 0, reg;
+-#define HQD_N_REGS (35+4)
+-#define DUMP_REG(addr) do { \
+- if (WARN_ON_ONCE(i >= HQD_N_REGS)) \
+- break; \
+- (*dump)[i][0] = (addr) << 2; \
+- (*dump)[i++][1] = RREG32(addr); \
+- } while (0)
+-
+- *dump = kmalloc(HQD_N_REGS*2*sizeof(uint32_t), GFP_KERNEL);
+- if (*dump == NULL)
+- return -ENOMEM;
+-
+- acquire_queue(kgd, pipe_id, queue_id);
+-
+- DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE0);
+- DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE1);
+- DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE2);
+- DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE3);
+-
+- for (reg = mmCP_MQD_BASE_ADDR; reg <= mmCP_MQD_CONTROL; reg++)
+- DUMP_REG(reg);
+-
+- release_queue(kgd);
+-
+- WARN_ON_ONCE(i != HQD_N_REGS);
+- *n_regs = i;
+-
+- return 0;
+-}
+-
+-static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
+- uint32_t __user *wptr, struct mm_struct *mm)
++static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd)
+ {
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ struct cik_sdma_rlc_registers *m;
+ unsigned long end_jiffies;
+ uint32_t sdma_base_addr;
+- uint32_t data;
+
+ m = get_sdma_mqd(mqd);
+ sdma_base_addr = get_sdma_base_addr(m);
+
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL,
+- m->sdma_rlc_rb_cntl & (~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK));
+-
+- while (true) {
+- temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS);
+- if (temp & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK)
+- break;
+- if (timeout == 0)
+- return -ETIME;
+- msleep(10);
+- timeout -= 10;
+- }
+- if (m->sdma_engine_id) {
+- data = RREG32(mmSDMA1_GFX_CONTEXT_CNTL);
+- data = REG_SET_FIELD(data, SDMA1_GFX_CONTEXT_CNTL,
+- RESUME_CTX, 0);
+- WREG32(mmSDMA1_GFX_CONTEXT_CNTL, data);
+- } else {
+- data = RREG32(mmSDMA0_GFX_CONTEXT_CNTL);
+- data = REG_SET_FIELD(data, SDMA0_GFX_CONTEXT_CNTL,
+- RESUME_CTX, 0);
+- WREG32(mmSDMA0_GFX_CONTEXT_CNTL, data);
+- }
+-
+- data = REG_SET_FIELD(m->sdma_rlc_doorbell, SDMA0_RLC0_DOORBELL,
+- ENABLE, 1);
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, data);
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR, m->sdma_rlc_rb_rptr);
+- if (read_user_wptr(mm, wptr, data))
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR, data);
+- else
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR,
+- m->sdma_rlc_rb_rptr);
+-
++ WREG32(sdma_base_addr + mmSDMA0_RLC0_VIRTUAL_ADDR,
++ m->sdma_rlc_virtual_addr);
++ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE,
++ m->sdma_rlc_rb_base);
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_VIRTUAL_ADDR,
+ m->sdma_rlc_virtual_addr);
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE, m->sdma_rlc_rb_base);
+
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE_HI,
+ m->sdma_rlc_rb_base_hi);
+@@ -523,35 +333,11 @@ static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
+ m->sdma_rlc_rb_rptr_addr_lo);
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_HI,
+ m->sdma_rlc_rb_rptr_addr_hi);
+- data = REG_SET_FIELD(m->sdma_rlc_rb_cntl, SDMA0_RLC0_RB_CNTL,
+- RB_ENABLE, 1);
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, data);
+- return 0;
+-}
++ WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL,
++ m->sdma_rlc_doorbell);
+
+-static int kgd_hqd_sdma_dump(struct kgd_dev *kgd,
+- uint32_t engine_id, uint32_t queue_id,
+- uint32_t (**dump)[2], uint32_t *n_regs)
+-{
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+- uint32_t sdma_offset = engine_id * SDMA1_REGISTER_OFFSET +
+- queue_id * KFD_CIK_SDMA_QUEUE_OFFSET;
+- uint32_t i = 0, reg;
+-#undef HQD_N_REGS
+-#define HQD_N_REGS (19+4)
+-
+- *dump = kmalloc(HQD_N_REGS*2*sizeof(uint32_t), GFP_KERNEL);
+- if (*dump == NULL)
+- return -ENOMEM;
+-
+- for (reg = mmSDMA0_RLC0_RB_CNTL; reg <= mmSDMA0_RLC0_DOORBELL; reg++)
+- DUMP_REG(sdma_offset + reg);
+- for (reg = mmSDMA0_RLC0_VIRTUAL_ADDR; reg <= mmSDMA0_RLC0_WATERMARK;
+- reg++)
+- DUMP_REG(sdma_offset + reg);
+-
+- WARN_ON_ONCE(i != HQD_N_REGS);
+- *n_regs = i;
++ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL,
++ m->sdma_rlc_rb_cntl);
+
+ return 0;
+ }
+@@ -596,99 +382,30 @@ static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
+ return false;
+ }
+
+-static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
+- enum kfd_preempt_type reset_type,
++static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
+ unsigned int utimeout, uint32_t pipe_id,
+ uint32_t queue_id)
+ {
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ uint32_t temp;
+- enum hqd_dequeue_request_type type;
+- unsigned long flags, end_jiffies;
+- int retry;
++ int timeout = utimeout;
+
+ acquire_queue(kgd, pipe_id, queue_id);
+ WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
+
+- switch (reset_type) {
+- case KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN:
+- type = DRAIN_PIPE;
+- break;
+- case KFD_PREEMPT_TYPE_WAVEFRONT_RESET:
+- type = RESET_WAVES;
+- break;
+- default:
+- type = DRAIN_PIPE;
+- break;
+- }
+-
+- /* Workaround: If IQ timer is active and the wait time is close to or
+- * equal to 0, dequeueing is not safe. Wait until either the wait time
+- * is larger or timer is cleared. Also, ensure that IQ_REQ_PEND is
+- * cleared before continuing. Also, ensure wait times are set to at
+- * least 0x3.
+- */
+- local_irq_save(flags);
+- preempt_disable();
+- retry = 5000; /* wait for 500 usecs at maximum */
+- while (true) {
+- temp = RREG32(mmCP_HQD_IQ_TIMER);
+- if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, PROCESSING_IQ)) {
+- pr_debug("HW is processing IQ\n");
+- goto loop;
+- }
+- if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, ACTIVE)) {
+- if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, RETRY_TYPE)
+- == 3) /* SEM-rearm is safe */
+- break;
+- /* Wait time 3 is safe for CP, but our MMIO read/write
+- * time is close to 1 microsecond, so check for 10 to
+- * leave more buffer room
+- */
+- if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, WAIT_TIME)
+- >= 10)
+- break;
+- pr_debug("IQ timer is active\n");
+- } else
+- break;
+-loop:
+- if (!retry) {
+- pr_err("CP HQD IQ timer status time out\n");
+- break;
+- }
+- ndelay(100);
+- --retry;
+- }
+- retry = 1000;
+- while (true) {
+- temp = RREG32(mmCP_HQD_DEQUEUE_REQUEST);
+- if (!(temp & CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_MASK))
+- break;
+- pr_debug("Dequeue request is pending\n");
+-
+- if (!retry) {
+- pr_err("CP HQD dequeue request time out\n");
+- break;
+- }
+- ndelay(100);
+- --retry;
+- }
+- local_irq_restore(flags);
+- preempt_enable();
+-
+- WREG32(mmCP_HQD_DEQUEUE_REQUEST, type);
++ WREG32(mmCP_HQD_DEQUEUE_REQUEST, reset_type);
+
+- end_jiffies = (utimeout * HZ / 1000) + jiffies;
+ while (true) {
+ temp = RREG32(mmCP_HQD_ACTIVE);
+- if (!(temp & CP_HQD_ACTIVE__ACTIVE_MASK))
++ if (temp & CP_HQD_ACTIVE__ACTIVE_MASK)
+ break;
+- if (time_after(jiffies, end_jiffies)) {
+- pr_err("cp queue preemption time out\n");
++ if (timeout <= 0) {
++ pr_err("kfd: cp queue preemption time out.\n");
+ release_queue(kgd);
+ return -ETIME;
+ }
+- usleep_range(500, 1000);
++ msleep(20);
++ timeout -= 20;
+ }
+
+ release_queue(kgd);
+@@ -702,7 +419,7 @@ static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
+ struct cik_sdma_rlc_registers *m;
+ uint32_t sdma_base_addr;
+ uint32_t temp;
+- unsigned long end_jiffies = (utimeout * HZ / 1000) + jiffies;
++ int timeout = utimeout;
+
+ m = get_sdma_mqd(mqd);
+ sdma_base_addr = get_sdma_base_addr(m);
+@@ -713,19 +430,18 @@ static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
+
+ while (true) {
+ temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS);
+- if (temp & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK)
++ if (temp & SDMA0_STATUS_REG__RB_CMD_IDLE__SHIFT)
+ break;
+- if (time_after(jiffies, end_jiffies))
++ if (timeout <= 0)
+ return -ETIME;
+- usleep_range(500, 1000);
++ msleep(20);
++ timeout -= 20;
+ }
+
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, 0);
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL,
+- RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL) |
+- SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK);
+-
+- m->sdma_rlc_rb_rptr = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR);
++ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR, 0);
++ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR, 0);
++ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE, 0);
+
+ return 0;
+ }
+@@ -744,9 +460,8 @@ static int kgd_address_watch_disable(struct kgd_dev *kgd)
+
+ /* Turning off this address until we set all the registers */
+ for (i = 0; i < MAX_WATCH_ADDRESSES; i++)
+- WREG32(watchRegs[i * ADDRESS_WATCH_REG_MAX
+- + ADDRESS_WATCH_REG_CNTL],
+- cntl.u32All);
++ WREG32(watchRegs[i * ADDRESS_WATCH_REG_MAX +
++ ADDRESS_WATCH_REG_CNTL], cntl.u32All);
+
+ return 0;
+ }
+@@ -764,24 +479,20 @@ static int kgd_address_watch_execute(struct kgd_dev *kgd,
+
+ /* Turning off this watch point until we set all the registers */
+ cntl.bitfields.valid = 0;
+- WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX
+- + ADDRESS_WATCH_REG_CNTL],
+- cntl.u32All);
++ WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX +
++ ADDRESS_WATCH_REG_CNTL], cntl.u32All);
+
+- WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX
+- + ADDRESS_WATCH_REG_ADDR_HI],
+- addr_hi);
++ WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX +
++ ADDRESS_WATCH_REG_ADDR_HI], addr_hi);
+
+- WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX
+- + ADDRESS_WATCH_REG_ADDR_LO],
+- addr_lo);
++ WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX +
++ ADDRESS_WATCH_REG_ADDR_LO], addr_lo);
+
+ /* Enable the watch point */
+ cntl.bitfields.valid = 1;
+
+- WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX
+- + ADDRESS_WATCH_REG_CNTL],
+- cntl.u32All);
++ WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX +
++ ADDRESS_WATCH_REG_CNTL], cntl.u32All);
+
+ return 0;
+ }
+@@ -835,7 +546,7 @@ static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd,
+ struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+
+ reg = RREG32(mmATC_VMID0_PASID_MAPPING + vmid);
+- return reg & ATC_VMID0_PASID_MAPPING__PASID_MASK;
++ return reg & ATC_VMID0_PASID_MAPPING__VALID_MASK;
+ }
+
+ static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid)
+@@ -845,90 +556,52 @@ static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid)
+ WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);
+ }
+
+-static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid)
+-{
+- struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+- int vmid;
+-
+- for (vmid = 0; vmid < 16; vmid++) {
+- if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid))
+- continue;
+- if (RREG32(mmATC_VMID0_PASID_MAPPING + vmid) &
+- ATC_VMID0_PASID_MAPPING__VALID_MASK) {
+- if ((RREG32(mmATC_VMID0_PASID_MAPPING + vmid) &
+- ATC_VMID0_PASID_MAPPING__PASID_MASK) == pasid) {
+- WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);
+- break;
+- }
+- }
+- }
+-
+- return 0;
+-}
+-
+-static int write_config_static_mem(struct kgd_dev *kgd, bool swizzle_enable,
+- uint8_t element_size, uint8_t index_stride, uint8_t mtype)
+-{
+- uint32_t reg;
+- struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+-
+- reg = swizzle_enable << SH_STATIC_MEM_CONFIG__SWIZZLE_ENABLE__SHIFT |
+- element_size << SH_STATIC_MEM_CONFIG__ELEMENT_SIZE__SHIFT |
+- index_stride << SH_STATIC_MEM_CONFIG__INDEX_STRIDE__SHIFT |
+- mtype << SH_STATIC_MEM_CONFIG__PRIVATE_MTYPE__SHIFT;
+-
+- WREG32(mmSH_STATIC_MEM_CONFIG, reg);
+- return 0;
+-}
+-static int alloc_memory_of_scratch(struct kgd_dev *kgd,
+- uint64_t va, uint32_t vmid)
+-{
+- struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+-
+- lock_srbm(kgd, 0, 0, 0, vmid);
+- WREG32(mmSH_HIDDEN_PRIVATE_BASE_VMID, va);
+- unlock_srbm(kgd);
+-
+- return 0;
+-}
+-
+-
+ static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type)
+ {
+ struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+ const union amdgpu_firmware_header *hdr;
+
++ BUG_ON(kgd == NULL);
++
+ switch (type) {
+ case KGD_ENGINE_PFP:
+- hdr = (const union amdgpu_firmware_header *)adev->gfx.pfp_fw->data;
++ hdr = (const union amdgpu_firmware_header *)
++ adev->gfx.pfp_fw->data;
+ break;
+
+ case KGD_ENGINE_ME:
+- hdr = (const union amdgpu_firmware_header *)adev->gfx.me_fw->data;
++ hdr = (const union amdgpu_firmware_header *)
++ adev->gfx.me_fw->data;
+ break;
+
+ case KGD_ENGINE_CE:
+- hdr = (const union amdgpu_firmware_header *)adev->gfx.ce_fw->data;
++ hdr = (const union amdgpu_firmware_header *)
++ adev->gfx.ce_fw->data;
+ break;
+
+ case KGD_ENGINE_MEC1:
+- hdr = (const union amdgpu_firmware_header *)adev->gfx.mec_fw->data;
++ hdr = (const union amdgpu_firmware_header *)
++ adev->gfx.mec_fw->data;
+ break;
+
+ case KGD_ENGINE_MEC2:
+- hdr = (const union amdgpu_firmware_header *)adev->gfx.mec2_fw->data;
++ hdr = (const union amdgpu_firmware_header *)
++ adev->gfx.mec2_fw->data;
+ break;
+
+ case KGD_ENGINE_RLC:
+- hdr = (const union amdgpu_firmware_header *)adev->gfx.rlc_fw->data;
++ hdr = (const union amdgpu_firmware_header *)
++ adev->gfx.rlc_fw->data;
+ break;
+
+ case KGD_ENGINE_SDMA1:
+- hdr = (const union amdgpu_firmware_header *)adev->sdma.instance[0].fw->data;
++ hdr = (const union amdgpu_firmware_header *)
++ adev->sdma.instance[0].fw->data;
+ break;
+
+ case KGD_ENGINE_SDMA2:
+- hdr = (const union amdgpu_firmware_header *)adev->sdma.instance[1].fw->data;
++ hdr = (const union amdgpu_firmware_header *)
++ adev->sdma.instance[1].fw->data;
+ break;
+
+ default:
+@@ -942,42 +615,3 @@ static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type)
+ return hdr->common.ucode_version;
+ }
+
+-static void set_num_of_requests(struct kgd_dev *dev, uint8_t num_of_req)
+-{
+- uint32_t value;
+- struct amdgpu_device *adev = get_amdgpu_device(dev);
+-
+- value = RREG32(mmATC_ATS_DEBUG);
+- value &= ~ATC_ATS_DEBUG__NUM_REQUESTS_AT_ERR_MASK;
+- value |= (num_of_req << ATC_ATS_DEBUG__NUM_REQUESTS_AT_ERR__SHIFT);
+-
+- WREG32(mmATC_ATS_DEBUG, value);
+-}
+-
+-static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
+- uint32_t page_table_base)
+-{
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+- /* TODO: Don't use hardcoded VMIDs */
+- if (vmid < 8 || vmid > 15) {
+- pr_err("trying to set page table base for wrong VMID\n");
+- return;
+- }
+- WREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vmid - 8, page_table_base);
+-}
+-
+- /**
+- * read_vmid_from_vmfault_reg - read vmid from register
+- *
+- * adev: amdgpu_device pointer
+- * @vmid: vmid pointer
+- * read vmid from register (CIK).
+- */
+-static uint32_t read_vmid_from_vmfault_reg(struct kgd_dev *kgd)
+-{
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+-
+- uint32_t status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS);
+-
+- return REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, VMID);
+-}
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
+old mode 100755
+new mode 100644
+index 2ff10e9..133d066
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
+@@ -20,9 +20,6 @@
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+-#undef pr_fmt
+-#define pr_fmt(fmt) "kfd2kgd: " fmt
+-
+ #include <linux/module.h>
+ #include <linux/fdtable.h>
+ #include <linux/uaccess.h>
+@@ -31,7 +28,7 @@
+ #include "amdgpu.h"
+ #include "amdgpu_amdkfd.h"
+ #include "amdgpu_ucode.h"
+-#include "amdgpu_amdkfd_gfx_v8.h"
++#include "gfx_v8_0.h"
+ #include "gca/gfx_8_0_sh_mask.h"
+ #include "gca/gfx_8_0_d.h"
+ #include "gca/gfx_8_0_enum.h"
+@@ -42,31 +39,7 @@
+ #include "vi_structs.h"
+ #include "vid.h"
+
+-enum hqd_dequeue_request_type {
+- NO_ACTION = 0,
+- DRAIN_PIPE,
+- RESET_WAVES,
+- SAVE_WAVES
+-};
+-
+-static const uint32_t watchRegs[MAX_WATCH_ADDRESSES * ADDRESS_WATCH_REG_MAX] = {
+- mmTCP_WATCH0_ADDR_H, mmTCP_WATCH0_ADDR_L, mmTCP_WATCH0_CNTL,
+- mmTCP_WATCH1_ADDR_H, mmTCP_WATCH1_ADDR_L, mmTCP_WATCH1_CNTL,
+- mmTCP_WATCH2_ADDR_H, mmTCP_WATCH2_ADDR_L, mmTCP_WATCH2_CNTL,
+- mmTCP_WATCH3_ADDR_H, mmTCP_WATCH3_ADDR_L, mmTCP_WATCH3_CNTL
+-};
+-
+-
+-struct vi_sdma_mqd;
+-
+-static int create_process_gpumem(struct kgd_dev *kgd, uint64_t va, size_t size,
+- void *vm, struct kgd_mem **mem);
+-static void destroy_process_gpumem(struct kgd_dev *kgd, struct kgd_mem *mem);
+-
+-static int open_graphic_handle(struct kgd_dev *kgd, uint64_t va, void *vm,
+- int fd, uint32_t handle, struct kgd_mem **mem);
+-
+-static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type);
++struct cik_sdma_rlc_registers;
+
+ /*
+ * Register access functions
+@@ -82,26 +55,17 @@ static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
+ uint32_t hpd_size, uint64_t hpd_gpu_addr);
+ static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id);
+ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
+- uint32_t queue_id, uint32_t __user *wptr,
+- uint32_t wptr_shift, uint32_t wptr_mask,
+- struct mm_struct *mm);
+-static int kgd_hqd_dump(struct kgd_dev *kgd,
+- uint32_t pipe_id, uint32_t queue_id,
+- uint32_t (**dump)[2], uint32_t *n_regs);
+-static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
+- uint32_t __user *wptr, struct mm_struct *mm);
+-static int kgd_hqd_sdma_dump(struct kgd_dev *kgd,
+- uint32_t engine_id, uint32_t queue_id,
+- uint32_t (**dump)[2], uint32_t *n_regs);
++ uint32_t queue_id, uint32_t __user *wptr);
++static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd);
+ static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
+ uint32_t pipe_id, uint32_t queue_id);
+ static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd);
+-static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
+- enum kfd_preempt_type reset_type,
++static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
+ unsigned int utimeout, uint32_t pipe_id,
+ uint32_t queue_id);
+ static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
+ unsigned int utimeout);
++static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid);
+ static int kgd_address_watch_disable(struct kgd_dev *kgd);
+ static int kgd_address_watch_execute(struct kgd_dev *kgd,
+ unsigned int watch_point_id,
+@@ -120,61 +84,20 @@ static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd,
+ static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd,
+ uint8_t vmid);
+ static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid);
+-static void set_num_of_requests(struct kgd_dev *kgd,
+- uint8_t num_of_requests);
+-static int alloc_memory_of_scratch(struct kgd_dev *kgd,
+- uint64_t va, uint32_t vmid);
+-static int write_config_static_mem(struct kgd_dev *kgd, bool swizzle_enable,
+- uint8_t element_size, uint8_t index_stride, uint8_t mtype);
+-static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
+- uint32_t page_table_base);
+-static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid);
+-
+-/* Because of REG_GET_FIELD() being used, we put this function in the
+- * asic specific file.
+- */
+-static int amdgpu_amdkfd_get_tile_config(struct kgd_dev *kgd,
+- struct tile_config *config)
+-{
+- struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+-
+- config->gb_addr_config = adev->gfx.config.gb_addr_config;
+- config->num_banks = REG_GET_FIELD(adev->gfx.config.mc_arb_ramcfg,
+- MC_ARB_RAMCFG, NOOFBANK);
+- config->num_ranks = REG_GET_FIELD(adev->gfx.config.mc_arb_ramcfg,
+- MC_ARB_RAMCFG, NOOFRANKS);
+-
+- config->tile_config_ptr = adev->gfx.config.tile_mode_array;
+- config->num_tile_configs =
+- ARRAY_SIZE(adev->gfx.config.tile_mode_array);
+- config->macro_tile_config_ptr =
+- adev->gfx.config.macrotile_mode_array;
+- config->num_macro_tile_configs =
+- ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);
+-
+- return 0;
+-}
++static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type);
+
+ static const struct kfd2kgd_calls kfd2kgd = {
+ .init_gtt_mem_allocation = alloc_gtt_mem,
+ .free_gtt_mem = free_gtt_mem,
+- .get_local_mem_info = get_local_mem_info,
++ .get_vmem_size = get_vmem_size,
+ .get_gpu_clock_counter = get_gpu_clock_counter,
+ .get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz,
+- .create_process_vm = amdgpu_amdkfd_gpuvm_create_process_vm,
+- .destroy_process_vm = amdgpu_amdkfd_gpuvm_destroy_process_vm,
+- .create_process_gpumem = create_process_gpumem,
+- .destroy_process_gpumem = destroy_process_gpumem,
+- .get_process_page_dir = amdgpu_amdkfd_gpuvm_get_process_page_dir,
+- .open_graphic_handle = open_graphic_handle,
+ .program_sh_mem_settings = kgd_program_sh_mem_settings,
+ .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
+ .init_pipeline = kgd_init_pipeline,
+ .init_interrupts = kgd_init_interrupts,
+ .hqd_load = kgd_hqd_load,
+ .hqd_sdma_load = kgd_hqd_sdma_load,
+- .hqd_dump = kgd_hqd_dump,
+- .hqd_sdma_dump = kgd_hqd_sdma_dump,
+ .hqd_is_occupied = kgd_hqd_is_occupied,
+ .hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied,
+ .hqd_destroy = kgd_hqd_destroy,
+@@ -188,56 +111,14 @@ static const struct kfd2kgd_calls kfd2kgd = {
+ .get_atc_vmid_pasid_mapping_valid =
+ get_atc_vmid_pasid_mapping_valid,
+ .write_vmid_invalidate_request = write_vmid_invalidate_request,
+- .invalidate_tlbs = invalidate_tlbs,
+- .sync_memory = amdgpu_amdkfd_gpuvm_sync_memory,
+- .alloc_memory_of_gpu = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu,
+- .free_memory_of_gpu = amdgpu_amdkfd_gpuvm_free_memory_of_gpu,
+- .map_memory_to_gpu = amdgpu_amdkfd_gpuvm_map_memory_to_gpu,
+- .unmap_memory_to_gpu = amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu,
+- .get_fw_version = get_fw_version,
+- .set_num_of_requests = set_num_of_requests,
+- .get_cu_info = get_cu_info,
+- .alloc_memory_of_scratch = alloc_memory_of_scratch,
+- .write_config_static_mem = write_config_static_mem,
+- .mmap_bo = amdgpu_amdkfd_gpuvm_mmap_bo,
+- .map_gtt_bo_to_kernel = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel,
+- .set_vm_context_page_table_base = set_vm_context_page_table_base,
+- .pin_get_sg_table_bo = amdgpu_amdkfd_gpuvm_pin_get_sg_table,
+- .unpin_put_sg_table_bo = amdgpu_amdkfd_gpuvm_unpin_put_sg_table,
+- .get_dmabuf_info = amdgpu_amdkfd_get_dmabuf_info,
+- .import_dmabuf = amdgpu_amdkfd_gpuvm_import_dmabuf,
+- .export_dmabuf = amdgpu_amdkfd_gpuvm_export_dmabuf,
+- .get_vm_fault_info = amdgpu_amdkfd_gpuvm_get_vm_fault_info,
+- .submit_ib = amdgpu_amdkfd_submit_ib,
+- .get_tile_config = amdgpu_amdkfd_get_tile_config,
+- .restore_process_bos = amdgpu_amdkfd_gpuvm_restore_process_bos,
+- .copy_mem_to_mem = amdgpu_amdkfd_copy_mem_to_mem,
+- .get_vram_usage = amdgpu_amdkfd_get_vram_usage
++ .get_fw_version = get_fw_version
+ };
+
+-struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions()
++struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void)
+ {
+ return (struct kfd2kgd_calls *)&kfd2kgd;
+ }
+
+-static int create_process_gpumem(struct kgd_dev *kgd, uint64_t va, size_t size,
+- void *vm, struct kgd_mem **mem)
+-{
+- return 0;
+-}
+-
+-/* Destroys the GPU allocation and frees the kgd_mem structure */
+-static void destroy_process_gpumem(struct kgd_dev *kgd, struct kgd_mem *mem)
+-{
+-
+-}
+-
+-static int open_graphic_handle(struct kgd_dev *kgd, uint64_t va, void *vm,
+- int fd, uint32_t handle, struct kgd_mem **mem)
+-{
+- return 0;
+-}
+-
+ static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd)
+ {
+ return (struct amdgpu_device *)kgd;
+@@ -266,7 +147,7 @@ static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id,
+ {
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+
+- uint32_t mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
++ uint32_t mec = (++pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
+ uint32_t pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
+
+ lock_srbm(kgd, mec, pipe, queue_id, 0);
+@@ -335,28 +216,21 @@ static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id)
+ uint32_t mec;
+ uint32_t pipe;
+
+- mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
++ mec = (++pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
+ pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
+
+ lock_srbm(kgd, mec, pipe, 0, 0);
+
+- WREG32(mmCPC_INT_CNTL, CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK |
+- CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK);
++ WREG32(mmCPC_INT_CNTL, CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK);
+
+ unlock_srbm(kgd);
+
+ return 0;
+ }
+
+-static inline uint32_t get_sdma_base_addr(struct vi_sdma_mqd *m)
++static inline uint32_t get_sdma_base_addr(struct cik_sdma_rlc_registers *m)
+ {
+- uint32_t retval;
+-
+- retval = m->sdma_engine_id * SDMA1_REGISTER_OFFSET +
+- m->sdma_queue_id * KFD_VI_SDMA_QUEUE_OFFSET;
+- pr_debug("sdma base address: 0x%x\n", retval);
+-
+- return retval;
++ return 0;
+ }
+
+ static inline struct vi_mqd *get_mqd(void *mqd)
+@@ -364,224 +238,33 @@ static inline struct vi_mqd *get_mqd(void *mqd)
+ return (struct vi_mqd *)mqd;
+ }
+
+-static inline struct vi_sdma_mqd *get_sdma_mqd(void *mqd)
++static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
+ {
+- return (struct vi_sdma_mqd *)mqd;
++ return (struct cik_sdma_rlc_registers *)mqd;
+ }
+
+ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
+- uint32_t queue_id, uint32_t __user *wptr,
+- uint32_t wptr_shift, uint32_t wptr_mask,
+- struct mm_struct *mm)
++ uint32_t queue_id, uint32_t __user *wptr)
+ {
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ struct vi_mqd *m;
+- uint32_t *mqd_hqd;
+- uint32_t reg, wptr_val, data;
+- bool valid_wptr = false;
++ uint32_t shadow_wptr, valid_wptr;
++ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+
+ m = get_mqd(mqd);
+
+- acquire_queue(kgd, pipe_id, queue_id);
+-
+- /* HIQ is set during driver init period with vmid set to 0. For SRIOV
+- * world switching support let the RLC know about the HIQ.
+- *
+- * Workaround: This causes reboots on CZ. Disable this on CZ, which
+- * doesn't support SRIOV anyway.
+- */
+- if (m->cp_hqd_vmid == 0 &&
+- adev->asic_type != CHIP_CARRIZO) {
+- uint32_t value, mec, pipe;
+-
+- mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
+- pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
+-
+- pr_debug("kfd: set HIQ, mec:%d, pipe:%d, queue:%d.\n",
+- mec, pipe, queue_id);
+- value = RREG32(mmRLC_CP_SCHEDULERS);
+- value = REG_SET_FIELD(value, RLC_CP_SCHEDULERS, scheduler1,
+- ((mec << 5) | (pipe << 3) | queue_id | 0x80));
+- WREG32(mmRLC_CP_SCHEDULERS, value);
+- }
+-
+- /* HQD registers extend from CP_MQD_BASE_ADDR to CP_HQD_EOP_WPTR_MEM. */
+- mqd_hqd = &m->cp_mqd_base_addr_lo;
++ valid_wptr = copy_from_user(&shadow_wptr, wptr, sizeof(shadow_wptr));
++ if (valid_wptr == 0)
++ m->cp_hqd_pq_wptr = shadow_wptr;
+
+- for (reg = mmCP_MQD_BASE_ADDR; reg <= mmCP_HQD_EOP_CONTROL; reg++)
+- WREG32(reg, mqd_hqd[reg - mmCP_MQD_BASE_ADDR]);
+-
+- /* Tonga errata: EOP RPTR/WPTR should be left unmodified.
+- * This is safe since EOP RPTR==WPTR for any inactive HQD
+- * on ASICs that do not support context-save.
+- * EOP writes/reads can start anywhere in the ring.
+- */
+- if (get_amdgpu_device(kgd)->asic_type != CHIP_TONGA) {
+- WREG32(mmCP_HQD_EOP_RPTR, m->cp_hqd_eop_rptr);
+- WREG32(mmCP_HQD_EOP_WPTR, m->cp_hqd_eop_wptr);
+- WREG32(mmCP_HQD_EOP_WPTR_MEM, m->cp_hqd_eop_wptr_mem);
+- }
+-
+- for (reg = mmCP_HQD_EOP_EVENTS; reg <= mmCP_HQD_ERROR; reg++)
+- WREG32(reg, mqd_hqd[reg - mmCP_MQD_BASE_ADDR]);
+-
+- /* Copy userspace write pointer value to register.
+- * Activate doorbell logic to monitor subsequent changes.
+- */
+- data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control,
+- CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
+- WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, data);
+-
+- /* read_user_ptr may take the mm->mmap_sem.
+- * release srbm_mutex to avoid circular dependency between
+- * srbm_mutex->mm_sem->reservation_ww_class_mutex->srbm_mutex.
+- */
+- release_queue(kgd);
+- valid_wptr = read_user_wptr(mm, wptr, wptr_val);
+ acquire_queue(kgd, pipe_id, queue_id);
+- if (valid_wptr)
+- WREG32(mmCP_HQD_PQ_WPTR, (wptr_val << wptr_shift) & wptr_mask);
+-
+- data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1);
+- WREG32(mmCP_HQD_ACTIVE, data);
+-
++ gfx_v8_0_mqd_commit(adev, mqd);
+ release_queue(kgd);
+
+ return 0;
+ }
+
+-static int kgd_hqd_dump(struct kgd_dev *kgd,
+- uint32_t pipe_id, uint32_t queue_id,
+- uint32_t (**dump)[2], uint32_t *n_regs)
++static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd)
+ {
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+- uint32_t i = 0, reg;
+-#define HQD_N_REGS (54+4)
+-#define DUMP_REG(addr) do { \
+- if (WARN_ON_ONCE(i >= HQD_N_REGS)) \
+- break; \
+- (*dump)[i][0] = (addr) << 2; \
+- (*dump)[i++][1] = RREG32(addr); \
+- } while (0)
+-
+- *dump = kmalloc(HQD_N_REGS*2*sizeof(uint32_t), GFP_KERNEL);
+- if (*dump == NULL)
+- return -ENOMEM;
+-
+- acquire_queue(kgd, pipe_id, queue_id);
+-
+- DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE0);
+- DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE1);
+- DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE2);
+- DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE3);
+-
+- for (reg = mmCP_MQD_BASE_ADDR; reg <= mmCP_HQD_EOP_DONES; reg++)
+- DUMP_REG(reg);
+-
+- release_queue(kgd);
+-
+- WARN_ON_ONCE(i != HQD_N_REGS);
+- *n_regs = i;
+-
+- return 0;
+-}
+-
+-static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
+- uint32_t __user *wptr, struct mm_struct *mm)
+-{
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+- struct vi_sdma_mqd *m;
+- uint32_t sdma_base_addr;
+- uint32_t temp, timeout = 2000;
+- uint32_t data;
+-
+- m = get_sdma_mqd(mqd);
+- sdma_base_addr = get_sdma_base_addr(m);
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL,
+- m->sdmax_rlcx_rb_cntl & (~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK));
+-
+- while (true) {
+- temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS);
+- if (temp & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK)
+- break;
+- if (timeout == 0)
+- return -ETIME;
+- msleep(10);
+- timeout -= 10;
+- }
+- if (m->sdma_engine_id) {
+- data = RREG32(mmSDMA1_GFX_CONTEXT_CNTL);
+- data = REG_SET_FIELD(data, SDMA1_GFX_CONTEXT_CNTL,
+- RESUME_CTX, 0);
+- WREG32(mmSDMA1_GFX_CONTEXT_CNTL, data);
+- } else {
+- data = RREG32(mmSDMA0_GFX_CONTEXT_CNTL);
+- data = REG_SET_FIELD(data, SDMA0_GFX_CONTEXT_CNTL,
+- RESUME_CTX, 0);
+- WREG32(mmSDMA0_GFX_CONTEXT_CNTL, data);
+- }
+-
+- data = REG_SET_FIELD(m->sdmax_rlcx_doorbell, SDMA0_RLC0_DOORBELL,
+- ENABLE, 1);
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, data);
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR, m->sdmax_rlcx_rb_rptr);
+-
+- if (read_user_wptr(mm, wptr, data))
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR, data);
+- else
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR,
+- m->sdmax_rlcx_rb_rptr);
+-
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_VIRTUAL_ADDR,
+- m->sdmax_rlcx_virtual_addr);
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE, m->sdmax_rlcx_rb_base);
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE_HI,
+- m->sdmax_rlcx_rb_base_hi);
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_LO,
+- m->sdmax_rlcx_rb_rptr_addr_lo);
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_HI,
+- m->sdmax_rlcx_rb_rptr_addr_hi);
+-
+- data = REG_SET_FIELD(m->sdmax_rlcx_rb_cntl, SDMA0_RLC0_RB_CNTL,
+- RB_ENABLE, 1);
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, data);
+-
+- return 0;
+-}
+-
+-static int kgd_hqd_sdma_dump(struct kgd_dev *kgd,
+- uint32_t engine_id, uint32_t queue_id,
+- uint32_t (**dump)[2], uint32_t *n_regs)
+-{
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+- uint32_t sdma_offset = engine_id * SDMA1_REGISTER_OFFSET +
+- queue_id * KFD_VI_SDMA_QUEUE_OFFSET;
+- uint32_t i = 0, reg;
+-#undef HQD_N_REGS
+-#define HQD_N_REGS (19+4+2+3+7)
+-
+- *dump = kmalloc(HQD_N_REGS*2*sizeof(uint32_t), GFP_KERNEL);
+- if (*dump == NULL)
+- return -ENOMEM;
+-
+- for (reg = mmSDMA0_RLC0_RB_CNTL; reg <= mmSDMA0_RLC0_DOORBELL; reg++)
+- DUMP_REG(sdma_offset + reg);
+- for (reg = mmSDMA0_RLC0_VIRTUAL_ADDR; reg <= mmSDMA0_RLC0_WATERMARK;
+- reg++)
+- DUMP_REG(sdma_offset + reg);
+- for (reg = mmSDMA0_RLC0_CSA_ADDR_LO; reg <= mmSDMA0_RLC0_CSA_ADDR_HI;
+- reg++)
+- DUMP_REG(sdma_offset + reg);
+- for (reg = mmSDMA0_RLC0_IB_SUB_REMAIN; reg <= mmSDMA0_RLC0_DUMMY_REG;
+- reg++)
+- DUMP_REG(sdma_offset + reg);
+- for (reg = mmSDMA0_RLC0_MIDCMD_DATA0; reg <= mmSDMA0_RLC0_MIDCMD_CNTL;
+- reg++)
+- DUMP_REG(sdma_offset + reg);
+-
+- WARN_ON_ONCE(i != HQD_N_REGS);
+- *n_regs = i;
+-
+ return 0;
+ }
+
+@@ -610,7 +293,7 @@ static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
+ static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
+ {
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+- struct vi_sdma_mqd *m;
++ struct cik_sdma_rlc_registers *m;
+ uint32_t sdma_base_addr;
+ uint32_t sdma_rlc_rb_cntl;
+
+@@ -625,102 +308,29 @@ static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
+ return false;
+ }
+
+-static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
+- enum kfd_preempt_type reset_type,
++static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
+ unsigned int utimeout, uint32_t pipe_id,
+ uint32_t queue_id)
+ {
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ uint32_t temp;
+- enum hqd_dequeue_request_type type;
+- unsigned long flags, end_jiffies;
+- int retry;
+- struct vi_mqd *m = get_mqd(mqd);
++ int timeout = utimeout;
+
+ acquire_queue(kgd, pipe_id, queue_id);
+
+- if (m->cp_hqd_vmid == 0)
+- WREG32_FIELD(RLC_CP_SCHEDULERS, scheduler1, 0);
+-
+- switch (reset_type) {
+- case KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN:
+- type = DRAIN_PIPE;
+- break;
+- case KFD_PREEMPT_TYPE_WAVEFRONT_RESET:
+- type = RESET_WAVES;
+- break;
+- default:
+- type = DRAIN_PIPE;
+- break;
+- }
++ WREG32(mmCP_HQD_DEQUEUE_REQUEST, reset_type);
+
+- /* Workaround: If IQ timer is active and the wait time is close to or
+- * equal to 0, dequeueing is not safe. Wait until either the wait time
+- * is larger or timer is cleared. Also, ensure that IQ_REQ_PEND is
+- * cleared before continuing. Also, ensure wait times are set to at
+- * least 0x3.
+- */
+- local_irq_save(flags);
+- preempt_disable();
+- retry = 5000; /* wait for 500 usecs at maximum */
+- while (true) {
+- temp = RREG32(mmCP_HQD_IQ_TIMER);
+- if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, PROCESSING_IQ)) {
+- pr_debug("HW is processing IQ\n");
+- goto loop;
+- }
+- if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, ACTIVE)) {
+- if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, RETRY_TYPE)
+- == 3) /* SEM-rearm is safe */
+- break;
+- /* Wait time 3 is safe for CP, but our MMIO read/write
+- * time is close to 1 microsecond, so check for 10 to
+- * leave more buffer room
+- */
+- if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, WAIT_TIME)
+- >= 10)
+- break;
+- pr_debug("IQ timer is active\n");
+- } else
+- break;
+-loop:
+- if (!retry) {
+- pr_err("CP HQD IQ timer status time out\n");
+- break;
+- }
+- ndelay(100);
+- --retry;
+- }
+- retry = 1000;
+- while (true) {
+- temp = RREG32(mmCP_HQD_DEQUEUE_REQUEST);
+- if (!(temp & CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_MASK))
+- break;
+- pr_debug("Dequeue request is pending\n");
+-
+- if (!retry) {
+- pr_err("CP HQD dequeue request time out\n");
+- break;
+- }
+- ndelay(100);
+- --retry;
+- }
+- local_irq_restore(flags);
+- preempt_enable();
+-
+- WREG32(mmCP_HQD_DEQUEUE_REQUEST, type);
+-
+- end_jiffies = (utimeout * HZ / 1000) + jiffies;
+ while (true) {
+ temp = RREG32(mmCP_HQD_ACTIVE);
+- if (!(temp & CP_HQD_ACTIVE__ACTIVE_MASK))
++ if (temp & CP_HQD_ACTIVE__ACTIVE_MASK)
+ break;
+- if (time_after(jiffies, end_jiffies)) {
+- pr_err("cp queue preemption time out.\n");
++ if (timeout <= 0) {
++ pr_err("kfd: cp queue preemption time out.\n");
+ release_queue(kgd);
+ return -ETIME;
+ }
+- usleep_range(500, 1000);
++ msleep(20);
++ timeout -= 20;
+ }
+
+ release_queue(kgd);
+@@ -731,10 +341,10 @@ static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
+ unsigned int utimeout)
+ {
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+- struct vi_sdma_mqd *m;
++ struct cik_sdma_rlc_registers *m;
+ uint32_t sdma_base_addr;
+ uint32_t temp;
+- unsigned long end_jiffies = (utimeout * HZ / 1000) + jiffies;
++ int timeout = utimeout;
+
+ m = get_sdma_mqd(mqd);
+ sdma_base_addr = get_sdma_base_addr(m);
+@@ -745,19 +355,18 @@ static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
+
+ while (true) {
+ temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS);
+- if (temp & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK)
++ if (temp & SDMA0_STATUS_REG__RB_CMD_IDLE__SHIFT)
+ break;
+- if (time_after(jiffies, end_jiffies))
++ if (timeout <= 0)
+ return -ETIME;
+- usleep_range(500, 1000);
++ msleep(20);
++ timeout -= 20;
+ }
+
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, 0);
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL,
+- RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL) |
+- SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK);
+-
+- m->sdmax_rlcx_rb_rptr = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR);
++ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR, 0);
++ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR, 0);
++ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE, 0);
+
+ return 0;
+ }
+@@ -779,7 +388,7 @@ static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd,
+ struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+
+ reg = RREG32(mmATC_VMID0_PASID_MAPPING + vmid);
+- return reg & ATC_VMID0_PASID_MAPPING__PASID_MASK;
++ return reg & ATC_VMID0_PASID_MAPPING__VALID_MASK;
+ }
+
+ static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid)
+@@ -789,83 +398,8 @@ static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid)
+ WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);
+ }
+
+-/*
+- * FIXME: Poliars test failed with this package, FIJI works fine
+- * From the CP spec it does not official support the invalidation
+- * with the specified pasid in the package, so disable it for V8
+- *
+- */
+-#ifdef V8_SUPPORT_IT_OFFICIAL
+-static int invalidate_tlbs_with_kiq(struct amdgpu_device *adev, uint16_t pasid)
+-{
+- signed long r;
+- struct dma_fence *f;
+- struct amdgpu_ring *ring = &adev->gfx.kiq.ring;
+-
+- mutex_lock(&adev->gfx.kiq.ring_mutex);
+- amdgpu_ring_alloc(ring, 12); /* fence + invalidate_tlbs package*/
+- amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
+- amdgpu_ring_write(ring,
+- PACKET3_INVALIDATE_TLBS_DST_SEL(1) |
+- PACKET3_INVALIDATE_TLBS_PASID(pasid));
+- amdgpu_fence_emit(ring, &f);
+- amdgpu_ring_commit(ring);
+- mutex_unlock(&adev->gfx.kiq.ring_mutex);
+-
+- r = dma_fence_wait(f, false);
+- if (r)
+- DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+- dma_fence_put(f);
+-
+- return r;
+-}
+-#endif
+-static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid)
+-{
+- struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+- int vmid;
+-
+-#ifdef V8_SUPPORT_IT_OFFICIAL
+- struct amdgpu_ring *ring = &adev->gfx.kiq.ring;
+-
+- if (ring->ready)
+- return invalidate_tlbs_with_kiq(adev, pasid);
+-#endif
+-
+- for (vmid = 0; vmid < 16; vmid++) {
+- if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid))
+- continue;
+- if (RREG32(mmATC_VMID0_PASID_MAPPING + vmid) &
+- ATC_VMID0_PASID_MAPPING__VALID_MASK) {
+- if ((RREG32(mmATC_VMID0_PASID_MAPPING + vmid) &
+- ATC_VMID0_PASID_MAPPING__PASID_MASK) == pasid) {
+- WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);
+- break;
+- }
+- }
+- }
+-
+- return 0;
+-}
+-
+ static int kgd_address_watch_disable(struct kgd_dev *kgd)
+ {
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+- union TCP_WATCH_CNTL_BITS cntl;
+- unsigned int i;
+-
+- cntl.u32All = 0;
+-
+- cntl.bitfields.valid = 0;
+- cntl.bitfields.mask = ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK;
+- cntl.bitfields.atc = 1;
+-
+- /* Turning off this address until we set all the registers */
+- for (i = 0; i < MAX_WATCH_ADDRESSES; i++)
+- WREG32(watchRegs[i * ADDRESS_WATCH_REG_MAX
+- + ADDRESS_WATCH_REG_CNTL],
+- cntl.u32All);
+-
+ return 0;
+ }
+
+@@ -875,32 +409,6 @@ static int kgd_address_watch_execute(struct kgd_dev *kgd,
+ uint32_t addr_hi,
+ uint32_t addr_lo)
+ {
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+- union TCP_WATCH_CNTL_BITS cntl;
+-
+- cntl.u32All = cntl_val;
+-
+- /* Turning off this watch point until we set all the registers */
+- cntl.bitfields.valid = 0;
+- WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX
+- + ADDRESS_WATCH_REG_CNTL],
+- cntl.u32All);
+-
+- WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX
+- + ADDRESS_WATCH_REG_ADDR_HI],
+- addr_hi);
+-
+- WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX
+- + ADDRESS_WATCH_REG_ADDR_LO],
+- addr_lo);
+-
+- /* Enable the watch point */
+- cntl.bitfields.valid = 1;
+-
+- WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX
+- + ADDRESS_WATCH_REG_CNTL],
+- cntl.u32All);
+-
+ return 0;
+ }
+
+@@ -933,32 +441,6 @@ static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd,
+ unsigned int watch_point_id,
+ unsigned int reg_offset)
+ {
+- return watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + reg_offset];
+-}
+-
+-static int write_config_static_mem(struct kgd_dev *kgd, bool swizzle_enable,
+- uint8_t element_size, uint8_t index_stride, uint8_t mtype)
+-{
+- uint32_t reg;
+- struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+-
+- reg = swizzle_enable << SH_STATIC_MEM_CONFIG__SWIZZLE_ENABLE__SHIFT |
+- element_size << SH_STATIC_MEM_CONFIG__ELEMENT_SIZE__SHIFT |
+- index_stride << SH_STATIC_MEM_CONFIG__INDEX_STRIDE__SHIFT |
+- mtype << SH_STATIC_MEM_CONFIG__PRIVATE_MTYPE__SHIFT;
+-
+- WREG32(mmSH_STATIC_MEM_CONFIG, reg);
+- return 0;
+-}
+-static int alloc_memory_of_scratch(struct kgd_dev *kgd,
+- uint64_t va, uint32_t vmid)
+-{
+- struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+-
+- lock_srbm(kgd, 0, 0, 0, vmid);
+- WREG32(mmSH_HIDDEN_PRIVATE_BASE_VMID, va);
+- unlock_srbm(kgd);
+-
+ return 0;
+ }
+
+@@ -967,45 +449,47 @@ static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type)
+ struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+ const union amdgpu_firmware_header *hdr;
+
++ BUG_ON(kgd == NULL);
++
+ switch (type) {
+ case KGD_ENGINE_PFP:
+ hdr = (const union amdgpu_firmware_header *)
+- adev->gfx.pfp_fw->data;
++ adev->gfx.pfp_fw->data;
+ break;
+
+ case KGD_ENGINE_ME:
+ hdr = (const union amdgpu_firmware_header *)
+- adev->gfx.me_fw->data;
++ adev->gfx.me_fw->data;
+ break;
+
+ case KGD_ENGINE_CE:
+ hdr = (const union amdgpu_firmware_header *)
+- adev->gfx.ce_fw->data;
++ adev->gfx.ce_fw->data;
+ break;
+
+ case KGD_ENGINE_MEC1:
+ hdr = (const union amdgpu_firmware_header *)
+- adev->gfx.mec_fw->data;
++ adev->gfx.mec_fw->data;
+ break;
+
+ case KGD_ENGINE_MEC2:
+ hdr = (const union amdgpu_firmware_header *)
+- adev->gfx.mec2_fw->data;
++ adev->gfx.mec2_fw->data;
+ break;
+
+ case KGD_ENGINE_RLC:
+ hdr = (const union amdgpu_firmware_header *)
+- adev->gfx.rlc_fw->data;
++ adev->gfx.rlc_fw->data;
+ break;
+
+ case KGD_ENGINE_SDMA1:
+ hdr = (const union amdgpu_firmware_header *)
+- adev->sdma.instance[0].fw->data;
++ adev->sdma.instance[0].fw->data;
+ break;
+
+ case KGD_ENGINE_SDMA2:
+ hdr = (const union amdgpu_firmware_header *)
+- adev->sdma.instance[1].fw->data;
++ adev->sdma.instance[1].fw->data;
+ break;
+
+ default:
+@@ -1018,21 +502,3 @@ static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type)
+ /* Only 12 bit in use*/
+ return hdr->common.ucode_version;
+ }
+-
+-static void set_num_of_requests(struct kgd_dev *kgd,
+- uint8_t num_of_requests)
+-{
+- pr_debug("This is a stub\n");
+-}
+-
+-static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
+- uint32_t page_table_base)
+-{
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+- /* TODO: Don't use hardcoded VMIDs */
+- if (vmid < 8 || vmid > 15) {
+- pr_err("trying to set page table base for wrong VMID\n");
+- return;
+- }
+- WREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vmid - 8, page_table_base);
+-}
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
+deleted file mode 100644
+index edbae19..0000000
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
++++ /dev/null
+@@ -1,1227 +0,0 @@
+-/*
+- * Copyright 2014 Advanced Micro Devices, Inc.
+- *
+- * Permission is hereby granted, free of charge, to any person obtaining a
+- * copy of this software and associated documentation files (the "Software"),
+- * to deal in the Software without restriction, including without limitation
+- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+- * and/or sell copies of the Software, and to permit persons to whom the
+- * Software is furnished to do so, subject to the following conditions:
+- *
+- * The above copyright notice and this permission notice shall be included in
+- * all copies or substantial portions of the Software.
+- *
+- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+- * OTHER DEALINGS IN THE SOFTWARE.
+- */
+-#undef pr_fmt
+-#define pr_fmt(fmt) "kfd2kgd: " fmt
+-
+-#include <linux/module.h>
+-#include <linux/fdtable.h>
+-#include <linux/uaccess.h>
+-#include <linux/firmware.h>
+-#include <drm/drmP.h>
+-#include "amdgpu.h"
+-#include "amdgpu_amdkfd.h"
+-#include "amdgpu_ucode.h"
+-#include "amdgpu_amdkfd_gfx_v8.h"
+-#include "vega10/soc15ip.h"
+-#include "vega10/GC/gc_9_0_offset.h"
+-#include "vega10/GC/gc_9_0_sh_mask.h"
+-#include "vega10/vega10_enum.h"
+-#include "vega10/SDMA0/sdma0_4_0_offset.h"
+-#include "vega10/SDMA0/sdma0_4_0_sh_mask.h"
+-#include "vega10/SDMA1/sdma1_4_0_offset.h"
+-#include "vega10/SDMA1/sdma1_4_0_sh_mask.h"
+-#include "vega10/ATHUB/athub_1_0_offset.h"
+-#include "vega10/ATHUB/athub_1_0_sh_mask.h"
+-#include "vega10/OSSSYS/osssys_4_0_offset.h"
+-#include "vega10/OSSSYS/osssys_4_0_sh_mask.h"
+-#include "soc15_common.h"
+-#include "v9_structs.h"
+-#include "soc15.h"
+-#include "soc15d.h"
+-
+-/* HACK: MMHUB and GC both have VM-related register with the same
+- * names but different offsets. Define the MMHUB register we need here
+- * with a prefix. A proper solution would be to move the functions
+- * programming these registers into gfx_v9_0.c and mmhub_v1_0.c
+- * respectively.
+- */
+-#define mmMMHUB_VM_INVALIDATE_ENG16_REQ 0x06f3
+-#define mmMMHUB_VM_INVALIDATE_ENG16_REQ_BASE_IDX 0
+-
+-#define mmMMHUB_VM_INVALIDATE_ENG16_ACK 0x0705
+-#define mmMMHUB_VM_INVALIDATE_ENG16_ACK_BASE_IDX 0
+-
+-#define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32 0x072b
+-#define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX 0
+-#define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32 0x072c
+-#define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX 0
+-
+-#define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32 0x074b
+-#define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32_BASE_IDX 0
+-#define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32 0x074c
+-#define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32_BASE_IDX 0
+-
+-#define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32 0x076b
+-#define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32_BASE_IDX 0
+-#define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32 0x076c
+-#define mmMMHUB_VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32_BASE_IDX 0
+-
+-#define mmMMHUB_VM_INVALIDATE_ENG16_ADDR_RANGE_LO32 0x0727
+-#define mmMMHUB_VM_INVALIDATE_ENG16_ADDR_RANGE_LO32_BASE_IDX 0
+-#define mmMMHUB_VM_INVALIDATE_ENG16_ADDR_RANGE_HI32 0x0728
+-#define mmMMHUB_VM_INVALIDATE_ENG16_ADDR_RANGE_HI32_BASE_IDX 0
+-
+-enum hqd_dequeue_request_type {
+- NO_ACTION = 0,
+- DRAIN_PIPE,
+- RESET_WAVES,
+- SAVE_WAVES
+-};
+-
+-static const uint32_t watchRegs[MAX_WATCH_ADDRESSES * ADDRESS_WATCH_REG_MAX] = {
+- mmTCP_WATCH0_ADDR_H, mmTCP_WATCH0_ADDR_L, mmTCP_WATCH0_CNTL,
+- mmTCP_WATCH1_ADDR_H, mmTCP_WATCH1_ADDR_L, mmTCP_WATCH1_CNTL,
+- mmTCP_WATCH2_ADDR_H, mmTCP_WATCH2_ADDR_L, mmTCP_WATCH2_CNTL,
+- mmTCP_WATCH3_ADDR_H, mmTCP_WATCH3_ADDR_L, mmTCP_WATCH3_CNTL
+-};
+-
+-
+-static int create_process_gpumem(struct kgd_dev *kgd, uint64_t va, size_t size,
+- void *vm, struct kgd_mem **mem);
+-static void destroy_process_gpumem(struct kgd_dev *kgd, struct kgd_mem *mem);
+-
+-static int open_graphic_handle(struct kgd_dev *kgd, uint64_t va, void *vm,
+- int fd, uint32_t handle, struct kgd_mem **mem);
+-
+-static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type);
+-
+-/*
+- * Register access functions
+- */
+-
+-static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
+- uint32_t sh_mem_config,
+- uint32_t sh_mem_ape1_base, uint32_t sh_mem_ape1_limit,
+- uint32_t sh_mem_bases);
+-static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
+- unsigned int vmid);
+-static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
+- uint32_t hpd_size, uint64_t hpd_gpu_addr);
+-static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id);
+-static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
+- uint32_t queue_id, uint32_t __user *wptr,
+- uint32_t wptr_shift, uint32_t wptr_mask,
+- struct mm_struct *mm);
+-static int kgd_hqd_dump(struct kgd_dev *kgd,
+- uint32_t pipe_id, uint32_t queue_id,
+- uint32_t (**dump)[2], uint32_t *n_regs);
+-static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
+- uint32_t __user *wptr, struct mm_struct *mm);
+-static int kgd_hqd_sdma_dump(struct kgd_dev *kgd,
+- uint32_t engine_id, uint32_t queue_id,
+- uint32_t (**dump)[2], uint32_t *n_regs);
+-static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
+- uint32_t pipe_id, uint32_t queue_id);
+-static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd);
+-static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
+- enum kfd_preempt_type reset_type,
+- unsigned int utimeout, uint32_t pipe_id,
+- uint32_t queue_id);
+-static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
+- unsigned int utimeout);
+-static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid);
+-static uint32_t get_watch_base_addr(void);
+-static int kgd_address_watch_disable(struct kgd_dev *kgd);
+-static int kgd_address_watch_execute(struct kgd_dev *kgd,
+- unsigned int watch_point_id,
+- uint32_t cntl_val,
+- uint32_t addr_hi,
+- uint32_t addr_lo);
+-static int kgd_wave_control_execute(struct kgd_dev *kgd,
+- uint32_t gfx_index_val,
+- uint32_t sq_cmd);
+-static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd,
+- unsigned int watch_point_id,
+- unsigned int reg_offset);
+-
+-static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd,
+- uint8_t vmid);
+-static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd,
+- uint8_t vmid);
+-static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid);
+-static void set_num_of_requests(struct kgd_dev *kgd,
+- uint8_t num_of_requests);
+-static int alloc_memory_of_scratch(struct kgd_dev *kgd,
+- uint64_t va, uint32_t vmid);
+-static int write_config_static_mem(struct kgd_dev *kgd, bool swizzle_enable,
+- uint8_t element_size, uint8_t index_stride, uint8_t mtype);
+-static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
+- uint32_t page_table_base);
+-static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid);
+-
+-/* Because of REG_GET_FIELD() being used, we put this function in the
+- * asic specific file.
+- */
+-static int amdgpu_amdkfd_get_tile_config(struct kgd_dev *kgd,
+- struct tile_config *config)
+-{
+- struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+-
+- config->gb_addr_config = adev->gfx.config.gb_addr_config;
+-#if 0
+-/* TODO - confirm REG_GET_FIELD x2, should be OK as is... but
+- * MC_ARB_RAMCFG register doesn't exist on Vega10 - initial amdgpu
+- * changes commented out related code, doing the same here for now but
+- * need to sync with Ken et al
+- */
+- config->num_banks = REG_GET_FIELD(adev->gfx.config.mc_arb_ramcfg,
+- MC_ARB_RAMCFG, NOOFBANK);
+- config->num_ranks = REG_GET_FIELD(adev->gfx.config.mc_arb_ramcfg,
+- MC_ARB_RAMCFG, NOOFRANKS);
+-#endif
+-
+- config->tile_config_ptr = adev->gfx.config.tile_mode_array;
+- config->num_tile_configs =
+- ARRAY_SIZE(adev->gfx.config.tile_mode_array);
+- config->macro_tile_config_ptr =
+- adev->gfx.config.macrotile_mode_array;
+- config->num_macro_tile_configs =
+- ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);
+-
+- return 0;
+-}
+-
+-static const struct kfd2kgd_calls kfd2kgd = {
+- .init_gtt_mem_allocation = alloc_gtt_mem,
+- .free_gtt_mem = free_gtt_mem,
+- .get_local_mem_info = get_local_mem_info,
+- .get_gpu_clock_counter = get_gpu_clock_counter,
+- .get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz,
+- .create_process_vm = amdgpu_amdkfd_gpuvm_create_process_vm,
+- .destroy_process_vm = amdgpu_amdkfd_gpuvm_destroy_process_vm,
+- .create_process_gpumem = create_process_gpumem,
+- .destroy_process_gpumem = destroy_process_gpumem,
+- .get_process_page_dir = amdgpu_amdkfd_gpuvm_get_process_page_dir,
+- .open_graphic_handle = open_graphic_handle,
+- .program_sh_mem_settings = kgd_program_sh_mem_settings,
+- .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
+- .init_pipeline = kgd_init_pipeline,
+- .init_interrupts = kgd_init_interrupts,
+- .hqd_load = kgd_hqd_load,
+- .hqd_sdma_load = kgd_hqd_sdma_load,
+- .hqd_dump = kgd_hqd_dump,
+- .hqd_sdma_dump = kgd_hqd_sdma_dump,
+- .hqd_is_occupied = kgd_hqd_is_occupied,
+- .hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied,
+- .hqd_destroy = kgd_hqd_destroy,
+- .hqd_sdma_destroy = kgd_hqd_sdma_destroy,
+- .address_watch_disable = kgd_address_watch_disable,
+- .address_watch_execute = kgd_address_watch_execute,
+- .wave_control_execute = kgd_wave_control_execute,
+- .address_watch_get_offset = kgd_address_watch_get_offset,
+- .get_atc_vmid_pasid_mapping_pasid =
+- get_atc_vmid_pasid_mapping_pasid,
+- .get_atc_vmid_pasid_mapping_valid =
+- get_atc_vmid_pasid_mapping_valid,
+- .write_vmid_invalidate_request = write_vmid_invalidate_request,
+- .invalidate_tlbs = invalidate_tlbs,
+- .sync_memory = amdgpu_amdkfd_gpuvm_sync_memory,
+- .alloc_memory_of_gpu = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu,
+- .free_memory_of_gpu = amdgpu_amdkfd_gpuvm_free_memory_of_gpu,
+- .map_memory_to_gpu = amdgpu_amdkfd_gpuvm_map_memory_to_gpu,
+- .unmap_memory_to_gpu = amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu,
+- .get_fw_version = get_fw_version,
+- .set_num_of_requests = set_num_of_requests,
+- .get_cu_info = get_cu_info,
+- .alloc_memory_of_scratch = alloc_memory_of_scratch,
+- .write_config_static_mem = write_config_static_mem,
+- .mmap_bo = amdgpu_amdkfd_gpuvm_mmap_bo,
+- .map_gtt_bo_to_kernel = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel,
+- .set_vm_context_page_table_base = set_vm_context_page_table_base,
+- .pin_get_sg_table_bo = amdgpu_amdkfd_gpuvm_pin_get_sg_table,
+- .unpin_put_sg_table_bo = amdgpu_amdkfd_gpuvm_unpin_put_sg_table,
+- .get_dmabuf_info = amdgpu_amdkfd_get_dmabuf_info,
+- .import_dmabuf = amdgpu_amdkfd_gpuvm_import_dmabuf,
+- .export_dmabuf = amdgpu_amdkfd_gpuvm_export_dmabuf,
+- .get_vm_fault_info = amdgpu_amdkfd_gpuvm_get_vm_fault_info,
+- .submit_ib = amdgpu_amdkfd_submit_ib,
+- .get_tile_config = amdgpu_amdkfd_get_tile_config,
+- .restore_process_bos = amdgpu_amdkfd_gpuvm_restore_process_bos,
+- .copy_mem_to_mem = amdgpu_amdkfd_copy_mem_to_mem,
+- .get_vram_usage = amdgpu_amdkfd_get_vram_usage
+-};
+-
+-struct kfd2kgd_calls *amdgpu_amdkfd_gfx_9_0_get_functions()
+-{
+- return (struct kfd2kgd_calls *)&kfd2kgd;
+-}
+-
+-static int create_process_gpumem(struct kgd_dev *kgd, uint64_t va, size_t size,
+- void *vm, struct kgd_mem **mem)
+-{
+- return 0;
+-}
+-
+-/* Destroys the GPU allocation and frees the kgd_mem structure */
+-static void destroy_process_gpumem(struct kgd_dev *kgd, struct kgd_mem *mem)
+-{
+-
+-}
+-
+-static int open_graphic_handle(struct kgd_dev *kgd, uint64_t va, void *vm,
+- int fd, uint32_t handle, struct kgd_mem **mem)
+-{
+- return 0;
+-}
+-
+-static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd)
+-{
+- return (struct amdgpu_device *)kgd;
+-}
+-
+-static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe,
+- uint32_t queue, uint32_t vmid)
+-{
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+-
+- mutex_lock(&adev->srbm_mutex);
+- soc15_grbm_select(adev, mec, pipe, queue, vmid);
+-}
+-
+-static void unlock_srbm(struct kgd_dev *kgd)
+-{
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+-
+- soc15_grbm_select(adev, 0, 0, 0, 0);
+- mutex_unlock(&adev->srbm_mutex);
+-}
+-
+-static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id,
+- uint32_t queue_id)
+-{
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+-
+- uint32_t mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
+- uint32_t pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
+-
+- lock_srbm(kgd, mec, pipe, queue_id, 0);
+-}
+-
+-static uint32_t get_queue_mask(struct amdgpu_device *adev,
+- uint32_t pipe_id, uint32_t queue_id)
+-{
+- unsigned int bit = (pipe_id * adev->gfx.mec.num_pipe_per_mec +
+- queue_id) & 31;
+-
+- return ((uint32_t)1) << bit;
+-}
+-
+-static void release_queue(struct kgd_dev *kgd)
+-{
+- unlock_srbm(kgd);
+-}
+-
+-static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
+- uint32_t sh_mem_config,
+- uint32_t sh_mem_ape1_base,
+- uint32_t sh_mem_ape1_limit,
+- uint32_t sh_mem_bases)
+-{
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+-
+- lock_srbm(kgd, 0, 0, 0, vmid);
+-
+- WREG32(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_CONFIG), sh_mem_config);
+- WREG32(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_BASES), sh_mem_bases);
+- /* APE1 no longer exists on GFX9 */
+-
+- unlock_srbm(kgd);
+-}
+-
+-static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
+- unsigned int vmid)
+-{
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+-
+- /*
+- * We have to assume that there is no outstanding mapping.
+- * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0 because
+- * a mapping is in progress or because a mapping finished
+- * and the SW cleared it.
+- * So the protocol is to always wait & clear.
+- */
+- uint32_t pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid |
+- ATC_VMID0_PASID_MAPPING__VALID_MASK;
+-
+- /*
+- * need to do this twice, once for gfx and once for mmhub
+- * for ATC add 16 to VMID for mmhub, for IH different registers.
+- * ATC_VMID0..15 registers are separate from ATC_VMID16..31.
+- */
+-
+- WREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) + vmid,
+- pasid_mapping);
+-
+- while (!(RREG32(SOC15_REG_OFFSET(
+- ATHUB, 0,
+- mmATC_VMID_PASID_MAPPING_UPDATE_STATUS)) &
+- (1U << vmid)))
+- cpu_relax();
+-
+- WREG32(SOC15_REG_OFFSET(ATHUB, 0,
+- mmATC_VMID_PASID_MAPPING_UPDATE_STATUS),
+- 1U << vmid);
+-
+- /* Mapping vmid to pasid also for IH block */
+- WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid,
+- pasid_mapping);
+-
+- WREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID16_PASID_MAPPING) + vmid,
+- pasid_mapping);
+-
+- while (!(RREG32(SOC15_REG_OFFSET(
+- ATHUB, 0,
+- mmATC_VMID_PASID_MAPPING_UPDATE_STATUS)) &
+- (1U << (vmid + 16))))
+- cpu_relax();
+-
+- WREG32(SOC15_REG_OFFSET(ATHUB, 0,
+- mmATC_VMID_PASID_MAPPING_UPDATE_STATUS),
+- 1U << (vmid + 16));
+-
+- /* Mapping vmid to pasid also for IH block */
+- WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid,
+- pasid_mapping);
+- return 0;
+-}
+-
+-static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
+- uint32_t hpd_size, uint64_t hpd_gpu_addr)
+-{
+- /* amdgpu owns the per-pipe state */
+- return 0;
+-}
+-
+-/* TODO - RING0 form of field is obsolete, seems to date back to SI
+- * but still works
+- */
+-
+-static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id)
+-{
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+- uint32_t mec;
+- uint32_t pipe;
+-
+- mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
+- pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
+-
+- lock_srbm(kgd, mec, pipe, 0, 0);
+-
+- WREG32(SOC15_REG_OFFSET(GC, 0, mmCPC_INT_CNTL),
+- CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK |
+- CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK);
+-
+- unlock_srbm(kgd);
+-
+- return 0;
+-}
+-
+-static uint32_t get_sdma_base_addr(unsigned int engine_id,
+- unsigned int queue_id)
+-{
+- static const uint32_t base[2] = {
+- SOC15_REG_OFFSET(SDMA0, 0,
+- mmSDMA0_RLC0_RB_CNTL) - mmSDMA0_RLC0_RB_CNTL,
+- SOC15_REG_OFFSET(SDMA1, 0,
+- mmSDMA1_RLC0_RB_CNTL) - mmSDMA1_RLC0_RB_CNTL
+- };
+- uint32_t retval;
+-
+- retval = base[engine_id] + queue_id * (mmSDMA0_RLC1_RB_CNTL -
+- mmSDMA0_RLC0_RB_CNTL);
+-
+- pr_debug("sdma base address: 0x%x\n", retval);
+-
+- return retval;
+-}
+-
+-static uint32_t get_watch_base_addr(void)
+-{
+- uint32_t retval = SOC15_REG_OFFSET(GC, 0, mmTCP_WATCH0_ADDR_H) -
+- mmTCP_WATCH0_ADDR_H;
+-
+- pr_debug("kfd: reg watch base address: 0x%x\n", retval);
+-
+- return retval;
+-}
+-
+-static inline struct v9_mqd *get_mqd(void *mqd)
+-{
+- return (struct v9_mqd *)mqd;
+-}
+-
+-static inline struct v9_sdma_mqd *get_sdma_mqd(void *mqd)
+-{
+- return (struct v9_sdma_mqd *)mqd;
+-}
+-
+-static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
+- uint32_t queue_id, uint32_t __user *wptr,
+- uint32_t wptr_shift, uint32_t wptr_mask,
+- struct mm_struct *mm)
+-{
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+- struct v9_mqd *m;
+- uint32_t *mqd_hqd;
+- uint32_t reg, hqd_base, data;
+-
+- m = get_mqd(mqd);
+-
+- acquire_queue(kgd, pipe_id, queue_id);
+-
+- /* HIQ is set during driver init period with vmid set to 0*/
+- if (m->cp_hqd_vmid == 0) {
+- uint32_t value, mec, pipe;
+-
+- mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
+- pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
+-
+- pr_debug("kfd: set HIQ, mec:%d, pipe:%d, queue:%d.\n",
+- mec, pipe, queue_id);
+- value = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CP_SCHEDULERS));
+- value = REG_SET_FIELD(value, RLC_CP_SCHEDULERS, scheduler1,
+- ((mec << 5) | (pipe << 3) | queue_id | 0x80));
+- WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CP_SCHEDULERS), value);
+- }
+-
+- /* HQD registers extend from CP_MQD_BASE_ADDR to CP_HQD_EOP_WPTR_MEM. */
+- mqd_hqd = &m->cp_mqd_base_addr_lo;
+- hqd_base = SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR);
+-
+- for (reg = hqd_base;
+- reg <= SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI); reg++)
+- WREG32(reg, mqd_hqd[reg - hqd_base]);
+-
+-
+- /* Activate doorbell logic before triggering WPTR poll. */
+- data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control,
+- CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
+- WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL), data);
+-
+- if (wptr) {
+- /* Don't read wptr with get_user because the user
+- * context may not be accessible (if this function
+- * runs in a work queue). Instead trigger a one-shot
+- * polling read from memory in the CP. This assumes
+- * that wptr is GPU-accessible in the queue's VMID via
+- * ATC or SVM. WPTR==RPTR before starting the poll so
+- * the CP starts fetching new commands from the right
+- * place.
+- *
+- * Guessing a 64-bit WPTR from a 32-bit RPTR is a bit
+- * tricky. Assume that the queue didn't overflow. The
+- * number of valid bits in the 32-bit RPTR depends on
+- * the queue size. The remaining bits are taken from
+- * the saved 64-bit WPTR. If the WPTR wrapped, add the
+- * queue size.
+- */
+- uint32_t queue_size =
+- 2 << REG_GET_FIELD(m->cp_hqd_pq_control,
+- CP_HQD_PQ_CONTROL, QUEUE_SIZE);
+- uint64_t guessed_wptr = m->cp_hqd_pq_rptr & (queue_size - 1);
+-
+- if ((m->cp_hqd_pq_wptr_lo & (queue_size - 1)) < guessed_wptr)
+- guessed_wptr += queue_size;
+- guessed_wptr += m->cp_hqd_pq_wptr_lo & ~(queue_size - 1);
+- guessed_wptr += (uint64_t)m->cp_hqd_pq_wptr_hi << 32;
+-
+- WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_LO),
+- lower_32_bits(guessed_wptr));
+- WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI),
+- upper_32_bits(guessed_wptr));
+- WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR),
+- lower_32_bits((uint64_t)wptr));
+- WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI),
+- upper_32_bits((uint64_t)wptr));
+- WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PQ_WPTR_POLL_CNTL1),
+- get_queue_mask(adev, pipe_id, queue_id));
+- }
+-
+- /* Start the EOP fetcher */
+- WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_EOP_RPTR),
+- REG_SET_FIELD(m->cp_hqd_eop_rptr,
+- CP_HQD_EOP_RPTR, INIT_FETCHER, 1));
+-
+- data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1);
+- WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE), data);
+-
+- release_queue(kgd);
+-
+- return 0;
+-}
+-
+-static int kgd_hqd_dump(struct kgd_dev *kgd,
+- uint32_t pipe_id, uint32_t queue_id,
+- uint32_t (**dump)[2], uint32_t *n_regs)
+-{
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+- uint32_t i = 0, reg;
+-#define HQD_N_REGS 56
+-#define DUMP_REG(addr) do { \
+- if (WARN_ON_ONCE(i >= HQD_N_REGS)) \
+- break; \
+- (*dump)[i][0] = (addr) << 2; \
+- (*dump)[i++][1] = RREG32(addr); \
+- } while (0)
+-
+- *dump = kmalloc(HQD_N_REGS*2*sizeof(uint32_t), GFP_KERNEL);
+- if (*dump == NULL)
+- return -ENOMEM;
+-
+- acquire_queue(kgd, pipe_id, queue_id);
+-
+- for (reg = SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR);
+- reg <= SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI); reg++)
+- DUMP_REG(reg);
+-
+- release_queue(kgd);
+-
+- WARN_ON_ONCE(i != HQD_N_REGS);
+- *n_regs = i;
+-
+- return 0;
+-}
+-
+-static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
+- uint32_t __user *wptr, struct mm_struct *mm)
+-{
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+- struct v9_sdma_mqd *m;
+- uint32_t sdma_base_addr, sdmax_gfx_context_cntl;
+- uint32_t temp, timeout = 2000;
+- uint32_t data;
+- uint64_t data64;
+- uint64_t __user *wptr64 = (uint64_t __user *)wptr;
+-
+- m = get_sdma_mqd(mqd);
+- sdma_base_addr = get_sdma_base_addr(m->sdma_engine_id,
+- m->sdma_queue_id);
+- sdmax_gfx_context_cntl = m->sdma_engine_id ?
+- SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_GFX_CONTEXT_CNTL) :
+- SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_GFX_CONTEXT_CNTL);
+-
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL,
+- m->sdmax_rlcx_rb_cntl & (~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK));
+-
+- while (true) {
+- temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS);
+- if (temp & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK)
+- break;
+- if (timeout == 0)
+- return -ETIME;
+- msleep(10);
+- timeout -= 10;
+- }
+- data = RREG32(sdmax_gfx_context_cntl);
+- data = REG_SET_FIELD(data, SDMA0_GFX_CONTEXT_CNTL,
+- RESUME_CTX, 0);
+- WREG32(sdmax_gfx_context_cntl, data);
+-
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL_OFFSET,
+- m->sdmax_rlcx_doorbell_offset);
+-
+- data = REG_SET_FIELD(m->sdmax_rlcx_doorbell, SDMA0_RLC0_DOORBELL,
+- ENABLE, 1);
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, data);
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR, m->sdmax_rlcx_rb_rptr);
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_HI,
+- m->sdmax_rlcx_rb_rptr_hi);
+-
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_MINOR_PTR_UPDATE, 1);
+- if (read_user_wptr(mm, wptr64, data64)) {
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR,
+- lower_32_bits(data64));
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR_HI,
+- upper_32_bits(data64));
+- } else {
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR,
+- m->sdmax_rlcx_rb_rptr);
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR_HI,
+- m->sdmax_rlcx_rb_rptr_hi);
+- }
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_MINOR_PTR_UPDATE, 0);
+-
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE, m->sdmax_rlcx_rb_base);
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE_HI,
+- m->sdmax_rlcx_rb_base_hi);
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_LO,
+- m->sdmax_rlcx_rb_rptr_addr_lo);
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_HI,
+- m->sdmax_rlcx_rb_rptr_addr_hi);
+-
+- data = REG_SET_FIELD(m->sdmax_rlcx_rb_cntl, SDMA0_RLC0_RB_CNTL,
+- RB_ENABLE, 1);
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, data);
+-
+- return 0;
+-}
+-
+-static int kgd_hqd_sdma_dump(struct kgd_dev *kgd,
+- uint32_t engine_id, uint32_t queue_id,
+- uint32_t (**dump)[2], uint32_t *n_regs)
+-{
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+- uint32_t sdma_base_addr = get_sdma_base_addr(engine_id, queue_id);
+- uint32_t i = 0, reg;
+-#undef HQD_N_REGS
+-#define HQD_N_REGS (19+6+7+10)
+-
+- *dump = kmalloc(HQD_N_REGS*2*sizeof(uint32_t), GFP_KERNEL);
+- if (*dump == NULL)
+- return -ENOMEM;
+-
+- for (reg = mmSDMA0_RLC0_RB_CNTL; reg <= mmSDMA0_RLC0_DOORBELL; reg++)
+- DUMP_REG(sdma_base_addr + reg);
+- for (reg = mmSDMA0_RLC0_STATUS; reg <= mmSDMA0_RLC0_CSA_ADDR_HI; reg++)
+- DUMP_REG(sdma_base_addr + reg);
+- for (reg = mmSDMA0_RLC0_IB_SUB_REMAIN;
+- reg <= mmSDMA0_RLC0_MINOR_PTR_UPDATE; reg++)
+- DUMP_REG(sdma_base_addr + reg);
+- for (reg = mmSDMA0_RLC0_MIDCMD_DATA0;
+- reg <= mmSDMA0_RLC0_MIDCMD_CNTL; reg++)
+- DUMP_REG(sdma_base_addr + reg);
+-
+- WARN_ON_ONCE(i != HQD_N_REGS);
+- *n_regs = i;
+-
+- return 0;
+-}
+-
+-static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
+- uint32_t pipe_id, uint32_t queue_id)
+-{
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+- uint32_t act;
+- bool retval = false;
+- uint32_t low, high;
+-
+- acquire_queue(kgd, pipe_id, queue_id);
+- act = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE));
+- if (act) {
+- low = lower_32_bits(queue_address >> 8);
+- high = upper_32_bits(queue_address >> 8);
+-
+- if (low == RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_BASE)) &&
+- high == RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_BASE_HI)))
+- retval = true;
+- }
+- release_queue(kgd);
+- return retval;
+-}
+-
+-static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
+-{
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+- struct v9_sdma_mqd *m;
+- uint32_t sdma_base_addr;
+- uint32_t sdma_rlc_rb_cntl;
+-
+- m = get_sdma_mqd(mqd);
+- sdma_base_addr = get_sdma_base_addr(m->sdma_engine_id,
+- m->sdma_queue_id);
+-
+- sdma_rlc_rb_cntl = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL);
+-
+- if (sdma_rlc_rb_cntl & SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK)
+- return true;
+-
+- return false;
+-}
+-
+-static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
+- enum kfd_preempt_type reset_type,
+- unsigned int utimeout, uint32_t pipe_id,
+- uint32_t queue_id)
+-{
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+- enum hqd_dequeue_request_type type;
+- unsigned long end_jiffies;
+- uint32_t temp;
+- struct v9_mqd *m = get_mqd(mqd);
+-
+-#if 0
+- unsigned long flags;
+- int retry;
+-#endif
+-
+- acquire_queue(kgd, pipe_id, queue_id);
+-
+- if (m->cp_hqd_vmid == 0)
+- WREG32_FIELD15(GC, 0, RLC_CP_SCHEDULERS, scheduler1, 0);
+-
+- switch (reset_type) {
+- case KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN:
+- type = DRAIN_PIPE;
+- break;
+- case KFD_PREEMPT_TYPE_WAVEFRONT_RESET:
+- type = RESET_WAVES;
+- break;
+- default:
+- type = DRAIN_PIPE;
+- break;
+- }
+-
+-#if 0 /* Is this still needed? */
+- /* Workaround: If IQ timer is active and the wait time is close to or
+- * equal to 0, dequeueing is not safe. Wait until either the wait time
+- * is larger or timer is cleared. Also, ensure that IQ_REQ_PEND is
+- * cleared before continuing. Also, ensure wait times are set to at
+- * least 0x3.
+- */
+- local_irq_save(flags);
+- preempt_disable();
+- retry = 5000; /* wait for 500 usecs at maximum */
+- while (true) {
+- temp = RREG32(mmCP_HQD_IQ_TIMER);
+- if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, PROCESSING_IQ)) {
+- pr_debug("HW is processing IQ\n");
+- goto loop;
+- }
+- if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, ACTIVE)) {
+- if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, RETRY_TYPE)
+- == 3) /* SEM-rearm is safe */
+- break;
+- /* Wait time 3 is safe for CP, but our MMIO read/write
+- * time is close to 1 microsecond, so check for 10 to
+- * leave more buffer room
+- */
+- if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, WAIT_TIME)
+- >= 10)
+- break;
+- pr_debug("IQ timer is active\n");
+- } else
+- break;
+-loop:
+- if (!retry) {
+- pr_err("CP HQD IQ timer status time out\n");
+- break;
+- }
+- ndelay(100);
+- --retry;
+- }
+- retry = 1000;
+- while (true) {
+- temp = RREG32(mmCP_HQD_DEQUEUE_REQUEST);
+- if (!(temp & CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_MASK))
+- break;
+- pr_debug("Dequeue request is pending\n");
+-
+- if (!retry) {
+- pr_err("CP HQD dequeue request time out\n");
+- break;
+- }
+- ndelay(100);
+- --retry;
+- }
+- local_irq_restore(flags);
+- preempt_enable();
+-#endif
+-
+- WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_DEQUEUE_REQUEST), type);
+-
+- end_jiffies = (utimeout * HZ / 1000) + jiffies;
+- while (true) {
+- temp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE));
+- if (!(temp & CP_HQD_ACTIVE__ACTIVE_MASK))
+- break;
+- if (time_after(jiffies, end_jiffies)) {
+- pr_err("cp queue preemption time out.\n");
+- release_queue(kgd);
+- return -ETIME;
+- }
+- usleep_range(500, 1000);
+- }
+-
+- release_queue(kgd);
+- return 0;
+-}
+-
+-static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
+- unsigned int utimeout)
+-{
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+- struct v9_sdma_mqd *m;
+- uint32_t sdma_base_addr;
+- uint32_t temp;
+- unsigned long end_jiffies = (utimeout * HZ / 1000) + jiffies;
+-
+- m = get_sdma_mqd(mqd);
+- sdma_base_addr = get_sdma_base_addr(m->sdma_engine_id,
+- m->sdma_queue_id);
+-
+- temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL);
+- temp = temp & ~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK;
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, temp);
+-
+- while (true) {
+- temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS);
+- if (temp & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK)
+- break;
+- if (time_after(jiffies, end_jiffies))
+- return -ETIME;
+- usleep_range(500, 1000);
+- }
+-
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, 0);
+- WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL,
+- RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL) |
+- SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK);
+-
+- m->sdmax_rlcx_rb_rptr = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR);
+- m->sdmax_rlcx_rb_rptr_hi =
+- RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_HI);
+-
+- return 0;
+-}
+-
+-static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd,
+- uint8_t vmid)
+-{
+- uint32_t reg;
+- struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+-
+- reg = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING)
+- + vmid);
+- return reg & ATC_VMID0_PASID_MAPPING__VALID_MASK;
+-}
+-
+-static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd,
+- uint8_t vmid)
+-{
+- uint32_t reg;
+- struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+-
+- reg = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING)
+- + vmid);
+- return reg & ATC_VMID0_PASID_MAPPING__PASID_MASK;
+-}
+-
+-static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid)
+-{
+- struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+- uint32_t req = (1 << vmid) |
+- (1 << VM_INVALIDATE_ENG16_REQ__FLUSH_TYPE__SHIFT) | /* light */
+- VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PTES_MASK |
+- VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE0_MASK |
+- VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE1_MASK |
+- VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE2_MASK |
+- VM_INVALIDATE_ENG16_REQ__INVALIDATE_L1_PTES_MASK;
+-
+- spin_lock(&adev->tlb_invalidation_lock);
+-
+- /* Use light weight invalidation.
+- *
+- * TODO 1: agree on the right set of invalidation registers for
+- * KFD use. Use the last one for now. Invalidate both GC and
+- * MMHUB.
+- *
+- * TODO 2: support range-based invalidation, requires kfg2kgd
+- * interface change
+- */
+- WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_INVALIDATE_ENG16_ADDR_RANGE_LO32),
+- 0xffffffff);
+- WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_INVALIDATE_ENG16_ADDR_RANGE_HI32),
+- 0x0000001f);
+-
+- WREG32(SOC15_REG_OFFSET(MMHUB, 0,
+- mmMMHUB_VM_INVALIDATE_ENG16_ADDR_RANGE_LO32),
+- 0xffffffff);
+- WREG32(SOC15_REG_OFFSET(MMHUB, 0,
+- mmMMHUB_VM_INVALIDATE_ENG16_ADDR_RANGE_HI32),
+- 0x0000001f);
+-
+- WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_INVALIDATE_ENG16_REQ), req);
+-
+- WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmMMHUB_VM_INVALIDATE_ENG16_REQ),
+- req);
+-
+- while (!(RREG32(SOC15_REG_OFFSET(GC, 0, mmVM_INVALIDATE_ENG16_ACK)) &
+- (1 << vmid)))
+- cpu_relax();
+-
+- while (!(RREG32(SOC15_REG_OFFSET(MMHUB, 0,
+- mmMMHUB_VM_INVALIDATE_ENG16_ACK)) &
+- (1 << vmid)))
+- cpu_relax();
+-
+- spin_unlock(&adev->tlb_invalidation_lock);
+-
+-}
+-
+-static int invalidate_tlbs_with_kiq(struct amdgpu_device *adev, uint16_t pasid)
+-{
+- signed long r;
+- struct dma_fence *f;
+- struct amdgpu_ring *ring = &adev->gfx.kiq.ring;
+-
+- mutex_lock(&adev->gfx.kiq.ring_mutex);
+- amdgpu_ring_alloc(ring, 12); /* fence + invalidate_tlbs package*/
+- amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
+- amdgpu_ring_write(ring,
+- PACKET3_INVALIDATE_TLBS_DST_SEL(1) |
+- PACKET3_INVALIDATE_TLBS_ALL_HUB(1) |
+- PACKET3_INVALIDATE_TLBS_PASID(pasid) |
+- PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(2));
+- amdgpu_fence_emit(ring, &f);
+- amdgpu_ring_commit(ring);
+- mutex_unlock(&adev->gfx.kiq.ring_mutex);
+-
+- r = dma_fence_wait(f, false);
+- if (r)
+- DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+- dma_fence_put(f);
+-
+- return r;
+-}
+-
+-static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid)
+-{
+- struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+- int vmid;
+- struct amdgpu_ring *ring = &adev->gfx.kiq.ring;
+-
+- if (ring->ready)
+- return invalidate_tlbs_with_kiq(adev, pasid);
+-
+- for (vmid = 0; vmid < 16; vmid++) {
+- if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid))
+- continue;
+- if (get_atc_vmid_pasid_mapping_valid(kgd, vmid)) {
+- if (get_atc_vmid_pasid_mapping_pasid(kgd, vmid)
+- == pasid) {
+- write_vmid_invalidate_request(kgd, vmid);
+- break;
+- }
+- }
+- }
+-
+- return 0;
+-}
+-
+-static int kgd_address_watch_disable(struct kgd_dev *kgd)
+-{
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+- union TCP_WATCH_CNTL_BITS cntl;
+- unsigned int i;
+- uint32_t watch_base_addr;
+-
+- cntl.u32All = 0;
+-
+- cntl.bitfields.valid = 0;
+- cntl.bitfields.mask = ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK;
+- cntl.bitfields.atc = 1;
+-
+- watch_base_addr = get_watch_base_addr();
+- /* Turning off this address until we set all the registers */
+- for (i = 0; i < MAX_WATCH_ADDRESSES; i++)
+- WREG32(watch_base_addr +
+- watchRegs[i * ADDRESS_WATCH_REG_MAX +
+- ADDRESS_WATCH_REG_CNTL],
+- cntl.u32All);
+-
+- return 0;
+-}
+-
+-static int kgd_address_watch_execute(struct kgd_dev *kgd,
+- unsigned int watch_point_id,
+- uint32_t cntl_val,
+- uint32_t addr_hi,
+- uint32_t addr_lo)
+-{
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+- union TCP_WATCH_CNTL_BITS cntl;
+- uint32_t watch_base_addr;
+-
+- watch_base_addr = get_watch_base_addr();
+- cntl.u32All = cntl_val;
+-
+- /* Turning off this watch point until we set all the registers */
+- cntl.bitfields.valid = 0;
+- WREG32(watch_base_addr + watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + ADDRESS_WATCH_REG_CNTL],
+- cntl.u32All);
+-
+- WREG32(watch_base_addr + watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + ADDRESS_WATCH_REG_ADDR_HI],
+- addr_hi);
+-
+- WREG32(watch_base_addr + watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + ADDRESS_WATCH_REG_ADDR_LO],
+- addr_lo);
+-
+- /* Enable the watch point */
+- cntl.bitfields.valid = 1;
+-
+- WREG32(watch_base_addr +
+- watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX +
+- ADDRESS_WATCH_REG_CNTL],
+- cntl.u32All);
+-
+- return 0;
+-}
+-
+-static int kgd_wave_control_execute(struct kgd_dev *kgd,
+- uint32_t gfx_index_val,
+- uint32_t sq_cmd)
+-{
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+- uint32_t data = 0;
+-
+- mutex_lock(&adev->grbm_idx_mutex);
+-
+- WREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX), gfx_index_val);
+- WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_CMD), sq_cmd);
+-
+- data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
+- INSTANCE_BROADCAST_WRITES, 1);
+- data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
+- SH_BROADCAST_WRITES, 1);
+- data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
+- SE_BROADCAST_WRITES, 1);
+-
+- WREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX), data);
+- mutex_unlock(&adev->grbm_idx_mutex);
+-
+- return 0;
+-}
+-
+-static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd,
+- unsigned int watch_point_id,
+- unsigned int reg_offset)
+-{
+- return get_watch_base_addr() +
+- watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + reg_offset];
+-}
+-
+-static int write_config_static_mem(struct kgd_dev *kgd, bool swizzle_enable,
+- uint8_t element_size, uint8_t index_stride, uint8_t mtype)
+-{
+- /* No longer needed on GFXv9. These values are now hard-coded,
+- * except for the MTYPE which comes from the page table.
+- */
+-
+- return 0;
+-}
+-static int alloc_memory_of_scratch(struct kgd_dev *kgd,
+- uint64_t va, uint32_t vmid)
+-{
+- /* No longer needed on GFXv9. The scratch base address is
+- * passed to the shader by the CP. It's the user mode driver's
+- * responsibility.
+- */
+-
+- return 0;
+-}
+-
+-/* FIXME: Does this need to be ASIC-specific code? */
+-static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type)
+-{
+- struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+- const union amdgpu_firmware_header *hdr;
+-
+- switch (type) {
+- case KGD_ENGINE_PFP:
+- hdr = (const union amdgpu_firmware_header *)adev->gfx.pfp_fw->data;
+- break;
+-
+- case KGD_ENGINE_ME:
+- hdr = (const union amdgpu_firmware_header *)adev->gfx.me_fw->data;
+- break;
+-
+- case KGD_ENGINE_CE:
+- hdr = (const union amdgpu_firmware_header *)adev->gfx.ce_fw->data;
+- break;
+-
+- case KGD_ENGINE_MEC1:
+- hdr = (const union amdgpu_firmware_header *)adev->gfx.mec_fw->data;
+- break;
+-
+- case KGD_ENGINE_MEC2:
+- hdr = (const union amdgpu_firmware_header *)adev->gfx.mec2_fw->data;
+- break;
+-
+- case KGD_ENGINE_RLC:
+- hdr = (const union amdgpu_firmware_header *)adev->gfx.rlc_fw->data;
+- break;
+-
+- case KGD_ENGINE_SDMA1:
+- hdr = (const union amdgpu_firmware_header *)adev->sdma.instance[0].fw->data;
+- break;
+-
+- case KGD_ENGINE_SDMA2:
+- hdr = (const union amdgpu_firmware_header *)adev->sdma.instance[1].fw->data;
+- break;
+-
+- default:
+- return 0;
+- }
+-
+- if (hdr == NULL)
+- return 0;
+-
+- /* Only 12 bit in use*/
+- return hdr->common.ucode_version;
+-}
+-
+-static void set_num_of_requests(struct kgd_dev *kgd,
+- uint8_t num_of_requests)
+-{
+- pr_debug("This is a stub\n");
+-}
+-
+-static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
+- uint32_t page_table_base)
+-{
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+- uint64_t base = (uint64_t)page_table_base << PAGE_SHIFT |
+- AMDGPU_PTE_VALID;
+-
+- /* TODO: Don't use hardcoded VMIDs */
+- if (vmid < 8 || vmid > 15) {
+- pr_err("trying to set page table base for wrong VMID %u\n",
+- vmid);
+- return;
+- }
+-
+- /* TODO: take advantage of per-process address space size. For
+- * now, all processes share the same address space size, like
+- * on GFX8 and older.
+- */
+- WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmMMHUB_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32) + (vmid*2), 0);
+- WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmMMHUB_VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32) + (vmid*2), 0);
+-
+- WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmMMHUB_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32) + (vmid*2),
+- lower_32_bits(adev->vm_manager.max_pfn - 1));
+- WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmMMHUB_VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32) + (vmid*2),
+- upper_32_bits(adev->vm_manager.max_pfn - 1));
+-
+- WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmMMHUB_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32) + (vmid*2), lower_32_bits(base));
+- WREG32(SOC15_REG_OFFSET(MMHUB, 0, mmMMHUB_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32) + (vmid*2), upper_32_bits(base));
+-
+- WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32) + (vmid*2), 0);
+- WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32) + (vmid*2), 0);
+-
+- WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32) + (vmid*2),
+- lower_32_bits(adev->vm_manager.max_pfn - 1));
+- WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32) + (vmid*2),
+- upper_32_bits(adev->vm_manager.max_pfn - 1));
+-
+- WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32) + (vmid*2), lower_32_bits(base));
+- WREG32(SOC15_REG_OFFSET(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32) + (vmid*2), upper_32_bits(base));
+-}
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+deleted file mode 100644
+index 7df892d..0000000
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
++++ /dev/null
+@@ -1,2578 +0,0 @@
+-/*
+- * Copyright 2014 Advanced Micro Devices, Inc.
+- *
+- * Permission is hereby granted, free of charge, to any person obtaining a
+- * copy of this software and associated documentation files (the "Software"),
+- * to deal in the Software without restriction, including without limitation
+- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+- * and/or sell copies of the Software, and to permit persons to whom the
+- * Software is furnished to do so, subject to the following conditions:
+- *
+- * The above copyright notice and this permission notice shall be included in
+- * all copies or substantial portions of the Software.
+- *
+- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+- * OTHER DEALINGS IN THE SOFTWARE.
+- */
+-
+-#undef pr_fmt
+-#define pr_fmt(fmt) "kfd2kgd: " fmt
+-
+-#include <linux/module.h>
+-#include <linux/fdtable.h>
+-#include <linux/uaccess.h>
+-#include <linux/firmware.h>
+-#include <linux/list.h>
+-#include <linux/sched/mm.h>
+-#include <drm/drmP.h>
+-#include <linux/dma-buf.h>
+-#include <linux/pagemap.h>
+-#include "amdgpu_amdkfd.h"
+-#include "amdgpu_ucode.h"
+-#include "gca/gfx_8_0_sh_mask.h"
+-#include "gca/gfx_8_0_d.h"
+-#include "gca/gfx_8_0_enum.h"
+-#include "oss/oss_3_0_sh_mask.h"
+-#include "oss/oss_3_0_d.h"
+-#include "gmc/gmc_8_1_sh_mask.h"
+-#include "gmc/gmc_8_1_d.h"
+-
+-/* Special VM and GART address alignment needed for VI pre-Fiji due to
+- * a HW bug.
+- */
+-#define VI_BO_SIZE_ALIGN (0x8000)
+-
+-/* BO flag to indicate a KFD userptr BO */
+-#define AMDGPU_AMDKFD_USERPTR_BO (1ULL << 63)
+-
+-/* Impose limit on how much memory KFD can use */
+-struct kfd_mem_usage_limit {
+- uint64_t max_system_mem_limit;
+- uint64_t max_userptr_mem_limit;
+- int64_t system_mem_used;
+- int64_t userptr_mem_used;
+- spinlock_t mem_limit_lock;
+-};
+-
+-static struct kfd_mem_usage_limit kfd_mem_limit;
+-
+-/* Struct used for amdgpu_amdkfd_bo_validate */
+-struct amdgpu_vm_parser {
+- uint32_t domain;
+- bool wait;
+-};
+-
+-static const char * const domain_bit_to_string[] = {
+- "CPU",
+- "GTT",
+- "VRAM",
+- "GDS",
+- "GWS",
+- "OA"
+-};
+-
+-#define domain_string(domain) domain_bit_to_string[ffs(domain)-1]
+-
+-static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work);
+-
+-
+-static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd)
+-{
+- return (struct amdgpu_device *)kgd;
+-}
+-
+-static bool check_if_add_bo_to_vm(struct amdgpu_vm *avm,
+- struct kgd_mem *mem)
+-{
+- struct kfd_bo_va_list *entry;
+-
+- list_for_each_entry(entry, &mem->bo_va_list, bo_list)
+- if (entry->bo_va->base.vm == avm)
+- return false;
+-
+- return true;
+-}
+-
+-/* Set memory usage limits. Current, limits are
+- * System (kernel) memory - 15/16th System RAM
+- * Userptr memory - 15/16th System RAM
+- */
+-void amdgpu_amdkfd_gpuvm_init_mem_limits(void)
+-{
+- struct sysinfo si;
+- uint64_t mem;
+-
+- si_meminfo(&si);
+- mem = si.totalram - si.totalhigh;
+- mem *= si.mem_unit;
+-
+- spin_lock_init(&kfd_mem_limit.mem_limit_lock);
+- kfd_mem_limit.max_system_mem_limit = mem - (mem >> 4); /* 15/16 */
+- kfd_mem_limit.max_userptr_mem_limit = mem - (mem >> 4); /* 15/16 */
+- pr_debug("Kernel memory limit %lluM, userptr limit %lluM\n",
+- (kfd_mem_limit.max_system_mem_limit >> 20),
+- (kfd_mem_limit.max_userptr_mem_limit >> 20));
+-}
+-
+-static int amdgpu_amdkfd_reserve_system_mem_limit(struct amdgpu_device *adev,
+- uint64_t size, u32 domain)
+-{
+- size_t acc_size;
+- int ret = 0;
+-
+- acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size,
+- sizeof(struct amdgpu_bo));
+-
+- spin_lock(&kfd_mem_limit.mem_limit_lock);
+- if (domain == AMDGPU_GEM_DOMAIN_GTT) {
+- if (kfd_mem_limit.system_mem_used + (acc_size + size) >
+- kfd_mem_limit.max_system_mem_limit) {
+- ret = -ENOMEM;
+- goto err_no_mem;
+- }
+- kfd_mem_limit.system_mem_used += (acc_size + size);
+- } else if (domain == AMDGPU_GEM_DOMAIN_CPU) {
+- if ((kfd_mem_limit.system_mem_used + acc_size >
+- kfd_mem_limit.max_system_mem_limit) ||
+- (kfd_mem_limit.userptr_mem_used + (size + acc_size) >
+- kfd_mem_limit.max_userptr_mem_limit)) {
+- ret = -ENOMEM;
+- goto err_no_mem;
+- }
+- kfd_mem_limit.system_mem_used += acc_size;
+- kfd_mem_limit.userptr_mem_used += size;
+- }
+-err_no_mem:
+- spin_unlock(&kfd_mem_limit.mem_limit_lock);
+- return ret;
+-}
+-
+-static void unreserve_system_mem_limit(struct amdgpu_device *adev,
+- uint64_t size, u32 domain)
+-{
+- size_t acc_size;
+-
+- acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size,
+- sizeof(struct amdgpu_bo));
+-
+- spin_lock(&kfd_mem_limit.mem_limit_lock);
+- if (domain == AMDGPU_GEM_DOMAIN_GTT) {
+- kfd_mem_limit.system_mem_used -= (acc_size + size);
+- } else if (domain == AMDGPU_GEM_DOMAIN_CPU) {
+- kfd_mem_limit.system_mem_used -= acc_size;
+- kfd_mem_limit.userptr_mem_used -= size;
+- }
+- WARN_ONCE(kfd_mem_limit.system_mem_used < 0,
+- "kfd system memory accounting unbalanced");
+- WARN_ONCE(kfd_mem_limit.userptr_mem_used < 0,
+- "kfd userptr memory accounting unbalanced");
+-
+- spin_unlock(&kfd_mem_limit.mem_limit_lock);
+-}
+-
+-void amdgpu_amdkfd_unreserve_system_memory_limit(struct amdgpu_bo *bo)
+-{
+- spin_lock(&kfd_mem_limit.mem_limit_lock);
+-
+- if (bo->flags & AMDGPU_AMDKFD_USERPTR_BO) {
+- kfd_mem_limit.system_mem_used -= bo->tbo.acc_size;
+- kfd_mem_limit.userptr_mem_used -= amdgpu_bo_size(bo);
+- } else if (bo->preferred_domains == AMDGPU_GEM_DOMAIN_GTT) {
+- kfd_mem_limit.system_mem_used -=
+- (bo->tbo.acc_size + amdgpu_bo_size(bo));
+- }
+- WARN_ONCE(kfd_mem_limit.system_mem_used < 0,
+- "kfd system memory accounting unbalanced");
+- WARN_ONCE(kfd_mem_limit.userptr_mem_used < 0,
+- "kfd userptr memory accounting unbalanced");
+-
+- spin_unlock(&kfd_mem_limit.mem_limit_lock);
+-}
+-
+-
+-/* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence(s) from BO's
+- * reservation object.
+- *
+- * @bo: [IN] Remove eviction fence(s) from this BO
+- * @ef: [IN] If ef is specified, then this eviction fence is removed if it
+- * is present in the shared list.
+- * @ef_list: [OUT] Returns list of eviction fences. These fences are removed
+- * from BO's reservation object shared list.
+- * @ef_count: [OUT] Number of fences in ef_list.
+- *
+- * NOTE: If called with ef_list, then amdgpu_amdkfd_add_eviction_fence must be
+- * called to restore the eviction fences and to avoid memory leak. This is
+- * useful for shared BOs.
+- * NOTE: Must be called with BO reserved i.e. bo->tbo.resv->lock held.
+- */
+-static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo,
+- struct amdgpu_amdkfd_fence *ef,
+- struct amdgpu_amdkfd_fence ***ef_list,
+- unsigned int *ef_count)
+-{
+- struct reservation_object_list *fobj;
+- struct reservation_object *resv;
+- unsigned int i = 0, j = 0, k = 0, shared_count;
+- unsigned int count = 0;
+- struct amdgpu_amdkfd_fence **fence_list;
+-
+- if (!ef && !ef_list)
+- return -EINVAL;
+-
+- if (ef_list) {
+- *ef_list = NULL;
+- *ef_count = 0;
+- }
+-
+- resv = bo->tbo.resv;
+- fobj = reservation_object_get_list(resv);
+-
+- if (!fobj)
+- return 0;
+-
+- preempt_disable();
+- write_seqcount_begin(&resv->seq);
+-
+- /* Go through all the shared fences in the resevation object. If
+- * ef is specified and it exists in the list, remove it and reduce the
+- * count. If ef is not specified, then get the count of eviction fences
+- * present.
+- */
+- shared_count = fobj->shared_count;
+- for (i = 0; i < shared_count; ++i) {
+- struct dma_fence *f;
+-
+- f = rcu_dereference_protected(fobj->shared[i],
+- reservation_object_held(resv));
+-
+- if (ef) {
+- if (f->context == ef->base.context) {
+- dma_fence_put(f);
+- fobj->shared_count--;
+- } else
+- RCU_INIT_POINTER(fobj->shared[j++], f);
+-
+- } else if (to_amdgpu_amdkfd_fence(f))
+- count++;
+- }
+- write_seqcount_end(&resv->seq);
+- preempt_enable();
+-
+- if (ef || !count)
+- return 0;
+-
+- /* Alloc memory for count number of eviction fence pointers. Fill the
+- * ef_list array and ef_count
+- */
+-
+- fence_list = kcalloc(count, sizeof(struct amdgpu_amdkfd_fence *),
+- GFP_KERNEL);
+- if (!fence_list)
+- return -ENOMEM;
+-
+- preempt_disable();
+- write_seqcount_begin(&resv->seq);
+-
+- j = 0;
+- for (i = 0; i < shared_count; ++i) {
+- struct dma_fence *f;
+- struct amdgpu_amdkfd_fence *efence;
+-
+- f = rcu_dereference_protected(fobj->shared[i],
+- reservation_object_held(resv));
+-
+- efence = to_amdgpu_amdkfd_fence(f);
+- if (efence) {
+- fence_list[k++] = efence;
+- fobj->shared_count--;
+- } else
+- RCU_INIT_POINTER(fobj->shared[j++], f);
+- }
+-
+- write_seqcount_end(&resv->seq);
+- preempt_enable();
+-
+- *ef_list = fence_list;
+- *ef_count = k;
+-
+- return 0;
+-}
+-
+-/* amdgpu_amdkfd_add_eviction_fence - Adds eviction fence(s) back into BO's
+- * reservation object.
+- *
+- * @bo: [IN] Add eviction fences to this BO
+- * @ef_list: [IN] List of eviction fences to be added
+- * @ef_count: [IN] Number of fences in ef_list.
+- *
+- * NOTE: Must call amdgpu_amdkfd_remove_eviction_fence before calling this
+- * function.
+- */
+-static void amdgpu_amdkfd_add_eviction_fence(struct amdgpu_bo *bo,
+- struct amdgpu_amdkfd_fence **ef_list,
+- unsigned int ef_count)
+-{
+- int i;
+-
+- if (!ef_list || !ef_count)
+- return;
+-
+- for (i = 0; i < ef_count; i++) {
+- amdgpu_bo_fence(bo, &ef_list[i]->base, true);
+- /* Readding the fence takes an additional reference. Drop that
+- * reference.
+- */
+- dma_fence_put(&ef_list[i]->base);
+- }
+-
+- kfree(ef_list);
+-}
+-
+-static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain,
+- bool wait)
+-{
+- int ret;
+-
+- if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm),
+- "Called with userptr BO"))
+- return -EINVAL;
+-
+- amdgpu_ttm_placement_from_domain(bo, domain);
+-
+- ret = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
+- if (ret)
+- goto validate_fail;
+- if (wait) {
+- struct amdgpu_amdkfd_fence **ef_list;
+- unsigned int ef_count;
+-
+- ret = amdgpu_amdkfd_remove_eviction_fence(bo, NULL, &ef_list,
+- &ef_count);
+- if (ret)
+- goto validate_fail;
+-
+- ttm_bo_wait(&bo->tbo, false, false);
+- amdgpu_amdkfd_add_eviction_fence(bo, ef_list, ef_count);
+- }
+-
+-validate_fail:
+- return ret;
+-}
+-
+-static int amdgpu_amdkfd_validate(void *param, struct amdgpu_bo *bo)
+-{
+- struct amdgpu_vm_parser *p = param;
+-
+- return amdgpu_amdkfd_bo_validate(bo, p->domain, p->wait);
+-}
+-
+-/* vm_validate_pt_pd_bos - Validate page table and directory BOs
+- *
+- * Also updates page directory entries so we don't need to do this
+- * again later until the page directory is validated again (e.g. after
+- * an eviction or allocating new page tables).
+- */
+-static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm)
+-{
+- struct amdgpu_bo *pd = vm->root.base.bo;
+- struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);
+- struct amdgpu_vm_parser param;
+- int ret;
+-
+- param.domain = AMDGPU_GEM_DOMAIN_VRAM;
+- param.wait = false;
+-
+- ret = amdgpu_vm_validate_pt_bos(adev, vm, amdgpu_amdkfd_validate,
+- &param);
+- if (ret) {
+- pr_err("amdgpu: failed to validate PT BOs\n");
+- return ret;
+- }
+-
+- ret = amdgpu_amdkfd_validate(&param, pd);
+- if (ret) {
+- pr_err("amdgpu: failed to validate PD\n");
+- return ret;
+- }
+-
+- ret = amdgpu_vm_update_directories(adev, vm);
+- if (ret != 0)
+- return ret;
+-
+- return 0;
+-}
+-
+-/* add_bo_to_vm - Add a BO to a VM
+- *
+- * Everything that needs to bo done only once when a BO is first added
+- * to a VM. It can later be mapped and unmapped many times without
+- * repeating these steps.
+- *
+- * 1. Allocate and initialize BO VA entry data structure
+- * 2. Add BO to the VM
+- * 3. Determine ASIC-specific PTE flags
+- * 4. Alloc page tables and directories if needed
+- * 4a. Validate new page tables and directories and update directories
+- */
+-static int add_bo_to_vm(struct amdgpu_device *adev, struct kgd_mem *mem,
+- struct amdgpu_vm *avm, bool is_aql,
+- struct kfd_bo_va_list **p_bo_va_entry)
+-{
+- int ret;
+- struct kfd_bo_va_list *bo_va_entry;
+- struct amdkfd_vm *kvm = container_of(avm,
+- struct amdkfd_vm, base);
+- struct amdgpu_bo *pd = avm->root.base.bo;
+- struct amdgpu_bo *bo = mem->bo;
+- uint64_t va = mem->va;
+- struct list_head *list_bo_va = &mem->bo_va_list;
+- unsigned long bo_size = bo->tbo.mem.size;
+-
+- if (!va) {
+- pr_err("Invalid VA when adding BO to VM\n");
+- return -EINVAL;
+- }
+-
+- if (is_aql)
+- va += bo_size;
+-
+- bo_va_entry = kzalloc(sizeof(*bo_va_entry), GFP_KERNEL);
+- if (!bo_va_entry)
+- return -ENOMEM;
+-
+- pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va,
+- va + bo_size, avm);
+-
+- /* Add BO to VM internal data structures*/
+- bo_va_entry->bo_va = amdgpu_vm_bo_add(adev, avm, bo);
+- if (bo_va_entry->bo_va == NULL) {
+- ret = -EINVAL;
+- pr_err("Failed to add BO object to VM. ret == %d\n",
+- ret);
+- goto err_vmadd;
+- }
+-
+- bo_va_entry->va = va;
+- bo_va_entry->pte_flags = amdgpu_vm_get_pte_flags(adev,
+- mem->mapping_flags);
+- bo_va_entry->kgd_dev = (void *)adev;
+- list_add(&bo_va_entry->bo_list, list_bo_va);
+-
+- if (p_bo_va_entry)
+- *p_bo_va_entry = bo_va_entry;
+-
+- /* Allocate new page tables if neeeded and validate
+- * them. Clearing of new page tables and validate need to wait
+- * on move fences. We don't want that to trigger the eviction
+- * fence, so remove it temporarily.
+- */
+- amdgpu_amdkfd_remove_eviction_fence(pd,
+- kvm->process_info->eviction_fence,
+- NULL, NULL);
+-
+- ret = amdgpu_vm_alloc_pts(adev, avm, va, amdgpu_bo_size(bo));
+- if (ret) {
+- pr_err("Failed to allocate pts, err=%d\n", ret);
+- goto err_alloc_pts;
+- }
+-
+- ret = vm_validate_pt_pd_bos(avm);
+- if (ret != 0) {
+- pr_err("validate_pt_pd_bos() failed\n");
+- goto err_alloc_pts;
+- }
+-
+- /* Add the eviction fence back */
+- amdgpu_bo_fence(pd, &kvm->process_info->eviction_fence->base, true);
+-
+- return 0;
+-
+-err_alloc_pts:
+- amdgpu_bo_fence(pd, &kvm->process_info->eviction_fence->base, true);
+- amdgpu_vm_bo_rmv(adev, bo_va_entry->bo_va);
+- list_del(&bo_va_entry->bo_list);
+-err_vmadd:
+- kfree(bo_va_entry);
+- return ret;
+-}
+-
+-static void remove_bo_from_vm(struct amdgpu_device *adev,
+- struct kfd_bo_va_list *entry, unsigned long size)
+-{
+- pr_debug("\t remove VA 0x%llx - 0x%llx in entry %p\n",
+- entry->va,
+- entry->va + size, entry);
+- amdgpu_vm_bo_rmv(adev, entry->bo_va);
+- list_del(&entry->bo_list);
+- kfree(entry);
+-}
+-
+-static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem *mem,
+- struct amdkfd_process_info *process_info,
+- bool userptr)
+-{
+- struct ttm_validate_buffer *entry = &mem->validate_list;
+- struct amdgpu_bo *bo = mem->bo;
+-
+- INIT_LIST_HEAD(&entry->head);
+- entry->shared = true;
+- entry->bo = &bo->tbo;
+- mutex_lock(&process_info->lock);
+- if (userptr)
+- list_add_tail(&entry->head, &process_info->userptr_valid_list);
+- else
+- list_add_tail(&entry->head, &process_info->kfd_bo_list);
+- mutex_unlock(&process_info->lock);
+-}
+-
+-/* Initializes user pages. It registers the MMU notifier and validates
+- * the userptr BO in the GTT domain.
+- *
+- * The BO must already be on the userptr_valid_list. Otherwise an
+- * eviction and restore may happen that leaves the new BO unmapped
+- * with the user mode queues running.
+- *
+- * Takes the process_info->lock to protect against concurrent restore
+- * workers.
+- *
+- * Returns 0 for success, negative errno for errors.
+- */
+-static int init_user_pages(struct kgd_mem *mem, struct mm_struct *mm,
+- uint64_t user_addr)
+-{
+- struct amdkfd_process_info *process_info = mem->process_info;
+- struct amdgpu_bo *bo = mem->bo;
+- int ret = 0;
+-
+- mutex_lock(&process_info->lock);
+-
+- ret = amdgpu_ttm_tt_set_userptr(bo->tbo.ttm, user_addr, 0);
+- if (ret) {
+- pr_err("%s: Failed to set userptr: %d\n", __func__, ret);
+- goto out;
+- }
+-
+- ret = amdgpu_mn_register(bo, user_addr);
+- if (ret) {
+- pr_err("%s: Failed to register MMU notifier: %d\n",
+- __func__, ret);
+- goto out;
+- }
+-
+- /* If no restore worker is running concurrently, user_pages
+- * should not be allocated
+- */
+- WARN(mem->user_pages, "Leaking user_pages array");
+-
+-#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)
+- mem->user_pages = drm_calloc_large(bo->tbo.ttm->num_pages,
+- sizeof(struct page *));
+-#else
+- mem->user_pages = kvmalloc_array(bo->tbo.ttm->num_pages,
+- sizeof(struct page *),
+- GFP_KERNEL | __GFP_ZERO);
+-#endif
+- if (!mem->user_pages) {
+- pr_err("%s: Failed to allocate pages array\n", __func__);
+- ret = -ENOMEM;
+- goto unregister_out;
+- }
+-
+- ret = amdgpu_ttm_tt_get_user_pages(bo->tbo.ttm, mem->user_pages);
+- if (ret) {
+- pr_err("%s: Failed to get user pages: %d\n", __func__, ret);
+- goto free_out;
+- }
+-
+- amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, mem->user_pages);
+-
+- ret = amdgpu_bo_reserve(bo, true);
+- if (ret) {
+- pr_err("%s: Failed to reserve BO\n", __func__);
+- goto release_out;
+- }
+- amdgpu_ttm_placement_from_domain(bo, mem->domain);
+- ret = ttm_bo_validate(&bo->tbo, &bo->placement,
+- true, false);
+- if (ret)
+- pr_err("%s: failed to validate BO\n", __func__);
+- amdgpu_bo_unreserve(bo);
+-
+-release_out:
+- if (ret)
+- release_pages(mem->user_pages, bo->tbo.ttm->num_pages, 0);
+-free_out:
+-#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)
+- drm_free_large(mem->user_pages);
+-#else
+- kvfree(mem->user_pages);
+-#endif
+- mem->user_pages = NULL;
+-unregister_out:
+- if (ret)
+- amdgpu_mn_unregister(bo);
+-out:
+- mutex_unlock(&process_info->lock);
+- return ret;
+-}
+-
+-static int __map_bo_to_kernel(struct amdgpu_bo *bo, u32 domain, void **kptr)
+-{
+- int ret;
+-
+- ret = amdgpu_bo_reserve(bo, true);
+- if (ret) {
+- pr_err("Failed to reserve bo. ret %d\n", ret);
+- return ret;
+- }
+-
+- ret = amdgpu_bo_pin(bo, domain, NULL);
+- if (ret) {
+- pr_err("Failed to pin bo. ret %d\n", ret);
+- goto pin_failed;
+- }
+-
+- ret = amdgpu_bo_kmap(bo, kptr);
+- if (ret) {
+- pr_err("Failed to map bo to kernel. ret %d\n", ret);
+- goto kmap_failed;
+- }
+-
+- amdgpu_bo_unreserve(bo);
+-
+- return ret;
+-
+-kmap_failed:
+- amdgpu_bo_unpin(bo);
+-pin_failed:
+- amdgpu_bo_unreserve(bo);
+-
+- return ret;
+-}
+-
+-static int __alloc_memory_of_gpu(struct kgd_dev *kgd, uint64_t va,
+- uint64_t size, void *vm, struct kgd_mem **mem,
+- uint64_t *offset, u32 domain, u64 flags,
+- struct sg_table *sg, bool aql_queue,
+- bool readonly, bool execute, bool coherent, bool no_sub,
+- bool userptr)
+-{
+- struct amdgpu_device *adev;
+- int ret;
+- struct amdgpu_bo *bo;
+- uint64_t user_addr = 0;
+- int byte_align;
+- u32 alloc_domain;
+- uint32_t mapping_flags;
+- struct amdkfd_vm *kfd_vm = (struct amdkfd_vm *)vm;
+-
+- if (aql_queue)
+- size = size >> 1;
+- if (userptr) {
+- if (!offset || !*offset)
+- return -EINVAL;
+- user_addr = *offset;
+- }
+-
+- adev = get_amdgpu_device(kgd);
+- byte_align = (adev->family == AMDGPU_FAMILY_VI &&
+- adev->asic_type != CHIP_FIJI &&
+- adev->asic_type != CHIP_POLARIS10 &&
+- adev->asic_type != CHIP_POLARIS11) ?
+- VI_BO_SIZE_ALIGN : 1;
+-
+- *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
+- if (*mem == NULL) {
+- ret = -ENOMEM;
+- goto err;
+- }
+- INIT_LIST_HEAD(&(*mem)->bo_va_list);
+- mutex_init(&(*mem)->lock);
+- (*mem)->coherent = coherent;
+- (*mem)->no_substitute = no_sub;
+- (*mem)->aql_queue = aql_queue;
+-
+- mapping_flags = AMDGPU_VM_PAGE_READABLE;
+- if (!readonly)
+- mapping_flags |= AMDGPU_VM_PAGE_WRITEABLE;
+- if (execute)
+- mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE;
+- if (coherent)
+- mapping_flags |= AMDGPU_VM_MTYPE_UC;
+- else
+- mapping_flags |= AMDGPU_VM_MTYPE_NC;
+-
+- (*mem)->mapping_flags = mapping_flags;
+-
+- alloc_domain = userptr ? AMDGPU_GEM_DOMAIN_CPU : domain;
+-
+- amdgpu_sync_create(&(*mem)->sync);
+-
+- ret = amdgpu_amdkfd_reserve_system_mem_limit(adev, size, alloc_domain);
+- if (ret) {
+- pr_err("Insufficient system memory\n");
+- goto err_bo_create;
+- }
+-
+- pr_debug("\t create BO VA 0x%llx size 0x%llx domain %s\n",
+- va, size, domain_string(alloc_domain));
+-
+- /* Allocate buffer object. Userptr objects need to start out
+- * in the CPU domain, get moved to GTT when pinned.
+- */
+- ret = amdgpu_bo_create(adev, size, byte_align, false,
+- alloc_domain,
+- flags, sg, NULL, 0, &bo);
+- if (ret != 0) {
+- pr_err("Failed to create BO on domain %s. ret %d\n",
+- domain_string(alloc_domain), ret);
+- unreserve_system_mem_limit(adev, size, alloc_domain);
+- goto err_bo_create;
+- }
+- bo->kfd_bo = *mem;
+- (*mem)->bo = bo;
+- if (userptr)
+- bo->flags |= AMDGPU_AMDKFD_USERPTR_BO;
+-
+- (*mem)->va = va;
+- (*mem)->domain = domain;
+- (*mem)->mapped_to_gpu_memory = 0;
+- (*mem)->process_info = kfd_vm->process_info;
+- add_kgd_mem_to_kfd_bo_list(*mem, kfd_vm->process_info, userptr);
+-
+- if (userptr) {
+- ret = init_user_pages(*mem, current->mm, user_addr);
+- if (ret) {
+- mutex_lock(&kfd_vm->process_info->lock);
+- list_del(&(*mem)->validate_list.head);
+- mutex_unlock(&kfd_vm->process_info->lock);
+- goto allocate_init_user_pages_failed;
+- }
+- }
+-
+- if (offset)
+- *offset = amdgpu_bo_mmap_offset(bo);
+-
+- return 0;
+-
+-allocate_init_user_pages_failed:
+- amdgpu_bo_unref(&bo);
+-err_bo_create:
+- kfree(*mem);
+-err:
+- return ret;
+-}
+-
+-/* Reserving a BO and its page table BOs must happen atomically to
+- * avoid deadlocks. When updating userptrs we need to temporarily
+- * back-off the reservation and then reacquire it. Track all the
+- * reservation info in a context structure. Buffers can be mapped to
+- * multiple VMs simultaneously (buffers being restored on multiple
+- * GPUs).
+- */
+-struct bo_vm_reservation_context {
+- struct amdgpu_bo_list_entry kfd_bo;
+- unsigned int n_vms;
+- struct amdgpu_bo_list_entry *vm_pd;
+- struct ww_acquire_ctx ticket;
+- struct list_head list, duplicates;
+- struct amdgpu_sync *sync;
+- bool reserved;
+-};
+-
+-/**
+- * reserve_bo_and_vm - reserve a BO and a VM unconditionally.
+- * @mem: KFD BO structure.
+- * @vm: the VM to reserve.
+- * @ctx: the struct that will be used in unreserve_bo_and_vms().
+- */
+-static int reserve_bo_and_vm(struct kgd_mem *mem,
+- struct amdgpu_vm *vm,
+- struct bo_vm_reservation_context *ctx)
+-{
+- struct amdgpu_bo *bo = mem->bo;
+- int ret;
+-
+- WARN_ON(!vm);
+-
+- ctx->reserved = false;
+- ctx->n_vms = 1;
+- ctx->sync = &mem->sync;
+-
+- INIT_LIST_HEAD(&ctx->list);
+- INIT_LIST_HEAD(&ctx->duplicates);
+-
+- ctx->vm_pd = kzalloc(sizeof(struct amdgpu_bo_list_entry)
+- * ctx->n_vms, GFP_KERNEL);
+- if (ctx->vm_pd == NULL)
+- return -ENOMEM;
+-
+- ctx->kfd_bo.robj = bo;
+- ctx->kfd_bo.priority = 0;
+- ctx->kfd_bo.tv.bo = &bo->tbo;
+- ctx->kfd_bo.tv.shared = true;
+- ctx->kfd_bo.user_pages = NULL;
+- list_add(&ctx->kfd_bo.tv.head, &ctx->list);
+-
+- amdgpu_vm_get_pd_bo(vm, &ctx->list, &ctx->vm_pd[0]);
+-
+- ret = ttm_eu_reserve_buffers(&ctx->ticket, &ctx->list,
+- false, &ctx->duplicates);
+- if (!ret)
+- ctx->reserved = true;
+- else
+- pr_err("Failed to reserve buffers in ttm\n");
+-
+- if (ret) {
+- kfree(ctx->vm_pd);
+- ctx->vm_pd = NULL;
+- }
+-
+- return ret;
+-}
+-
+-enum VA_TYPE {
+- VA_NOT_MAPPED = 0,
+- VA_MAPPED,
+- VA_DO_NOT_CARE,
+-};
+-
+-/**
+- * reserve_bo_and_vm - reserve a BO and some VMs that the BO has been added
+- * to, conditionally based on map_type.
+- * @mem: KFD BO structure.
+- * @vm: the VM to reserve. If NULL, then all VMs associated with the BO
+- * is used. Otherwise, a single VM associated with the BO.
+- * @map_type: the mapping status that will be used to filter the VMs.
+- * @ctx: the struct that will be used in unreserve_bo_and_vms().
+- */
+-static int reserve_bo_and_cond_vms(struct kgd_mem *mem,
+- struct amdgpu_vm *vm, enum VA_TYPE map_type,
+- struct bo_vm_reservation_context *ctx)
+-{
+- struct amdgpu_bo *bo = mem->bo;
+- struct kfd_bo_va_list *entry;
+- unsigned int i;
+- int ret;
+-
+- ctx->reserved = false;
+- ctx->n_vms = 0;
+- ctx->vm_pd = NULL;
+- ctx->sync = &mem->sync;
+-
+- INIT_LIST_HEAD(&ctx->list);
+- INIT_LIST_HEAD(&ctx->duplicates);
+-
+- list_for_each_entry(entry, &mem->bo_va_list, bo_list) {
+- if ((vm && vm != entry->bo_va->base.vm) ||
+- (entry->is_mapped != map_type
+- && map_type != VA_DO_NOT_CARE))
+- continue;
+-
+- ctx->n_vms++;
+- }
+-
+- if (ctx->n_vms != 0) {
+- ctx->vm_pd = kzalloc(sizeof(struct amdgpu_bo_list_entry)
+- * ctx->n_vms, GFP_KERNEL);
+- if (ctx->vm_pd == NULL)
+- return -ENOMEM;
+- }
+-
+- ctx->kfd_bo.robj = bo;
+- ctx->kfd_bo.priority = 0;
+- ctx->kfd_bo.tv.bo = &bo->tbo;
+- ctx->kfd_bo.tv.shared = true;
+- ctx->kfd_bo.user_pages = NULL;
+- list_add(&ctx->kfd_bo.tv.head, &ctx->list);
+-
+- i = 0;
+- list_for_each_entry(entry, &mem->bo_va_list, bo_list) {
+- if ((vm && vm != entry->bo_va->base.vm) ||
+- (entry->is_mapped != map_type
+- && map_type != VA_DO_NOT_CARE))
+- continue;
+-
+- amdgpu_vm_get_pd_bo(entry->bo_va->base.vm, &ctx->list,
+- &ctx->vm_pd[i]);
+- i++;
+- }
+-
+- ret = ttm_eu_reserve_buffers(&ctx->ticket, &ctx->list,
+- false, &ctx->duplicates);
+- if (!ret)
+- ctx->reserved = true;
+- else
+- pr_err("Failed to reserve buffers in ttm.\n");
+-
+- if (ret) {
+- kfree(ctx->vm_pd);
+- ctx->vm_pd = NULL;
+- }
+-
+- return ret;
+-}
+-
+-static int unreserve_bo_and_vms(struct bo_vm_reservation_context *ctx,
+- bool wait, bool intr)
+-{
+- int ret = 0;
+-
+- if (wait)
+- ret = amdgpu_sync_wait(ctx->sync, intr);
+-
+- if (ctx->reserved)
+- ttm_eu_backoff_reservation(&ctx->ticket, &ctx->list);
+- kfree(ctx->vm_pd);
+-
+- ctx->sync = NULL;
+-
+- ctx->reserved = false;
+- ctx->vm_pd = NULL;
+-
+- return ret;
+-}
+-
+-static int unmap_bo_from_gpuvm(struct amdgpu_device *adev,
+- struct kfd_bo_va_list *entry,
+- struct amdgpu_sync *sync)
+-{
+- struct amdgpu_bo_va *bo_va = entry->bo_va;
+- struct amdgpu_vm *vm = bo_va->base.vm;
+- struct amdkfd_vm *kvm = container_of(vm, struct amdkfd_vm, base);
+- struct amdgpu_bo *pd = vm->root.base.bo;
+-
+- /* Remove eviction fence from PD (and thereby from PTs too as they
+- * share the resv. object. Otherwise during PT update job (see
+- * amdgpu_vm_bo_update_mapping), eviction fence will get added to
+- * job->sync object
+- */
+- amdgpu_amdkfd_remove_eviction_fence(pd,
+- kvm->process_info->eviction_fence,
+- NULL, NULL);
+- amdgpu_vm_bo_unmap(adev, bo_va, entry->va);
+-
+- amdgpu_vm_clear_freed(adev, vm, &bo_va->last_pt_update);
+-
+- /* Add the eviction fence back */
+- amdgpu_bo_fence(pd, &kvm->process_info->eviction_fence->base, true);
+-
+- amdgpu_sync_fence(adev, sync, bo_va->last_pt_update);
+-
+- /* Sync objects can't handle multiple GPUs (contexts) updating
+- * sync->last_vm_update. Fortunately we don't need it for
+- * KFD's purposes, so we can just drop that fence.
+- */
+- if (sync->last_vm_update) {
+- dma_fence_put(sync->last_vm_update);
+- sync->last_vm_update = NULL;
+- }
+-
+- return 0;
+-}
+-
+-static int update_gpuvm_pte(struct amdgpu_device *adev,
+- struct kfd_bo_va_list *entry,
+- struct amdgpu_sync *sync)
+-{
+- int ret;
+- struct amdgpu_vm *vm;
+- struct amdgpu_bo_va *bo_va;
+- struct amdgpu_bo *bo;
+-
+- bo_va = entry->bo_va;
+- vm = bo_va->base.vm;
+- bo = bo_va->base.bo;
+-
+- /* Update the page tables */
+- ret = amdgpu_vm_bo_update(adev, bo_va, false);
+- if (ret != 0) {
+- pr_err("amdgpu_vm_bo_update failed\n");
+- return ret;
+- }
+-
+- amdgpu_sync_fence(adev, sync, bo_va->last_pt_update);
+-
+- /* Sync objects can't handle multiple GPUs (contexts) updating
+- * sync->last_vm_update. Fortunately we don't need it for
+- * KFD's purposes, so we can just drop that fence.
+- */
+- if (sync->last_vm_update) {
+- dma_fence_put(sync->last_vm_update);
+- sync->last_vm_update = NULL;
+- }
+-
+- return 0;
+-}
+-
+-static int map_bo_to_gpuvm(struct amdgpu_device *adev,
+- struct kfd_bo_va_list *entry, struct amdgpu_sync *sync,
+- bool no_update_pte)
+-{
+- int ret;
+-
+- /* Set virtual address for the allocation */
+- ret = amdgpu_vm_bo_map(adev, entry->bo_va, entry->va, 0,
+- amdgpu_bo_size(entry->bo_va->base.bo), entry->pte_flags);
+- if (ret != 0) {
+- pr_err("Failed to map VA 0x%llx in vm. ret %d\n",
+- entry->va, ret);
+- return ret;
+- }
+-
+- if (no_update_pte)
+- return 0;
+-
+- ret = update_gpuvm_pte(adev, entry, sync);
+- if (ret != 0) {
+- pr_err("update_gpuvm_pte() failed\n");
+- goto update_gpuvm_pte_failed;
+- }
+-
+- return 0;
+-
+-update_gpuvm_pte_failed:
+- unmap_bo_from_gpuvm(adev, entry, sync);
+- return ret;
+-}
+-
+-static struct sg_table *create_doorbell_sg(uint64_t addr, uint32_t size)
+-{
+- struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
+-
+- if (!sg)
+- return NULL;
+- if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
+- kfree(sg);
+- return NULL;
+- }
+- sg->sgl->dma_address = addr;
+- sg->sgl->length = size;
+-#ifdef CONFIG_NEED_SG_DMA_LENGTH
+- sg->sgl->dma_length = size;
+-#endif
+- return sg;
+-}
+-
+-int amdgpu_amdkfd_gpuvm_sync_memory(
+- struct kgd_dev *kgd, struct kgd_mem *mem, bool intr)
+-{
+- int ret = 0;
+- struct amdgpu_sync sync;
+- struct amdgpu_device *adev;
+-
+- adev = get_amdgpu_device(kgd);
+- amdgpu_sync_create(&sync);
+-
+- mutex_lock(&mem->lock);
+- amdgpu_sync_clone(adev, &mem->sync, &sync);
+- mutex_unlock(&mem->lock);
+-
+- ret = amdgpu_sync_wait(&sync, intr);
+- amdgpu_sync_free(&sync);
+- return ret;
+-}
+-
+-#define BOOL_TO_STR(b) (b == true) ? "true" : "false"
+-
+-int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
+- struct kgd_dev *kgd, uint64_t va, uint64_t size,
+- void *vm, struct kgd_mem **mem,
+- uint64_t *offset, uint32_t flags)
+-{
+- bool aql_queue, public, readonly, execute, coherent, no_sub, userptr;
+- u64 alloc_flag;
+- uint32_t domain;
+- uint64_t *temp_offset;
+- struct sg_table *sg = NULL;
+-
+- if (!(flags & ALLOC_MEM_FLAGS_NONPAGED)) {
+- pr_err("current hw doesn't support paged memory\n");
+- return -EINVAL;
+- }
+-
+- domain = 0;
+- alloc_flag = 0;
+- temp_offset = NULL;
+-
+- aql_queue = (flags & ALLOC_MEM_FLAGS_AQL_QUEUE_MEM) ? true : false;
+- public = (flags & ALLOC_MEM_FLAGS_PUBLIC) ? true : false;
+- readonly = (flags & ALLOC_MEM_FLAGS_READONLY) ? true : false;
+- execute = (flags & ALLOC_MEM_FLAGS_EXECUTE_ACCESS) ? true : false;
+- coherent = (flags & ALLOC_MEM_FLAGS_COHERENT) ? true : false;
+- no_sub = (flags & ALLOC_MEM_FLAGS_NO_SUBSTITUTE) ? true : false;
+- userptr = (flags & ALLOC_MEM_FLAGS_USERPTR) ? true : false;
+-
+- /*
+- * Check on which domain to allocate BO
+- */
+- if (flags & ALLOC_MEM_FLAGS_VRAM) {
+- domain = AMDGPU_GEM_DOMAIN_VRAM;
+- alloc_flag = AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
+- if (public) {
+- alloc_flag = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
+- temp_offset = offset;
+- }
+- alloc_flag |= AMDGPU_GEM_CREATE_VRAM_CLEARED;
+- } else if (flags & (ALLOC_MEM_FLAGS_GTT | ALLOC_MEM_FLAGS_USERPTR)) {
+- domain = AMDGPU_GEM_DOMAIN_GTT;
+- alloc_flag = 0;
+- temp_offset = offset;
+- } else if (flags & ALLOC_MEM_FLAGS_DOORBELL) {
+- domain = AMDGPU_GEM_DOMAIN_GTT;
+- alloc_flag = 0;
+- temp_offset = offset;
+- if (size > UINT_MAX)
+- return -EINVAL;
+- sg = create_doorbell_sg(*offset, size);
+- if (!sg)
+- return -ENOMEM;
+- }
+-
+- if (offset && !userptr)
+- *offset = 0;
+-
+- pr_debug("Allocate VA 0x%llx - 0x%llx domain %s aql %s\n",
+- va, va + size, domain_string(domain),
+- BOOL_TO_STR(aql_queue));
+-
+- pr_debug("\t alloc_flag 0x%llx public %s readonly %s execute %s coherent %s no_sub %s\n",
+- alloc_flag, BOOL_TO_STR(public),
+- BOOL_TO_STR(readonly), BOOL_TO_STR(execute),
+- BOOL_TO_STR(coherent), BOOL_TO_STR(no_sub));
+-
+- return __alloc_memory_of_gpu(kgd, va, size, vm, mem,
+- temp_offset, domain,
+- alloc_flag, sg,
+- aql_queue, readonly, execute,
+- coherent, no_sub, userptr);
+-}
+-
+-int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
+- struct kgd_dev *kgd, struct kgd_mem *mem, void *vm)
+-{
+- struct amdgpu_device *adev;
+- struct kfd_bo_va_list *entry, *tmp;
+- struct bo_vm_reservation_context ctx;
+- int ret = 0;
+- struct ttm_validate_buffer *bo_list_entry;
+- struct amdkfd_process_info *process_info;
+- unsigned long bo_size;
+-
+- adev = get_amdgpu_device(kgd);
+- process_info = ((struct amdkfd_vm *)vm)->process_info;
+-
+- bo_size = mem->bo->tbo.mem.size;
+-
+- mutex_lock(&mem->lock);
+-
+- if (mem->mapped_to_gpu_memory > 0) {
+- pr_err("BO VA 0x%llx size 0x%lx is already mapped to vm %p.\n",
+- mem->va, bo_size, vm);
+- mutex_unlock(&mem->lock);
+- return -EBUSY;
+- }
+-
+- mutex_unlock(&mem->lock);
+- /* lock is not needed after this, since mem is unused and will
+- * be freed anyway
+- */
+-
+- /* No more MMU notifiers */
+- amdgpu_mn_unregister(mem->bo);
+-
+- /* Make sure restore workers don't access the BO any more */
+- bo_list_entry = &mem->validate_list;
+- mutex_lock(&process_info->lock);
+- list_del(&bo_list_entry->head);
+- mutex_unlock(&process_info->lock);
+-
+- /* Free user pages if necessary */
+- if (mem->user_pages) {
+- pr_debug("%s: Freeing user_pages array\n", __func__);
+- if (mem->user_pages[0])
+- release_pages(mem->user_pages,
+- mem->bo->tbo.ttm->num_pages, 0);
+-#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)
+- drm_free_large(mem->user_pages);
+-#else
+- kvfree(mem->user_pages);
+-#endif
+- }
+-
+- ret = reserve_bo_and_cond_vms(mem, NULL, VA_DO_NOT_CARE, &ctx);
+- if (unlikely(ret != 0))
+- return ret;
+-
+- /* The eviction fence should be removed by the last unmap.
+- * TODO: Log an error condition if the bo still has the eviction fence
+- * attached
+- */
+- amdgpu_amdkfd_remove_eviction_fence(mem->bo,
+- process_info->eviction_fence,
+- NULL, NULL);
+- pr_debug("Release VA 0x%llx - 0x%llx\n", mem->va,
+- mem->va + bo_size * (1 + mem->aql_queue));
+-
+- /* Remove from VM internal data structures */
+- list_for_each_entry_safe(entry, tmp, &mem->bo_va_list, bo_list) {
+- remove_bo_from_vm((struct amdgpu_device *)entry->kgd_dev,
+- entry, bo_size);
+- }
+-
+- ret = unreserve_bo_and_vms(&ctx, false, false);
+-
+- /* Free the sync object */
+- amdgpu_sync_free(&mem->sync);
+-
+- /* If the SG is not NULL, it's one we created for a doorbell
+- * BO. We need to free it.
+- */
+- if (mem->bo->tbo.sg) {
+- sg_free_table(mem->bo->tbo.sg);
+- kfree(mem->bo->tbo.sg);
+- }
+-
+- /* Free the BO*/
+- amdgpu_bo_unref(&mem->bo);
+- kfree(mem);
+-
+- return ret;
+-}
+-
+-int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
+- struct kgd_dev *kgd, struct kgd_mem *mem, void *vm)
+-{
+- struct amdgpu_device *adev;
+- int ret;
+- struct amdgpu_bo *bo;
+- uint32_t domain;
+- struct kfd_bo_va_list *entry;
+- struct bo_vm_reservation_context ctx;
+- struct kfd_bo_va_list *bo_va_entry = NULL;
+- struct kfd_bo_va_list *bo_va_entry_aql = NULL;
+- struct amdkfd_vm *kfd_vm = (struct amdkfd_vm *)vm;
+- unsigned long bo_size;
+- bool is_invalid_userptr;
+-
+- adev = get_amdgpu_device(kgd);
+-
+- /* Make sure restore is not running concurrently. Since we
+- * don't map invalid userptr BOs, we rely on the next restore
+- * worker to do the mapping
+- */
+- mutex_lock(&mem->process_info->lock);
+-
+- /* Lock mmap-sem. If we find an invalid userptr BO, we can be
+- * sure that the MMU notifier is no longer running
+- * concurrently and the queues are actually stopped
+- */
+- down_read(&current->mm->mmap_sem);
+- is_invalid_userptr = atomic_read(&mem->invalid);
+- up_read(&current->mm->mmap_sem);
+-
+- mutex_lock(&mem->lock);
+-
+- bo = mem->bo;
+-
+- if (!bo) {
+- pr_err("Invalid BO when mapping memory to GPU\n");
+- return -EINVAL;
+- }
+-
+- domain = mem->domain;
+- bo_size = bo->tbo.mem.size;
+-
+- pr_debug("Map VA 0x%llx - 0x%llx to vm %p domain %s\n",
+- mem->va,
+- mem->va + bo_size * (1 + mem->aql_queue),
+- vm, domain_string(domain));
+-
+- ret = reserve_bo_and_vm(mem, vm, &ctx);
+- if (unlikely(ret != 0))
+- goto bo_reserve_failed;
+-
+- /* Userptr can be marked as "not invalid", but not actually be
+- * validated yet (still in the system domain). In that case
+- * the queues are still stopped and we can leave mapping for
+- * the next restore worker
+- */
+- if (bo->tbo.mem.mem_type == TTM_PL_SYSTEM)
+- is_invalid_userptr = true;
+-
+- if (check_if_add_bo_to_vm((struct amdgpu_vm *)vm, mem)) {
+- ret = add_bo_to_vm(adev, mem, (struct amdgpu_vm *)vm, false,
+- &bo_va_entry);
+- if (ret != 0)
+- goto add_bo_to_vm_failed;
+- if (mem->aql_queue) {
+- ret = add_bo_to_vm(adev, mem, (struct amdgpu_vm *)vm,
+- true, &bo_va_entry_aql);
+- if (ret != 0)
+- goto add_bo_to_vm_failed_aql;
+- }
+- }
+-
+- if (mem->mapped_to_gpu_memory == 0 &&
+- !amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {
+- /* Validate BO only once. The eviction fence gets added to BO
+- * the first time it is mapped. Validate will wait for all
+- * background evictions to complete.
+- */
+- ret = amdgpu_amdkfd_bo_validate(bo, domain, true);
+- if (ret) {
+- pr_debug("Validate failed\n");
+- goto map_bo_to_gpuvm_failed;
+- }
+- }
+-
+- list_for_each_entry(entry, &mem->bo_va_list, bo_list) {
+- if (entry->bo_va->base.vm == vm && !entry->is_mapped) {
+- pr_debug("\t map VA 0x%llx - 0x%llx in entry %p\n",
+- entry->va, entry->va + bo_size,
+- entry);
+-
+- ret = map_bo_to_gpuvm(adev, entry, ctx.sync,
+- is_invalid_userptr);
+- if (ret != 0) {
+- pr_err("Failed to map radeon bo to gpuvm\n");
+- goto map_bo_to_gpuvm_failed;
+- }
+- entry->is_mapped = true;
+- mem->mapped_to_gpu_memory++;
+- pr_debug("\t INC mapping count %d\n",
+- mem->mapped_to_gpu_memory);
+- }
+- }
+-
+- if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) == NULL)
+- amdgpu_bo_fence(bo,
+- &kfd_vm->process_info->eviction_fence->base,
+- true);
+- ret = unreserve_bo_and_vms(&ctx, false, false);
+-
+- mutex_unlock(&mem->process_info->lock);
+- mutex_unlock(&mem->lock);
+- return ret;
+-
+-map_bo_to_gpuvm_failed:
+- if (bo_va_entry_aql)
+- remove_bo_from_vm(adev, bo_va_entry_aql, bo_size);
+-add_bo_to_vm_failed_aql:
+- if (bo_va_entry)
+- remove_bo_from_vm(adev, bo_va_entry, bo_size);
+-add_bo_to_vm_failed:
+- unreserve_bo_and_vms(&ctx, false, false);
+-bo_reserve_failed:
+- mutex_unlock(&mem->process_info->lock);
+- mutex_unlock(&mem->lock);
+- return ret;
+-}
+-
+-static u64 get_vm_pd_gpu_offset(void *vm)
+-{
+- struct amdgpu_vm *avm = (struct amdgpu_vm *) vm;
+- struct amdgpu_device *adev =
+- amdgpu_ttm_adev(avm->root.base.bo->tbo.bdev);
+- u64 offset;
+-
+- BUG_ON(avm == NULL);
+-
+- amdgpu_bo_reserve(avm->root.base.bo, false);
+-
+- offset = amdgpu_bo_gpu_offset(avm->root.base.bo);
+-
+- amdgpu_bo_unreserve(avm->root.base.bo);
+-
+- /* On some ASICs the FB doesn't start at 0. Adjust FB offset
+- * to an actual MC address.
+- */
+- if (adev->gart.gart_funcs->get_vm_pde)
+- offset = amdgpu_gart_get_vm_pde(adev, offset);
+-
+- return offset;
+-}
+-
+-int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, void **vm,
+- void **process_info,
+- struct dma_fence **ef)
+-{
+- int ret;
+- struct amdkfd_vm *new_vm;
+- struct amdkfd_process_info *info;
+- struct amdgpu_device *adev = get_amdgpu_device(kgd);
+-
+- new_vm = kzalloc(sizeof(*new_vm), GFP_KERNEL);
+- if (new_vm == NULL)
+- return -ENOMEM;
+-
+- /* Initialize the VM context, allocate the page directory and zero it */
+- ret = amdgpu_vm_init(adev, &new_vm->base, AMDGPU_VM_CONTEXT_COMPUTE);
+- if (ret != 0) {
+- pr_err("Failed init vm ret %d\n", ret);
+- /* Undo everything related to the new VM context */
+- goto vm_init_fail;
+- }
+- new_vm->adev = adev;
+-
+- if (!*process_info) {
+- info = kzalloc(sizeof(*info), GFP_KERNEL);
+- if (!info) {
+- pr_err("Failed to create amdkfd_process_info");
+- ret = -ENOMEM;
+- goto alloc_process_info_fail;
+- }
+-
+- mutex_init(&info->lock);
+- INIT_LIST_HEAD(&info->vm_list_head);
+- INIT_LIST_HEAD(&info->kfd_bo_list);
+- INIT_LIST_HEAD(&info->userptr_valid_list);
+- INIT_LIST_HEAD(&info->userptr_inval_list);
+-
+- info->eviction_fence =
+- amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1),
+- current->mm);
+- if (info->eviction_fence == NULL) {
+- pr_err("Failed to create eviction fence\n");
+- goto create_evict_fence_fail;
+- }
+-
+- info->pid = get_task_pid(current->group_leader,
+- PIDTYPE_PID);
+- atomic_set(&info->evicted_bos, 0);
+- INIT_DELAYED_WORK(&info->work,
+- amdgpu_amdkfd_restore_userptr_worker);
+-
+- *process_info = info;
+- *ef = dma_fence_get(&info->eviction_fence->base);
+- }
+-
+- new_vm->process_info = *process_info;
+-
+- mutex_lock(&new_vm->process_info->lock);
+- list_add_tail(&new_vm->vm_list_node,
+- &(new_vm->process_info->vm_list_head));
+- new_vm->process_info->n_vms++;
+- mutex_unlock(&new_vm->process_info->lock);
+-
+- *vm = (void *) new_vm;
+-
+- pr_debug("Created process vm %p\n", *vm);
+-
+- return ret;
+-
+-create_evict_fence_fail:
+- kfree(info);
+-alloc_process_info_fail:
+- amdgpu_vm_fini(adev, &new_vm->base);
+-vm_init_fail:
+- kfree(new_vm);
+- return ret;
+-
+-}
+-
+-void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev *kgd, void *vm)
+-{
+- struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+- struct amdkfd_vm *kfd_vm = (struct amdkfd_vm *) vm;
+- struct amdgpu_vm *avm = &kfd_vm->base;
+- struct amdgpu_bo *pd;
+- struct amdkfd_process_info *process_info;
+-
+- if (WARN_ON(!kgd || !vm))
+- return;
+-
+- pr_debug("Destroying process vm %p\n", vm);
+- /* Release eviction fence from PD */
+- pd = avm->root.base.bo;
+- amdgpu_bo_reserve(pd, false);
+- amdgpu_bo_fence(pd, NULL, false);
+- amdgpu_bo_unreserve(pd);
+-
+- process_info = kfd_vm->process_info;
+-
+- mutex_lock(&process_info->lock);
+- process_info->n_vms--;
+- list_del(&kfd_vm->vm_list_node);
+- mutex_unlock(&process_info->lock);
+-
+- /* Release per-process resources */
+- if (!process_info->n_vms) {
+- WARN_ON(!list_empty(&process_info->kfd_bo_list));
+- WARN_ON(!list_empty(&process_info->userptr_valid_list));
+- WARN_ON(!list_empty(&process_info->userptr_inval_list));
+-
+- dma_fence_put(&process_info->eviction_fence->base);
+- cancel_delayed_work_sync(&process_info->work);
+- put_pid(process_info->pid);
+- kfree(process_info);
+- }
+-
+- /* Release the VM context */
+- amdgpu_vm_fini(adev, avm);
+- kfree(vm);
+-}
+-
+-uint32_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm)
+-{
+- return get_vm_pd_gpu_offset(vm) >> AMDGPU_GPU_PAGE_SHIFT;
+-}
+-
+-int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd,
+- struct kfd_vm_fault_info *mem)
+-{
+- struct amdgpu_device *adev;
+-
+- adev = (struct amdgpu_device *) kgd;
+- if (atomic_read(&adev->mc.vm_fault_info_updated) == 1) {
+- *mem = *adev->mc.vm_fault_info;
+- mb();
+- atomic_set(&adev->mc.vm_fault_info_updated, 0);
+- }
+- return 0;
+-}
+-
+-static bool is_mem_on_local_device(struct kgd_dev *kgd,
+- struct list_head *bo_va_list, void *vm)
+-{
+- struct kfd_bo_va_list *entry;
+-
+- list_for_each_entry(entry, bo_va_list, bo_list) {
+- if (entry->kgd_dev == kgd && entry->bo_va->base.vm == vm)
+- return true;
+- }
+-
+- return false;
+-}
+-
+-int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
+- struct kgd_dev *kgd, struct kgd_mem *mem, void *vm)
+-{
+- struct kfd_bo_va_list *entry;
+- struct amdgpu_device *adev;
+- unsigned int mapped_before;
+- int ret = 0;
+- struct bo_vm_reservation_context ctx;
+- struct amdkfd_process_info *process_info;
+- unsigned long bo_size;
+-
+- adev = (struct amdgpu_device *) kgd;
+- process_info = ((struct amdkfd_vm *)vm)->process_info;
+-
+- bo_size = mem->bo->tbo.mem.size;
+-
+- mutex_lock(&mem->lock);
+-
+- /*
+- * Make sure that this BO mapped on KGD before unmappping it
+- */
+- if (!is_mem_on_local_device(kgd, &mem->bo_va_list, vm)) {
+- ret = -EINVAL;
+- goto out;
+- }
+-
+- if (mem->mapped_to_gpu_memory == 0) {
+- pr_debug("BO VA 0x%llx size 0x%lx is not mapped to vm %p\n",
+- mem->va, bo_size, vm);
+- ret = -EINVAL;
+- goto out;
+- }
+- mapped_before = mem->mapped_to_gpu_memory;
+-
+- ret = reserve_bo_and_cond_vms(mem, vm, VA_MAPPED, &ctx);
+- if (unlikely(ret != 0))
+- goto out;
+-
+- pr_debug("Unmap VA 0x%llx - 0x%llx from vm %p\n",
+- mem->va,
+- mem->va + bo_size * (1 + mem->aql_queue),
+- vm);
+-
+- list_for_each_entry(entry, &mem->bo_va_list, bo_list) {
+- if (entry->bo_va->base.vm == vm && entry->is_mapped) {
+- pr_debug("\t unmap VA 0x%llx - 0x%llx from entry %p\n",
+- entry->va,
+- entry->va + bo_size,
+- entry);
+-
+- ret = unmap_bo_from_gpuvm(adev, entry, ctx.sync);
+- if (ret == 0) {
+- entry->is_mapped = false;
+- } else {
+- pr_err("failed to unmap VA 0x%llx\n",
+- mem->va);
+- goto unreserve_out;
+- }
+-
+- mem->mapped_to_gpu_memory--;
+- pr_debug("\t DEC mapping count %d\n",
+- mem->mapped_to_gpu_memory);
+- }
+- }
+-
+- /* If BO is unmapped from all VMs, unfence it. It can be evicted if
+- * required.
+- */
+- if (mem->mapped_to_gpu_memory == 0 &&
+- !amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm))
+- amdgpu_amdkfd_remove_eviction_fence(mem->bo,
+- process_info->eviction_fence,
+- NULL, NULL);
+-
+- if (mapped_before == mem->mapped_to_gpu_memory) {
+- pr_debug("BO VA 0x%llx size 0x%lx is not mapped to vm %p\n",
+- mem->va, bo_size, vm);
+- ret = -EINVAL;
+- }
+-
+-unreserve_out:
+- unreserve_bo_and_vms(&ctx, false, false);
+-out:
+- mutex_unlock(&mem->lock);
+- return ret;
+-}
+-
+-int amdgpu_amdkfd_gpuvm_mmap_bo(struct kgd_dev *kgd, struct vm_area_struct *vma)
+-{
+- struct amdgpu_device *adev;
+-
+- adev = get_amdgpu_device(kgd);
+- if (!adev) {
+- pr_err("Could not get amdgpu device in %s\n", __func__);
+- return -ENODEV;
+- }
+-
+- return amdgpu_bo_mmap(NULL, vma, &adev->mman.bdev);
+-}
+-
+-int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd,
+- struct kgd_mem *mem, void **kptr)
+-{
+- int ret;
+- struct amdgpu_bo *bo = mem->bo;
+-
+- if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {
+- pr_err("userptr can't be mapped to kernel\n");
+- return -EINVAL;
+- }
+-
+- /* delete kgd_mem from kfd_bo_list to avoid re-validating
+- * this BO in BO's restoring after eviction.
+- */
+- mutex_lock(&mem->process_info->lock);
+-
+- list_del_init(&mem->validate_list.head);
+-
+- ret = __map_bo_to_kernel(bo, AMDGPU_GEM_DOMAIN_GTT, kptr);
+- if (!ret)
+- mem->kptr = *kptr;
+-
+- mutex_unlock(&mem->process_info->lock);
+-
+- return ret;
+-}
+-
+-static int pin_bo_wo_map(struct kgd_mem *mem)
+-{
+- struct amdgpu_bo *bo = mem->bo;
+- int ret = 0;
+-
+- ret = amdgpu_bo_reserve(bo, false);
+- if (unlikely(ret != 0))
+- return ret;
+-
+- ret = amdgpu_bo_pin(bo, mem->domain, NULL);
+- amdgpu_bo_unreserve(bo);
+-
+- return ret;
+-}
+-
+-static void unpin_bo_wo_map(struct kgd_mem *mem)
+-{
+- struct amdgpu_bo *bo = mem->bo;
+- int ret = 0;
+-
+- ret = amdgpu_bo_reserve(bo, false);
+- if (unlikely(ret != 0))
+- return;
+-
+- amdgpu_bo_unpin(bo);
+- amdgpu_bo_unreserve(bo);
+-}
+-
+-#define AMD_GPU_PAGE_SHIFT PAGE_SHIFT
+-#define AMD_GPU_PAGE_SIZE (_AC(1, UL) << AMD_GPU_PAGE_SHIFT)
+-
+-static int get_sg_table(struct amdgpu_device *adev,
+- struct kgd_mem *mem, uint64_t offset,
+- uint64_t size, struct sg_table **ret_sg)
+-{
+- struct amdgpu_bo *bo = mem->bo;
+- struct sg_table *sg = NULL;
+- unsigned long bus_addr;
+- unsigned int chunks;
+- unsigned int i;
+- struct scatterlist *s;
+- uint64_t offset_in_page;
+- unsigned int page_size;
+- int ret;
+-
+- sg = kmalloc(sizeof(*sg), GFP_KERNEL);
+- if (!sg) {
+- ret = -ENOMEM;
+- goto out;
+- }
+-
+- if (bo->preferred_domains == AMDGPU_GEM_DOMAIN_VRAM)
+- page_size = AMD_GPU_PAGE_SIZE;
+- else
+- page_size = PAGE_SIZE;
+-
+-
+- offset_in_page = offset & (page_size - 1);
+- chunks = (size + offset_in_page + page_size - 1)
+- / page_size;
+-
+- ret = sg_alloc_table(sg, chunks, GFP_KERNEL);
+- if (unlikely(ret))
+- goto out;
+-
+- if (bo->preferred_domains == AMDGPU_GEM_DOMAIN_VRAM) {
+- bus_addr = bo->tbo.offset + adev->mc.aper_base + offset;
+-
+- for_each_sg(sg->sgl, s, sg->orig_nents, i) {
+- uint64_t chunk_size, length;
+-
+- chunk_size = page_size - offset_in_page;
+- length = min(size, chunk_size);
+-
+- sg_set_page(s, NULL, length, offset_in_page);
+- s->dma_address = bus_addr;
+- s->dma_length = length;
+-
+- size -= length;
+- offset_in_page = 0;
+- bus_addr += length;
+- }
+- } else {
+- struct page **pages;
+- unsigned int cur_page;
+-
+- pages = bo->tbo.ttm->pages;
+-
+- cur_page = offset / page_size;
+- for_each_sg(sg->sgl, s, sg->orig_nents, i) {
+- uint64_t chunk_size, length;
+-
+- chunk_size = page_size - offset_in_page;
+- length = min(size, chunk_size);
+-
+- sg_set_page(s, pages[cur_page], length, offset_in_page);
+- s->dma_address = page_to_phys(pages[cur_page]);
+- s->dma_length = length;
+-
+- size -= length;
+- offset_in_page = 0;
+- cur_page++;
+- }
+- }
+-
+- *ret_sg = sg;
+- return 0;
+-out:
+- kfree(sg);
+- *ret_sg = NULL;
+- return ret;
+-}
+-
+-int amdgpu_amdkfd_gpuvm_pin_get_sg_table(struct kgd_dev *kgd,
+- struct kgd_mem *mem, uint64_t offset,
+- uint64_t size, struct sg_table **ret_sg)
+-{
+- int ret;
+- struct amdgpu_device *adev;
+-
+- ret = pin_bo_wo_map(mem);
+- if (unlikely(ret != 0))
+- return ret;
+-
+- adev = get_amdgpu_device(kgd);
+-
+- ret = get_sg_table(adev, mem, offset, size, ret_sg);
+- if (ret)
+- unpin_bo_wo_map(mem);
+-
+- return ret;
+-}
+-
+-void amdgpu_amdkfd_gpuvm_unpin_put_sg_table(
+- struct kgd_mem *mem, struct sg_table *sg)
+-{
+- sg_free_table(sg);
+- kfree(sg);
+-
+- unpin_bo_wo_map(mem);
+-}
+-
+-int amdgpu_amdkfd_gpuvm_import_dmabuf(struct kgd_dev *kgd,
+- struct dma_buf *dma_buf,
+- uint64_t va, void *vm,
+- struct kgd_mem **mem, uint64_t *size,
+- uint64_t *mmap_offset)
+-{
+- struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+- struct drm_gem_object *obj;
+- struct amdgpu_bo *bo;
+- struct amdkfd_vm *kfd_vm = (struct amdkfd_vm *)vm;
+-
+- if (dma_buf->ops != &drm_gem_prime_dmabuf_ops)
+- /* Can't handle non-graphics buffers */
+- return -EINVAL;
+-
+- obj = dma_buf->priv;
+- if (obj->dev->dev_private != adev)
+- /* Can't handle buffers from other devices */
+- return -EINVAL;
+-
+- bo = gem_to_amdgpu_bo(obj);
+- if (!(bo->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM |
+- AMDGPU_GEM_DOMAIN_GTT |
+- AMDGPU_GEM_DOMAIN_DGMA)))
+- /* Only VRAM and GTT BOs are supported */
+- return -EINVAL;
+-
+- *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
+- if (*mem == NULL)
+- return -ENOMEM;
+-
+- if (size)
+- *size = amdgpu_bo_size(bo);
+-
+- if (mmap_offset)
+- *mmap_offset = amdgpu_bo_mmap_offset(bo);
+-
+- INIT_LIST_HEAD(&(*mem)->bo_va_list);
+- mutex_init(&(*mem)->lock);
+- (*mem)->mapping_flags =
+- AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE |
+- AMDGPU_VM_PAGE_EXECUTABLE | AMDGPU_VM_MTYPE_NC;
+-
+- (*mem)->bo = amdgpu_bo_ref(bo);
+- (*mem)->va = va;
+- if (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM)
+- (*mem)->domain = AMDGPU_GEM_DOMAIN_VRAM;
+- else if (bo->preferred_domains & AMDGPU_GEM_DOMAIN_GTT)
+- (*mem)->domain = AMDGPU_GEM_DOMAIN_GTT;
+- else
+- (*mem)->domain = AMDGPU_GEM_DOMAIN_DGMA;
+- (*mem)->mapped_to_gpu_memory = 0;
+- (*mem)->process_info = kfd_vm->process_info;
+- add_kgd_mem_to_kfd_bo_list(*mem, kfd_vm->process_info, false);
+- amdgpu_sync_create(&(*mem)->sync);
+-
+- return 0;
+-}
+-
+-int amdgpu_amdkfd_gpuvm_export_dmabuf(struct kgd_dev *kgd, void *vm,
+- struct kgd_mem *mem,
+- struct dma_buf **dmabuf)
+-{
+- struct amdgpu_device *adev = NULL;
+- struct amdgpu_bo *bo = NULL;
+- struct drm_gem_object *gobj = NULL;
+-
+- if (!dmabuf || !kgd || !vm || !mem)
+- return -EINVAL;
+-
+- adev = get_amdgpu_device(kgd);
+- bo = mem->bo;
+-
+- gobj = amdgpu_gem_prime_foreign_bo(adev, bo);
+- if (gobj == NULL) {
+- pr_err("Export BO failed. Unable to find/create GEM object\n");
+- return -EINVAL;
+- }
+-
+- *dmabuf = amdgpu_gem_prime_export(adev->ddev, gobj, 0);
+- return 0;
+-}
+-
+-static int process_validate_vms(struct amdkfd_process_info *process_info)
+-{
+- struct amdkfd_vm *peer_vm;
+- int ret;
+-
+- list_for_each_entry(peer_vm, &process_info->vm_list_head,
+- vm_list_node) {
+- ret = vm_validate_pt_pd_bos(&peer_vm->base);
+- if (ret)
+- return ret;
+- }
+-
+- return 0;
+-}
+-
+-/* Evict a userptr BO by stopping the queues if necessary
+- *
+- * Runs in MMU notifier, may be in RECLAIM_FS context. This means it
+- * cannot do any memory allocations, and cannot take any locks that
+- * are held elsewhere while allocating memory. Therefore this is as
+- * simple as possible, using atomic counters.
+- *
+- * It doesn't do anything to the BO itself. The real work happens in
+- * restore, where we get updated page addresses. This function only
+- * ensures that GPU access to the BO is stopped.
+- */
+-int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem,
+- struct mm_struct *mm)
+-{
+- struct amdkfd_process_info *process_info = mem->process_info;
+- int invalid, evicted_bos;
+- int r = 0;
+-
+- invalid = atomic_inc_return(&mem->invalid);
+- evicted_bos = atomic_inc_return(&process_info->evicted_bos);
+- if (evicted_bos == 1) {
+- /* First eviction, stop the queues */
+- r = kgd2kfd->quiesce_mm(NULL, mm);
+- if (r != 0)
+- pr_err("Failed to quiesce KFD\n");
+- schedule_delayed_work(&process_info->work, 1);
+- }
+-
+- return r;
+-}
+-
+-/* Update invalid userptr BOs
+- *
+- * Moves invalidated (evicted) userptr BOs from userptr_valid_list to
+- * userptr_inval_list and updates user pages for all BOs that have
+- * been invalidated since their last update.
+- */
+-static int update_invalid_user_pages(struct amdkfd_process_info *process_info,
+- struct mm_struct *mm)
+-{
+- struct kgd_mem *mem, *tmp_mem;
+- struct amdgpu_bo *bo;
+- int invalid, ret;
+-
+- /* Move all invalidated BOs to the userptr_inval_list and
+- * release their user pages by migration to the CPU domain
+- */
+- list_for_each_entry_safe(mem, tmp_mem,
+- &process_info->userptr_valid_list,
+- validate_list.head) {
+- if (!atomic_read(&mem->invalid))
+- continue; /* BO is still valid */
+-
+- bo = mem->bo;
+-
+- if (amdgpu_bo_reserve(bo, true))
+- return -EAGAIN;
+- amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
+- ret = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
+- amdgpu_bo_unreserve(bo);
+- if (ret) {
+- pr_err("%s: Failed to invalidate userptr BO\n",
+- __func__);
+- return -EAGAIN;
+- }
+-
+- list_move_tail(&mem->validate_list.head,
+- &process_info->userptr_inval_list);
+- }
+-
+- if (list_empty(&process_info->userptr_inval_list))
+- return 0; /* All evicted userptr BOs were freed */
+-
+- /* Go through userptr_inval_list and update any invalid user_pages */
+- list_for_each_entry(mem, &process_info->userptr_inval_list,
+- validate_list.head) {
+- invalid = atomic_read(&mem->invalid);
+- if (!invalid)
+- /* BO hasn't been invalidated since the last
+- * revalidation attempt. Keep its BO list.
+- */
+- continue;
+-
+- bo = mem->bo;
+-
+- if (!mem->user_pages) {
+-#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)
+- mem->user_pages =
+- drm_calloc_large(bo->tbo.ttm->num_pages,
+- sizeof(struct page *));
+-#else
+- mem->user_pages =
+- kvmalloc_array(bo->tbo.ttm->num_pages,
+- sizeof(struct page *),
+- GFP_KERNEL | __GFP_ZERO);
+-#endif
+- if (!mem->user_pages) {
+- pr_err("%s: Failed to allocate pages array\n",
+- __func__);
+- return -ENOMEM;
+- }
+- } else if (mem->user_pages[0]) {
+- release_pages(mem->user_pages,
+- bo->tbo.ttm->num_pages, 0);
+- }
+-
+- /* Get updated user pages */
+- ret = amdgpu_ttm_tt_get_user_pages(bo->tbo.ttm,
+- mem->user_pages);
+- if (ret) {
+- mem->user_pages[0] = NULL;
+- pr_info("%s: Failed to get user pages: %d\n",
+- __func__, ret);
+- /* Pretend it succeeded. It will fail later
+- * with a VM fault if the GPU tries to access
+- * it. Better than hanging indefinitely with
+- * stalled user mode queues.
+- */
+- }
+-
+- /* Mark the BO as valid unless it was invalidated
+- * again concurrently
+- */
+- if (atomic_cmpxchg(&mem->invalid, invalid, 0) != invalid)
+- return -EAGAIN;
+- }
+- return 0;
+-}
+-
+-/* Validate invalid userptr BOs
+- *
+- * Validates BOs on the userptr_inval_list, and moves them back to the
+- * userptr_valid_list. Also updates GPUVM page tables with new page
+- * addresses and waits for the page table updates to complete.
+- */
+-static int validate_invalid_user_pages(struct amdkfd_process_info *process_info)
+-{
+- struct amdgpu_bo_list_entry *pd_bo_list_entries;
+- struct list_head resv_list, duplicates;
+- struct ww_acquire_ctx ticket;
+- struct amdgpu_sync sync;
+-
+- struct amdkfd_vm *peer_vm;
+- struct kgd_mem *mem, *tmp_mem;
+- struct amdgpu_bo *bo;
+- int i, ret;
+-
+- pd_bo_list_entries = kcalloc(process_info->n_vms,
+- sizeof(struct amdgpu_bo_list_entry),
+- GFP_KERNEL);
+- if (!pd_bo_list_entries) {
+- pr_err("%s: Failed to allocate PD BO list entries\n", __func__);
+- return -ENOMEM;
+- }
+-
+- INIT_LIST_HEAD(&resv_list);
+- INIT_LIST_HEAD(&duplicates);
+-
+- /* Get all the page directory BOs that need to be reserved */
+- i = 0;
+- list_for_each_entry(peer_vm, &process_info->vm_list_head,
+- vm_list_node)
+- amdgpu_vm_get_pd_bo(&peer_vm->base, &resv_list,
+- &pd_bo_list_entries[i++]);
+- /* Add the userptr_inval_list entries to resv_list */
+- list_for_each_entry(mem, &process_info->userptr_inval_list,
+- validate_list.head) {
+- list_add_tail(&mem->resv_list.head, &resv_list);
+- mem->resv_list.bo = mem->validate_list.bo;
+- mem->resv_list.shared = mem->validate_list.shared;
+- }
+-
+- /* Reserve all BOs and page tables for validation */
+- ret = ttm_eu_reserve_buffers(&ticket, &resv_list, false, &duplicates);
+- WARN(!list_empty(&duplicates), "Duplicates should be empty");
+- if (ret)
+- goto out;
+-
+- amdgpu_sync_create(&sync);
+-
+- /* Avoid triggering eviction fences when unmapping invalid
+- * userptr BOs (waits for all fences, doesn't use
+- * FENCE_OWNER_VM)
+- */
+- list_for_each_entry(peer_vm, &process_info->vm_list_head,
+- vm_list_node)
+- amdgpu_amdkfd_remove_eviction_fence(peer_vm->base.root.base.bo,
+- process_info->eviction_fence,
+- NULL, NULL);
+-
+- ret = process_validate_vms(process_info);
+- if (ret)
+- goto unreserve_out;
+-
+- /* Validate BOs and update GPUVM page tables */
+- list_for_each_entry_safe(mem, tmp_mem,
+- &process_info->userptr_inval_list,
+- validate_list.head) {
+- struct kfd_bo_va_list *bo_va_entry;
+-
+- bo = mem->bo;
+-
+- /* Copy pages array and validate the BO if we got user pages */
+- if (mem->user_pages[0]) {
+- amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm,
+- mem->user_pages);
+- amdgpu_ttm_placement_from_domain(bo, mem->domain);
+- ret = ttm_bo_validate(&bo->tbo, &bo->placement,
+- false, false);
+- if (ret) {
+- pr_err("%s: failed to validate BO\n", __func__);
+- goto unreserve_out;
+- }
+- }
+-
+- /* Validate succeeded, now the BO owns the pages, free
+- * our copy of the pointer array. Put this BO back on
+- * the userptr_valid_list. If we need to revalidate
+- * it, we need to start from scratch.
+- */
+-#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)
+- drm_free_large(mem->user_pages);
+-#else
+- kvfree(mem->user_pages);
+-#endif
+- mem->user_pages = NULL;
+- list_move_tail(&mem->validate_list.head,
+- &process_info->userptr_valid_list);
+-
+- /* Update mapping. If the BO was not validated
+- * (because we couldn't get user pages), this will
+- * clear the page table entries, which will result in
+- * VM faults if the GPU tries to access the invalid
+- * memory.
+- */
+- list_for_each_entry(bo_va_entry, &mem->bo_va_list, bo_list) {
+- if (!bo_va_entry->is_mapped)
+- continue;
+-
+- ret = update_gpuvm_pte((struct amdgpu_device *)
+- bo_va_entry->kgd_dev,
+- bo_va_entry, &sync);
+- if (ret) {
+- pr_err("%s: update PTE failed\n", __func__);
+- /* make sure this gets validated again */
+- atomic_inc(&mem->invalid);
+- goto unreserve_out;
+- }
+- }
+- }
+-unreserve_out:
+- list_for_each_entry(peer_vm, &process_info->vm_list_head,
+- vm_list_node)
+- amdgpu_bo_fence(peer_vm->base.root.base.bo,
+- &process_info->eviction_fence->base, true);
+- ttm_eu_backoff_reservation(&ticket, &resv_list);
+- amdgpu_sync_wait(&sync, false);
+- amdgpu_sync_free(&sync);
+-out:
+- kfree(pd_bo_list_entries);
+-
+- return ret;
+-}
+-
+-/* Worker callback to restore evicted userptr BOs
+- *
+- * Tries to update and validate all userptr BOs. If successful and no
+- * concurrent evictions happened, the queues are restarted. Otherwise,
+- * reschedule for another attempt later.
+- */
+-static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work)
+-{
+- struct delayed_work *dwork = to_delayed_work(work);
+- struct amdkfd_process_info *process_info =
+- container_of(dwork, struct amdkfd_process_info, work);
+- struct task_struct *usertask;
+- struct mm_struct *mm;
+- int evicted_bos;
+-
+- evicted_bos = atomic_read(&process_info->evicted_bos);
+- if (!evicted_bos)
+- return;
+-
+- /* Reference task and mm in case of concurrent process termination */
+- usertask = get_pid_task(process_info->pid, PIDTYPE_PID);
+- if (!usertask)
+- return;
+- mm = get_task_mm(usertask);
+- if (!mm) {
+- put_task_struct(usertask);
+- return;
+- }
+-
+- mutex_lock(&process_info->lock);
+-
+- if (update_invalid_user_pages(process_info, mm))
+- goto unlock_out;
+- /* userptr_inval_list can be empty if all evicted userptr BOs
+- * have been freed. In that case there is nothing to validate
+- * and we can just restart the queues.
+- */
+- if (!list_empty(&process_info->userptr_inval_list)) {
+- if (atomic_read(&process_info->evicted_bos) != evicted_bos)
+- goto unlock_out; /* Concurrent eviction, try again */
+-
+- if (validate_invalid_user_pages(process_info))
+- goto unlock_out;
+- }
+- /* Final check for concurrent evicton and atomic update. If
+- * another eviction happens after successful update, it will
+- * be a first eviction that calls quiesce_mm. The eviction
+- * reference counting inside KFD will handle this case.
+- */
+- if (atomic_cmpxchg(&process_info->evicted_bos, evicted_bos, 0) !=
+- evicted_bos)
+- goto unlock_out;
+- evicted_bos = 0;
+- if (kgd2kfd->resume_mm(NULL, mm)) {
+- pr_err("%s: Failed to resume KFD\n", __func__);
+- /* No recovery from this failure. Probably the CP is
+- * hanging. No point trying again.
+- */
+- }
+-unlock_out:
+- mutex_unlock(&process_info->lock);
+- mmput(mm);
+- put_task_struct(usertask);
+-
+- /* If validation failed, reschedule another attempt */
+- if (evicted_bos)
+- schedule_delayed_work(&process_info->work, 1);
+-}
+-
+-/** amdgpu_amdkfd_gpuvm_restore_process_bos - Restore all BOs for the given
+- * KFD process identified by process_info
+- *
+- * @process_info: amdkfd_process_info of the KFD process
+- *
+- * After memory eviction, restore thread calls this function. The function
+- * should be called when the Process is still valid. BO restore involves -
+- *
+- * 1. Release old eviction fence and create new one
+- * 2. Get two copies of PD BO list from all the VMs. Keep one copy as pd_list.
+- * 3 Use the second PD list and kfd_bo_list to create a list (ctx.list) of
+- * BOs that need to be reserved.
+- * 4. Reserve all the BOs
+- * 5. Validate of PD and PT BOs.
+- * 6. Validate all KFD BOs using kfd_bo_list and Map them and add new fence
+- * 7. Add fence to all PD and PT BOs.
+- * 8. Unreserve all BOs
+- */
+-
+-int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence **ef)
+-{
+- struct amdgpu_bo_list_entry *pd_bo_list;
+- struct amdkfd_process_info *process_info = info;
+- struct amdkfd_vm *peer_vm;
+- struct kgd_mem *mem;
+- struct bo_vm_reservation_context ctx;
+- struct amdgpu_amdkfd_fence *new_fence;
+- int ret = 0, i;
+- struct list_head duplicate_save;
+- struct amdgpu_sync sync_obj;
+-
+- INIT_LIST_HEAD(&duplicate_save);
+- INIT_LIST_HEAD(&ctx.list);
+- INIT_LIST_HEAD(&ctx.duplicates);
+-
+- pd_bo_list = kcalloc(process_info->n_vms,
+- sizeof(struct amdgpu_bo_list_entry),
+- GFP_KERNEL);
+- if (pd_bo_list == NULL)
+- return -ENOMEM;
+-
+- i = 0;
+- mutex_lock(&process_info->lock);
+- list_for_each_entry(peer_vm, &process_info->vm_list_head,
+- vm_list_node)
+- amdgpu_vm_get_pd_bo(&peer_vm->base, &ctx.list,
+- &pd_bo_list[i++]);
+-
+- /* Reserve all BOs and page tables/directory. Add all BOs from
+- * kfd_bo_list to ctx.list
+- */
+- list_for_each_entry(mem, &process_info->kfd_bo_list,
+- validate_list.head) {
+-
+- list_add_tail(&mem->resv_list.head, &ctx.list);
+- mem->resv_list.bo = mem->validate_list.bo;
+- mem->resv_list.shared = mem->validate_list.shared;
+- }
+-
+- ret = ttm_eu_reserve_buffers(&ctx.ticket, &ctx.list,
+- false, &duplicate_save);
+- if (ret) {
+- pr_debug("Memory eviction: TTM Reserve Failed. Try again\n");
+- goto ttm_reserve_fail;
+- }
+-
+- amdgpu_sync_create(&sync_obj);
+- ctx.sync = &sync_obj;
+-
+- /* Validate PDs and PTs */
+- ret = process_validate_vms(process_info);
+- if (ret)
+- goto validate_map_fail;
+-
+- /* Wait for PD/PTs validate to finish */
+- /* FIXME: I think this isn't needed */
+- list_for_each_entry(peer_vm, &process_info->vm_list_head,
+- vm_list_node) {
+- struct amdgpu_bo *bo = peer_vm->base.root.base.bo;
+-
+- ttm_bo_wait(&bo->tbo, false, false);
+- }
+-
+- /* Validate BOs and map them to GPUVM (update VM page tables). */
+- list_for_each_entry(mem, &process_info->kfd_bo_list,
+- validate_list.head) {
+-
+- struct amdgpu_bo *bo = mem->bo;
+- uint32_t domain = mem->domain;
+- struct kfd_bo_va_list *bo_va_entry;
+-
+- ret = amdgpu_amdkfd_bo_validate(bo, domain, false);
+- if (ret) {
+- pr_debug("Memory eviction: Validate BOs failed. Try again\n");
+- goto validate_map_fail;
+- }
+-
+- list_for_each_entry(bo_va_entry, &mem->bo_va_list,
+- bo_list) {
+- ret = update_gpuvm_pte((struct amdgpu_device *)
+- bo_va_entry->kgd_dev,
+- bo_va_entry,
+- ctx.sync);
+- if (ret) {
+- pr_debug("Memory eviction: update PTE failed. Try again\n");
+- goto validate_map_fail;
+- }
+- }
+- }
+-
+- amdgpu_sync_wait(ctx.sync, false);
+-
+- /* Release old eviction fence and create new one, because fence only
+- * goes from unsignaled to signaled, fence cannot be reused.
+- * Use context and mm from the old fence.
+- */
+- new_fence = amdgpu_amdkfd_fence_create(
+- process_info->eviction_fence->base.context,
+- process_info->eviction_fence->mm);
+- if (!new_fence) {
+- pr_err("Failed to create eviction fence\n");
+- ret = -ENOMEM;
+- goto validate_map_fail;
+- }
+- dma_fence_put(&process_info->eviction_fence->base);
+- process_info->eviction_fence = new_fence;
+- *ef = dma_fence_get(&new_fence->base);
+-
+- /* Wait for validate to finish and attach new eviction fence */
+- list_for_each_entry(mem, &process_info->kfd_bo_list,
+- validate_list.head)
+- ttm_bo_wait(&mem->bo->tbo, false, false);
+- list_for_each_entry(mem, &process_info->kfd_bo_list,
+- validate_list.head)
+- amdgpu_bo_fence(mem->bo,
+- &process_info->eviction_fence->base, true);
+-
+- /* Attach eviction fence to PD / PT BOs */
+- list_for_each_entry(peer_vm, &process_info->vm_list_head,
+- vm_list_node) {
+- struct amdgpu_bo *bo = peer_vm->base.root.base.bo;
+-
+- amdgpu_bo_fence(bo, &process_info->eviction_fence->base, true);
+- }
+-validate_map_fail:
+- ttm_eu_backoff_reservation(&ctx.ticket, &ctx.list);
+- amdgpu_sync_free(&sync_obj);
+-ttm_reserve_fail:
+- mutex_unlock(&process_info->lock);
+-evict_fence_fail:
+- kfree(pd_bo_list);
+- return ret;
+-}
+-
+-int amdgpu_amdkfd_copy_mem_to_mem(struct kgd_dev *kgd, struct kgd_mem *src_mem,
+- uint64_t src_offset, struct kgd_mem *dst_mem,
+- uint64_t dst_offset, uint64_t size,
+- struct dma_fence **f, uint64_t *actual_size)
+-{
+- struct amdgpu_device *adev = NULL;
+- struct ttm_mem_reg *src = NULL, *dst = NULL;
+- struct ttm_buffer_object *src_ttm_bo, *dst_ttm_bo;
+- struct drm_mm_node *src_mm, *dst_mm;
+- struct amdgpu_ring *ring;
+- struct ww_acquire_ctx ticket;
+- struct list_head list;
+- struct ttm_validate_buffer resv_list[2];
+- uint64_t src_start, dst_start;
+- uint64_t src_left, dst_left, cur_copy_size, total_copy_size = 0;
+- struct dma_fence *fence = NULL;
+- int r;
+-
+- if (!kgd || !src_mem || !dst_mem)
+- return -EINVAL;
+-
+- if (actual_size)
+- *actual_size = 0;
+-
+- adev = get_amdgpu_device(kgd);
+- src_ttm_bo = &src_mem->bo->tbo;
+- dst_ttm_bo = &dst_mem->bo->tbo;
+- src = &src_ttm_bo->mem;
+- dst = &dst_ttm_bo->mem;
+- src_mm = (struct drm_mm_node *)src->mm_node;
+- dst_mm = (struct drm_mm_node *)dst->mm_node;
+-
+- ring = adev->mman.buffer_funcs_ring;
+-
+- INIT_LIST_HEAD(&list);
+-
+- resv_list[0].bo = src_ttm_bo;
+- resv_list[0].shared = true;
+- resv_list[1].bo = dst_ttm_bo;
+- resv_list[1].shared = true;
+-
+- list_add_tail(&resv_list[0].head, &list);
+- list_add_tail(&resv_list[1].head, &list);
+-
+- if (!ring->ready) {
+- pr_err("Trying to move memory with ring turned off.\n");
+- return -EINVAL;
+- }
+-
+- r = ttm_eu_reserve_buffers(&ticket, &list, false, NULL);
+- if (r) {
+- pr_err("Copy buffer failed. Unable to reserve bo (%d)\n", r);
+- return r;
+- }
+-
+- switch (src->mem_type) {
+- case TTM_PL_TT:
+- r = amdgpu_ttm_bind(src_ttm_bo, src);
+- if (r) {
+- DRM_ERROR("Copy failed. Cannot bind to gart\n");
+- goto copy_fail;
+- }
+- break;
+- case TTM_PL_VRAM:
+- /* VRAM could be scattered. Find the node in which the offset
+- * belongs to
+- */
+- while (src_offset >= (src_mm->size << PAGE_SHIFT)) {
+- src_offset -= (src_mm->size << PAGE_SHIFT);
+- ++src_mm;
+- }
+- break;
+- default:
+- DRM_ERROR("Unknown placement %d\n", src->mem_type);
+- r = -EINVAL;
+- goto copy_fail;
+- }
+- src_start = src_mm->start << PAGE_SHIFT;
+- src_start += src_ttm_bo->bdev->man[src->mem_type].gpu_offset;
+- src_start += src_offset;
+- src_left = (src_mm->size << PAGE_SHIFT) - src_offset;
+-
+- switch (dst->mem_type) {
+- case TTM_PL_TT:
+- r = amdgpu_ttm_bind(dst_ttm_bo, dst);
+- if (r) {
+- DRM_ERROR("Copy failed. Cannot bind to gart\n");
+- goto copy_fail;
+- }
+- break;
+- case TTM_PL_VRAM:
+- while (dst_offset >= (dst_mm->size << PAGE_SHIFT)) {
+- dst_offset -= (dst_mm->size << PAGE_SHIFT);
+- ++dst_mm;
+- }
+- break;
+- default:
+- DRM_ERROR("Unknown placement %d\n", dst->mem_type);
+- r = -EINVAL;
+- goto copy_fail;
+- }
+- dst_start = dst_mm->start << PAGE_SHIFT;
+- dst_start += dst_ttm_bo->bdev->man[dst->mem_type].gpu_offset;
+- dst_start += dst_offset;
+- dst_left = (dst_mm->size << PAGE_SHIFT) - dst_offset;
+-
+- do {
+- struct dma_fence *next;
+-
+- /* src_left/dst_left: amount of space left in the current node
+- * Copy minimum of (src_left, dst_left, amount of bytes left to
+- * copy)
+- */
+- cur_copy_size = min3(src_left, dst_left,
+- (size - total_copy_size));
+-
+- r = amdgpu_copy_buffer(ring, src_start, dst_start,
+- cur_copy_size, NULL, &next, false, false);
+- if (r)
+- break;
+-
+- /* Just keep the last fence */
+- dma_fence_put(fence);
+- fence = next;
+-
+- total_copy_size += cur_copy_size;
+- /* Required amount of bytes copied. Done. */
+- if (total_copy_size >= size)
+- break;
+-
+- /* If end of src or dst node is reached, move to next node */
+- src_left -= cur_copy_size;
+- if (!src_left) {
+- ++src_mm;
+- src_start = src_mm->start << PAGE_SHIFT;
+- src_start +=
+- src_ttm_bo->bdev->man[src->mem_type].gpu_offset;
+- src_left = src_mm->size << PAGE_SHIFT;
+- } else
+- src_start += cur_copy_size;
+-
+- dst_left -= cur_copy_size;
+- if (!dst_left) {
+- ++dst_mm;
+- dst_start = dst_mm->start << PAGE_SHIFT;
+- dst_start +=
+- dst_ttm_bo->bdev->man[dst->mem_type].gpu_offset;
+- dst_left = dst_mm->size << PAGE_SHIFT;
+- } else
+- dst_start += cur_copy_size;
+-
+- } while (total_copy_size < size);
+-
+- /* Failure could occur after partial copy. So fill in amount copied
+- * and fence, still fill-in
+- */
+- if (actual_size)
+- *actual_size = total_copy_size;
+-
+- if (fence) {
+- amdgpu_bo_fence(src_mem->bo, fence, true);
+- amdgpu_bo_fence(dst_mem->bo, fence, true);
+- }
+-
+- if (f)
+- *f = fence;
+-
+-copy_fail:
+- ttm_eu_backoff_reservation(&ticket, &list);
+- return r;
+-}
+-
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+index ff6f90a..5ad0580 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+@@ -27,7 +27,9 @@
+ #include <linux/pagemap.h>
+ #include <drm/drmP.h>
+ #include <drm/amdgpu_drm.h>
++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0)
+ #include <drm/drm_syncobj.h>
++#endif
+ #include "amdgpu.h"
+ #include "amdgpu_trace.h"
+
+@@ -38,7 +40,7 @@ static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p,
+ struct drm_gem_object *gobj;
+ unsigned long size;
+
+- gobj = drm_gem_object_lookup(p->filp, data->handle);
++ gobj = kcl_drm_gem_object_lookup(p->adev->ddev, p->filp, data->handle);
+ if (gobj == NULL)
+ return -EINVAL;
+
+@@ -54,7 +56,7 @@ static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p,
+
+ *offset = data->offset;
+
+- drm_gem_object_put_unlocked(gobj);
++ kcl_drm_gem_object_put_unlocked(gobj);
+
+ if (amdgpu_ttm_tt_get_usermm(p->uf_entry.robj->tbo.ttm)) {
+ amdgpu_bo_unref(&p->uf_entry.robj);
+@@ -90,7 +92,7 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
+ }
+
+ /* get chunks */
+- chunk_array_user = u64_to_user_ptr(cs->in.chunks);
++ chunk_array_user = kcl_u64_to_user_ptr(cs->in.chunks);
+ if (copy_from_user(chunk_array, chunk_array_user,
+ sizeof(uint64_t)*cs->in.num_chunks)) {
+ ret = -EFAULT;
+@@ -110,7 +112,7 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
+ struct drm_amdgpu_cs_chunk user_chunk;
+ uint32_t __user *cdata;
+
+- chunk_ptr = u64_to_user_ptr(chunk_array[i]);
++ chunk_ptr = kcl_u64_to_user_ptr(chunk_array[i]);
+ if (copy_from_user(&user_chunk, chunk_ptr,
+ sizeof(struct drm_amdgpu_cs_chunk))) {
+ ret = -EFAULT;
+@@ -121,9 +123,13 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
+ p->chunks[i].length_dw = user_chunk.length_dw;
+
+ size = p->chunks[i].length_dw;
+- cdata = u64_to_user_ptr(user_chunk.chunk_data);
++ cdata = kcl_u64_to_user_ptr(user_chunk.chunk_data);
+
++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)
++ p->chunks[i].kdata = drm_malloc_ab(size, sizeof(uint32_t));
++#else
+ p->chunks[i].kdata = kvmalloc_array(size, sizeof(uint32_t), GFP_KERNEL);
++#endif
+ if (p->chunks[i].kdata == NULL) {
+ ret = -ENOMEM;
+ i--;
+@@ -155,8 +161,10 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
+ break;
+
+ case AMDGPU_CHUNK_ID_DEPENDENCIES:
++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0)
+ case AMDGPU_CHUNK_ID_SYNCOBJ_IN:
+ case AMDGPU_CHUNK_ID_SYNCOBJ_OUT:
++#endif
+ break;
+
+ default:
+@@ -178,7 +186,11 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
+ i = p->nchunks - 1;
+ free_partial_kdata:
+ for (; i >= 0; i--)
++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)
++ drm_free_large(p->chunks[i].kdata);
++#else
+ kvfree(p->chunks[i].kdata);
++#endif
+ kfree(p->chunks);
+ p->chunks = NULL;
+ p->nchunks = 0;
+@@ -477,16 +489,11 @@ static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p,
+ return -EPERM;
+
+ /* Check if we have user pages and nobody bound the BO already */
+- if (amdgpu_ttm_tt_userptr_needs_pages(bo->tbo.ttm) &&
+- lobj->user_pages) {
+- amdgpu_ttm_placement_from_domain(bo,
+- AMDGPU_GEM_DOMAIN_CPU);
+- r = ttm_bo_validate(&bo->tbo, &bo->placement, true,
+- false);
+- if (r)
+- return r;
+- amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm,
+- lobj->user_pages);
++ if (lobj->user_pages && bo->tbo.ttm->state != tt_bound) {
++ size_t size = sizeof(struct page *);
++
++ size *= bo->tbo.ttm->num_pages;
++ memcpy(bo->tbo.ttm->pages, lobj->user_pages, size);
+ binding_userptr = true;
+ }
+
+@@ -498,7 +505,11 @@ static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p,
+ return r;
+
+ if (binding_userptr) {
++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)
++ drm_free_large(lobj->user_pages);
++#else
+ kvfree(lobj->user_pages);
++#endif
+ lobj->user_pages = NULL;
+ }
+ }
+@@ -511,6 +522,7 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
+ struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
+ struct amdgpu_bo_list_entry *e;
+ struct list_head duplicates;
++ bool need_mmap_lock = false;
+ unsigned i, tries = 10;
+ int r;
+
+@@ -518,9 +530,9 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
+
+ p->bo_list = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle);
+ if (p->bo_list) {
++ need_mmap_lock = p->bo_list->first_userptr !=
++ p->bo_list->num_entries;
+ amdgpu_bo_list_get_list(p->bo_list, &p->validated);
+- if (p->bo_list->first_userptr != p->bo_list->num_entries)
+- p->mn = amdgpu_mn_get(p->adev);
+ }
+
+ INIT_LIST_HEAD(&duplicates);
+@@ -529,6 +541,9 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
+ if (p->uf_entry.robj && !p->uf_entry.robj->parent)
+ list_add(&p->uf_entry.tv.head, &p->validated);
+
++ if (need_mmap_lock)
++ down_read(&current->mm->mmap_sem);
++
+ while (1) {
+ struct list_head need_pages;
+ unsigned i;
+@@ -548,25 +563,27 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
+ INIT_LIST_HEAD(&need_pages);
+ for (i = p->bo_list->first_userptr;
+ i < p->bo_list->num_entries; ++i) {
+- struct amdgpu_bo *bo;
+
+ e = &p->bo_list->array[i];
+- bo = e->robj;
+-
+- if (amdgpu_ttm_tt_userptr_invalidated(bo->tbo.ttm,
++
++ if (amdgpu_ttm_tt_userptr_invalidated(e->robj->tbo.ttm,
+ &e->user_invalidated) && e->user_pages) {
+
+ /* We acquired a page array, but somebody
+ * invalidated it. Free it and try again
+ */
+ release_pages(e->user_pages,
+- bo->tbo.ttm->num_pages,
++ e->robj->tbo.ttm->num_pages,
+ false);
++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)
++ drm_free_large(e->user_pages);
++#else
+ kvfree(e->user_pages);
++#endif
+ e->user_pages = NULL;
+ }
+
+- if (amdgpu_ttm_tt_userptr_needs_pages(bo->tbo.ttm) &&
++ if (e->robj->tbo.ttm->state != tt_bound &&
+ !e->user_pages) {
+ list_del(&e->tv.head);
+ list_add(&e->tv.head, &need_pages);
+@@ -592,9 +609,14 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
+ list_for_each_entry(e, &need_pages, tv.head) {
+ struct ttm_tt *ttm = e->robj->tbo.ttm;
+
++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)
++ e->user_pages = drm_calloc_large(ttm->num_pages,
++ sizeof(struct page*));
++#else
+ e->user_pages = kvmalloc_array(ttm->num_pages,
+ sizeof(struct page*),
+ GFP_KERNEL | __GFP_ZERO);
++#endif
+ if (!e->user_pages) {
+ r = -ENOMEM;
+ DRM_ERROR("calloc failure in %s\n", __func__);
+@@ -604,7 +626,11 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
+ r = amdgpu_ttm_tt_get_user_pages(ttm, e->user_pages);
+ if (r) {
+ DRM_ERROR("amdgpu_ttm_tt_get_user_pages failed.\n");
++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)
++ drm_free_large(e->user_pages);
++#else
+ kvfree(e->user_pages);
++#endif
+ e->user_pages = NULL;
+ goto error_free_pages;
+ }
+@@ -643,6 +669,9 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
+
+ amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved,
+ p->bytes_moved_vis);
++ fpriv->vm.last_eviction_counter =
++ atomic64_read(&p->adev->num_evictions);
++
+ if (p->bo_list) {
+ struct amdgpu_bo *gds = p->bo_list->gds_obj;
+ struct amdgpu_bo *gws = p->bo_list->gws_obj;
+@@ -683,6 +712,9 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
+
+ error_free_pages:
+
++ if (need_mmap_lock)
++ up_read(&current->mm->mmap_sem);
++
+ if (p->bo_list) {
+ for (i = p->bo_list->first_userptr;
+ i < p->bo_list->num_entries; ++i) {
+@@ -694,7 +726,11 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
+ release_pages(e->user_pages,
+ e->robj->tbo.ttm->num_pages,
+ false);
++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)
++ drm_free_large(e->user_pages);
++#else
+ kvfree(e->user_pages);
++#endif
+ }
+ }
+
+@@ -729,13 +765,19 @@ static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error,
+ {
+ unsigned i;
+
+- if (error && backoff)
++ if (!error)
++ ttm_eu_fence_buffer_objects(&parser->ticket,
++ &parser->validated,
++ parser->fence);
++ else if (backoff)
+ ttm_eu_backoff_reservation(&parser->ticket,
+ &parser->validated);
+
++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0)
+ for (i = 0; i < parser->num_post_dep_syncobjs; i++)
+ drm_syncobj_put(parser->post_dep_syncobjs[i]);
+ kfree(parser->post_dep_syncobjs);
++#endif
+
+ dma_fence_put(parser->fence);
+
+@@ -745,7 +787,11 @@ static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error,
+ amdgpu_bo_list_put(parser->bo_list);
+
+ for (i = 0; i < parser->nchunks; i++)
++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)
++ drm_free_large(parser->chunks[i].kdata);
++#else
+ kvfree(parser->chunks[i].kdata);
++#endif
+ kfree(parser->chunks);
+ if (parser->job)
+ amdgpu_job_free(parser->job);
+@@ -765,6 +811,10 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p)
+ if (r)
+ return r;
+
++ r = amdgpu_sync_fence(adev, &p->job->sync, vm->last_dir_update);
++ if (r)
++ return r;
++
+ r = amdgpu_vm_clear_freed(adev, vm, NULL);
+ if (r)
+ return r;
+@@ -818,13 +868,7 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p)
+
+ }
+
+- r = amdgpu_vm_handle_moved(adev, vm);
+- if (r)
+- return r;
+-
+- r = amdgpu_sync_fence(adev, &p->job->sync, vm->last_update);
+- if (r)
+- return r;
++ r = amdgpu_vm_clear_moved(adev, vm, &p->job->sync);
+
+ if (amdgpu_vm_debug && p->bo_list) {
+ /* Invalidate all BOs to test for userspace bugs */
+@@ -834,7 +878,7 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p)
+ if (!bo)
+ continue;
+
+- amdgpu_vm_bo_invalidate(adev, bo, false);
++ amdgpu_vm_bo_invalidate(adev, bo);
+ }
+ }
+
+@@ -859,7 +903,7 @@ static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev,
+ }
+
+ if (p->job->vm) {
+- p->job->vm_pd_addr = amdgpu_bo_gpu_offset(vm->root.base.bo);
++ p->job->vm_pd_addr = amdgpu_bo_gpu_offset(vm->root.bo);
+
+ r = amdgpu_bo_vm_update_pte(p);
+ if (r)
+@@ -927,11 +971,11 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
+ uint64_t offset;
+ uint8_t *kptr;
+
+- r = amdgpu_cs_find_mapping(parser, chunk_ib->va_start,
+- &aobj, &m);
+- if (r) {
++ m = amdgpu_cs_find_mapping(parser, chunk_ib->va_start,
++ &aobj);
++ if (!aobj) {
+ DRM_ERROR("IB va_start is invalid\n");
+- return r;
++ return -EINVAL;
+ }
+
+ if ((chunk_ib->va_start + chunk_ib->ib_bytes) >
+@@ -1029,12 +1073,13 @@ static int amdgpu_cs_process_fence_dep(struct amdgpu_cs_parser *p,
+ return 0;
+ }
+
++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0)
+ static int amdgpu_syncobj_lookup_and_add_to_sync(struct amdgpu_cs_parser *p,
+ uint32_t handle)
+ {
+ int r;
+ struct dma_fence *fence;
+- r = drm_syncobj_find_fence(p->filp, handle, &fence);
++ r = drm_syncobj_fence_get(p->filp, handle, &fence);
+ if (r)
+ return r;
+
+@@ -1089,6 +1134,7 @@ static int amdgpu_cs_process_syncobj_out_dep(struct amdgpu_cs_parser *p,
+ }
+ return 0;
+ }
++#endif
+
+ static int amdgpu_cs_dependencies(struct amdgpu_device *adev,
+ struct amdgpu_cs_parser *p)
+@@ -1104,6 +1150,7 @@ static int amdgpu_cs_dependencies(struct amdgpu_device *adev,
+ r = amdgpu_cs_process_fence_dep(p, chunk);
+ if (r)
+ return r;
++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0)
+ } else if (chunk->chunk_id == AMDGPU_CHUNK_ID_SYNCOBJ_IN) {
+ r = amdgpu_cs_process_syncobj_in_dep(p, chunk);
+ if (r)
+@@ -1112,12 +1159,14 @@ static int amdgpu_cs_dependencies(struct amdgpu_device *adev,
+ r = amdgpu_cs_process_syncobj_out_dep(p, chunk);
+ if (r)
+ return r;
++#endif
+ }
+ }
+
+ return amdgpu_sem_add_cs(p->ctx, p->job->ring, &p->job->dep_sync);
+ }
+
++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0)
+ static void amdgpu_cs_post_dependencies(struct amdgpu_cs_parser *p)
+ {
+ int i;
+@@ -1125,6 +1174,7 @@ static void amdgpu_cs_post_dependencies(struct amdgpu_cs_parser *p)
+ for (i = 0; i < p->num_post_dep_syncobjs; ++i)
+ drm_syncobj_replace_fence(p->post_dep_syncobjs[i], p->fence);
+ }
++#endif
+
+ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
+ union drm_amdgpu_cs *cs)
+@@ -1132,29 +1182,14 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
+ struct amdgpu_ring *ring = p->job->ring;
+ struct amd_sched_entity *entity = &p->ctx->rings[ring->idx].entity;
+ struct amdgpu_job *job;
+- unsigned i;
+ int r;
+
+- amdgpu_mn_lock(p->mn);
+- if (p->bo_list) {
+- for (i = p->bo_list->first_userptr;
+- i < p->bo_list->num_entries; ++i) {
+- struct amdgpu_bo *bo = p->bo_list->array[i].robj;
+-
+- if (amdgpu_ttm_tt_userptr_needs_pages(bo->tbo.ttm)) {
+- amdgpu_mn_unlock(p->mn);
+- return -ERESTARTSYS;
+- }
+- }
+- }
+-
+ job = p->job;
+ p->job = NULL;
+
+ r = amd_sched_job_init(&job->base, &ring->sched, entity, p->filp);
+ if (r) {
+ amdgpu_job_free(job);
+- amdgpu_mn_unlock(p->mn);
+ return r;
+ }
+
+@@ -1162,18 +1197,17 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
+ job->fence_ctx = entity->fence_context;
+ p->fence = dma_fence_get(&job->base.s_fence->finished);
+
++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0)
+ amdgpu_cs_post_dependencies(p);
++#endif
+
+ cs->out.handle = amdgpu_ctx_add_fence(p->ctx, ring, p->fence);
+ job->uf_sequence = cs->out.handle;
+ amdgpu_job_free_resources(job);
++ amdgpu_cs_parser_fini(p, 0, true);
+
+ trace_amdgpu_cs_ioctl(job);
+ amd_sched_entity_push_job(&job->base);
+-
+- ttm_eu_fence_buffer_objects(&p->ticket, &p->validated, p->fence);
+- amdgpu_mn_unlock(p->mn);
+-
+ return 0;
+ }
+
+@@ -1228,7 +1262,10 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
+ goto out;
+
+ r = amdgpu_cs_submit(&parser, cs);
++ if (r)
++ goto out;
+
++ return 0;
+ out:
+ amdgpu_cs_parser_fini(&parser, r, reserved_buffers);
+ return r;
+@@ -1274,7 +1311,7 @@ int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data,
+ if (IS_ERR(fence))
+ r = PTR_ERR(fence);
+ else if (fence) {
+- r = dma_fence_wait_timeout(fence, true, timeout);
++ r = kcl_fence_wait_timeout(fence, true, timeout);
+ dma_fence_put(fence);
+ } else
+ r = 1;
+@@ -1349,7 +1386,7 @@ static int amdgpu_cs_wait_all_fences(struct amdgpu_device *adev,
+ else if (!fence)
+ continue;
+
+- r = dma_fence_wait_timeout(fence, true, timeout);
++ r = kcl_fence_wait_timeout(fence, true, timeout);
+ dma_fence_put(fence);
+ if (r < 0)
+ return r;
+@@ -1401,13 +1438,12 @@ static int amdgpu_cs_wait_any_fence(struct amdgpu_device *adev,
+ array[i] = fence;
+ } else { /* NULL, the fence has been already signaled */
+ r = 1;
+- first = i;
+ goto out;
+ }
+ }
+
+- r = dma_fence_wait_any_timeout(array, fence_count, true, timeout,
+- &first);
++ r = kcl_fence_wait_any_timeout(array, fence_count, true, timeout,
++ &first);
+ if (r < 0)
+ goto err_free_fence_array;
+
+@@ -1452,7 +1488,7 @@ int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data,
+ if (fences == NULL)
+ return -ENOMEM;
+
+- fences_user = u64_to_user_ptr(wait->in.fences);
++ fences_user = kcl_u64_to_user_ptr(wait->in.fences);
+ if (copy_from_user(fences, fences_user,
+ sizeof(struct drm_amdgpu_fence) * fence_count)) {
+ r = -EFAULT;
+@@ -1481,36 +1517,78 @@ int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data,
+ * virtual memory address. Returns allocation structure when found, NULL
+ * otherwise.
+ */
+-int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
+- uint64_t addr, struct amdgpu_bo **bo,
+- struct amdgpu_bo_va_mapping **map)
++struct amdgpu_bo_va_mapping *
++amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
++ uint64_t addr, struct amdgpu_bo **bo)
+ {
+- struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
+- struct amdgpu_vm *vm = &fpriv->vm;
+ struct amdgpu_bo_va_mapping *mapping;
+- int r;
++ unsigned i;
++
++ if (!parser->bo_list)
++ return NULL;
+
+ addr /= AMDGPU_GPU_PAGE_SIZE;
+
+- mapping = amdgpu_vm_bo_lookup_mapping(vm, addr);
+- if (!mapping || !mapping->bo_va || !mapping->bo_va->base.bo)
+- return -EINVAL;
++ for (i = 0; i < parser->bo_list->num_entries; i++) {
++ struct amdgpu_bo_list_entry *lobj;
+
+- *bo = mapping->bo_va->base.bo;
+- *map = mapping;
++ lobj = &parser->bo_list->array[i];
++ if (!lobj->bo_va)
++ continue;
+
+- /* Double check that the BO is reserved by this CS */
+- if (READ_ONCE((*bo)->tbo.resv->lock.ctx) != &parser->ticket)
+- return -EINVAL;
++ list_for_each_entry(mapping, &lobj->bo_va->valids, list) {
++ if (mapping->start > addr ||
++ addr > mapping->last)
++ continue;
+
+- r = amdgpu_ttm_bind(&(*bo)->tbo, &(*bo)->tbo.mem);
+- if (unlikely(r))
+- return r;
++ *bo = lobj->bo_va->base.bo;
++ return mapping;
++ }
++
++ list_for_each_entry(mapping, &lobj->bo_va->invalids, list) {
++ if (mapping->start > addr ||
++ addr > mapping->last)
++ continue;
+
+- if ((*bo)->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
++ *bo = lobj->bo_va->base.bo;
++ return mapping;
++ }
++ }
++
++ return NULL;
++}
++
++/**
++ * amdgpu_cs_sysvm_access_required - make BOs accessible by the system VM
++ *
++ * @parser: command submission parser context
++ *
++ * Helper for UVD/VCE VM emulation, make sure BOs are accessible by the system VM.
++ */
++int amdgpu_cs_sysvm_access_required(struct amdgpu_cs_parser *parser)
++{
++ unsigned i;
++ int r;
++
++ if (!parser->bo_list)
+ return 0;
+
+- (*bo)->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
+- amdgpu_ttm_placement_from_domain(*bo, (*bo)->allowed_domains);
+- return ttm_bo_validate(&(*bo)->tbo, &(*bo)->placement, false, false);
++ for (i = 0; i < parser->bo_list->num_entries; i++) {
++ struct amdgpu_bo *bo = parser->bo_list->array[i].robj;
++
++ r = amdgpu_ttm_bind(&bo->tbo, &bo->tbo.mem);
++ if (unlikely(r))
++ return r;
++
++ if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
++ continue;
++
++ bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
++ amdgpu_ttm_placement_from_domain(bo, bo->allowed_domains);
++ r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
++ if (unlikely(r))
++ return r;
++ }
++
++ return 0;
+ }
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+index f032e87..37398e3 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+@@ -404,15 +404,6 @@ void amdgpu_pci_config_reset(struct amdgpu_device *adev)
+ */
+ static int amdgpu_doorbell_init(struct amdgpu_device *adev)
+ {
+- /* No doorbell on SI hardware generation */
+- if (adev->asic_type < CHIP_BONAIRE) {
+- adev->doorbell.base = 0;
+- adev->doorbell.size = 0;
+- adev->doorbell.num_doorbells = 0;
+- adev->doorbell.ptr = NULL;
+- return 0;
+- }
+-
+ /* doorbell bar mapping */
+ adev->doorbell.base = pci_resource_start(adev->pdev, 2);
+ adev->doorbell.size = pci_resource_len(adev->pdev, 2);
+@@ -2130,8 +2121,9 @@ int amdgpu_device_init(struct amdgpu_device *adev,
+ DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
+ DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
+
+- /* doorbell bar mapping */
+- amdgpu_doorbell_init(adev);
++ if (adev->asic_type >= CHIP_BONAIRE)
++ /* doorbell bar mapping */
++ amdgpu_doorbell_init(adev);
+
+ /* io port mapping */
+ for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
+@@ -2348,7 +2340,9 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
+ amdgpu_atombios_fini(adev);
+ kfree(adev->bios);
+ adev->bios = NULL;
++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)
+ if (!pci_is_thunderbolt_attached(adev->pdev))
++#endif
+ vga_switcheroo_unregister_client(adev->pdev);
+ if (adev->flags & AMD_IS_PX)
+ vga_switcheroo_fini_domain_pm_ops(adev->dev);
+@@ -2358,7 +2352,8 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
+ adev->rio_mem = NULL;
+ iounmap(adev->rmmio);
+ adev->rmmio = NULL;
+- amdgpu_doorbell_fini(adev);
++ if (adev->asic_type >= CHIP_BONAIRE)
++ amdgpu_doorbell_fini(adev);
+ amdgpu_debugfs_regs_cleanup(adev);
+ }
+
+@@ -3159,6 +3154,27 @@ int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
+ return 0;
+ }
+
++#if defined(BUILD_AS_DKMS) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)
++void amdgpu_debugfs_cleanup(struct drm_minor *minor)
++{
++ struct drm_info_node *node, *tmp;
++
++ if (!&minor->debugfs_root)
++ return 0;
++
++ mutex_lock(&minor->debugfs_lock);
++ list_for_each_entry_safe(node, tmp,
++ &minor->debugfs_list, list) {
++ debugfs_remove(node->dent);
++ list_del(&node->list);
++ kfree(node);
++ }
++ mutex_unlock(&minor->debugfs_lock);
++
++ return 0;
++}
++#endif
++
+ #if defined(CONFIG_DEBUG_FS)
+
+ static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
+@@ -3570,7 +3586,10 @@ static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
+
+ valuesize = sizeof(values);
+ if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->read_sensor)
+- r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
++ r = adev->powerplay.pp_funcs->read_sensor(adev->powerplay.pp_handle, idx, &values[0], &valuesize);
++ else if (adev->pm.funcs && adev->pm.funcs->read_sensor)
++ r = adev->pm.funcs->read_sensor(adev, idx, &values[0],
++ &valuesize);
+ else
+ return -EINVAL;
+
+@@ -3594,7 +3613,7 @@ static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
+ static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
+ size_t size, loff_t *pos)
+ {
+- struct amdgpu_device *adev = f->f_inode->i_private;
++ struct amdgpu_device *adev = file_inode(f)->i_private;
+ int r, x;
+ ssize_t result=0;
+ uint32_t offset, se, sh, cu, wave, simd, data[32];
+@@ -3644,7 +3663,8 @@ static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
+ static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
+ size_t size, loff_t *pos)
+ {
+- struct amdgpu_device *adev = f->f_inode->i_private;
++ struct amdgpu_device *adev = file_inode(f)->i_private;
++
+ int r;
+ ssize_t result = 0;
+ uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
+index 0d22259..12a4a78 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
+@@ -427,6 +427,7 @@ struct amdgpu_pm {
+ struct amdgpu_dpm dpm;
+ const struct firmware *fw; /* SMC firmware */
+ uint32_t fw_version;
++ const struct amdgpu_dpm_funcs *funcs;
+ uint32_t pcie_gen_mask;
+ uint32_t pcie_mlw_mask;
+ struct amd_pp_display_configuration pm_display_cfg;/* set by dc */
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+index 2be2e05..0720358 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+@@ -69,10 +69,9 @@
+ * - 3.17.0 - Add AMDGPU_NUM_VRAM_CPU_PAGE_FAULTS.
+ * - 3.18.0 - Export gpu always on cu bitmap
+ * - 3.19.0 - Add support for UVD MJPEG decode
+- * - 3.20.0 - Add support for local BOs
+ */
+ #define KMS_DRIVER_MAJOR 3
+-#define KMS_DRIVER_MINOR 20
++#define KMS_DRIVER_MINOR 19
+ #define KMS_DRIVER_PATCHLEVEL 0
+
+ int amdgpu_vram_limit = 0;
+@@ -124,7 +123,6 @@ int amdgpu_cntl_sb_buf_per_se = 0;
+ int amdgpu_param_buf_per_se = 0;
+ int amdgpu_job_hang_limit = 0;
+ int amdgpu_lbpw = -1;
+-int amdgpu_compute_multipipe = -1;
+
+ MODULE_PARM_DESC(vramlimit, "Restrict VRAM for testing, in megabytes");
+ module_param_named(vramlimit, amdgpu_vram_limit, int, 0600);
+@@ -274,9 +272,6 @@ module_param_named(job_hang_limit, amdgpu_job_hang_limit, int ,0444);
+ MODULE_PARM_DESC(lbpw, "Load Balancing Per Watt (LBPW) support (1 = enable, 0 = disable, -1 = auto)");
+ module_param_named(lbpw, amdgpu_lbpw, int, 0444);
+
+-MODULE_PARM_DESC(compute_multipipe, "Force compute queues to be spread across pipes (1 = enable, 0 = disable, -1 = auto)");
+-module_param_named(compute_multipipe, amdgpu_compute_multipipe, int, 0444);
+-
+ #ifdef CONFIG_DRM_AMDGPU_SI
+
+ int amdgpu_si_support = 1;
+@@ -822,10 +817,8 @@ static struct drm_driver kms_driver = {
+ .open = amdgpu_driver_open_kms,
+ .postclose = amdgpu_driver_postclose_kms,
+ .lastclose = amdgpu_driver_lastclose_kms,
++ .set_busid = drm_pci_set_busid,
+ .unload = amdgpu_driver_unload_kms,
+- .get_vblank_counter = amdgpu_get_vblank_counter_kms,
+- .enable_vblank = amdgpu_enable_vblank_kms,
+- .disable_vblank = amdgpu_disable_vblank_kms,
+ .get_vblank_timestamp = drm_calc_vbltimestamp_from_scanoutpos,
+ .get_scanout_position = amdgpu_get_crtc_scanout_position,
+ #if defined(CONFIG_DEBUG_FS)
+@@ -841,6 +834,7 @@ static struct drm_driver kms_driver = {
+ .gem_close_object = amdgpu_gem_object_close,
+ .dumb_create = amdgpu_mode_dumb_create,
+ .dumb_map_offset = amdgpu_mode_dumb_mmap,
++ .dumb_destroy = drm_gem_dumb_destroy,
+ .fops = &amdgpu_driver_kms_fops,
+
+ .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
+@@ -931,4 +925,3 @@ module_exit(amdgpu_exit);
+ MODULE_AUTHOR(DRIVER_AUTHOR);
+ MODULE_DESCRIPTION(DRIVER_DESC);
+ MODULE_LICENSE("GPL and additional rights");
+-MODULE_VERSION("17.50.2.13");
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+index 3d08c6f..fdb9d85 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+@@ -44,12 +44,20 @@
+ * This is the main unload function for KMS (all asics).
+ * Returns 0 on success.
+ */
++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)
++int amdgpu_driver_unload_kms(struct drm_device *dev)
++#else
+ void amdgpu_driver_unload_kms(struct drm_device *dev)
++#endif
+ {
+ struct amdgpu_device *adev = dev->dev_private;
+
+ if (adev == NULL)
++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)
++ return 0;
++#else
+ return;
++#endif
+
+ if (adev->rmmio == NULL)
+ goto done_free;
+@@ -71,6 +79,9 @@ void amdgpu_driver_unload_kms(struct drm_device *dev)
+ done_free:
+ kfree(adev);
+ dev->dev_private = NULL;
++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)
++ return 0;
++#endif
+ }
+
+ /**
+@@ -129,8 +140,12 @@ int amdgpu_driver_load_kms(struct drm_device *dev, unsigned long flags)
+ amdgpu_has_atpx() &&
+ (amdgpu_is_atpx_hybrid() ||
+ amdgpu_has_atpx_dgpu_power_cntl()) &&
++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)
++ ((flags & AMD_IS_APU) == 0))
++#else
+ ((flags & AMD_IS_APU) == 0) &&
+ !pci_is_thunderbolt_attached(dev->pdev))
++#endif
+ flags |= AMD_IS_PX;
+
+ /* amdgpu_device_init should report only fatal error
+@@ -1052,6 +1067,72 @@ void amdgpu_disable_vblank_kms(struct drm_device *dev, unsigned int pipe)
+ amdgpu_irq_put(adev, &adev->crtc_irq, idx);
+ }
+
++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0)
++/**
++ * amdgpu_get_vblank_timestamp_kms - get vblank timestamp
++ *
++ * @dev: drm dev pointer
++ * @crtc: crtc to get the timestamp for
++ * @max_error: max error
++ * @vblank_time: time value
++ * @flags: flags passed to the driver
++ *
++ * Gets the timestamp on the requested crtc based on the
++ * scanout position. (all asics).
++ * Returns postive status flags on success, negative error on failure.
++ */
++int amdgpu_get_vblank_timestamp_kms(struct drm_device *dev, unsigned int pipe,
++ int *max_error,
++ struct timeval *vblank_time,
++ unsigned flags)
++{
++ struct drm_crtc *crtc;
++ struct amdgpu_device *adev = dev->dev_private;
++
++ if (pipe >= dev->num_crtcs) {
++ DRM_ERROR("Invalid crtc %u\n", pipe);
++ return -EINVAL;
++ }
++
++ /* Get associated drm_crtc: */
++ crtc = &adev->mode_info.crtcs[pipe]->base;
++ if (!crtc) {
++ /* This can occur on driver load if some component fails to
++ * initialize completely and driver is unloaded */
++ DRM_ERROR("Uninitialized crtc %d\n", pipe);
++ return -EINVAL;
++ }
++
++ /* Helper routine in DRM core does all the work: */
++ return kcl_drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
++ vblank_time, flags,
++ crtc, &crtc->hwmode);
++}
++#endif
++
++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0)
++const struct drm_ioctl_desc amdgpu_ioctls_kms[] = {
++ DRM_IOCTL_DEF_DRV(AMDGPU_GEM_CREATE, amdgpu_gem_create_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
++ DRM_IOCTL_DEF_DRV(AMDGPU_CTX, amdgpu_ctx_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
++ DRM_IOCTL_DEF_DRV(AMDGPU_BO_LIST, amdgpu_bo_list_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
++ /* KMS */
++ DRM_IOCTL_DEF_DRV(AMDGPU_GEM_MMAP, amdgpu_gem_mmap_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
++ DRM_IOCTL_DEF_DRV(AMDGPU_GEM_WAIT_IDLE, amdgpu_gem_wait_idle_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
++ DRM_IOCTL_DEF_DRV(AMDGPU_CS, amdgpu_cs_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
++ DRM_IOCTL_DEF_DRV(AMDGPU_INFO, amdgpu_info_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
++ DRM_IOCTL_DEF_DRV(AMDGPU_WAIT_CS, amdgpu_cs_wait_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
++ DRM_IOCTL_DEF_DRV(AMDGPU_WAIT_FENCES, amdgpu_cs_wait_fences_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
++ DRM_IOCTL_DEF_DRV(AMDGPU_GEM_METADATA, amdgpu_gem_metadata_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
++ DRM_IOCTL_DEF_DRV(AMDGPU_GEM_VA, amdgpu_gem_va_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
++ DRM_IOCTL_DEF_DRV(AMDGPU_GEM_OP, amdgpu_gem_op_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
++ DRM_IOCTL_DEF_DRV(AMDGPU_GEM_USERPTR, amdgpu_gem_userptr_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
++ DRM_IOCTL_DEF_DRV(AMDGPU_GEM_FIND_BO, amdgpu_gem_find_bo_by_cpu_mapping_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
++ DRM_IOCTL_DEF_DRV(AMDGPU_FREESYNC, amdgpu_freesync_ioctl, DRM_MASTER|DRM_UNLOCKED),
++ DRM_IOCTL_DEF_DRV(AMDGPU_GEM_FIND_BO, amdgpu_gem_find_bo_by_cpu_mapping_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
++ DRM_IOCTL_DEF_DRV(AMDGPU_GEM_DGMA, amdgpu_gem_dgma_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
++ DRM_IOCTL_DEF_DRV(AMDGPU_SEM, amdgpu_sem_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
++};
++#else
+ const struct drm_ioctl_desc amdgpu_ioctls_kms[] = {
+ DRM_IOCTL_DEF_DRV(AMDGPU_GEM_CREATE, amdgpu_gem_create_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(AMDGPU_CTX, amdgpu_ctx_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+@@ -1073,6 +1154,7 @@ const struct drm_ioctl_desc amdgpu_ioctls_kms[] = {
+ DRM_IOCTL_DEF_DRV(AMDGPU_GEM_DGMA, amdgpu_gem_dgma_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(AMDGPU_SEM, amdgpu_sem_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+ };
++#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) */
+ const int amdgpu_max_kms_ioctl = ARRAY_SIZE(amdgpu_ioctls_kms);
+
+ /*
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
+index d25ec38..430c622 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
+@@ -50,10 +50,8 @@ struct amdgpu_mn {
+ struct hlist_node node;
+
+ /* objects protected by lock */
+- struct rw_semaphore lock;
+- struct rb_root_cached objects;
+- struct mutex read_lock;
+- atomic_t recursion;
++ struct mutex lock;
++ struct rb_root objects;
+ };
+
+ struct amdgpu_mn_node {
+@@ -76,17 +74,17 @@ static void amdgpu_mn_destroy(struct work_struct *work)
+ struct amdgpu_bo *bo, *next_bo;
+
+ mutex_lock(&adev->mn_lock);
+- down_write(&rmn->lock);
++ mutex_lock(&rmn->lock);
+ hash_del(&rmn->node);
+- rbtree_postorder_for_each_entry_safe(node, next_node,
+- &rmn->objects.rb_root, it.rb) {
++ rbtree_postorder_for_each_entry_safe(node, next_node, &rmn->objects,
++ it.rb) {
+ list_for_each_entry_safe(bo, next_bo, &node->bos, mn_list) {
+ bo->mn = NULL;
+ list_del_init(&bo->mn_list);
+ }
+ kfree(node);
+ }
+- up_write(&rmn->lock);
++ mutex_unlock(&rmn->lock);
+ mutex_unlock(&adev->mn_lock);
+ mmu_notifier_unregister_no_release(&rmn->mn, rmn->mm);
+ kfree(rmn);
+@@ -108,53 +106,6 @@ static void amdgpu_mn_release(struct mmu_notifier *mn,
+ schedule_work(&rmn->work);
+ }
+
+-
+-/**
+- * amdgpu_mn_lock - take the write side lock for this mn
+- */
+-void amdgpu_mn_lock(struct amdgpu_mn *mn)
+-{
+- if (mn)
+- down_write(&mn->lock);
+-}
+-
+-/**
+- * amdgpu_mn_unlock - drop the write side lock for this mn
+- */
+-void amdgpu_mn_unlock(struct amdgpu_mn *mn)
+-{
+- if (mn)
+- up_write(&mn->lock);
+-}
+-
+-/**
+- * amdgpu_mn_read_lock - take the rmn read lock
+- *
+- * @rmn: our notifier
+- *
+- * Take the rmn read side lock.
+- */
+-static void amdgpu_mn_read_lock(struct amdgpu_mn *rmn)
+-{
+- mutex_lock(&rmn->read_lock);
+- if (atomic_inc_return(&rmn->recursion) == 1)
+- down_read_non_owner(&rmn->lock);
+- mutex_unlock(&rmn->read_lock);
+-}
+-
+-/**
+- * amdgpu_mn_read_unlock - drop the rmn read lock
+- *
+- * @rmn: our notifier
+- *
+- * Drop the rmn read side lock.
+- */
+-static void amdgpu_mn_read_unlock(struct amdgpu_mn *rmn)
+-{
+- if (atomic_dec_return(&rmn->recursion) == 0)
+- up_read_non_owner(&rmn->lock);
+-}
+-
+ /**
+ * amdgpu_mn_invalidate_node - unmap all BOs of a node
+ *
+@@ -175,12 +126,23 @@ static void amdgpu_mn_invalidate_node(struct amdgpu_mn_node *node,
+ if (!amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm, start, end))
+ continue;
+
+- r = reservation_object_wait_timeout_rcu(bo->tbo.resv,
++ r = amdgpu_bo_reserve(bo, true);
++ if (r) {
++ DRM_ERROR("(%ld) failed to reserve user bo\n", r);
++ continue;
++ }
++
++ r = kcl_reservation_object_wait_timeout_rcu(bo->tbo.resv,
+ true, false, MAX_SCHEDULE_TIMEOUT);
+ if (r <= 0)
+ DRM_ERROR("(%ld) failed to wait for user bo\n", r);
+
+- amdgpu_ttm_tt_mark_user_pages(bo->tbo.ttm);
++ amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
++ r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
++ if (r)
++ DRM_ERROR("(%ld) failed to validate user bo\n", r);
++
++ amdgpu_bo_unreserve(bo);
+ }
+ }
+
+@@ -206,7 +168,7 @@ static void amdgpu_mn_invalidate_range_start(struct mmu_notifier *mn,
+ /* notification is exclusive, but interval is inclusive */
+ end -= 1;
+
+- amdgpu_mn_read_lock(rmn);
++ mutex_lock(&rmn->lock);
+
+ it = interval_tree_iter_first(&rmn->objects, start, end);
+ while (it) {
+@@ -218,33 +180,12 @@ static void amdgpu_mn_invalidate_range_start(struct mmu_notifier *mn,
+ amdgpu_mn_invalidate_node(node, start, end);
+ }
+
+- up_read(&rmn->lock);
+-}
+-
+-/**
+- * amdgpu_mn_invalidate_range_end - callback to notify about mm change
+- *
+- * @mn: our notifier
+- * @mn: the mm this callback is about
+- * @start: start of updated range
+- * @end: end of updated range
+- *
+- * Release the lock again to allow new command submissions.
+- */
+-static void amdgpu_mn_invalidate_range_end(struct mmu_notifier *mn,
+- struct mm_struct *mm,
+- unsigned long start,
+- unsigned long end)
+-{
+- struct amdgpu_mn *rmn = container_of(mn, struct amdgpu_mn, mn);
+-
+- amdgpu_mn_read_unlock(rmn);
++ mutex_unlock(&rmn->lock);
+ }
+
+ static const struct mmu_notifier_ops amdgpu_mn_ops = {
+ .release = amdgpu_mn_release,
+ .invalidate_range_start = amdgpu_mn_invalidate_range_start,
+- .invalidate_range_end = amdgpu_mn_invalidate_range_end,
+ };
+
+ /**
+@@ -254,19 +195,30 @@ static const struct mmu_notifier_ops amdgpu_mn_ops = {
+ *
+ * Creates a notifier context for current->mm.
+ */
+-struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev)
++static struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev)
+ {
+ struct mm_struct *mm = current->mm;
+ struct amdgpu_mn *rmn;
+ int r;
++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0)
++ struct hlist_node *node;
++#endif
+
+ mutex_lock(&adev->mn_lock);
++#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0)
++ down_write(&mm->mmap_sem);
++#else
+ if (down_write_killable(&mm->mmap_sem)) {
+ mutex_unlock(&adev->mn_lock);
+ return ERR_PTR(-EINTR);
+ }
++#endif
+
++#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0)
++ hash_for_each_possible(adev->mn_hash, rmn, node, node, (unsigned long)mm)
++#else
+ hash_for_each_possible(adev->mn_hash, rmn, node, (unsigned long)mm)
++#endif
+ if (rmn->mm == mm)
+ goto release_locks;
+
+@@ -279,10 +231,8 @@ struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev)
+ rmn->adev = adev;
+ rmn->mm = mm;
+ rmn->mn.ops = &amdgpu_mn_ops;
+- init_rwsem(&rmn->lock);
+- rmn->objects = RB_ROOT_CACHED;
+- mutex_init(&rmn->read_lock);
+- atomic_set(&rmn->recursion, 0);
++ mutex_init(&rmn->lock);
++ rmn->objects = RB_ROOT;
+
+ r = __mmu_notifier_register(&rmn->mn, mm);
+ if (r)
+@@ -328,7 +278,7 @@ int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
+
+ INIT_LIST_HEAD(&bos);
+
+- down_write(&rmn->lock);
++ mutex_lock(&rmn->lock);
+
+ while ((it = interval_tree_iter_first(&rmn->objects, addr, end))) {
+ kfree(node);
+@@ -340,9 +290,9 @@ int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
+ }
+
+ if (!node) {
+- node = kmalloc(sizeof(struct amdgpu_mn_node), GFP_NOIO);
++ node = kmalloc(sizeof(struct amdgpu_mn_node), GFP_KERNEL);
+ if (!node) {
+- up_write(&rmn->lock);
++ mutex_unlock(&rmn->lock);
+ return -ENOMEM;
+ }
+ }
+@@ -357,7 +307,7 @@ int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
+
+ interval_tree_insert(&node->it, &rmn->objects);
+
+- up_write(&rmn->lock);
++ mutex_unlock(&rmn->lock);
+
+ return 0;
+ }
+@@ -383,7 +333,7 @@ void amdgpu_mn_unregister(struct amdgpu_bo *bo)
+ return;
+ }
+
+- down_write(&rmn->lock);
++ mutex_lock(&rmn->lock);
+
+ /* save the next list entry for later */
+ head = bo->mn_list.next;
+@@ -398,7 +348,6 @@ void amdgpu_mn_unregister(struct amdgpu_bo *bo)
+ kfree(node);
+ }
+
+- up_write(&rmn->lock);
++ mutex_unlock(&rmn->lock);
+ mutex_unlock(&adev->mn_lock);
+ }
+-
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+index f421505..fb6c3d6 100755
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+@@ -36,7 +36,6 @@
+ #include <drm/drm_cache.h>
+ #include "amdgpu.h"
+ #include "amdgpu_trace.h"
+-#include "amdgpu_amdkfd.h"
+
+ static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo)
+ {
+@@ -47,9 +46,8 @@ static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo)
+
+ if (bo->tbo.mem.mem_type == AMDGPU_PL_DGMA_IMPORT)
+ kfree(tbo->mem.bus.addr);
+- if (bo->kfd_bo)
+- amdgpu_amdkfd_unreserve_system_memory_limit(bo);
+ amdgpu_bo_kunmap(bo);
++ drm_gem_object_release(&bo->gem_base);
+
+ if (bo->gem_base.import_attach)
+ drm_prime_gem_destroy(&bo->gem_base, bo->tbo.sg);
+@@ -70,12 +68,11 @@ bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)
+ return false;
+ }
+
+-void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
++static void amdgpu_ttm_placement_init(struct amdgpu_device *adev,
++ struct ttm_placement *placement,
++ struct ttm_place *places,
++ u32 domain, u64 flags)
+ {
+- struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
+- struct ttm_placement *placement = &abo->placement;
+- struct ttm_place *places = abo->placements;
+- u64 flags = abo->flags;
+ u32 c = 0, i;
+
+ if ((domain & AMDGPU_GEM_DOMAIN_DGMA) && amdgpu_direct_gma_size) {
+@@ -178,6 +175,27 @@ void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
+ placement->busy_placement = places;
+ }
+
++void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
++{
++ struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
++
++ amdgpu_ttm_placement_init(adev, &abo->placement, abo->placements,
++ domain, abo->flags);
++}
++
++static void amdgpu_fill_placement_to_bo(struct amdgpu_bo *bo,
++ struct ttm_placement *placement)
++{
++ BUG_ON(placement->num_placement > (AMDGPU_GEM_DOMAIN_MAX + 1));
++
++ memcpy(bo->placements, placement->placement,
++ placement->num_placement * sizeof(struct ttm_place));
++ bo->placement.num_placement = placement->num_placement;
++ bo->placement.num_busy_placement = placement->num_busy_placement;
++ bo->placement.placement = bo->placements;
++ bo->placement.busy_placement = bo->placements;
++}
++
+ /**
+ * amdgpu_bo_create_reserved - create reserved BO for kernel use
+ *
+@@ -309,13 +327,14 @@ void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
+ *cpu_addr = NULL;
+ }
+
+-static int amdgpu_bo_do_create(struct amdgpu_device *adev,
+- unsigned long size, int byte_align,
+- bool kernel, u32 domain, u64 flags,
+- struct sg_table *sg,
+- struct reservation_object *resv,
+- uint64_t init_value,
+- struct amdgpu_bo **bo_ptr)
++int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
++ unsigned long size, int byte_align,
++ bool kernel, u32 domain, u64 flags,
++ struct sg_table *sg,
++ struct ttm_placement *placement,
++ struct reservation_object *resv,
++ uint64_t init_value,
++ struct amdgpu_bo **bo_ptr)
+ {
+ struct amdgpu_bo *bo;
+ enum ttm_bo_type type;
+@@ -342,10 +361,13 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev,
+ bo = kzalloc(sizeof(struct amdgpu_bo), GFP_KERNEL);
+ if (bo == NULL)
+ return -ENOMEM;
+-
++ r = drm_gem_object_init(adev->ddev, &bo->gem_base, size);
++ if (unlikely(r)) {
++ kfree(bo);
++ return r;
++ }
+ INIT_LIST_HEAD(&bo->shadow_list);
+ INIT_LIST_HEAD(&bo->va);
+- INIT_LIST_HEAD(&bo->gem_objects);
+ bo->preferred_domains = domain & (AMDGPU_GEM_DOMAIN_VRAM |
+ AMDGPU_GEM_DOMAIN_GTT |
+ AMDGPU_GEM_DOMAIN_CPU |
+@@ -388,17 +410,13 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev,
+ bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
+ #endif
+
+- bo->tbo.bdev = &adev->mman.bdev;
+- amdgpu_ttm_placement_from_domain(bo, domain);
++ amdgpu_fill_placement_to_bo(bo, placement);
++ /* Kernel allocation are uninterruptible */
+
+ initial_bytes_moved = atomic64_read(&adev->num_bytes_moved);
+- /* Kernel allocation are uninterruptible */
+ r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, type,
+ &bo->placement, page_align, !kernel, NULL,
+ acc_size, sg, resv, &amdgpu_ttm_bo_destroy);
+- if (unlikely(r != 0))
+- return r;
+-
+ bytes_moved = atomic64_read(&adev->num_bytes_moved) -
+ initial_bytes_moved;
+ if (adev->mc.visible_vram_size < adev->mc.real_vram_size &&
+@@ -408,6 +426,9 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev,
+ else
+ amdgpu_cs_report_moved_bytes(adev, bytes_moved, 0);
+
++ if (unlikely(r != 0))
++ return r;
++
+ if (domain & AMDGPU_GEM_DOMAIN_DGMA && adev->ssg.enabled)
+ bo->tbo.ssg_can_map = true;
+
+@@ -422,9 +443,13 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev,
+ if (unlikely(r))
+ goto fail_unreserve;
+
++#if defined(BUILD_AS_DKMS)
++ dma_fence_wait(fence, false);
++#else
+ amdgpu_bo_fence(bo, fence, false);
+ dma_fence_put(bo->tbo.moving);
+ bo->tbo.moving = dma_fence_get(fence);
++#endif
+ dma_fence_put(fence);
+ }
+ if (!resv)
+@@ -459,17 +484,27 @@ static int amdgpu_bo_create_shadow(struct amdgpu_device *adev,
+ unsigned long size, int byte_align,
+ struct amdgpu_bo *bo)
+ {
++ struct ttm_placement placement = {0};
++ struct ttm_place placements[AMDGPU_GEM_DOMAIN_MAX + 1];
+ int r;
+
+ if (bo->shadow)
+ return 0;
+
+- r = amdgpu_bo_do_create(adev, size, byte_align, true,
+- AMDGPU_GEM_DOMAIN_GTT,
+- AMDGPU_GEM_CREATE_CPU_GTT_USWC |
+- AMDGPU_GEM_CREATE_SHADOW,
+- NULL, bo->tbo.resv, 0,
+- &bo->shadow);
++ memset(&placements, 0, sizeof(placements));
++ amdgpu_ttm_placement_init(adev, &placement, placements,
++ AMDGPU_GEM_DOMAIN_GTT,
++ AMDGPU_GEM_CREATE_CPU_GTT_USWC |
++ AMDGPU_GEM_CREATE_SHADOW);
++
++ r = amdgpu_bo_create_restricted(adev, size, byte_align, true,
++ AMDGPU_GEM_DOMAIN_GTT,
++ AMDGPU_GEM_CREATE_CPU_GTT_USWC |
++ AMDGPU_GEM_CREATE_SHADOW,
++ NULL, &placement,
++ bo->tbo.resv,
++ 0,
++ &bo->shadow);
+ if (!r) {
+ bo->shadow->parent = amdgpu_bo_ref(bo);
+ mutex_lock(&adev->shadow_list_lock);
+@@ -491,11 +526,18 @@ int amdgpu_bo_create(struct amdgpu_device *adev,
+ uint64_t init_value,
+ struct amdgpu_bo **bo_ptr)
+ {
++ struct ttm_placement placement = {0};
++ struct ttm_place placements[AMDGPU_GEM_DOMAIN_MAX + 1];
+ uint64_t parent_flags = flags & ~AMDGPU_GEM_CREATE_SHADOW;
+ int r;
+
+- r = amdgpu_bo_do_create(adev, size, byte_align, kernel, domain,
+- parent_flags, sg, resv, init_value, bo_ptr);
++ memset(&placements, 0, sizeof(placements));
++ amdgpu_ttm_placement_init(adev, &placement, placements,
++ domain, parent_flags);
++
++ r = amdgpu_bo_create_restricted(adev, size, byte_align, kernel, domain,
++ parent_flags, sg, &placement, resv,
++ init_value, bo_ptr);
+ if (r)
+ return r;
+
+@@ -931,7 +973,7 @@ void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
+ return;
+
+ abo = container_of(bo, struct amdgpu_bo, tbo);
+- amdgpu_vm_bo_invalidate(adev, abo, evict);
++ amdgpu_vm_bo_invalidate(adev, abo);
+
+ amdgpu_bo_kunmap(abo);
+
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+index f73dba5..024e5cb 100755
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+@@ -35,7 +35,6 @@
+
+ /* bo virtual addresses in a vm */
+ struct amdgpu_bo_va_mapping {
+- struct amdgpu_bo_va *bo_va;
+ struct list_head list;
+ struct rb_node rb;
+ uint64_t start;
+@@ -50,17 +49,12 @@ struct amdgpu_bo_va {
+ struct amdgpu_vm_bo_base base;
+
+ /* protected by bo being reserved */
+- unsigned ref_count;
+-
+- /* all other members protected by the VM PD being reserved */
+ struct dma_fence *last_pt_update;
++ unsigned ref_count;
+
+ /* mappings for this bo_va */
+ struct list_head invalids;
+ struct list_head valids;
+-
+- /* If the mappings are cleared or filled */
+- bool cleared;
+ };
+
+ struct amdgpu_bo {
+@@ -78,18 +72,16 @@ struct amdgpu_bo {
+ void *metadata;
+ u32 metadata_size;
+ unsigned prime_shared_count;
+- /* GEM objects refereing to this BO */
+- struct list_head gem_objects;
+-
+ /* list of all virtual address to which this bo is associated to */
+ struct list_head va;
+ /* Constant after initialization */
++ struct drm_gem_object gem_base;
+ struct amdgpu_bo *parent;
+ struct amdgpu_bo *shadow;
+
+ struct ttm_bo_kmap_obj dma_buf_vmap;
+ struct amdgpu_mn *mn;
+- struct kgd_mem *kfd_bo;
++ struct kfd_process_device *pdd;
+
+ union {
+ struct list_head mn_list;
+@@ -207,6 +199,14 @@ int amdgpu_bo_create(struct amdgpu_device *adev,
+ struct reservation_object *resv,
+ uint64_t init_value,
+ struct amdgpu_bo **bo_ptr);
++int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
++ unsigned long size, int byte_align,
++ bool kernel, u32 domain, u64 flags,
++ struct sg_table *sg,
++ struct ttm_placement *placement,
++ struct reservation_object *resv,
++ uint64_t init_value,
++ struct amdgpu_bo **bo_ptr);
+ int amdgpu_bo_create_reserved(struct amdgpu_device *adev,
+ unsigned long size, int align,
+ u32 domain, struct amdgpu_bo **bo_ptr,
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+index 90adff8..06b824c 100644
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+@@ -136,8 +136,7 @@ void amdgpu_ring_commit(struct amdgpu_ring *ring)
+ if (ring->funcs->end_use)
+ ring->funcs->end_use(ring);
+
+- if (ring->funcs->type != AMDGPU_RING_TYPE_KIQ)
+- amdgpu_ring_lru_touch(ring->adev, ring);
++ amdgpu_ring_lru_touch(ring->adev, ring);
+ }
+
+ /**
+@@ -382,7 +381,7 @@ void amdgpu_ring_lru_touch(struct amdgpu_device *adev, struct amdgpu_ring *ring)
+ static ssize_t amdgpu_debugfs_ring_read(struct file *f, char __user *buf,
+ size_t size, loff_t *pos)
+ {
+- struct amdgpu_ring *ring = file_inode(f)->i_private;
++ struct amdgpu_ring *ring = (struct amdgpu_ring*)kcl_file_private(f);
+ int r, i;
+ uint32_t value, result, early[3];
+
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+index af8e544..322d2529 100755
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+@@ -36,7 +36,6 @@
+ /* some special values for the owner field */
+ #define AMDGPU_FENCE_OWNER_UNDEFINED ((void*)0ul)
+ #define AMDGPU_FENCE_OWNER_VM ((void*)1ul)
+-#define AMDGPU_FENCE_OWNER_KFD ((void *)2ul)
+
+ #define AMDGPU_FENCE_FLAG_64BIT (1 << 0)
+ #define AMDGPU_FENCE_FLAG_INT (1 << 1)
+diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
+index 7ee8247..8492a26 100755
+--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
+@@ -31,7 +31,6 @@
+ #include <drm/drmP.h>
+ #include "amdgpu.h"
+ #include "amdgpu_trace.h"
+-#include "amdgpu_amdkfd.h"
+
+ struct amdgpu_sync_entry {
+ struct hlist_node node;
+@@ -85,20 +84,11 @@ static bool amdgpu_sync_same_dev(struct amdgpu_device *adev,
+ */
+ static void *amdgpu_sync_get_owner(struct dma_fence *f)
+ {
+- struct amd_sched_fence *s_fence;
+- struct amdgpu_amdkfd_fence *kfd_fence;
+-
+- if (f == NULL)
+- return AMDGPU_FENCE_OWNER_UNDEFINED;
++ struct amd_sched_fence *s_fence = to_amd_sched_fence(f);
+
+- s_fence = to_amd_sched_fence(f);
+ if (s_fence)
+ return s_fence->owner;
+
+- kfd_fence = to_amdgpu_amdkfd_fence(f);
+- if (kfd_fence)
+- return AMDGPU_FENCE_OWNER_KFD;
+-
+ return AMDGPU_FENCE_OWNER_UNDEFINED;
+ }
+
+@@ -180,9 +170,7 @@ int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync,
+ * @sync: sync object to add fences from reservation object to
+ * @resv: reservation object with embedded fence
+ * @shared: true if we should only sync to the exclusive fence
+- *
+- * Sync to the fence except if it is KFD eviction fence and owner is
+- * AMDGPU_FENCE_OWNER_VM.
++ * Sync to the fence
+ */
+ int amdgpu_sync_resv(struct amdgpu_device *adev,
+ struct amdgpu_sync *sync,
+@@ -209,15 +197,12 @@ int amdgpu_sync_resv(struct amdgpu_device *adev,
+ for (i = 0; i < flist->shared_count; ++i) {
+ f = rcu_dereference_protected(flist->shared[i],
+ reservation_object_held(resv));
+- fence_owner = amdgpu_sync_get_owner(f);
+- if (fence_owner == AMDGPU_FENCE_OWNER_KFD &&
+- owner != AMDGPU_FENCE_OWNER_UNDEFINED)
+- continue;
+
+ if (amdgpu_sync_same_dev(adev, f)) {
+ /* VM updates are only interesting
+ * for other VM updates and moves.
+ */
++ fence_owner = amdgpu_sync_get_owner(f);
+ if ((owner != AMDGPU_FENCE_OWNER_UNDEFINED) &&
+ (fence_owner != AMDGPU_FENCE_OWNER_UNDEFINED) &&
+ ((owner == AMDGPU_FENCE_OWNER_VM) !=
+diff --git a/drivers/gpu/drm/amd/amdgpu/vid.h b/drivers/gpu/drm/amd/amdgpu/vid.h
+index d09592a..a648525 100755
+--- a/drivers/gpu/drm/amd/amdgpu/vid.h
++++ b/drivers/gpu/drm/amd/amdgpu/vid.h
+@@ -27,8 +27,6 @@
+ #define SDMA1_REGISTER_OFFSET 0x200 /* not a register */
+ #define SDMA_MAX_INSTANCE 2
+
+-#define KFD_VI_SDMA_QUEUE_OFFSET 0x80 /* not a register */
+-
+ /* crtc instance offsets */
+ #define CRTC0_REGISTER_OFFSET (0x1b9c - 0x1b9c)
+ #define CRTC1_REGISTER_OFFSET (0x1d9c - 0x1b9c)
+@@ -369,10 +367,6 @@
+ * x=0: tmz_begin
+ * x=1: tmz_end
+ */
+-#define PACKET3_INVALIDATE_TLBS 0x98
+-# define PACKET3_INVALIDATE_TLBS_DST_SEL(x) ((x) << 0)
+-# define PACKET3_INVALIDATE_TLBS_PASID(x) ((x) << 5)
+-
+ #define PACKET3_SET_RESOURCES 0xA0
+ /* 1. header
+ * 2. CONTROL
+diff --git a/drivers/gpu/drm/amd/amdkfd/Kconfig b/drivers/gpu/drm/amd/amdkfd/Kconfig
+index 95be0dd..e13c67c 100644
+--- a/drivers/gpu/drm/amd/amdkfd/Kconfig
++++ b/drivers/gpu/drm/amd/amdkfd/Kconfig
+@@ -4,7 +4,6 @@
+
+ config HSA_AMD
+ tristate "HSA kernel driver for AMD GPU devices"
+- depends on (DRM_RADEON || DRM_AMDGPU) && (X86_64 || PPC64 || ARM64)
+- select DRM_AMDGPU_USERPTR
++ depends on (DRM_RADEON || DRM_AMDGPU) && AMD_IOMMU_V2 && X86_64
+ help
+ Enable this if you want to use HSA features on AMD GPU devices.
+diff --git a/drivers/gpu/drm/amd/amdkfd/Makefile b/drivers/gpu/drm/amd/amdkfd/Makefile
+old mode 100755
+new mode 100644
+index dba08ec..b400d56
+--- a/drivers/gpu/drm/amd/amdkfd/Makefile
++++ b/drivers/gpu/drm/amd/amdkfd/Makefile
+@@ -1,28 +1,19 @@
+-# SPDX-License-Identifier: GPL-2.0
+ #
+ # Makefile for Heterogenous System Architecture support for AMD GPU devices
+ #
+
+-FULL_AMD_PATH=$(src)/..
+-
+-ccflags-y := -I$(FULL_AMD_PATH)/include/ \
+- -I$(FULL_AMD_PATH)/include/asic_reg
++ccflags-y := -Idrivers/gpu/drm/amd/include/ \
++ -Idrivers/gpu/drm/amd/include/asic_reg
+
+ amdkfd-y := kfd_module.o kfd_device.o kfd_chardev.o kfd_topology.o \
+ kfd_pasid.o kfd_doorbell.o kfd_flat_memory.o \
+ kfd_process.o kfd_queue.o kfd_mqd_manager.o \
+ kfd_mqd_manager_cik.o kfd_mqd_manager_vi.o \
+- kfd_mqd_manager_v9.o \
+ kfd_kernel_queue.o kfd_kernel_queue_cik.o \
+- kfd_kernel_queue_vi.o kfd_kernel_queue_v9.o \
+- kfd_packet_manager.o kfd_process_queue_manager.o \
+- kfd_device_queue_manager.o kfd_device_queue_manager_cik.o \
+- kfd_device_queue_manager_vi.o kfd_device_queue_manager_v9.o \
+- kfd_interrupt.o kfd_events.o cik_event_interrupt.o kfd_int_process_v9.o \
+- kfd_dbgdev.o kfd_dbgmgr.o kfd_flat_memory.o kfd_crat.o kfd_rdma.o \
+- kfd_peerdirect.o kfd_ipc.o
+-
+-amdkfd-$(CONFIG_DEBUG_FS) += kfd_debugfs.o
++ kfd_kernel_queue_vi.o kfd_packet_manager.o \
++ kfd_process_queue_manager.o kfd_device_queue_manager.o \
++ kfd_device_queue_manager_cik.o kfd_device_queue_manager_vi.o \
++ kfd_interrupt.o kfd_events.o cik_event_interrupt.o \
++ kfd_dbgdev.o kfd_dbgmgr.o
+
+ obj-$(CONFIG_HSA_AMD) += amdkfd.o
+-
+diff --git a/drivers/gpu/drm/amd/amdkfd/backport/Makefile b/drivers/gpu/drm/amd/amdkfd/backport/Makefile
+deleted file mode 100644
+index 6a3845e..0000000
+--- a/drivers/gpu/drm/amd/amdkfd/backport/Makefile
++++ /dev/null
+@@ -1,7 +0,0 @@
+-
+-
+-LINUXINCLUDE := $(DKMS_INCLUDE_PREFIX) $(LINUXINCLUDE)
+-
+-ccflags-y += \
+- -I$(AMDKFD_FULL_PATH) \
+- -include backport/backport.h
+diff --git a/drivers/gpu/drm/amd/amdkfd/backport/backport.h b/drivers/gpu/drm/amd/amdkfd/backport/backport.h
+deleted file mode 100644
+index e1f8c1d..0000000
+--- a/drivers/gpu/drm/amd/amdkfd/backport/backport.h
++++ /dev/null
+@@ -1,6 +0,0 @@
+-#ifndef AMDKFD_BACKPORT_H
+-#define AMDKFD_BACKPORT_H
+-
+-#include <linux/version.h>
+-
+-#endif
+diff --git a/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c b/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c
+index 00536a1..211fc48 100644
+--- a/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c
++++ b/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c
+@@ -24,89 +24,40 @@
+ #include "kfd_events.h"
+ #include "cik_int.h"
+
+-static bool is_cpc_vm_fault(struct kfd_dev *dev,
+- const uint32_t *ih_ring_entry)
+-{
+- const struct cik_ih_ring_entry *ihre =
+- (const struct cik_ih_ring_entry *)ih_ring_entry;
+-
+- if ((ihre->source_id == CIK_INTSRC_GFX_PAGE_INV_FAULT ||
+- ihre->source_id == CIK_INTSRC_GFX_MEM_PROT_FAULT) &&
+- ihre->vmid >= dev->vm_info.first_vmid_kfd &&
+- ihre->vmid <= dev->vm_info.last_vmid_kfd)
+- return true;
+- return false;
+-}
+-
+ static bool cik_event_interrupt_isr(struct kfd_dev *dev,
+- const uint32_t *ih_ring_entry,
+- uint32_t *patched_ihre,
+- bool *patched_flag)
++ const uint32_t *ih_ring_entry)
+ {
++ unsigned int pasid;
+ const struct cik_ih_ring_entry *ihre =
+ (const struct cik_ih_ring_entry *)ih_ring_entry;
+- const struct kfd2kgd_calls *f2g = dev->kfd2kgd;
+- struct cik_ih_ring_entry *tmp_ihre =
+- (struct cik_ih_ring_entry *) patched_ihre;
+
+- /* This workaround is due to HW/FW limitation on Hawaii that
+- * VMID and PASID are not written into ih_ring_entry
+- */
+- if ((ihre->source_id == CIK_INTSRC_GFX_PAGE_INV_FAULT ||
+- ihre->source_id == CIK_INTSRC_GFX_MEM_PROT_FAULT) &&
+- dev->device_info->asic_family == CHIP_HAWAII) {
+- *patched_flag = true;
+- *tmp_ihre = *ihre;
++ pasid = (ihre->ring_id & 0xffff0000) >> 16;
+
+- tmp_ihre->vmid = f2g->read_vmid_from_vmfault_reg(dev->kgd);
+- tmp_ihre->pasid = f2g->get_atc_vmid_pasid_mapping_pasid(
+- dev->kgd, tmp_ihre->vmid);
+- return (tmp_ihre->pasid != 0) &&
+- tmp_ihre->vmid >= dev->vm_info.first_vmid_kfd &&
+- tmp_ihre->vmid <= dev->vm_info.last_vmid_kfd;
+- }
+ /* Do not process in ISR, just request it to be forwarded to WQ. */
+- return (ihre->pasid != 0) &&
++ return (pasid != 0) &&
+ (ihre->source_id == CIK_INTSRC_CP_END_OF_PIPE ||
+- ihre->source_id == CIK_INTSRC_SDMA_TRAP ||
+ ihre->source_id == CIK_INTSRC_SQ_INTERRUPT_MSG ||
+- ihre->source_id == CIK_INTSRC_CP_BAD_OPCODE ||
+- is_cpc_vm_fault(dev, ih_ring_entry));
++ ihre->source_id == CIK_INTSRC_CP_BAD_OPCODE);
+ }
+
+ static void cik_event_interrupt_wq(struct kfd_dev *dev,
+ const uint32_t *ih_ring_entry)
+ {
++ unsigned int pasid;
+ const struct cik_ih_ring_entry *ihre =
+ (const struct cik_ih_ring_entry *)ih_ring_entry;
+
+- if (ihre->pasid == 0)
++ pasid = (ihre->ring_id & 0xffff0000) >> 16;
++
++ if (pasid == 0)
+ return;
+
+ if (ihre->source_id == CIK_INTSRC_CP_END_OF_PIPE)
+- kfd_signal_event_interrupt(ihre->pasid, 0, 0);
+- else if (ihre->source_id == CIK_INTSRC_SDMA_TRAP)
+- kfd_signal_event_interrupt(ihre->pasid, 0, 0);
++ kfd_signal_event_interrupt(pasid, 0, 0);
+ else if (ihre->source_id == CIK_INTSRC_SQ_INTERRUPT_MSG)
+- kfd_signal_event_interrupt(ihre->pasid, ihre->data & 0xFF, 8);
++ kfd_signal_event_interrupt(pasid, ihre->data & 0xFF, 8);
+ else if (ihre->source_id == CIK_INTSRC_CP_BAD_OPCODE)
+- kfd_signal_hw_exception_event(ihre->pasid);
+- else if (ihre->source_id == CIK_INTSRC_GFX_PAGE_INV_FAULT ||
+- ihre->source_id == CIK_INTSRC_GFX_MEM_PROT_FAULT) {
+- struct kfd_vm_fault_info info;
+-
+- kfd_process_vm_fault(dev->dqm, ihre->pasid);
+-
+- memset(&info, 0, sizeof(info));
+- dev->kfd2kgd->get_vm_fault_info(dev->kgd, &info);
+- if (!info.page_addr && !info.status)
+- return;
+-
+- if (info.vmid == ihre->vmid)
+- kfd_signal_vm_fault_event(dev, ihre->pasid, &info);
+- else
+- kfd_signal_vm_fault_event(dev, ihre->pasid, NULL);
+- }
++ kfd_signal_hw_exception_event(pasid);
+ }
+
+ const struct kfd_event_interrupt_class event_interrupt_class_cik = {
+diff --git a/drivers/gpu/drm/amd/amdkfd/cik_int.h b/drivers/gpu/drm/amd/amdkfd/cik_int.h
+index ff8255d..79a16d2 100644
+--- a/drivers/gpu/drm/amd/amdkfd/cik_int.h
++++ b/drivers/gpu/drm/amd/amdkfd/cik_int.h
+@@ -26,32 +26,16 @@
+ #include <linux/types.h>
+
+ struct cik_ih_ring_entry {
+- uint32_t source_id:8;
+- uint32_t reserved1:8;
+- uint32_t reserved2:16;
+-
+- uint32_t data:28;
+- uint32_t reserved3:4;
+-
+- /* pipeid, meid and unused3 are officially called RINGID,
+- * but for our purposes, they always decode into pipe and ME.
+- */
+- uint32_t pipeid:2;
+- uint32_t meid:2;
+- uint32_t reserved4:4;
+- uint32_t vmid:8;
+- uint32_t pasid:16;
+-
+- uint32_t reserved5;
++ uint32_t source_id;
++ uint32_t data;
++ uint32_t ring_id;
++ uint32_t reserved;
+ };
+
+ #define CIK_INTSRC_DEQUEUE_COMPLETE 0xC6
+ #define CIK_INTSRC_CP_END_OF_PIPE 0xB5
+ #define CIK_INTSRC_CP_BAD_OPCODE 0xB7
+ #define CIK_INTSRC_SQ_INTERRUPT_MSG 0xEF
+-#define CIK_INTSRC_GFX_PAGE_INV_FAULT 0x92
+-#define CIK_INTSRC_GFX_MEM_PROT_FAULT 0x93
+-#define CIK_INTSRC_SDMA_TRAP 0xE0
+
+ #endif
+
+diff --git a/drivers/gpu/drm/amd/amdkfd/cik_regs.h b/drivers/gpu/drm/amd/amdkfd/cik_regs.h
+index 37ce6dd..48769d1 100644
+--- a/drivers/gpu/drm/amd/amdkfd/cik_regs.h
++++ b/drivers/gpu/drm/amd/amdkfd/cik_regs.h
+@@ -33,8 +33,7 @@
+ #define APE1_MTYPE(x) ((x) << 7)
+
+ /* valid for both DEFAULT_MTYPE and APE1_MTYPE */
+-#define MTYPE_CACHED_NV 0
+-#define MTYPE_CACHED 1
++#define MTYPE_CACHED 0
+ #define MTYPE_NONCACHED 3
+
+ #define DEFAULT_CP_HQD_PERSISTENT_STATE (0x33U << 8)
+diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_carrizo.h b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_carrizo.h
+deleted file mode 100644
+index d5d1331..0000000
+--- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_carrizo.h
++++ /dev/null
+@@ -1,1384 +0,0 @@
+-/*
+- * Copyright 2015 Advanced Micro Devices, Inc.
+- *
+- * Permission is hereby granted, free of charge, to any person obtaining a
+- * copy of this software and associated documentation files (the "Software"),
+- * to deal in the Software without restriction, including without limitation
+- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+- * and/or sell copies of the Software, and to permit persons to whom the
+- * Software is furnished to do so, subject to the following conditions:
+- *
+- * The above copyright notice and this permission notice shall be included in
+- * all copies or substantial portions of the Software.
+- *
+- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+- * OTHER DEALINGS IN THE SOFTWARE.
+- */
+-
+-#if 0
+-HW (VI) source code for CWSR trap handler
+-#Version 18 + multiple trap handler
+-
+-// this performance-optimal version was originally from Seven Xu at SRDC
+-
+-// Revison #18 --...
+-/* Rev History
+-** #1. Branch from gc dv. //gfxip/gfx8/main/src/test/suites/block/cs/sr/cs_trap_handler.sp3#1,#50, #51, #52-53(Skip, Already Fixed by PV), #54-56(merged),#57-58(mergerd, skiped-already fixed by PV)
+-** #4. SR Memory Layout:
+-** 1. VGPR-SGPR-HWREG-{LDS}
+-** 2. tba_hi.bits.26 - reconfigured as the first wave in tg bits, for defer Save LDS for a threadgroup.. performance concern..
+-** #5. Update: 1. Accurate g8sr_ts_save_d timestamp
+-** #6. Update: 1. Fix s_barrier usage; 2. VGPR s/r using swizzle buffer?(NoNeed, already matched the swizzle pattern, more investigation)
+-** #7. Update: 1. don't barrier if noLDS
+-** #8. Branch: 1. Branch to ver#0, which is very similar to gc dv version
+-** 2. Fix SQ issue by s_sleep 2
+-** #9. Update: 1. Fix scc restore failed issue, restore wave_status at last
+-** 2. optimize s_buffer save by burst 16sgprs...
+-** #10. Update 1. Optimize restore sgpr by busrt 16 sgprs.
+-** #11. Update 1. Add 2 more timestamp for debug version
+-** #12. Update 1. Add VGPR SR using DWx4, some case improve and some case drop performance
+-** #13. Integ 1. Always use MUBUF for PV trap shader...
+-** #14. Update 1. s_buffer_store soft clause...
+-** #15. Update 1. PERF - sclar write with glc:0/mtype0 to allow L2 combine. perf improvement a lot.
+-** #16. Update 1. PRRF - UNROLL LDS_DMA got 2500cycle save in IP tree
+-** #17. Update 1. FUNC - LDS_DMA has issues while ATC, replace with ds_read/buffer_store for save part[TODO restore part]
+-** 2. PERF - Save LDS before save VGPR to cover LDS save long latency...
+-** #18. Update 1. FUNC - Implicitly estore STATUS.VCCZ, which is not writable by s_setreg_b32
+-** 2. FUNC - Handle non-CWSR traps
+-*/
+-
+-var G8SR_WDMEM_HWREG_OFFSET = 0
+-var G8SR_WDMEM_SGPR_OFFSET = 128 // in bytes
+-
+-// Keep definition same as the app shader, These 2 time stamps are part of the app shader... Should before any Save and after restore.
+-
+-var G8SR_DEBUG_TIMESTAMP = 0
+-var G8SR_DEBUG_TS_SAVE_D_OFFSET = 40*4 // ts_save_d timestamp offset relative to SGPR_SR_memory_offset
+-var s_g8sr_ts_save_s = s[34:35] // save start
+-var s_g8sr_ts_sq_save_msg = s[36:37] // The save shader send SAVEWAVE msg to spi
+-var s_g8sr_ts_spi_wrexec = s[38:39] // the SPI write the sr address to SQ
+-var s_g8sr_ts_save_d = s[40:41] // save end
+-var s_g8sr_ts_restore_s = s[42:43] // restore start
+-var s_g8sr_ts_restore_d = s[44:45] // restore end
+-
+-var G8SR_VGPR_SR_IN_DWX4 = 0
+-var G8SR_SAVE_BUF_RSRC_WORD1_STRIDE_DWx4 = 0x00100000 // DWx4 stride is 4*4Bytes
+-var G8SR_RESTORE_BUF_RSRC_WORD1_STRIDE_DWx4 = G8SR_SAVE_BUF_RSRC_WORD1_STRIDE_DWx4
+-
+-
+-/*************************************************************************/
+-/* control on how to run the shader */
+-/*************************************************************************/
+-//any hack that needs to be made to run this code in EMU (either becasue various EMU code are not ready or no compute save & restore in EMU run)
+-var EMU_RUN_HACK = 0
+-var EMU_RUN_HACK_RESTORE_NORMAL = 0
+-var EMU_RUN_HACK_SAVE_NORMAL_EXIT = 0
+-var EMU_RUN_HACK_SAVE_SINGLE_WAVE = 0
+-var EMU_RUN_HACK_SAVE_FIRST_TIME = 0 //for interrupted restore in which the first save is through EMU_RUN_HACK
+-var EMU_RUN_HACK_SAVE_FIRST_TIME_TBA_LO = 0 //for interrupted restore in which the first save is through EMU_RUN_HACK
+-var EMU_RUN_HACK_SAVE_FIRST_TIME_TBA_HI = 0 //for interrupted restore in which the first save is through EMU_RUN_HACK
+-var SAVE_LDS = 1
+-var WG_BASE_ADDR_LO = 0x9000a000
+-var WG_BASE_ADDR_HI = 0x0
+-var WAVE_SPACE = 0x5000 //memory size that each wave occupies in workgroup state mem
+-var CTX_SAVE_CONTROL = 0x0
+-var CTX_RESTORE_CONTROL = CTX_SAVE_CONTROL
+-var SIM_RUN_HACK = 0 //any hack that needs to be made to run this code in SIM (either becasue various RTL code are not ready or no compute save & restore in RTL run)
+-var SGPR_SAVE_USE_SQC = 1 //use SQC D$ to do the write
+-var USE_MTBUF_INSTEAD_OF_MUBUF = 0 //becasue TC EMU curently asserts on 0 of // overload DFMT field to carry 4 more bits of stride for MUBUF opcodes
+-var SWIZZLE_EN = 0 //whether we use swizzled buffer addressing
+-
+-/**************************************************************************/
+-/* variables */
+-/**************************************************************************/
+-var SQ_WAVE_STATUS_INST_ATC_SHIFT = 23
+-var SQ_WAVE_STATUS_INST_ATC_MASK = 0x00800000
+-var SQ_WAVE_STATUS_SPI_PRIO_MASK = 0x00000006
+-
+-var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT = 12
+-var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE = 9
+-var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT = 8
+-var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE = 6
+-var SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT = 24
+-var SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE = 3 //FIXME sq.blk still has 4 bits at this time while SQ programming guide has 3 bits
+-
+-var SQ_WAVE_TRAPSTS_SAVECTX_MASK = 0x400
+-var SQ_WAVE_TRAPSTS_EXCE_MASK = 0x1FF // Exception mask
+-var SQ_WAVE_TRAPSTS_SAVECTX_SHIFT = 10
+-var SQ_WAVE_TRAPSTS_MEM_VIOL_MASK = 0x100
+-var SQ_WAVE_TRAPSTS_MEM_VIOL_SHIFT = 8
+-var SQ_WAVE_TRAPSTS_PRE_SAVECTX_MASK = 0x3FF
+-var SQ_WAVE_TRAPSTS_PRE_SAVECTX_SHIFT = 0x0
+-var SQ_WAVE_TRAPSTS_PRE_SAVECTX_SIZE = 10
+-var SQ_WAVE_TRAPSTS_POST_SAVECTX_MASK = 0xFFFFF800
+-var SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT = 11
+-var SQ_WAVE_TRAPSTS_POST_SAVECTX_SIZE = 21
+-
+-var SQ_WAVE_IB_STS_RCNT_SHIFT = 16 //FIXME
+-var SQ_WAVE_IB_STS_RCNT_SIZE = 4 //FIXME
+-var SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT = 15 //FIXME
+-var SQ_WAVE_IB_STS_FIRST_REPLAY_SIZE = 1 //FIXME
+-var SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK_NEG = 0x00007FFF //FIXME
+-
+-var SQ_BUF_RSRC_WORD1_ATC_SHIFT = 24
+-var SQ_BUF_RSRC_WORD3_MTYPE_SHIFT = 27
+-
+-
+-/* Save */
+-var S_SAVE_BUF_RSRC_WORD1_STRIDE = 0x00040000 //stride is 4 bytes
+-var S_SAVE_BUF_RSRC_WORD3_MISC = 0x00807FAC //SQ_SEL_X/Y/Z/W, BUF_NUM_FORMAT_FLOAT, (0 for MUBUF stride[17:14] when ADD_TID_ENABLE and BUF_DATA_FORMAT_32 for MTBUF), ADD_TID_ENABLE
+-
+-var S_SAVE_SPI_INIT_ATC_MASK = 0x08000000 //bit[27]: ATC bit
+-var S_SAVE_SPI_INIT_ATC_SHIFT = 27
+-var S_SAVE_SPI_INIT_MTYPE_MASK = 0x70000000 //bit[30:28]: Mtype
+-var S_SAVE_SPI_INIT_MTYPE_SHIFT = 28
+-var S_SAVE_SPI_INIT_FIRST_WAVE_MASK = 0x04000000 //bit[26]: FirstWaveInTG
+-var S_SAVE_SPI_INIT_FIRST_WAVE_SHIFT = 26
+-
+-var S_SAVE_PC_HI_RCNT_SHIFT = 28 //FIXME check with Brian to ensure all fields other than PC[47:0] can be used
+-var S_SAVE_PC_HI_RCNT_MASK = 0xF0000000 //FIXME
+-var S_SAVE_PC_HI_FIRST_REPLAY_SHIFT = 27 //FIXME
+-var S_SAVE_PC_HI_FIRST_REPLAY_MASK = 0x08000000 //FIXME
+-
+-var s_save_spi_init_lo = exec_lo
+-var s_save_spi_init_hi = exec_hi
+-
+- //tba_lo and tba_hi need to be saved/restored
+-var s_save_pc_lo = ttmp0 //{TTMP1, TTMP0} = {3¡¯h0,pc_rewind[3:0], HT[0],trapID[7:0], PC[47:0]}
+-var s_save_pc_hi = ttmp1
+-var s_save_exec_lo = ttmp2
+-var s_save_exec_hi = ttmp3
+-var s_save_status = ttmp4
+-var s_save_trapsts = ttmp5 //not really used until the end of the SAVE routine
+-var s_save_xnack_mask_lo = ttmp6
+-var s_save_xnack_mask_hi = ttmp7
+-var s_save_buf_rsrc0 = ttmp8
+-var s_save_buf_rsrc1 = ttmp9
+-var s_save_buf_rsrc2 = ttmp10
+-var s_save_buf_rsrc3 = ttmp11
+-
+-var s_save_mem_offset = tma_lo
+-var s_save_alloc_size = s_save_trapsts //conflict
+-var s_save_tmp = s_save_buf_rsrc2 //shared with s_save_buf_rsrc2 (conflict: should not use mem access with s_save_tmp at the same time)
+-var s_save_m0 = tma_hi
+-
+-/* Restore */
+-var S_RESTORE_BUF_RSRC_WORD1_STRIDE = S_SAVE_BUF_RSRC_WORD1_STRIDE
+-var S_RESTORE_BUF_RSRC_WORD3_MISC = S_SAVE_BUF_RSRC_WORD3_MISC
+-
+-var S_RESTORE_SPI_INIT_ATC_MASK = 0x08000000 //bit[27]: ATC bit
+-var S_RESTORE_SPI_INIT_ATC_SHIFT = 27
+-var S_RESTORE_SPI_INIT_MTYPE_MASK = 0x70000000 //bit[30:28]: Mtype
+-var S_RESTORE_SPI_INIT_MTYPE_SHIFT = 28
+-var S_RESTORE_SPI_INIT_FIRST_WAVE_MASK = 0x04000000 //bit[26]: FirstWaveInTG
+-var S_RESTORE_SPI_INIT_FIRST_WAVE_SHIFT = 26
+-
+-var S_RESTORE_PC_HI_RCNT_SHIFT = S_SAVE_PC_HI_RCNT_SHIFT
+-var S_RESTORE_PC_HI_RCNT_MASK = S_SAVE_PC_HI_RCNT_MASK
+-var S_RESTORE_PC_HI_FIRST_REPLAY_SHIFT = S_SAVE_PC_HI_FIRST_REPLAY_SHIFT
+-var S_RESTORE_PC_HI_FIRST_REPLAY_MASK = S_SAVE_PC_HI_FIRST_REPLAY_MASK
+-
+-var s_restore_spi_init_lo = exec_lo
+-var s_restore_spi_init_hi = exec_hi
+-
+-var s_restore_mem_offset = ttmp2
+-var s_restore_alloc_size = ttmp3
+-var s_restore_tmp = ttmp6 //tba_lo/hi need to be restored
+-var s_restore_mem_offset_save = s_restore_tmp //no conflict
+-
+-var s_restore_m0 = s_restore_alloc_size //no conflict
+-
+-var s_restore_mode = ttmp7
+-
+-var s_restore_pc_lo = ttmp0
+-var s_restore_pc_hi = ttmp1
+-var s_restore_exec_lo = tma_lo //no conflict
+-var s_restore_exec_hi = tma_hi //no conflict
+-var s_restore_status = ttmp4
+-var s_restore_trapsts = ttmp5
+-var s_restore_xnack_mask_lo = xnack_mask_lo
+-var s_restore_xnack_mask_hi = xnack_mask_hi
+-var s_restore_buf_rsrc0 = ttmp8
+-var s_restore_buf_rsrc1 = ttmp9
+-var s_restore_buf_rsrc2 = ttmp10
+-var s_restore_buf_rsrc3 = ttmp11
+-
+-/**************************************************************************/
+-/* trap handler entry points */
+-/**************************************************************************/
+-/* Shader Main*/
+-
+-shader main
+- asic(VI)
+- type(CS)
+-
+-
+- if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_RESTORE_NORMAL)) //hack to use trap_id for determining save/restore
+- //FIXME VCCZ un-init assertion s_getreg_b32 s_save_status, hwreg(HW_REG_STATUS) //save STATUS since we will change SCC
+- s_and_b32 s_save_tmp, s_save_pc_hi, 0xffff0000 //change SCC
+- s_cmp_eq_u32 s_save_tmp, 0x007e0000 //Save: trap_id = 0x7e. Restore: trap_id = 0x7f.
+- s_cbranch_scc0 L_JUMP_TO_RESTORE //do not need to recover STATUS here since we are going to RESTORE
+- //FIXME s_setreg_b32 hwreg(HW_REG_STATUS), s_save_status //need to recover STATUS since we are going to SAVE
+- s_branch L_SKIP_RESTORE //NOT restore, SAVE actually
+- else
+- s_branch L_SKIP_RESTORE //NOT restore. might be a regular trap or save
+- end
+-
+-L_JUMP_TO_RESTORE:
+- s_branch L_RESTORE //restore
+-
+-L_SKIP_RESTORE:
+-
+- s_getreg_b32 s_save_status, hwreg(HW_REG_STATUS) //save STATUS since we will change SCC
+- s_andn2_b32 s_save_status, s_save_status, SQ_WAVE_STATUS_SPI_PRIO_MASK //check whether this is for save
+- s_getreg_b32 s_save_trapsts, hwreg(HW_REG_TRAPSTS)
+- s_and_b32 s_save_trapsts, s_save_trapsts, SQ_WAVE_TRAPSTS_SAVECTX_MASK //check whether this is for save
+- s_cbranch_scc1 L_SAVE //this is the operation for save
+-
+- // ********* Handle non-CWSR traps *******************
+-if (!EMU_RUN_HACK)
+- /* read tba and tma for next level trap handler, ttmp4 is used as s_save_status */
+- s_load_dwordx4 [ttmp8,ttmp9,ttmp10, ttmp11], [tma_lo,tma_hi], 0
+- s_waitcnt lgkmcnt(0)
+- s_or_b32 ttmp7, ttmp8, ttmp9
+- s_cbranch_scc0 L_NO_NEXT_TRAP //next level trap handler not been set
+- s_setreg_b32 hwreg(HW_REG_STATUS), s_save_status //restore HW status(SCC)
+- s_setpc_b64 [ttmp8,ttmp9] //jump to next level trap handler
+-
+-L_NO_NEXT_TRAP:
+- s_getreg_b32 s_save_trapsts, hwreg(HW_REG_TRAPSTS)
+- s_and_b32 s_save_trapsts, s_save_trapsts, SQ_WAVE_TRAPSTS_EXCE_MASK // Check whether it is an exception
+- s_cbranch_scc1 L_EXCP_CASE // Exception, jump back to the shader program directly.
+- s_add_u32 ttmp0, ttmp0, 4 // S_TRAP case, add 4 to ttmp0
+- s_addc_u32 ttmp1, ttmp1, 0
+-L_EXCP_CASE:
+- s_and_b32 ttmp1, ttmp1, 0xFFFF
+- s_setreg_b32 hwreg(HW_REG_STATUS), s_save_status //restore HW status(SCC)
+- s_rfe_b64 [ttmp0, ttmp1]
+-end
+- // ********* End handling of non-CWSR traps *******************
+-
+-/**************************************************************************/
+-/* save routine */
+-/**************************************************************************/
+-
+-L_SAVE:
+-
+-if G8SR_DEBUG_TIMESTAMP
+- s_memrealtime s_g8sr_ts_save_s
+- s_waitcnt lgkmcnt(0) //FIXME, will cause xnack??
+-end
+-
+- //check whether there is mem_viol
+- s_getreg_b32 s_save_trapsts, hwreg(HW_REG_TRAPSTS)
+- s_and_b32 s_save_trapsts, s_save_trapsts, SQ_WAVE_TRAPSTS_MEM_VIOL_MASK
+- s_cbranch_scc0 L_NO_PC_REWIND
+-
+- //if so, need rewind PC assuming GDS operation gets NACKed
+- s_mov_b32 s_save_tmp, 0 //clear mem_viol bit
+- s_setreg_b32 hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_MEM_VIOL_SHIFT, 1), s_save_tmp //clear mem_viol bit
+- s_and_b32 s_save_pc_hi, s_save_pc_hi, 0x0000ffff //pc[47:32]
+- s_sub_u32 s_save_pc_lo, s_save_pc_lo, 8 //pc[31:0]-8
+- s_subb_u32 s_save_pc_hi, s_save_pc_hi, 0x0 // -scc
+-
+-L_NO_PC_REWIND:
+- s_mov_b32 s_save_tmp, 0 //clear saveCtx bit
+- s_setreg_b32 hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_SAVECTX_SHIFT, 1), s_save_tmp //clear saveCtx bit
+-
+- s_mov_b32 s_save_xnack_mask_lo, xnack_mask_lo //save XNACK_MASK
+- s_mov_b32 s_save_xnack_mask_hi, xnack_mask_hi //save XNACK must before any memory operation
+- s_getreg_b32 s_save_tmp, hwreg(HW_REG_IB_STS, SQ_WAVE_IB_STS_RCNT_SHIFT, SQ_WAVE_IB_STS_RCNT_SIZE) //save RCNT
+- s_lshl_b32 s_save_tmp, s_save_tmp, S_SAVE_PC_HI_RCNT_SHIFT
+- s_or_b32 s_save_pc_hi, s_save_pc_hi, s_save_tmp
+- s_getreg_b32 s_save_tmp, hwreg(HW_REG_IB_STS, SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT, SQ_WAVE_IB_STS_FIRST_REPLAY_SIZE) //save FIRST_REPLAY
+- s_lshl_b32 s_save_tmp, s_save_tmp, S_SAVE_PC_HI_FIRST_REPLAY_SHIFT
+- s_or_b32 s_save_pc_hi, s_save_pc_hi, s_save_tmp
+- s_getreg_b32 s_save_tmp, hwreg(HW_REG_IB_STS) //clear RCNT and FIRST_REPLAY in IB_STS
+- s_and_b32 s_save_tmp, s_save_tmp, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK_NEG
+-
+- s_setreg_b32 hwreg(HW_REG_IB_STS), s_save_tmp
+-
+- /* inform SPI the readiness and wait for SPI's go signal */
+- s_mov_b32 s_save_exec_lo, exec_lo //save EXEC and use EXEC for the go signal from SPI
+- s_mov_b32 s_save_exec_hi, exec_hi
+- s_mov_b64 exec, 0x0 //clear EXEC to get ready to receive
+-
+-if G8SR_DEBUG_TIMESTAMP
+- s_memrealtime s_g8sr_ts_sq_save_msg
+- s_waitcnt lgkmcnt(0)
+-end
+-
+- if (EMU_RUN_HACK)
+-
+- else
+- s_sendmsg sendmsg(MSG_SAVEWAVE) //send SPI a message and wait for SPI's write to EXEC
+- end
+-
+- L_SLEEP:
+- s_sleep 0x2 // sleep 1 (64clk) is not enough for 8 waves per SIMD, which will cause SQ hang, since the 7,8th wave could not get arbit to exec inst, while other waves are stuck into the sleep-loop and waiting for wrexec!=0
+-
+- if (EMU_RUN_HACK)
+-
+- else
+- s_cbranch_execz L_SLEEP
+- end
+-
+-if G8SR_DEBUG_TIMESTAMP
+- s_memrealtime s_g8sr_ts_spi_wrexec
+- s_waitcnt lgkmcnt(0)
+-end
+-
+- /* setup Resource Contants */
+- if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_SAVE_SINGLE_WAVE))
+- //calculate wd_addr using absolute thread id
+- v_readlane_b32 s_save_tmp, v9, 0
+- s_lshr_b32 s_save_tmp, s_save_tmp, 6
+- s_mul_i32 s_save_tmp, s_save_tmp, WAVE_SPACE
+- s_add_i32 s_save_spi_init_lo, s_save_tmp, WG_BASE_ADDR_LO
+- s_mov_b32 s_save_spi_init_hi, WG_BASE_ADDR_HI
+- s_and_b32 s_save_spi_init_hi, s_save_spi_init_hi, CTX_SAVE_CONTROL
+- else
+- end
+- if ((EMU_RUN_HACK) && (EMU_RUN_HACK_SAVE_SINGLE_WAVE))
+- s_add_i32 s_save_spi_init_lo, s_save_tmp, WG_BASE_ADDR_LO
+- s_mov_b32 s_save_spi_init_hi, WG_BASE_ADDR_HI
+- s_and_b32 s_save_spi_init_hi, s_save_spi_init_hi, CTX_SAVE_CONTROL
+- else
+- end
+-
+-
+- s_mov_b32 s_save_buf_rsrc0, s_save_spi_init_lo //base_addr_lo
+- s_and_b32 s_save_buf_rsrc1, s_save_spi_init_hi, 0x0000FFFF //base_addr_hi
+- s_or_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, S_SAVE_BUF_RSRC_WORD1_STRIDE
+- s_mov_b32 s_save_buf_rsrc2, 0 //NUM_RECORDS initial value = 0 (in bytes) although not neccessarily inited
+- s_mov_b32 s_save_buf_rsrc3, S_SAVE_BUF_RSRC_WORD3_MISC
+- s_and_b32 s_save_tmp, s_save_spi_init_hi, S_SAVE_SPI_INIT_ATC_MASK
+- s_lshr_b32 s_save_tmp, s_save_tmp, (S_SAVE_SPI_INIT_ATC_SHIFT-SQ_BUF_RSRC_WORD1_ATC_SHIFT) //get ATC bit into position
+- s_or_b32 s_save_buf_rsrc3, s_save_buf_rsrc3, s_save_tmp //or ATC
+- s_and_b32 s_save_tmp, s_save_spi_init_hi, S_SAVE_SPI_INIT_MTYPE_MASK
+- s_lshr_b32 s_save_tmp, s_save_tmp, (S_SAVE_SPI_INIT_MTYPE_SHIFT-SQ_BUF_RSRC_WORD3_MTYPE_SHIFT) //get MTYPE bits into position
+- s_or_b32 s_save_buf_rsrc3, s_save_buf_rsrc3, s_save_tmp //or MTYPE
+-
+- //FIXME right now s_save_m0/s_save_mem_offset use tma_lo/tma_hi (might need to save them before using them?)
+- s_mov_b32 s_save_m0, m0 //save M0
+-
+- /* global mem offset */
+- s_mov_b32 s_save_mem_offset, 0x0 //mem offset initial value = 0
+-
+-
+-
+-
+- /* save HW registers */
+- //////////////////////////////
+-
+- L_SAVE_HWREG:
+- // HWREG SR memory offset : size(VGPR)+size(SGPR)
+- get_vgpr_size_bytes(s_save_mem_offset)
+- get_sgpr_size_bytes(s_save_tmp)
+- s_add_u32 s_save_mem_offset, s_save_mem_offset, s_save_tmp
+-
+-
+- s_mov_b32 s_save_buf_rsrc2, 0x4 //NUM_RECORDS in bytes
+- if (SWIZZLE_EN)
+- s_add_u32 s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+- else
+- s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+- end
+-
+-
+- write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset) //M0
+-
+- if ((EMU_RUN_HACK) && (EMU_RUN_HACK_SAVE_FIRST_TIME))
+- s_add_u32 s_save_pc_lo, s_save_pc_lo, 4 //pc[31:0]+4
+- s_addc_u32 s_save_pc_hi, s_save_pc_hi, 0x0 //carry bit over
+- s_mov_b32 tba_lo, EMU_RUN_HACK_SAVE_FIRST_TIME_TBA_LO
+- s_mov_b32 tba_hi, EMU_RUN_HACK_SAVE_FIRST_TIME_TBA_HI
+- end
+-
+- write_hwreg_to_mem(s_save_pc_lo, s_save_buf_rsrc0, s_save_mem_offset) //PC
+- write_hwreg_to_mem(s_save_pc_hi, s_save_buf_rsrc0, s_save_mem_offset)
+- write_hwreg_to_mem(s_save_exec_lo, s_save_buf_rsrc0, s_save_mem_offset) //EXEC
+- write_hwreg_to_mem(s_save_exec_hi, s_save_buf_rsrc0, s_save_mem_offset)
+- write_hwreg_to_mem(s_save_status, s_save_buf_rsrc0, s_save_mem_offset) //STATUS
+-
+- //s_save_trapsts conflicts with s_save_alloc_size
+- s_getreg_b32 s_save_trapsts, hwreg(HW_REG_TRAPSTS)
+- write_hwreg_to_mem(s_save_trapsts, s_save_buf_rsrc0, s_save_mem_offset) //TRAPSTS
+-
+- write_hwreg_to_mem(s_save_xnack_mask_lo, s_save_buf_rsrc0, s_save_mem_offset) //XNACK_MASK_LO
+- write_hwreg_to_mem(s_save_xnack_mask_hi, s_save_buf_rsrc0, s_save_mem_offset) //XNACK_MASK_HI
+-
+- //use s_save_tmp would introduce conflict here between s_save_tmp and s_save_buf_rsrc2
+- s_getreg_b32 s_save_m0, hwreg(HW_REG_MODE) //MODE
+- write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)
+- write_hwreg_to_mem(tba_lo, s_save_buf_rsrc0, s_save_mem_offset) //TBA_LO
+- write_hwreg_to_mem(tba_hi, s_save_buf_rsrc0, s_save_mem_offset) //TBA_HI
+-
+-
+-
+- /* the first wave in the threadgroup */
+- // save fist_wave bits in tba_hi unused bit.26
+- s_and_b32 s_save_tmp, s_save_spi_init_hi, S_SAVE_SPI_INIT_FIRST_WAVE_MASK // extract fisrt wave bit
+- //s_or_b32 tba_hi, s_save_tmp, tba_hi // save first wave bit to tba_hi.bits[26]
+- s_mov_b32 s_save_exec_hi, 0x0
+- s_or_b32 s_save_exec_hi, s_save_tmp, s_save_exec_hi // save first wave bit to s_save_exec_hi.bits[26]
+-
+-
+- /* save SGPRs */
+- // Save SGPR before LDS save, then the s0 to s4 can be used during LDS save...
+- //////////////////////////////
+-
+- // SGPR SR memory offset : size(VGPR)
+- get_vgpr_size_bytes(s_save_mem_offset)
+- // TODO, change RSRC word to rearrange memory layout for SGPRS
+-
+- s_getreg_b32 s_save_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE) //spgr_size
+- s_add_u32 s_save_alloc_size, s_save_alloc_size, 1
+- s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 4 //Number of SGPRs = (sgpr_size + 1) * 16 (non-zero value)
+-
+- if (SGPR_SAVE_USE_SQC)
+- s_lshl_b32 s_save_buf_rsrc2, s_save_alloc_size, 2 //NUM_RECORDS in bytes
+- else
+- s_lshl_b32 s_save_buf_rsrc2, s_save_alloc_size, 8 //NUM_RECORDS in bytes (64 threads)
+- end
+-
+- if (SWIZZLE_EN)
+- s_add_u32 s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+- else
+- s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+- end
+-
+-
+- // backup s_save_buf_rsrc0,1 to s_save_pc_lo/hi, since write_16sgpr_to_mem function will change the rsrc0
+- //s_mov_b64 s_save_pc_lo, s_save_buf_rsrc0
+- s_mov_b64 s_save_xnack_mask_lo, s_save_buf_rsrc0
+- s_add_u32 s_save_buf_rsrc0, s_save_buf_rsrc0, s_save_mem_offset
+- s_addc_u32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0
+-
+- s_mov_b32 m0, 0x0 //SGPR initial index value =0
+- L_SAVE_SGPR_LOOP:
+- // SGPR is allocated in 16 SGPR granularity
+- s_movrels_b64 s0, s0 //s0 = s[0+m0], s1 = s[1+m0]
+- s_movrels_b64 s2, s2 //s2 = s[2+m0], s3 = s[3+m0]
+- s_movrels_b64 s4, s4 //s4 = s[4+m0], s5 = s[5+m0]
+- s_movrels_b64 s6, s6 //s6 = s[6+m0], s7 = s[7+m0]
+- s_movrels_b64 s8, s8 //s8 = s[8+m0], s9 = s[9+m0]
+- s_movrels_b64 s10, s10 //s10 = s[10+m0], s11 = s[11+m0]
+- s_movrels_b64 s12, s12 //s12 = s[12+m0], s13 = s[13+m0]
+- s_movrels_b64 s14, s14 //s14 = s[14+m0], s15 = s[15+m0]
+-
+- write_16sgpr_to_mem(s0, s_save_buf_rsrc0, s_save_mem_offset) //PV: the best performance should be using s_buffer_store_dwordx4
+- s_add_u32 m0, m0, 16 //next sgpr index
+- s_cmp_lt_u32 m0, s_save_alloc_size //scc = (m0 < s_save_alloc_size) ? 1 : 0
+- s_cbranch_scc1 L_SAVE_SGPR_LOOP //SGPR save is complete?
+- // restore s_save_buf_rsrc0,1
+- //s_mov_b64 s_save_buf_rsrc0, s_save_pc_lo
+- s_mov_b64 s_save_buf_rsrc0, s_save_xnack_mask_lo
+-
+-
+-
+-
+- /* save first 4 VGPR, then LDS save could use */
+- // each wave will alloc 4 vgprs at least...
+- /////////////////////////////////////////////////////////////////////////////////////
+-
+- s_mov_b32 s_save_mem_offset, 0
+- s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on
+- s_mov_b32 exec_hi, 0xFFFFFFFF
+-
+- if (SWIZZLE_EN)
+- s_add_u32 s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+- else
+- s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+- end
+-
+-
+- // VGPR Allocated in 4-GPR granularity
+-
+-if G8SR_VGPR_SR_IN_DWX4
+- // the const stride for DWx4 is 4*4 bytes
+- s_and_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0x0000FFFF // reset const stride to 0
+- s_or_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, G8SR_SAVE_BUF_RSRC_WORD1_STRIDE_DWx4 // const stride to 4*4 bytes
+-
+- buffer_store_dwordx4 v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
+-
+- s_and_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0x0000FFFF // reset const stride to 0
+- s_or_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, S_SAVE_BUF_RSRC_WORD1_STRIDE // reset const stride to 4 bytes
+-else
+- buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
+- buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256
+- buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256*2
+- buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256*3
+-end
+-
+-
+-
+- /* save LDS */
+- //////////////////////////////
+-
+- L_SAVE_LDS:
+-
+- // Change EXEC to all threads...
+- s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on
+- s_mov_b32 exec_hi, 0xFFFFFFFF
+-
+- s_getreg_b32 s_save_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE) //lds_size
+- s_and_b32 s_save_alloc_size, s_save_alloc_size, 0xFFFFFFFF //lds_size is zero?
+- s_cbranch_scc0 L_SAVE_LDS_DONE //no lds used? jump to L_SAVE_DONE
+-
+- s_barrier //LDS is used? wait for other waves in the same TG
+- //s_and_b32 s_save_tmp, tba_hi, S_SAVE_SPI_INIT_FIRST_WAVE_MASK //exec is still used here
+- s_and_b32 s_save_tmp, s_save_exec_hi, S_SAVE_SPI_INIT_FIRST_WAVE_MASK //exec is still used here
+- s_cbranch_scc0 L_SAVE_LDS_DONE
+-
+- // first wave do LDS save;
+-
+- s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 6 //LDS size in dwords = lds_size * 64dw
+- s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 2 //LDS size in bytes
+- s_mov_b32 s_save_buf_rsrc2, s_save_alloc_size //NUM_RECORDS in bytes
+-
+- // LDS at offset: size(VGPR)+SIZE(SGPR)+SIZE(HWREG)
+- //
+- get_vgpr_size_bytes(s_save_mem_offset)
+- get_sgpr_size_bytes(s_save_tmp)
+- s_add_u32 s_save_mem_offset, s_save_mem_offset, s_save_tmp
+- s_add_u32 s_save_mem_offset, s_save_mem_offset, get_hwreg_size_bytes()
+-
+-
+- if (SWIZZLE_EN)
+- s_add_u32 s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+- else
+- s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+- end
+-
+- s_mov_b32 m0, 0x0 //lds_offset initial value = 0
+-
+-
+-var LDS_DMA_ENABLE = 0
+-var UNROLL = 0
+-if UNROLL==0 && LDS_DMA_ENABLE==1
+- s_mov_b32 s3, 256*2
+- s_nop 0
+- s_nop 0
+- s_nop 0
+- L_SAVE_LDS_LOOP:
+- //TODO: looks the 2 buffer_store/load clause for s/r will hurt performance.???
+- if (SAVE_LDS) //SPI always alloc LDS space in 128DW granularity
+- buffer_store_lds_dword s_save_buf_rsrc0, s_save_mem_offset lds:1 // first 64DW
+- buffer_store_lds_dword s_save_buf_rsrc0, s_save_mem_offset lds:1 offset:256 // second 64DW
+- end
+-
+- s_add_u32 m0, m0, s3 //every buffer_store_lds does 256 bytes
+- s_add_u32 s_save_mem_offset, s_save_mem_offset, s3 //mem offset increased by 256 bytes
+- s_cmp_lt_u32 m0, s_save_alloc_size //scc=(m0 < s_save_alloc_size) ? 1 : 0
+- s_cbranch_scc1 L_SAVE_LDS_LOOP //LDS save is complete?
+-
+-elsif LDS_DMA_ENABLE==1 && UNROLL==1 // UNROOL , has ichace miss
+- // store from higest LDS address to lowest
+- s_mov_b32 s3, 256*2
+- s_sub_u32 m0, s_save_alloc_size, s3
+- s_add_u32 s_save_mem_offset, s_save_mem_offset, m0
+- s_lshr_b32 s_save_alloc_size, s_save_alloc_size, 9 // how many 128 trunks...
+- s_sub_u32 s_save_alloc_size, 128, s_save_alloc_size // store from higheset addr to lowest
+- s_mul_i32 s_save_alloc_size, s_save_alloc_size, 6*4 // PC offset increment, each LDS save block cost 6*4 Bytes instruction
+- s_add_u32 s_save_alloc_size, s_save_alloc_size, 3*4 //2is the below 2 inst...//s_addc and s_setpc
+- s_nop 0
+- s_nop 0
+- s_nop 0 //pad 3 dw to let LDS_DMA align with 64Bytes
+- s_getpc_b64 s[0:1] // reuse s[0:1], since s[0:1] already saved
+- s_add_u32 s0, s0,s_save_alloc_size
+- s_addc_u32 s1, s1, 0
+- s_setpc_b64 s[0:1]
+-
+-
+- for var i =0; i< 128; i++
+- // be careful to make here a 64Byte aligned address, which could improve performance...
+- buffer_store_lds_dword s_save_buf_rsrc0, s_save_mem_offset lds:1 offset:0 // first 64DW
+- buffer_store_lds_dword s_save_buf_rsrc0, s_save_mem_offset lds:1 offset:256 // second 64DW
+-
+- if i!=127
+- s_sub_u32 m0, m0, s3 // use a sgpr to shrink 2DW-inst to 1DW inst to improve performance , i.e. pack more LDS_DMA inst to one Cacheline
+- s_sub_u32 s_save_mem_offset, s_save_mem_offset, s3
+- end
+- end
+-
+-else // BUFFER_STORE
+- v_mbcnt_lo_u32_b32 v2, 0xffffffff, 0x0
+- v_mbcnt_hi_u32_b32 v3, 0xffffffff, v2 // tid
+- v_mul_i32_i24 v2, v3, 8 // tid*8
+- v_mov_b32 v3, 256*2
+- s_mov_b32 m0, 0x10000
+- s_mov_b32 s0, s_save_buf_rsrc3
+- s_and_b32 s_save_buf_rsrc3, s_save_buf_rsrc3, 0xFF7FFFFF // disable add_tid
+- s_or_b32 s_save_buf_rsrc3, s_save_buf_rsrc3, 0x58000 //DFMT
+-
+-L_SAVE_LDS_LOOP_VECTOR:
+- ds_read_b64 v[0:1], v2 //x =LDS[a], byte address
+- s_waitcnt lgkmcnt(0)
+- buffer_store_dwordx2 v[0:1], v2, s_save_buf_rsrc0, s_save_mem_offset offen:1 glc:1 slc:1
+-// s_waitcnt vmcnt(0)
+- v_add_u32 v2, vcc[0:1], v2, v3
+- v_cmp_lt_u32 vcc[0:1], v2, s_save_alloc_size
+- s_cbranch_vccnz L_SAVE_LDS_LOOP_VECTOR
+-
+- // restore rsrc3
+- s_mov_b32 s_save_buf_rsrc3, s0
+-
+-end
+-
+-L_SAVE_LDS_DONE:
+-
+-
+- /* save VGPRs - set the Rest VGPRs */
+- //////////////////////////////////////////////////////////////////////////////////////
+- L_SAVE_VGPR:
+- // VGPR SR memory offset: 0
+- // TODO rearrange the RSRC words to use swizzle for VGPR save...
+-
+- s_mov_b32 s_save_mem_offset, (0+256*4) // for the rest VGPRs
+- s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on
+- s_mov_b32 exec_hi, 0xFFFFFFFF
+-
+- s_getreg_b32 s_save_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE) //vpgr_size
+- s_add_u32 s_save_alloc_size, s_save_alloc_size, 1
+- s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 2 //Number of VGPRs = (vgpr_size + 1) * 4 (non-zero value) //FIXME for GFX, zero is possible
+- s_lshl_b32 s_save_buf_rsrc2, s_save_alloc_size, 8 //NUM_RECORDS in bytes (64 threads*4)
+- if (SWIZZLE_EN)
+- s_add_u32 s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+- else
+- s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+- end
+-
+-
+- // VGPR Allocated in 4-GPR granularity
+-
+-if G8SR_VGPR_SR_IN_DWX4
+- // the const stride for DWx4 is 4*4 bytes
+- s_and_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0x0000FFFF // reset const stride to 0
+- s_or_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, G8SR_SAVE_BUF_RSRC_WORD1_STRIDE_DWx4 // const stride to 4*4 bytes
+-
+- s_mov_b32 m0, 4 // skip first 4 VGPRs
+- s_cmp_lt_u32 m0, s_save_alloc_size
+- s_cbranch_scc0 L_SAVE_VGPR_LOOP_END // no more vgprs
+-
+- s_set_gpr_idx_on m0, 0x1 // This will change M0
+- s_add_u32 s_save_alloc_size, s_save_alloc_size, 0x1000 // because above inst change m0
+-L_SAVE_VGPR_LOOP:
+- v_mov_b32 v0, v0 // v0 = v[0+m0]
+- v_mov_b32 v1, v1
+- v_mov_b32 v2, v2
+- v_mov_b32 v3, v3
+-
+-
+- buffer_store_dwordx4 v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
+- s_add_u32 m0, m0, 4
+- s_add_u32 s_save_mem_offset, s_save_mem_offset, 256*4
+- s_cmp_lt_u32 m0, s_save_alloc_size
+- s_cbranch_scc1 L_SAVE_VGPR_LOOP //VGPR save is complete?
+- s_set_gpr_idx_off
+-L_SAVE_VGPR_LOOP_END:
+-
+- s_and_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0x0000FFFF // reset const stride to 0
+- s_or_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, S_SAVE_BUF_RSRC_WORD1_STRIDE // reset const stride to 4 bytes
+-else
+- // VGPR store using dw burst
+- s_mov_b32 m0, 0x4 //VGPR initial index value =0
+- s_cmp_lt_u32 m0, s_save_alloc_size
+- s_cbranch_scc0 L_SAVE_VGPR_END
+-
+-
+- s_set_gpr_idx_on m0, 0x1 //M0[7:0] = M0[7:0] and M0[15:12] = 0x1
+- s_add_u32 s_save_alloc_size, s_save_alloc_size, 0x1000 //add 0x1000 since we compare m0 against it later
+-
+- L_SAVE_VGPR_LOOP:
+- v_mov_b32 v0, v0 //v0 = v[0+m0]
+- v_mov_b32 v1, v1 //v0 = v[0+m0]
+- v_mov_b32 v2, v2 //v0 = v[0+m0]
+- v_mov_b32 v3, v3 //v0 = v[0+m0]
+-
+- if(USE_MTBUF_INSTEAD_OF_MUBUF)
+- tbuffer_store_format_x v0, v0, s_save_buf_rsrc0, s_save_mem_offset format:BUF_NUM_FORMAT_FLOAT format: BUF_DATA_FORMAT_32 slc:1 glc:1
+- else
+- buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
+- buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256
+- buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256*2
+- buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256*3
+- end
+-
+- s_add_u32 m0, m0, 4 //next vgpr index
+- s_add_u32 s_save_mem_offset, s_save_mem_offset, 256*4 //every buffer_store_dword does 256 bytes
+- s_cmp_lt_u32 m0, s_save_alloc_size //scc = (m0 < s_save_alloc_size) ? 1 : 0
+- s_cbranch_scc1 L_SAVE_VGPR_LOOP //VGPR save is complete?
+- s_set_gpr_idx_off
+-end
+-
+-L_SAVE_VGPR_END:
+-
+-
+-
+-
+-
+-
+- /* S_PGM_END_SAVED */ //FIXME graphics ONLY
+- if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_SAVE_NORMAL_EXIT))
+- s_and_b32 s_save_pc_hi, s_save_pc_hi, 0x0000ffff //pc[47:32]
+- s_add_u32 s_save_pc_lo, s_save_pc_lo, 4 //pc[31:0]+4
+- s_addc_u32 s_save_pc_hi, s_save_pc_hi, 0x0 //carry bit over
+- s_rfe_b64 s_save_pc_lo //Return to the main shader program
+- else
+- end
+-
+-// Save Done timestamp
+-if G8SR_DEBUG_TIMESTAMP
+- s_memrealtime s_g8sr_ts_save_d
+- // SGPR SR memory offset : size(VGPR)
+- get_vgpr_size_bytes(s_save_mem_offset)
+- s_add_u32 s_save_mem_offset, s_save_mem_offset, G8SR_DEBUG_TS_SAVE_D_OFFSET
+- s_waitcnt lgkmcnt(0) //FIXME, will cause xnack??
+- // Need reset rsrc2??
+- s_mov_b32 m0, s_save_mem_offset
+- s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+- s_buffer_store_dwordx2 s_g8sr_ts_save_d, s_save_buf_rsrc0, m0 glc:1
+-end
+-
+-
+- s_branch L_END_PGM
+-
+-
+-
+-/**************************************************************************/
+-/* restore routine */
+-/**************************************************************************/
+-
+-L_RESTORE:
+- /* Setup Resource Contants */
+- if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_RESTORE_NORMAL))
+- //calculate wd_addr using absolute thread id
+- v_readlane_b32 s_restore_tmp, v9, 0
+- s_lshr_b32 s_restore_tmp, s_restore_tmp, 6
+- s_mul_i32 s_restore_tmp, s_restore_tmp, WAVE_SPACE
+- s_add_i32 s_restore_spi_init_lo, s_restore_tmp, WG_BASE_ADDR_LO
+- s_mov_b32 s_restore_spi_init_hi, WG_BASE_ADDR_HI
+- s_and_b32 s_restore_spi_init_hi, s_restore_spi_init_hi, CTX_RESTORE_CONTROL
+- else
+- end
+-
+-if G8SR_DEBUG_TIMESTAMP
+- s_memrealtime s_g8sr_ts_restore_s
+- s_waitcnt lgkmcnt(0) //FIXME, will cause xnack??
+- // tma_lo/hi are sgpr 110, 111, which will not used for 112 SGPR allocated case...
+- s_mov_b32 s_restore_pc_lo, s_g8sr_ts_restore_s[0]
+- s_mov_b32 s_restore_pc_hi, s_g8sr_ts_restore_s[1] //backup ts to ttmp0/1, sicne exec will be finally restored..
+-end
+-
+-
+-
+- s_mov_b32 s_restore_buf_rsrc0, s_restore_spi_init_lo //base_addr_lo
+- s_and_b32 s_restore_buf_rsrc1, s_restore_spi_init_hi, 0x0000FFFF //base_addr_hi
+- s_or_b32 s_restore_buf_rsrc1, s_restore_buf_rsrc1, S_RESTORE_BUF_RSRC_WORD1_STRIDE
+- s_mov_b32 s_restore_buf_rsrc2, 0 //NUM_RECORDS initial value = 0 (in bytes)
+- s_mov_b32 s_restore_buf_rsrc3, S_RESTORE_BUF_RSRC_WORD3_MISC
+- s_and_b32 s_restore_tmp, s_restore_spi_init_hi, S_RESTORE_SPI_INIT_ATC_MASK
+- s_lshr_b32 s_restore_tmp, s_restore_tmp, (S_RESTORE_SPI_INIT_ATC_SHIFT-SQ_BUF_RSRC_WORD1_ATC_SHIFT) //get ATC bit into position
+- s_or_b32 s_restore_buf_rsrc3, s_restore_buf_rsrc3, s_restore_tmp //or ATC
+- s_and_b32 s_restore_tmp, s_restore_spi_init_hi, S_RESTORE_SPI_INIT_MTYPE_MASK
+- s_lshr_b32 s_restore_tmp, s_restore_tmp, (S_RESTORE_SPI_INIT_MTYPE_SHIFT-SQ_BUF_RSRC_WORD3_MTYPE_SHIFT) //get MTYPE bits into position
+- s_or_b32 s_restore_buf_rsrc3, s_restore_buf_rsrc3, s_restore_tmp //or MTYPE
+-
+- /* global mem offset */
+-// s_mov_b32 s_restore_mem_offset, 0x0 //mem offset initial value = 0
+-
+- /* the first wave in the threadgroup */
+- s_and_b32 s_restore_tmp, s_restore_spi_init_hi, S_RESTORE_SPI_INIT_FIRST_WAVE_MASK
+- s_cbranch_scc0 L_RESTORE_VGPR
+-
+- /* restore LDS */
+- //////////////////////////////
+- L_RESTORE_LDS:
+-
+- s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on //be consistent with SAVE although can be moved ahead
+- s_mov_b32 exec_hi, 0xFFFFFFFF
+-
+- s_getreg_b32 s_restore_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE) //lds_size
+- s_and_b32 s_restore_alloc_size, s_restore_alloc_size, 0xFFFFFFFF //lds_size is zero?
+- s_cbranch_scc0 L_RESTORE_VGPR //no lds used? jump to L_RESTORE_VGPR
+- s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 6 //LDS size in dwords = lds_size * 64dw
+- s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 2 //LDS size in bytes
+- s_mov_b32 s_restore_buf_rsrc2, s_restore_alloc_size //NUM_RECORDS in bytes
+-
+- // LDS at offset: size(VGPR)+SIZE(SGPR)+SIZE(HWREG)
+- //
+- get_vgpr_size_bytes(s_restore_mem_offset)
+- get_sgpr_size_bytes(s_restore_tmp)
+- s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp
+- s_add_u32 s_restore_mem_offset, s_restore_mem_offset, get_hwreg_size_bytes() //FIXME, Check if offset overflow???
+-
+-
+- if (SWIZZLE_EN)
+- s_add_u32 s_restore_buf_rsrc2, s_restore_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+- else
+- s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+- end
+- s_mov_b32 m0, 0x0 //lds_offset initial value = 0
+-
+- L_RESTORE_LDS_LOOP:
+- if (SAVE_LDS)
+- buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset lds:1 // first 64DW
+- buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset lds:1 offset:256 // second 64DW
+- end
+- s_add_u32 m0, m0, 256*2 // 128 DW
+- s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 256*2 //mem offset increased by 128DW
+- s_cmp_lt_u32 m0, s_restore_alloc_size //scc=(m0 < s_restore_alloc_size) ? 1 : 0
+- s_cbranch_scc1 L_RESTORE_LDS_LOOP //LDS restore is complete?
+-
+-
+- /* restore VGPRs */
+- //////////////////////////////
+- L_RESTORE_VGPR:
+- // VGPR SR memory offset : 0
+- s_mov_b32 s_restore_mem_offset, 0x0
+- s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on //be consistent with SAVE although can be moved ahead
+- s_mov_b32 exec_hi, 0xFFFFFFFF
+-
+- s_getreg_b32 s_restore_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE) //vpgr_size
+- s_add_u32 s_restore_alloc_size, s_restore_alloc_size, 1
+- s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 2 //Number of VGPRs = (vgpr_size + 1) * 4 (non-zero value)
+- s_lshl_b32 s_restore_buf_rsrc2, s_restore_alloc_size, 8 //NUM_RECORDS in bytes (64 threads*4)
+- if (SWIZZLE_EN)
+- s_add_u32 s_restore_buf_rsrc2, s_restore_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+- else
+- s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+- end
+-
+-if G8SR_VGPR_SR_IN_DWX4
+- get_vgpr_size_bytes(s_restore_mem_offset)
+- s_sub_u32 s_restore_mem_offset, s_restore_mem_offset, 256*4
+-
+- // the const stride for DWx4 is 4*4 bytes
+- s_and_b32 s_restore_buf_rsrc1, s_restore_buf_rsrc1, 0x0000FFFF // reset const stride to 0
+- s_or_b32 s_restore_buf_rsrc1, s_restore_buf_rsrc1, G8SR_RESTORE_BUF_RSRC_WORD1_STRIDE_DWx4 // const stride to 4*4 bytes
+-
+- s_mov_b32 m0, s_restore_alloc_size
+- s_set_gpr_idx_on m0, 0x8 // Note.. This will change m0
+-
+-L_RESTORE_VGPR_LOOP:
+- buffer_load_dwordx4 v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1
+- s_waitcnt vmcnt(0)
+- s_sub_u32 m0, m0, 4
+- v_mov_b32 v0, v0 // v[0+m0] = v0
+- v_mov_b32 v1, v1
+- v_mov_b32 v2, v2
+- v_mov_b32 v3, v3
+- s_sub_u32 s_restore_mem_offset, s_restore_mem_offset, 256*4
+- s_cmp_eq_u32 m0, 0x8000
+- s_cbranch_scc0 L_RESTORE_VGPR_LOOP
+- s_set_gpr_idx_off
+-
+- s_and_b32 s_restore_buf_rsrc1, s_restore_buf_rsrc1, 0x0000FFFF // reset const stride to 0
+- s_or_b32 s_restore_buf_rsrc1, s_restore_buf_rsrc1, S_RESTORE_BUF_RSRC_WORD1_STRIDE // const stride to 4*4 bytes
+-
+-else
+- // VGPR load using dw burst
+- s_mov_b32 s_restore_mem_offset_save, s_restore_mem_offset // restore start with v1, v0 will be the last
+- s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 256*4
+- s_mov_b32 m0, 4 //VGPR initial index value = 1
+- s_set_gpr_idx_on m0, 0x8 //M0[7:0] = M0[7:0] and M0[15:12] = 0x8
+- s_add_u32 s_restore_alloc_size, s_restore_alloc_size, 0x8000 //add 0x8000 since we compare m0 against it later
+-
+- L_RESTORE_VGPR_LOOP:
+- if(USE_MTBUF_INSTEAD_OF_MUBUF)
+- tbuffer_load_format_x v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset format:BUF_NUM_FORMAT_FLOAT format: BUF_DATA_FORMAT_32 slc:1 glc:1
+- else
+- buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1
+- buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:256
+- buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:256*2
+- buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:256*3
+- end
+- s_waitcnt vmcnt(0) //ensure data ready
+- v_mov_b32 v0, v0 //v[0+m0] = v0
+- v_mov_b32 v1, v1
+- v_mov_b32 v2, v2
+- v_mov_b32 v3, v3
+- s_add_u32 m0, m0, 4 //next vgpr index
+- s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 256*4 //every buffer_load_dword does 256 bytes
+- s_cmp_lt_u32 m0, s_restore_alloc_size //scc = (m0 < s_restore_alloc_size) ? 1 : 0
+- s_cbranch_scc1 L_RESTORE_VGPR_LOOP //VGPR restore (except v0) is complete?
+- s_set_gpr_idx_off
+- /* VGPR restore on v0 */
+- if(USE_MTBUF_INSTEAD_OF_MUBUF)
+- tbuffer_load_format_x v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save format:BUF_NUM_FORMAT_FLOAT format: BUF_DATA_FORMAT_32 slc:1 glc:1
+- else
+- buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1
+- buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1 offset:256
+- buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1 offset:256*2
+- buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1 offset:256*3
+- end
+-
+-end
+-
+- /* restore SGPRs */
+- //////////////////////////////
+-
+- // SGPR SR memory offset : size(VGPR)
+- get_vgpr_size_bytes(s_restore_mem_offset)
+- get_sgpr_size_bytes(s_restore_tmp)
+- s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp
+- s_sub_u32 s_restore_mem_offset, s_restore_mem_offset, 16*4 // restore SGPR from S[n] to S[0], by 16 sgprs group
+- // TODO, change RSRC word to rearrange memory layout for SGPRS
+-
+- s_getreg_b32 s_restore_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE) //spgr_size
+- s_add_u32 s_restore_alloc_size, s_restore_alloc_size, 1
+- s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 4 //Number of SGPRs = (sgpr_size + 1) * 16 (non-zero value)
+-
+- if (SGPR_SAVE_USE_SQC)
+- s_lshl_b32 s_restore_buf_rsrc2, s_restore_alloc_size, 2 //NUM_RECORDS in bytes
+- else
+- s_lshl_b32 s_restore_buf_rsrc2, s_restore_alloc_size, 8 //NUM_RECORDS in bytes (64 threads)
+- end
+- if (SWIZZLE_EN)
+- s_add_u32 s_restore_buf_rsrc2, s_restore_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+- else
+- s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+- end
+-
+- /* If 112 SGPRs ar allocated, 4 sgprs are not used TBA(108,109),TMA(110,111),
+- However, we are safe to restore these 4 SGPRs anyway, since TBA,TMA will later be restored by HWREG
+- */
+- s_mov_b32 m0, s_restore_alloc_size
+-
+- L_RESTORE_SGPR_LOOP:
+- read_16sgpr_from_mem(s0, s_restore_buf_rsrc0, s_restore_mem_offset) //PV: further performance improvement can be made
+- s_waitcnt lgkmcnt(0) //ensure data ready
+-
+- s_sub_u32 m0, m0, 16 // Restore from S[n] to S[0]
+-
+- s_movreld_b64 s0, s0 //s[0+m0] = s0
+- s_movreld_b64 s2, s2
+- s_movreld_b64 s4, s4
+- s_movreld_b64 s6, s6
+- s_movreld_b64 s8, s8
+- s_movreld_b64 s10, s10
+- s_movreld_b64 s12, s12
+- s_movreld_b64 s14, s14
+-
+- s_cmp_eq_u32 m0, 0 //scc = (m0 < s_restore_alloc_size) ? 1 : 0
+- s_cbranch_scc0 L_RESTORE_SGPR_LOOP //SGPR restore (except s0) is complete?
+-
+- /* restore HW registers */
+- //////////////////////////////
+- L_RESTORE_HWREG:
+-
+-
+-if G8SR_DEBUG_TIMESTAMP
+- s_mov_b32 s_g8sr_ts_restore_s[0], s_restore_pc_lo
+- s_mov_b32 s_g8sr_ts_restore_s[1], s_restore_pc_hi
+-end
+-
+- // HWREG SR memory offset : size(VGPR)+size(SGPR)
+- get_vgpr_size_bytes(s_restore_mem_offset)
+- get_sgpr_size_bytes(s_restore_tmp)
+- s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp
+-
+-
+- s_mov_b32 s_restore_buf_rsrc2, 0x4 //NUM_RECORDS in bytes
+- if (SWIZZLE_EN)
+- s_add_u32 s_restore_buf_rsrc2, s_restore_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+- else
+- s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+- end
+-
+- read_hwreg_from_mem(s_restore_m0, s_restore_buf_rsrc0, s_restore_mem_offset) //M0
+- read_hwreg_from_mem(s_restore_pc_lo, s_restore_buf_rsrc0, s_restore_mem_offset) //PC
+- read_hwreg_from_mem(s_restore_pc_hi, s_restore_buf_rsrc0, s_restore_mem_offset)
+- read_hwreg_from_mem(s_restore_exec_lo, s_restore_buf_rsrc0, s_restore_mem_offset) //EXEC
+- read_hwreg_from_mem(s_restore_exec_hi, s_restore_buf_rsrc0, s_restore_mem_offset)
+- read_hwreg_from_mem(s_restore_status, s_restore_buf_rsrc0, s_restore_mem_offset) //STATUS
+- read_hwreg_from_mem(s_restore_trapsts, s_restore_buf_rsrc0, s_restore_mem_offset) //TRAPSTS
+- read_hwreg_from_mem(xnack_mask_lo, s_restore_buf_rsrc0, s_restore_mem_offset) //XNACK_MASK_LO
+- read_hwreg_from_mem(xnack_mask_hi, s_restore_buf_rsrc0, s_restore_mem_offset) //XNACK_MASK_HI
+- read_hwreg_from_mem(s_restore_mode, s_restore_buf_rsrc0, s_restore_mem_offset) //MODE
+- read_hwreg_from_mem(tba_lo, s_restore_buf_rsrc0, s_restore_mem_offset) //TBA_LO
+- read_hwreg_from_mem(tba_hi, s_restore_buf_rsrc0, s_restore_mem_offset) //TBA_HI
+-
+- s_waitcnt lgkmcnt(0) //from now on, it is safe to restore STATUS and IB_STS
+-
+- s_and_b32 s_restore_pc_hi, s_restore_pc_hi, 0x0000ffff //pc[47:32] //Do it here in order not to affect STATUS
+-
+- //for normal save & restore, the saved PC points to the next inst to execute, no adjustment needs to be made, otherwise:
+- if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_RESTORE_NORMAL))
+- s_add_u32 s_restore_pc_lo, s_restore_pc_lo, 8 //pc[31:0]+8 //two back-to-back s_trap are used (first for save and second for restore)
+- s_addc_u32 s_restore_pc_hi, s_restore_pc_hi, 0x0 //carry bit over
+- end
+- if ((EMU_RUN_HACK) && (EMU_RUN_HACK_RESTORE_NORMAL))
+- s_add_u32 s_restore_pc_lo, s_restore_pc_lo, 4 //pc[31:0]+4 // save is hack through s_trap but restore is normal
+- s_addc_u32 s_restore_pc_hi, s_restore_pc_hi, 0x0 //carry bit over
+- end
+-
+- s_mov_b32 m0, s_restore_m0
+- s_mov_b32 exec_lo, s_restore_exec_lo
+- s_mov_b32 exec_hi, s_restore_exec_hi
+-
+- s_and_b32 s_restore_m0, SQ_WAVE_TRAPSTS_PRE_SAVECTX_MASK, s_restore_trapsts
+- s_setreg_b32 hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_PRE_SAVECTX_SHIFT, SQ_WAVE_TRAPSTS_PRE_SAVECTX_SIZE), s_restore_m0
+- s_and_b32 s_restore_m0, SQ_WAVE_TRAPSTS_POST_SAVECTX_MASK, s_restore_trapsts
+- s_lshr_b32 s_restore_m0, s_restore_m0, SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT
+- s_setreg_b32 hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT, SQ_WAVE_TRAPSTS_POST_SAVECTX_SIZE), s_restore_m0
+- //s_setreg_b32 hwreg(HW_REG_TRAPSTS), s_restore_trapsts //don't overwrite SAVECTX bit as it may be set through external SAVECTX during restore
+- s_setreg_b32 hwreg(HW_REG_MODE), s_restore_mode
+- //reuse s_restore_m0 as a temp register
+- s_and_b32 s_restore_m0, s_restore_pc_hi, S_SAVE_PC_HI_RCNT_MASK
+- s_lshr_b32 s_restore_m0, s_restore_m0, S_SAVE_PC_HI_RCNT_SHIFT
+- s_lshl_b32 s_restore_m0, s_restore_m0, SQ_WAVE_IB_STS_RCNT_SHIFT
+- s_mov_b32 s_restore_tmp, 0x0 //IB_STS is zero
+- s_or_b32 s_restore_tmp, s_restore_tmp, s_restore_m0
+- s_and_b32 s_restore_m0, s_restore_pc_hi, S_SAVE_PC_HI_FIRST_REPLAY_MASK
+- s_lshr_b32 s_restore_m0, s_restore_m0, S_SAVE_PC_HI_FIRST_REPLAY_SHIFT
+- s_lshl_b32 s_restore_m0, s_restore_m0, SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT
+- s_or_b32 s_restore_tmp, s_restore_tmp, s_restore_m0
+- s_and_b32 s_restore_m0, s_restore_status, SQ_WAVE_STATUS_INST_ATC_MASK
+- s_lshr_b32 s_restore_m0, s_restore_m0, SQ_WAVE_STATUS_INST_ATC_SHIFT
+- s_setreg_b32 hwreg(HW_REG_IB_STS), s_restore_tmp
+-
+- s_and_b64 exec, exec, exec // Restore STATUS.EXECZ, not writable by s_setreg_b32
+- s_and_b64 vcc, vcc, vcc // Restore STATUS.VCCZ, not writable by s_setreg_b32
+- s_setreg_b32 hwreg(HW_REG_STATUS), s_restore_status // SCC is included, which is changed by previous salu
+-
+- s_barrier //barrier to ensure the readiness of LDS before access attemps from any other wave in the same TG //FIXME not performance-optimal at this time
+-
+-if G8SR_DEBUG_TIMESTAMP
+- s_memrealtime s_g8sr_ts_restore_d
+- s_waitcnt lgkmcnt(0)
+-end
+-
+-// s_rfe_b64 s_restore_pc_lo //Return to the main shader program and resume execution
+- s_rfe_restore_b64 s_restore_pc_lo, s_restore_m0 // s_restore_m0[0] is used to set STATUS.inst_atc
+-
+-
+-/**************************************************************************/
+-/* the END */
+-/**************************************************************************/
+-L_END_PGM:
+- s_endpgm
+-
+-end
+-
+-
+-/**************************************************************************/
+-/* the helper functions */
+-/**************************************************************************/
+-
+-//Only for save hwreg to mem
+-function write_hwreg_to_mem(s, s_rsrc, s_mem_offset)
+- s_mov_b32 exec_lo, m0 //assuming exec_lo is not needed anymore from this point on
+- s_mov_b32 m0, s_mem_offset
+- s_buffer_store_dword s, s_rsrc, m0 glc:1
+- s_add_u32 s_mem_offset, s_mem_offset, 4
+- s_mov_b32 m0, exec_lo
+-end
+-
+-
+-// HWREG are saved before SGPRs, so all HWREG could be use.
+-function write_16sgpr_to_mem(s, s_rsrc, s_mem_offset)
+-
+- s_buffer_store_dwordx4 s[0], s_rsrc, 0 glc:1
+- s_buffer_store_dwordx4 s[4], s_rsrc, 16 glc:1
+- s_buffer_store_dwordx4 s[8], s_rsrc, 32 glc:1
+- s_buffer_store_dwordx4 s[12], s_rsrc, 48 glc:1
+- s_add_u32 s_rsrc[0], s_rsrc[0], 4*16
+- s_addc_u32 s_rsrc[1], s_rsrc[1], 0x0 // +scc
+-end
+-
+-
+-function read_hwreg_from_mem(s, s_rsrc, s_mem_offset)
+- s_buffer_load_dword s, s_rsrc, s_mem_offset glc:1
+- s_add_u32 s_mem_offset, s_mem_offset, 4
+-end
+-
+-function read_16sgpr_from_mem(s, s_rsrc, s_mem_offset)
+- s_buffer_load_dwordx16 s, s_rsrc, s_mem_offset glc:1
+- s_sub_u32 s_mem_offset, s_mem_offset, 4*16
+-end
+-
+-
+-
+-function get_lds_size_bytes(s_lds_size_byte)
+- // SQ LDS granularity is 64DW, while PGM_RSRC2.lds_size is in granularity 128DW
+- s_getreg_b32 s_lds_size_byte, hwreg(HW_REG_LDS_ALLOC, SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT, SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE) // lds_size
+- s_lshl_b32 s_lds_size_byte, s_lds_size_byte, 8 //LDS size in dwords = lds_size * 64 *4Bytes // granularity 64DW
+-end
+-
+-function get_vgpr_size_bytes(s_vgpr_size_byte)
+- s_getreg_b32 s_vgpr_size_byte, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE) //vpgr_size
+- s_add_u32 s_vgpr_size_byte, s_vgpr_size_byte, 1
+- s_lshl_b32 s_vgpr_size_byte, s_vgpr_size_byte, (2+8) //Number of VGPRs = (vgpr_size + 1) * 4 * 64 * 4 (non-zero value) //FIXME for GFX, zero is possible
+-end
+-
+-function get_sgpr_size_bytes(s_sgpr_size_byte)
+- s_getreg_b32 s_sgpr_size_byte, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE) //spgr_size
+- s_add_u32 s_sgpr_size_byte, s_sgpr_size_byte, 1
+- s_lshl_b32 s_sgpr_size_byte, s_sgpr_size_byte, 6 //Number of SGPRs = (sgpr_size + 1) * 16 *4 (non-zero value)
+-end
+-
+-function get_hwreg_size_bytes
+- return 128 //HWREG size 128 bytes
+-end
+-
+-
+-#endif
+-
+-static const uint32_t cwsr_trap_carrizo_hex[] = {
+- 0xbf820001, 0xbf820123,
+- 0xb8f4f802, 0x89748674,
+- 0xb8f5f803, 0x8675ff75,
+- 0x00000400, 0xbf850011,
+- 0xc00a1e37, 0x00000000,
+- 0xbf8c007f, 0x87777978,
+- 0xbf840002, 0xb974f802,
+- 0xbe801d78, 0xb8f5f803,
+- 0x8675ff75, 0x000001ff,
+- 0xbf850002, 0x80708470,
+- 0x82718071, 0x8671ff71,
+- 0x0000ffff, 0xb974f802,
+- 0xbe801f70, 0xb8f5f803,
+- 0x8675ff75, 0x00000100,
+- 0xbf840006, 0xbefa0080,
+- 0xb97a0203, 0x8671ff71,
+- 0x0000ffff, 0x80f08870,
+- 0x82f18071, 0xbefa0080,
+- 0xb97a0283, 0xbef60068,
+- 0xbef70069, 0xb8fa1c07,
+- 0x8e7a9c7a, 0x87717a71,
+- 0xb8fa03c7, 0x8e7a9b7a,
+- 0x87717a71, 0xb8faf807,
+- 0x867aff7a, 0x00007fff,
+- 0xb97af807, 0xbef2007e,
+- 0xbef3007f, 0xbefe0180,
+- 0xbf900004, 0xbf8e0002,
+- 0xbf88fffe, 0xbef8007e,
+- 0x8679ff7f, 0x0000ffff,
+- 0x8779ff79, 0x00040000,
+- 0xbefa0080, 0xbefb00ff,
+- 0x00807fac, 0x867aff7f,
+- 0x08000000, 0x8f7a837a,
+- 0x877b7a7b, 0x867aff7f,
+- 0x70000000, 0x8f7a817a,
+- 0x877b7a7b, 0xbeef007c,
+- 0xbeee0080, 0xb8ee2a05,
+- 0x806e816e, 0x8e6e8a6e,
+- 0xb8fa1605, 0x807a817a,
+- 0x8e7a867a, 0x806e7a6e,
+- 0xbefa0084, 0xbefa00ff,
+- 0x01000000, 0xbefe007c,
+- 0xbefc006e, 0xc0611bfc,
+- 0x0000007c, 0x806e846e,
+- 0xbefc007e, 0xbefe007c,
+- 0xbefc006e, 0xc0611c3c,
+- 0x0000007c, 0x806e846e,
+- 0xbefc007e, 0xbefe007c,
+- 0xbefc006e, 0xc0611c7c,
+- 0x0000007c, 0x806e846e,
+- 0xbefc007e, 0xbefe007c,
+- 0xbefc006e, 0xc0611cbc,
+- 0x0000007c, 0x806e846e,
+- 0xbefc007e, 0xbefe007c,
+- 0xbefc006e, 0xc0611cfc,
+- 0x0000007c, 0x806e846e,
+- 0xbefc007e, 0xbefe007c,
+- 0xbefc006e, 0xc0611d3c,
+- 0x0000007c, 0x806e846e,
+- 0xbefc007e, 0xb8f5f803,
+- 0xbefe007c, 0xbefc006e,
+- 0xc0611d7c, 0x0000007c,
+- 0x806e846e, 0xbefc007e,
+- 0xbefe007c, 0xbefc006e,
+- 0xc0611dbc, 0x0000007c,
+- 0x806e846e, 0xbefc007e,
+- 0xbefe007c, 0xbefc006e,
+- 0xc0611dfc, 0x0000007c,
+- 0x806e846e, 0xbefc007e,
+- 0xb8eff801, 0xbefe007c,
+- 0xbefc006e, 0xc0611bfc,
+- 0x0000007c, 0x806e846e,
+- 0xbefc007e, 0xbefe007c,
+- 0xbefc006e, 0xc0611b3c,
+- 0x0000007c, 0x806e846e,
+- 0xbefc007e, 0xbefe007c,
+- 0xbefc006e, 0xc0611b7c,
+- 0x0000007c, 0x806e846e,
+- 0xbefc007e, 0x867aff7f,
+- 0x04000000, 0xbef30080,
+- 0x8773737a, 0xb8ee2a05,
+- 0x806e816e, 0x8e6e8a6e,
+- 0xb8f51605, 0x80758175,
+- 0x8e758475, 0x8e7a8275,
+- 0xbefa00ff, 0x01000000,
+- 0xbef60178, 0x80786e78,
+- 0x82798079, 0xbefc0080,
+- 0xbe802b00, 0xbe822b02,
+- 0xbe842b04, 0xbe862b06,
+- 0xbe882b08, 0xbe8a2b0a,
+- 0xbe8c2b0c, 0xbe8e2b0e,
+- 0xc06b003c, 0x00000000,
+- 0xc06b013c, 0x00000010,
+- 0xc06b023c, 0x00000020,
+- 0xc06b033c, 0x00000030,
+- 0x8078c078, 0x82798079,
+- 0x807c907c, 0xbf0a757c,
+- 0xbf85ffeb, 0xbef80176,
+- 0xbeee0080, 0xbefe00c1,
+- 0xbeff00c1, 0xbefa00ff,
+- 0x01000000, 0xe0724000,
+- 0x6e1e0000, 0xe0724100,
+- 0x6e1e0100, 0xe0724200,
+- 0x6e1e0200, 0xe0724300,
+- 0x6e1e0300, 0xbefe00c1,
+- 0xbeff00c1, 0xb8f54306,
+- 0x8675c175, 0xbf84002c,
+- 0xbf8a0000, 0x867aff73,
+- 0x04000000, 0xbf840028,
+- 0x8e758675, 0x8e758275,
+- 0xbefa0075, 0xb8ee2a05,
+- 0x806e816e, 0x8e6e8a6e,
+- 0xb8fa1605, 0x807a817a,
+- 0x8e7a867a, 0x806e7a6e,
+- 0x806eff6e, 0x00000080,
+- 0xbefa00ff, 0x01000000,
+- 0xbefc0080, 0xd28c0002,
+- 0x000100c1, 0xd28d0003,
+- 0x000204c1, 0xd1060002,
+- 0x00011103, 0x7e0602ff,
+- 0x00000200, 0xbefc00ff,
+- 0x00010000, 0xbe80007b,
+- 0x867bff7b, 0xff7fffff,
+- 0x877bff7b, 0x00058000,
+- 0xd8ec0000, 0x00000002,
+- 0xbf8c007f, 0xe0765000,
+- 0x6e1e0002, 0x32040702,
+- 0xd0c9006a, 0x0000eb02,
+- 0xbf87fff7, 0xbefb0000,
+- 0xbeee00ff, 0x00000400,
+- 0xbefe00c1, 0xbeff00c1,
+- 0xb8f52a05, 0x80758175,
+- 0x8e758275, 0x8e7a8875,
+- 0xbefa00ff, 0x01000000,
+- 0xbefc0084, 0xbf0a757c,
+- 0xbf840015, 0xbf11017c,
+- 0x8075ff75, 0x00001000,
+- 0x7e000300, 0x7e020301,
+- 0x7e040302, 0x7e060303,
+- 0xe0724000, 0x6e1e0000,
+- 0xe0724100, 0x6e1e0100,
+- 0xe0724200, 0x6e1e0200,
+- 0xe0724300, 0x6e1e0300,
+- 0x807c847c, 0x806eff6e,
+- 0x00000400, 0xbf0a757c,
+- 0xbf85ffef, 0xbf9c0000,
+- 0xbf8200ca, 0xbef8007e,
+- 0x8679ff7f, 0x0000ffff,
+- 0x8779ff79, 0x00040000,
+- 0xbefa0080, 0xbefb00ff,
+- 0x00807fac, 0x8676ff7f,
+- 0x08000000, 0x8f768376,
+- 0x877b767b, 0x8676ff7f,
+- 0x70000000, 0x8f768176,
+- 0x877b767b, 0x8676ff7f,
+- 0x04000000, 0xbf84001e,
+- 0xbefe00c1, 0xbeff00c1,
+- 0xb8f34306, 0x8673c173,
+- 0xbf840019, 0x8e738673,
+- 0x8e738273, 0xbefa0073,
+- 0xb8f22a05, 0x80728172,
+- 0x8e728a72, 0xb8f61605,
+- 0x80768176, 0x8e768676,
+- 0x80727672, 0x8072ff72,
+- 0x00000080, 0xbefa00ff,
+- 0x01000000, 0xbefc0080,
+- 0xe0510000, 0x721e0000,
+- 0xe0510100, 0x721e0000,
+- 0x807cff7c, 0x00000200,
+- 0x8072ff72, 0x00000200,
+- 0xbf0a737c, 0xbf85fff6,
+- 0xbef20080, 0xbefe00c1,
+- 0xbeff00c1, 0xb8f32a05,
+- 0x80738173, 0x8e738273,
+- 0x8e7a8873, 0xbefa00ff,
+- 0x01000000, 0xbef60072,
+- 0x8072ff72, 0x00000400,
+- 0xbefc0084, 0xbf11087c,
+- 0x8073ff73, 0x00008000,
+- 0xe0524000, 0x721e0000,
+- 0xe0524100, 0x721e0100,
+- 0xe0524200, 0x721e0200,
+- 0xe0524300, 0x721e0300,
+- 0xbf8c0f70, 0x7e000300,
+- 0x7e020301, 0x7e040302,
+- 0x7e060303, 0x807c847c,
+- 0x8072ff72, 0x00000400,
+- 0xbf0a737c, 0xbf85ffee,
+- 0xbf9c0000, 0xe0524000,
+- 0x761e0000, 0xe0524100,
+- 0x761e0100, 0xe0524200,
+- 0x761e0200, 0xe0524300,
+- 0x761e0300, 0xb8f22a05,
+- 0x80728172, 0x8e728a72,
+- 0xb8f61605, 0x80768176,
+- 0x8e768676, 0x80727672,
+- 0x80f2c072, 0xb8f31605,
+- 0x80738173, 0x8e738473,
+- 0x8e7a8273, 0xbefa00ff,
+- 0x01000000, 0xbefc0073,
+- 0xc031003c, 0x00000072,
+- 0x80f2c072, 0xbf8c007f,
+- 0x80fc907c, 0xbe802d00,
+- 0xbe822d02, 0xbe842d04,
+- 0xbe862d06, 0xbe882d08,
+- 0xbe8a2d0a, 0xbe8c2d0c,
+- 0xbe8e2d0e, 0xbf06807c,
+- 0xbf84fff1, 0xb8f22a05,
+- 0x80728172, 0x8e728a72,
+- 0xb8f61605, 0x80768176,
+- 0x8e768676, 0x80727672,
+- 0xbefa0084, 0xbefa00ff,
+- 0x01000000, 0xc0211cfc,
+- 0x00000072, 0x80728472,
+- 0xc0211c3c, 0x00000072,
+- 0x80728472, 0xc0211c7c,
+- 0x00000072, 0x80728472,
+- 0xc0211bbc, 0x00000072,
+- 0x80728472, 0xc0211bfc,
+- 0x00000072, 0x80728472,
+- 0xc0211d3c, 0x00000072,
+- 0x80728472, 0xc0211d7c,
+- 0x00000072, 0x80728472,
+- 0xc0211a3c, 0x00000072,
+- 0x80728472, 0xc0211a7c,
+- 0x00000072, 0x80728472,
+- 0xc0211dfc, 0x00000072,
+- 0x80728472, 0xc0211b3c,
+- 0x00000072, 0x80728472,
+- 0xc0211b7c, 0x00000072,
+- 0x80728472, 0xbf8c007f,
+- 0x8671ff71, 0x0000ffff,
+- 0xbefc0073, 0xbefe006e,
+- 0xbeff006f, 0x867375ff,
+- 0x000003ff, 0xb9734803,
+- 0x867375ff, 0xfffff800,
+- 0x8f738b73, 0xb973a2c3,
+- 0xb977f801, 0x8673ff71,
+- 0xf0000000, 0x8f739c73,
+- 0x8e739073, 0xbef60080,
+- 0x87767376, 0x8673ff71,
+- 0x08000000, 0x8f739b73,
+- 0x8e738f73, 0x87767376,
+- 0x8673ff74, 0x00800000,
+- 0x8f739773, 0xb976f807,
+- 0x86fe7e7e, 0x86ea6a6a,
+- 0xb974f802, 0xbf8a0000,
+- 0x95807370, 0xbf810000,
+-};
+-
+diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx9.asm b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx9.asm
+deleted file mode 100644
+index ae2af3d..0000000
+--- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx9.asm
++++ /dev/null
+@@ -1,1388 +0,0 @@
+-/*
+- * Copyright 2016 Advanced Micro Devices, Inc.
+- *
+- * Permission is hereby granted, free of charge, to any person obtaining a
+- * copy of this software and associated documentation files (the "Software"),
+- * to deal in the Software without restriction, including without limitation
+- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+- * and/or sell copies of the Software, and to permit persons to whom the
+- * Software is furnished to do so, subject to the following conditions:
+- *
+- * The above copyright notice and this permission notice shall be included in
+- * all copies or substantial portions of the Software.
+- *
+- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+- * OTHER DEALINGS IN THE SOFTWARE.
+- */
+-
+-#if 0
+-HW (GFX9) source code for CWSR trap handler
+-#Version 18 + multiple trap handler
+-
+-// this performance-optimal version was originally from Seven Xu at SRDC
+-
+-// Revison #18 --...
+-/* Rev History
+-** #1. Branch from gc dv. //gfxip/gfx9/main/src/test/suites/block/cs/sr/cs_trap_handler.sp3#1,#50, #51, #52-53(Skip, Already Fixed by PV), #54-56(merged),#57-58(mergerd, skiped-already fixed by PV)
+-** #4. SR Memory Layout:
+-** 1. VGPR-SGPR-HWREG-{LDS}
+-** 2. tba_hi.bits.26 - reconfigured as the first wave in tg bits, for defer Save LDS for a threadgroup.. performance concern..
+-** #5. Update: 1. Accurate g8sr_ts_save_d timestamp
+-** #6. Update: 1. Fix s_barrier usage; 2. VGPR s/r using swizzle buffer?(NoNeed, already matched the swizzle pattern, more investigation)
+-** #7. Update: 1. don't barrier if noLDS
+-** #8. Branch: 1. Branch to ver#0, which is very similar to gc dv version
+-** 2. Fix SQ issue by s_sleep 2
+-** #9. Update: 1. Fix scc restore failed issue, restore wave_status at last
+-** 2. optimize s_buffer save by burst 16sgprs...
+-** #10. Update 1. Optimize restore sgpr by busrt 16 sgprs.
+-** #11. Update 1. Add 2 more timestamp for debug version
+-** #12. Update 1. Add VGPR SR using DWx4, some case improve and some case drop performance
+-** #13. Integ 1. Always use MUBUF for PV trap shader...
+-** #14. Update 1. s_buffer_store soft clause...
+-** #15. Update 1. PERF - sclar write with glc:0/mtype0 to allow L2 combine. perf improvement a lot.
+-** #16. Update 1. PRRF - UNROLL LDS_DMA got 2500cycle save in IP tree
+-** #17. Update 1. FUNC - LDS_DMA has issues while ATC, replace with ds_read/buffer_store for save part[TODO restore part]
+-** 2. PERF - Save LDS before save VGPR to cover LDS save long latency...
+-** #18. Update 1. FUNC - Implicitly estore STATUS.VCCZ, which is not writable by s_setreg_b32
+-** 2. FUNC - Handle non-CWSR traps
+-*/
+-
+-var G8SR_WDMEM_HWREG_OFFSET = 0
+-var G8SR_WDMEM_SGPR_OFFSET = 128 // in bytes
+-
+-// Keep definition same as the app shader, These 2 time stamps are part of the app shader... Should before any Save and after restore.
+-
+-var G8SR_DEBUG_TIMESTAMP = 0
+-var G8SR_DEBUG_TS_SAVE_D_OFFSET = 40*4 // ts_save_d timestamp offset relative to SGPR_SR_memory_offset
+-var s_g8sr_ts_save_s = s[34:35] // save start
+-var s_g8sr_ts_sq_save_msg = s[36:37] // The save shader send SAVEWAVE msg to spi
+-var s_g8sr_ts_spi_wrexec = s[38:39] // the SPI write the sr address to SQ
+-var s_g8sr_ts_save_d = s[40:41] // save end
+-var s_g8sr_ts_restore_s = s[42:43] // restore start
+-var s_g8sr_ts_restore_d = s[44:45] // restore end
+-
+-var G8SR_VGPR_SR_IN_DWX4 = 0
+-var G8SR_SAVE_BUF_RSRC_WORD1_STRIDE_DWx4 = 0x00100000 // DWx4 stride is 4*4Bytes
+-var G8SR_RESTORE_BUF_RSRC_WORD1_STRIDE_DWx4 = G8SR_SAVE_BUF_RSRC_WORD1_STRIDE_DWx4
+-
+-
+-/*************************************************************************/
+-/* control on how to run the shader */
+-/*************************************************************************/
+-//any hack that needs to be made to run this code in EMU (either becasue various EMU code are not ready or no compute save & restore in EMU run)
+-var EMU_RUN_HACK = 0
+-var EMU_RUN_HACK_RESTORE_NORMAL = 0
+-var EMU_RUN_HACK_SAVE_NORMAL_EXIT = 0
+-var EMU_RUN_HACK_SAVE_SINGLE_WAVE = 0
+-var EMU_RUN_HACK_SAVE_FIRST_TIME = 0 //for interrupted restore in which the first save is through EMU_RUN_HACK
+-var SAVE_LDS = 1
+-var WG_BASE_ADDR_LO = 0x9000a000
+-var WG_BASE_ADDR_HI = 0x0
+-var WAVE_SPACE = 0x5000 //memory size that each wave occupies in workgroup state mem
+-var CTX_SAVE_CONTROL = 0x0
+-var CTX_RESTORE_CONTROL = CTX_SAVE_CONTROL
+-var SIM_RUN_HACK = 0 //any hack that needs to be made to run this code in SIM (either becasue various RTL code are not ready or no compute save & restore in RTL run)
+-var SGPR_SAVE_USE_SQC = 1 //use SQC D$ to do the write
+-var USE_MTBUF_INSTEAD_OF_MUBUF = 0 //becasue TC EMU curently asserts on 0 of // overload DFMT field to carry 4 more bits of stride for MUBUF opcodes
+-var SWIZZLE_EN = 0 //whether we use swizzled buffer addressing
+-
+-/**************************************************************************/
+-/* variables */
+-/**************************************************************************/
+-var SQ_WAVE_STATUS_INST_ATC_SHIFT = 23
+-var SQ_WAVE_STATUS_INST_ATC_MASK = 0x00800000
+-var SQ_WAVE_STATUS_SPI_PRIO_MASK = 0x00000006
+-var SQ_WAVE_STATUS_HALT_MASK = 0x2000
+-
+-var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT = 12
+-var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE = 9
+-var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT = 8
+-var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE = 6
+-var SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT = 24
+-var SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE = 3 //FIXME sq.blk still has 4 bits at this time while SQ programming guide has 3 bits
+-
+-var SQ_WAVE_TRAPSTS_SAVECTX_MASK = 0x400
+-var SQ_WAVE_TRAPSTS_EXCE_MASK = 0x1FF // Exception mask
+-var SQ_WAVE_TRAPSTS_SAVECTX_SHIFT = 10
+-var SQ_WAVE_TRAPSTS_MEM_VIOL_MASK = 0x100
+-var SQ_WAVE_TRAPSTS_MEM_VIOL_SHIFT = 8
+-var SQ_WAVE_TRAPSTS_PRE_SAVECTX_MASK = 0x3FF
+-var SQ_WAVE_TRAPSTS_PRE_SAVECTX_SHIFT = 0x0
+-var SQ_WAVE_TRAPSTS_PRE_SAVECTX_SIZE = 10
+-var SQ_WAVE_TRAPSTS_POST_SAVECTX_MASK = 0xFFFFF800
+-var SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT = 11
+-var SQ_WAVE_TRAPSTS_POST_SAVECTX_SIZE = 21
+-var SQ_WAVE_TRAPSTS_ILLEGAL_INST_MASK = 0x800
+-
+-var SQ_WAVE_IB_STS_RCNT_SHIFT = 16 //FIXME
+-var SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT = 15 //FIXME
+-var SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK_NEG = 0x00007FFF //FIXME
+-
+-var SQ_BUF_RSRC_WORD1_ATC_SHIFT = 24
+-var SQ_BUF_RSRC_WORD3_MTYPE_SHIFT = 27
+-
+-
+-/* Save */
+-var S_SAVE_BUF_RSRC_WORD1_STRIDE = 0x00040000 //stride is 4 bytes
+-var S_SAVE_BUF_RSRC_WORD3_MISC = 0x00807FAC //SQ_SEL_X/Y/Z/W, BUF_NUM_FORMAT_FLOAT, (0 for MUBUF stride[17:14] when ADD_TID_ENABLE and BUF_DATA_FORMAT_32 for MTBUF), ADD_TID_ENABLE
+-
+-var S_SAVE_SPI_INIT_ATC_MASK = 0x08000000 //bit[27]: ATC bit
+-var S_SAVE_SPI_INIT_ATC_SHIFT = 27
+-var S_SAVE_SPI_INIT_MTYPE_MASK = 0x70000000 //bit[30:28]: Mtype
+-var S_SAVE_SPI_INIT_MTYPE_SHIFT = 28
+-var S_SAVE_SPI_INIT_FIRST_WAVE_MASK = 0x04000000 //bit[26]: FirstWaveInTG
+-var S_SAVE_SPI_INIT_FIRST_WAVE_SHIFT = 26
+-
+-var S_SAVE_PC_HI_RCNT_SHIFT = 28 //FIXME check with Brian to ensure all fields other than PC[47:0] can be used
+-var S_SAVE_PC_HI_RCNT_MASK = 0xF0000000 //FIXME
+-var S_SAVE_PC_HI_FIRST_REPLAY_SHIFT = 27 //FIXME
+-var S_SAVE_PC_HI_FIRST_REPLAY_MASK = 0x08000000 //FIXME
+-
+-var s_save_spi_init_lo = exec_lo
+-var s_save_spi_init_hi = exec_hi
+-
+-var s_save_pc_lo = ttmp0 //{TTMP1, TTMP0} = {3¡¯h0,pc_rewind[3:0], HT[0],trapID[7:0], PC[47:0]}
+-var s_save_pc_hi = ttmp1
+-var s_save_exec_lo = ttmp2
+-var s_save_exec_hi = ttmp3
+-var s_save_status = ttmp4
+-var s_save_trapsts = ttmp5 //not really used until the end of the SAVE routine
+-var s_save_xnack_mask_lo = ttmp6
+-var s_save_xnack_mask_hi = ttmp7
+-var s_save_buf_rsrc0 = ttmp8
+-var s_save_buf_rsrc1 = ttmp9
+-var s_save_buf_rsrc2 = ttmp10
+-var s_save_buf_rsrc3 = ttmp11
+-
+-var s_save_mem_offset = ttmp14
+-var s_save_alloc_size = s_save_trapsts //conflict
+-var s_save_tmp = s_save_buf_rsrc2 //shared with s_save_buf_rsrc2 (conflict: should not use mem access with s_save_tmp at the same time)
+-var s_save_m0 = ttmp15
+-
+-/* Restore */
+-var S_RESTORE_BUF_RSRC_WORD1_STRIDE = S_SAVE_BUF_RSRC_WORD1_STRIDE
+-var S_RESTORE_BUF_RSRC_WORD3_MISC = S_SAVE_BUF_RSRC_WORD3_MISC
+-
+-var S_RESTORE_SPI_INIT_ATC_MASK = 0x08000000 //bit[27]: ATC bit
+-var S_RESTORE_SPI_INIT_ATC_SHIFT = 27
+-var S_RESTORE_SPI_INIT_MTYPE_MASK = 0x70000000 //bit[30:28]: Mtype
+-var S_RESTORE_SPI_INIT_MTYPE_SHIFT = 28
+-var S_RESTORE_SPI_INIT_FIRST_WAVE_MASK = 0x04000000 //bit[26]: FirstWaveInTG
+-var S_RESTORE_SPI_INIT_FIRST_WAVE_SHIFT = 26
+-
+-var S_RESTORE_PC_HI_RCNT_SHIFT = S_SAVE_PC_HI_RCNT_SHIFT
+-var S_RESTORE_PC_HI_RCNT_MASK = S_SAVE_PC_HI_RCNT_MASK
+-var S_RESTORE_PC_HI_FIRST_REPLAY_SHIFT = S_SAVE_PC_HI_FIRST_REPLAY_SHIFT
+-var S_RESTORE_PC_HI_FIRST_REPLAY_MASK = S_SAVE_PC_HI_FIRST_REPLAY_MASK
+-
+-var s_restore_spi_init_lo = exec_lo
+-var s_restore_spi_init_hi = exec_hi
+-
+-var s_restore_mem_offset = ttmp12
+-var s_restore_alloc_size = ttmp3
+-var s_restore_tmp = ttmp6
+-var s_restore_mem_offset_save = s_restore_tmp //no conflict
+-
+-var s_restore_m0 = s_restore_alloc_size //no conflict
+-
+-var s_restore_mode = ttmp7
+-
+-var s_restore_pc_lo = ttmp0
+-var s_restore_pc_hi = ttmp1
+-var s_restore_exec_lo = ttmp14
+-var s_restore_exec_hi = ttmp15
+-var s_restore_status = ttmp4
+-var s_restore_trapsts = ttmp5
+-var s_restore_xnack_mask_lo = xnack_mask_lo
+-var s_restore_xnack_mask_hi = xnack_mask_hi
+-var s_restore_buf_rsrc0 = ttmp8
+-var s_restore_buf_rsrc1 = ttmp9
+-var s_restore_buf_rsrc2 = ttmp10
+-var s_restore_buf_rsrc3 = ttmp11
+-
+-/**************************************************************************/
+-/* trap handler entry points */
+-/**************************************************************************/
+-/* Shader Main*/
+-
+-shader main
+- asic(GFX9)
+- type(CS)
+-
+-
+- if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_RESTORE_NORMAL)) //hack to use trap_id for determining save/restore
+- //FIXME VCCZ un-init assertion s_getreg_b32 s_save_status, hwreg(HW_REG_STATUS) //save STATUS since we will change SCC
+- s_and_b32 s_save_tmp, s_save_pc_hi, 0xffff0000 //change SCC
+- s_cmp_eq_u32 s_save_tmp, 0x007e0000 //Save: trap_id = 0x7e. Restore: trap_id = 0x7f.
+- s_cbranch_scc0 L_JUMP_TO_RESTORE //do not need to recover STATUS here since we are going to RESTORE
+- //FIXME s_setreg_b32 hwreg(HW_REG_STATUS), s_save_status //need to recover STATUS since we are going to SAVE
+- s_branch L_SKIP_RESTORE //NOT restore, SAVE actually
+- else
+- s_branch L_SKIP_RESTORE //NOT restore. might be a regular trap or save
+- end
+-
+-L_JUMP_TO_RESTORE:
+- s_branch L_RESTORE //restore
+-
+-L_SKIP_RESTORE:
+-
+- s_getreg_b32 s_save_status, hwreg(HW_REG_STATUS) //save STATUS since we will change SCC
+- s_andn2_b32 s_save_status, s_save_status, SQ_WAVE_STATUS_SPI_PRIO_MASK //check whether this is for save
+- s_getreg_b32 s_save_trapsts, hwreg(HW_REG_TRAPSTS)
+- s_and_b32 ttmp8, s_save_trapsts, SQ_WAVE_TRAPSTS_SAVECTX_MASK //check whether this is for save
+- s_cbranch_scc1 L_SAVE //this is the operation for save
+-
+- // ********* Handle non-CWSR traps *******************
+-if (!EMU_RUN_HACK)
+- // Illegal instruction is a non-maskable exception which blocks context save.
+- // Halt the wavefront and return from the trap.
+- s_and_b32 ttmp8, s_save_trapsts, SQ_WAVE_TRAPSTS_ILLEGAL_INST_MASK
+- s_cbranch_scc1 L_HALT_WAVE
+-
+- // If STATUS.MEM_VIOL is asserted then we cannot fetch from the TMA.
+- // Instead, halt the wavefront and return from the trap.
+- s_and_b32 ttmp8, s_save_trapsts, SQ_WAVE_TRAPSTS_MEM_VIOL_MASK
+- s_cbranch_scc0 L_NO_MEM_VIOL
+-
+-L_HALT_WAVE:
+- s_or_b32 s_save_status, s_save_status, SQ_WAVE_STATUS_HALT_MASK
+- s_branch L_EXCP_CASE
+-
+-L_NO_MEM_VIOL:
+- /* read tba and tma for next level trap handler, ttmp4 is used as s_save_status */
+- s_getreg_b32 ttmp14,hwreg(HW_REG_SQ_SHADER_TMA_LO)
+- s_getreg_b32 ttmp15,hwreg(HW_REG_SQ_SHADER_TMA_HI)
+- s_lshl_b64 [ttmp14, ttmp15], [ttmp14, ttmp15], 0x8
+- s_load_dwordx4 [ttmp8, ttmp9, ttmp10, ttmp11], [ttmp14, ttmp15], 0
+- s_waitcnt lgkmcnt(0)
+- s_or_b32 ttmp7, ttmp8, ttmp9
+- s_cbranch_scc0 L_NO_NEXT_TRAP //next level trap handler not been set
+- s_setreg_b32 hwreg(HW_REG_STATUS), s_save_status //restore HW status(SCC)
+- s_setpc_b64 [ttmp8,ttmp9] //jump to next level trap handler
+-
+-L_NO_NEXT_TRAP:
+- s_getreg_b32 s_save_trapsts, hwreg(HW_REG_TRAPSTS)
+- s_and_b32 s_save_trapsts, s_save_trapsts, SQ_WAVE_TRAPSTS_EXCE_MASK // Check whether it is an exception
+- s_cbranch_scc1 L_EXCP_CASE // Exception, jump back to the shader program directly.
+- s_add_u32 ttmp0, ttmp0, 4 // S_TRAP case, add 4 to ttmp0
+- s_addc_u32 ttmp1, ttmp1, 0
+-L_EXCP_CASE:
+- s_and_b32 ttmp1, ttmp1, 0xFFFF
+- s_setreg_b32 hwreg(HW_REG_STATUS), s_save_status //restore HW status(SCC)
+- s_rfe_b64 [ttmp0, ttmp1]
+-end
+- // ********* End handling of non-CWSR traps *******************
+-
+-/**************************************************************************/
+-/* save routine */
+-/**************************************************************************/
+-
+-L_SAVE:
+-
+-if G8SR_DEBUG_TIMESTAMP
+- s_memrealtime s_g8sr_ts_save_s
+- s_waitcnt lgkmcnt(0) //FIXME, will cause xnack??
+-end
+-
+- //check whether there is mem_viol
+- s_getreg_b32 s_save_trapsts, hwreg(HW_REG_TRAPSTS)
+- s_and_b32 s_save_trapsts, s_save_trapsts, SQ_WAVE_TRAPSTS_MEM_VIOL_MASK
+- s_cbranch_scc0 L_NO_PC_REWIND
+-
+- //if so, need rewind PC assuming GDS operation gets NACKed
+- s_mov_b32 s_save_tmp, 0 //clear mem_viol bit
+- s_setreg_b32 hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_MEM_VIOL_SHIFT, 1), s_save_tmp //clear mem_viol bit
+- s_and_b32 s_save_pc_hi, s_save_pc_hi, 0x0000ffff //pc[47:32]
+- s_sub_u32 s_save_pc_lo, s_save_pc_lo, 8 //pc[31:0]-8
+- s_subb_u32 s_save_pc_hi, s_save_pc_hi, 0x0 // -scc
+-
+-L_NO_PC_REWIND:
+- s_mov_b32 s_save_tmp, 0 //clear saveCtx bit
+- s_setreg_b32 hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_SAVECTX_SHIFT, 1), s_save_tmp //clear saveCtx bit
+-
+- s_mov_b32 s_save_xnack_mask_lo, xnack_mask_lo //save XNACK_MASK
+- s_mov_b32 s_save_xnack_mask_hi, xnack_mask_hi //save XNACK must before any memory operation
+- s_getreg_b32 s_save_tmp, hwreg(HW_REG_IB_STS, SQ_WAVE_IB_STS_RCNT_SHIFT, SQ_WAVE_IB_STS_RCNT_SIZE) //save RCNT
+- s_lshl_b32 s_save_tmp, s_save_tmp, S_SAVE_PC_HI_RCNT_SHIFT
+- s_or_b32 s_save_pc_hi, s_save_pc_hi, s_save_tmp
+- s_getreg_b32 s_save_tmp, hwreg(HW_REG_IB_STS, SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT, SQ_WAVE_IB_STS_FIRST_REPLAY_SIZE) //save FIRST_REPLAY
+- s_lshl_b32 s_save_tmp, s_save_tmp, S_SAVE_PC_HI_FIRST_REPLAY_SHIFT
+- s_or_b32 s_save_pc_hi, s_save_pc_hi, s_save_tmp
+- s_getreg_b32 s_save_tmp, hwreg(HW_REG_IB_STS) //clear RCNT and FIRST_REPLAY in IB_STS
+- s_and_b32 s_save_tmp, s_save_tmp, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK_NEG
+-
+- s_setreg_b32 hwreg(HW_REG_IB_STS), s_save_tmp
+-
+- /* inform SPI the readiness and wait for SPI's go signal */
+- s_mov_b32 s_save_exec_lo, exec_lo //save EXEC and use EXEC for the go signal from SPI
+- s_mov_b32 s_save_exec_hi, exec_hi
+- s_mov_b64 exec, 0x0 //clear EXEC to get ready to receive
+-
+-if G8SR_DEBUG_TIMESTAMP
+- s_memrealtime s_g8sr_ts_sq_save_msg
+- s_waitcnt lgkmcnt(0)
+-end
+-
+- if (EMU_RUN_HACK)
+-
+- else
+- s_sendmsg sendmsg(MSG_SAVEWAVE) //send SPI a message and wait for SPI's write to EXEC
+- end
+-
+- L_SLEEP:
+- s_sleep 0x2 // sleep 1 (64clk) is not enough for 8 waves per SIMD, which will cause SQ hang, since the 7,8th wave could not get arbit to exec inst, while other waves are stuck into the sleep-loop and waiting for wrexec!=0
+-
+- if (EMU_RUN_HACK)
+-
+- else
+- s_cbranch_execz L_SLEEP
+- end
+-
+-if G8SR_DEBUG_TIMESTAMP
+- s_memrealtime s_g8sr_ts_spi_wrexec
+- s_waitcnt lgkmcnt(0)
+-end
+-
+- /* setup Resource Contants */
+- if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_SAVE_SINGLE_WAVE))
+- //calculate wd_addr using absolute thread id
+- v_readlane_b32 s_save_tmp, v9, 0
+- s_lshr_b32 s_save_tmp, s_save_tmp, 6
+- s_mul_i32 s_save_tmp, s_save_tmp, WAVE_SPACE
+- s_add_i32 s_save_spi_init_lo, s_save_tmp, WG_BASE_ADDR_LO
+- s_mov_b32 s_save_spi_init_hi, WG_BASE_ADDR_HI
+- s_and_b32 s_save_spi_init_hi, s_save_spi_init_hi, CTX_SAVE_CONTROL
+- else
+- end
+- if ((EMU_RUN_HACK) && (EMU_RUN_HACK_SAVE_SINGLE_WAVE))
+- s_add_i32 s_save_spi_init_lo, s_save_tmp, WG_BASE_ADDR_LO
+- s_mov_b32 s_save_spi_init_hi, WG_BASE_ADDR_HI
+- s_and_b32 s_save_spi_init_hi, s_save_spi_init_hi, CTX_SAVE_CONTROL
+- else
+- end
+-
+-
+- s_mov_b32 s_save_buf_rsrc0, s_save_spi_init_lo //base_addr_lo
+- s_and_b32 s_save_buf_rsrc1, s_save_spi_init_hi, 0x0000FFFF //base_addr_hi
+- s_or_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, S_SAVE_BUF_RSRC_WORD1_STRIDE
+- s_mov_b32 s_save_buf_rsrc2, 0 //NUM_RECORDS initial value = 0 (in bytes) although not neccessarily inited
+- s_mov_b32 s_save_buf_rsrc3, S_SAVE_BUF_RSRC_WORD3_MISC
+- s_and_b32 s_save_tmp, s_save_spi_init_hi, S_SAVE_SPI_INIT_ATC_MASK
+- s_lshr_b32 s_save_tmp, s_save_tmp, (S_SAVE_SPI_INIT_ATC_SHIFT-SQ_BUF_RSRC_WORD1_ATC_SHIFT) //get ATC bit into position
+- s_or_b32 s_save_buf_rsrc3, s_save_buf_rsrc3, s_save_tmp //or ATC
+- s_and_b32 s_save_tmp, s_save_spi_init_hi, S_SAVE_SPI_INIT_MTYPE_MASK
+- s_lshr_b32 s_save_tmp, s_save_tmp, (S_SAVE_SPI_INIT_MTYPE_SHIFT-SQ_BUF_RSRC_WORD3_MTYPE_SHIFT) //get MTYPE bits into position
+- s_or_b32 s_save_buf_rsrc3, s_save_buf_rsrc3, s_save_tmp //or MTYPE
+-
+- //FIXME right now s_save_m0/s_save_mem_offset use tma_lo/tma_hi (might need to save them before using them?)
+- s_mov_b32 s_save_m0, m0 //save M0
+-
+- /* global mem offset */
+- s_mov_b32 s_save_mem_offset, 0x0 //mem offset initial value = 0
+-
+-
+-
+-
+- /* save HW registers */
+- //////////////////////////////
+-
+- L_SAVE_HWREG:
+- // HWREG SR memory offset : size(VGPR)+size(SGPR)
+- get_vgpr_size_bytes(s_save_mem_offset)
+- get_sgpr_size_bytes(s_save_tmp)
+- s_add_u32 s_save_mem_offset, s_save_mem_offset, s_save_tmp
+-
+-
+- s_mov_b32 s_save_buf_rsrc2, 0x4 //NUM_RECORDS in bytes
+- if (SWIZZLE_EN)
+- s_add_u32 s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+- else
+- s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+- end
+-
+-
+- write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset) //M0
+-
+- if ((EMU_RUN_HACK) && (EMU_RUN_HACK_SAVE_FIRST_TIME))
+- s_add_u32 s_save_pc_lo, s_save_pc_lo, 4 //pc[31:0]+4
+- s_addc_u32 s_save_pc_hi, s_save_pc_hi, 0x0 //carry bit over
+- end
+-
+- write_hwreg_to_mem(s_save_pc_lo, s_save_buf_rsrc0, s_save_mem_offset) //PC
+- write_hwreg_to_mem(s_save_pc_hi, s_save_buf_rsrc0, s_save_mem_offset)
+- write_hwreg_to_mem(s_save_exec_lo, s_save_buf_rsrc0, s_save_mem_offset) //EXEC
+- write_hwreg_to_mem(s_save_exec_hi, s_save_buf_rsrc0, s_save_mem_offset)
+- write_hwreg_to_mem(s_save_status, s_save_buf_rsrc0, s_save_mem_offset) //STATUS
+-
+- //s_save_trapsts conflicts with s_save_alloc_size
+- s_getreg_b32 s_save_trapsts, hwreg(HW_REG_TRAPSTS)
+- write_hwreg_to_mem(s_save_trapsts, s_save_buf_rsrc0, s_save_mem_offset) //TRAPSTS
+-
+- write_hwreg_to_mem(s_save_xnack_mask_lo, s_save_buf_rsrc0, s_save_mem_offset) //XNACK_MASK_LO
+- write_hwreg_to_mem(s_save_xnack_mask_hi, s_save_buf_rsrc0, s_save_mem_offset) //XNACK_MASK_HI
+-
+- //use s_save_tmp would introduce conflict here between s_save_tmp and s_save_buf_rsrc2
+- s_getreg_b32 s_save_m0, hwreg(HW_REG_MODE) //MODE
+- write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)
+-
+-
+-
+- /* the first wave in the threadgroup */
+- s_and_b32 s_save_tmp, s_save_spi_init_hi, S_SAVE_SPI_INIT_FIRST_WAVE_MASK // extract fisrt wave bit
+- s_mov_b32 s_save_exec_hi, 0x0
+- s_or_b32 s_save_exec_hi, s_save_tmp, s_save_exec_hi // save first wave bit to s_save_exec_hi.bits[26]
+-
+-
+- /* save SGPRs */
+- // Save SGPR before LDS save, then the s0 to s4 can be used during LDS save...
+- //////////////////////////////
+-
+- // SGPR SR memory offset : size(VGPR)
+- get_vgpr_size_bytes(s_save_mem_offset)
+- // TODO, change RSRC word to rearrange memory layout for SGPRS
+-
+- s_getreg_b32 s_save_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE) //spgr_size
+- s_add_u32 s_save_alloc_size, s_save_alloc_size, 1
+- s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 4 //Number of SGPRs = (sgpr_size + 1) * 16 (non-zero value)
+-
+- if (SGPR_SAVE_USE_SQC)
+- s_lshl_b32 s_save_buf_rsrc2, s_save_alloc_size, 2 //NUM_RECORDS in bytes
+- else
+- s_lshl_b32 s_save_buf_rsrc2, s_save_alloc_size, 8 //NUM_RECORDS in bytes (64 threads)
+- end
+-
+- if (SWIZZLE_EN)
+- s_add_u32 s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+- else
+- s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+- end
+-
+-
+- // backup s_save_buf_rsrc0,1 to s_save_pc_lo/hi, since write_16sgpr_to_mem function will change the rsrc0
+- //s_mov_b64 s_save_pc_lo, s_save_buf_rsrc0
+- s_mov_b64 s_save_xnack_mask_lo, s_save_buf_rsrc0
+- s_add_u32 s_save_buf_rsrc0, s_save_buf_rsrc0, s_save_mem_offset
+- s_addc_u32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0
+-
+- s_mov_b32 m0, 0x0 //SGPR initial index value =0
+- s_nop 0x0 //Manually inserted wait states
+- L_SAVE_SGPR_LOOP:
+- // SGPR is allocated in 16 SGPR granularity
+- s_movrels_b64 s0, s0 //s0 = s[0+m0], s1 = s[1+m0]
+- s_movrels_b64 s2, s2 //s2 = s[2+m0], s3 = s[3+m0]
+- s_movrels_b64 s4, s4 //s4 = s[4+m0], s5 = s[5+m0]
+- s_movrels_b64 s6, s6 //s6 = s[6+m0], s7 = s[7+m0]
+- s_movrels_b64 s8, s8 //s8 = s[8+m0], s9 = s[9+m0]
+- s_movrels_b64 s10, s10 //s10 = s[10+m0], s11 = s[11+m0]
+- s_movrels_b64 s12, s12 //s12 = s[12+m0], s13 = s[13+m0]
+- s_movrels_b64 s14, s14 //s14 = s[14+m0], s15 = s[15+m0]
+-
+- write_16sgpr_to_mem(s0, s_save_buf_rsrc0, s_save_mem_offset) //PV: the best performance should be using s_buffer_store_dwordx4
+- s_add_u32 m0, m0, 16 //next sgpr index
+- s_cmp_lt_u32 m0, s_save_alloc_size //scc = (m0 < s_save_alloc_size) ? 1 : 0
+- s_cbranch_scc1 L_SAVE_SGPR_LOOP //SGPR save is complete?
+- // restore s_save_buf_rsrc0,1
+- //s_mov_b64 s_save_buf_rsrc0, s_save_pc_lo
+- s_mov_b64 s_save_buf_rsrc0, s_save_xnack_mask_lo
+-
+-
+-
+-
+- /* save first 4 VGPR, then LDS save could use */
+- // each wave will alloc 4 vgprs at least...
+- /////////////////////////////////////////////////////////////////////////////////////
+-
+- s_mov_b32 s_save_mem_offset, 0
+- s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on
+- s_mov_b32 exec_hi, 0xFFFFFFFF
+-
+- if (SWIZZLE_EN)
+- s_add_u32 s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+- else
+- s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+- end
+-
+-
+- // VGPR Allocated in 4-GPR granularity
+-
+-if G8SR_VGPR_SR_IN_DWX4
+- // the const stride for DWx4 is 4*4 bytes
+- s_and_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0x0000FFFF // reset const stride to 0
+- s_or_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, G8SR_SAVE_BUF_RSRC_WORD1_STRIDE_DWx4 // const stride to 4*4 bytes
+-
+- buffer_store_dwordx4 v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
+-
+- s_and_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0x0000FFFF // reset const stride to 0
+- s_or_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, S_SAVE_BUF_RSRC_WORD1_STRIDE // reset const stride to 4 bytes
+-else
+- buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
+- buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256
+- buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256*2
+- buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256*3
+-end
+-
+-
+-
+- /* save LDS */
+- //////////////////////////////
+-
+- L_SAVE_LDS:
+-
+- // Change EXEC to all threads...
+- s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on
+- s_mov_b32 exec_hi, 0xFFFFFFFF
+-
+- s_getreg_b32 s_save_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE) //lds_size
+- s_and_b32 s_save_alloc_size, s_save_alloc_size, 0xFFFFFFFF //lds_size is zero?
+- s_cbranch_scc0 L_SAVE_LDS_DONE //no lds used? jump to L_SAVE_DONE
+-
+- s_barrier //LDS is used? wait for other waves in the same TG
+- s_and_b32 s_save_tmp, s_save_exec_hi, S_SAVE_SPI_INIT_FIRST_WAVE_MASK //exec is still used here
+- s_cbranch_scc0 L_SAVE_LDS_DONE
+-
+- // first wave do LDS save;
+-
+- s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 6 //LDS size in dwords = lds_size * 64dw
+- s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 2 //LDS size in bytes
+- s_mov_b32 s_save_buf_rsrc2, s_save_alloc_size //NUM_RECORDS in bytes
+-
+- // LDS at offset: size(VGPR)+SIZE(SGPR)+SIZE(HWREG)
+- //
+- get_vgpr_size_bytes(s_save_mem_offset)
+- get_sgpr_size_bytes(s_save_tmp)
+- s_add_u32 s_save_mem_offset, s_save_mem_offset, s_save_tmp
+- s_add_u32 s_save_mem_offset, s_save_mem_offset, get_hwreg_size_bytes()
+-
+-
+- if (SWIZZLE_EN)
+- s_add_u32 s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+- else
+- s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+- end
+-
+- s_mov_b32 m0, 0x0 //lds_offset initial value = 0
+-
+-
+-var LDS_DMA_ENABLE = 0
+-var UNROLL = 0
+-if UNROLL==0 && LDS_DMA_ENABLE==1
+- s_mov_b32 s3, 256*2
+- s_nop 0
+- s_nop 0
+- s_nop 0
+- L_SAVE_LDS_LOOP:
+- //TODO: looks the 2 buffer_store/load clause for s/r will hurt performance.???
+- if (SAVE_LDS) //SPI always alloc LDS space in 128DW granularity
+- buffer_store_lds_dword s_save_buf_rsrc0, s_save_mem_offset lds:1 // first 64DW
+- buffer_store_lds_dword s_save_buf_rsrc0, s_save_mem_offset lds:1 offset:256 // second 64DW
+- end
+-
+- s_add_u32 m0, m0, s3 //every buffer_store_lds does 256 bytes
+- s_add_u32 s_save_mem_offset, s_save_mem_offset, s3 //mem offset increased by 256 bytes
+- s_cmp_lt_u32 m0, s_save_alloc_size //scc=(m0 < s_save_alloc_size) ? 1 : 0
+- s_cbranch_scc1 L_SAVE_LDS_LOOP //LDS save is complete?
+-
+-elsif LDS_DMA_ENABLE==1 && UNROLL==1 // UNROOL , has ichace miss
+- // store from higest LDS address to lowest
+- s_mov_b32 s3, 256*2
+- s_sub_u32 m0, s_save_alloc_size, s3
+- s_add_u32 s_save_mem_offset, s_save_mem_offset, m0
+- s_lshr_b32 s_save_alloc_size, s_save_alloc_size, 9 // how many 128 trunks...
+- s_sub_u32 s_save_alloc_size, 128, s_save_alloc_size // store from higheset addr to lowest
+- s_mul_i32 s_save_alloc_size, s_save_alloc_size, 6*4 // PC offset increment, each LDS save block cost 6*4 Bytes instruction
+- s_add_u32 s_save_alloc_size, s_save_alloc_size, 3*4 //2is the below 2 inst...//s_addc and s_setpc
+- s_nop 0
+- s_nop 0
+- s_nop 0 //pad 3 dw to let LDS_DMA align with 64Bytes
+- s_getpc_b64 s[0:1] // reuse s[0:1], since s[0:1] already saved
+- s_add_u32 s0, s0,s_save_alloc_size
+- s_addc_u32 s1, s1, 0
+- s_setpc_b64 s[0:1]
+-
+-
+- for var i =0; i< 128; i++
+- // be careful to make here a 64Byte aligned address, which could improve performance...
+- buffer_store_lds_dword s_save_buf_rsrc0, s_save_mem_offset lds:1 offset:0 // first 64DW
+- buffer_store_lds_dword s_save_buf_rsrc0, s_save_mem_offset lds:1 offset:256 // second 64DW
+-
+- if i!=127
+- s_sub_u32 m0, m0, s3 // use a sgpr to shrink 2DW-inst to 1DW inst to improve performance , i.e. pack more LDS_DMA inst to one Cacheline
+- s_sub_u32 s_save_mem_offset, s_save_mem_offset, s3
+- end
+- end
+-
+-else // BUFFER_STORE
+- v_mbcnt_lo_u32_b32 v2, 0xffffffff, 0x0
+- v_mbcnt_hi_u32_b32 v3, 0xffffffff, v2 // tid
+- v_mul_i32_i24 v2, v3, 8 // tid*8
+- v_mov_b32 v3, 256*2
+- s_mov_b32 m0, 0x10000
+- s_mov_b32 s0, s_save_buf_rsrc3
+- s_and_b32 s_save_buf_rsrc3, s_save_buf_rsrc3, 0xFF7FFFFF // disable add_tid
+- s_or_b32 s_save_buf_rsrc3, s_save_buf_rsrc3, 0x58000 //DFMT
+-
+-L_SAVE_LDS_LOOP_VECTOR:
+- ds_read_b64 v[0:1], v2 //x =LDS[a], byte address
+- s_waitcnt lgkmcnt(0)
+- buffer_store_dwordx2 v[0:1], v2, s_save_buf_rsrc0, s_save_mem_offset offen:1 glc:1 slc:1
+-// s_waitcnt vmcnt(0)
+-// v_add_u32 v2, vcc[0:1], v2, v3
+- v_add_u32 v2, v2, v3
+- v_cmp_lt_u32 vcc[0:1], v2, s_save_alloc_size
+- s_cbranch_vccnz L_SAVE_LDS_LOOP_VECTOR
+-
+- // restore rsrc3
+- s_mov_b32 s_save_buf_rsrc3, s0
+-
+-end
+-
+-L_SAVE_LDS_DONE:
+-
+-
+- /* save VGPRs - set the Rest VGPRs */
+- //////////////////////////////////////////////////////////////////////////////////////
+- L_SAVE_VGPR:
+- // VGPR SR memory offset: 0
+- // TODO rearrange the RSRC words to use swizzle for VGPR save...
+-
+- s_mov_b32 s_save_mem_offset, (0+256*4) // for the rest VGPRs
+- s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on
+- s_mov_b32 exec_hi, 0xFFFFFFFF
+-
+- s_getreg_b32 s_save_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE) //vpgr_size
+- s_add_u32 s_save_alloc_size, s_save_alloc_size, 1
+- s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 2 //Number of VGPRs = (vgpr_size + 1) * 4 (non-zero value) //FIXME for GFX, zero is possible
+- s_lshl_b32 s_save_buf_rsrc2, s_save_alloc_size, 8 //NUM_RECORDS in bytes (64 threads*4)
+- if (SWIZZLE_EN)
+- s_add_u32 s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+- else
+- s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+- end
+-
+-
+- // VGPR Allocated in 4-GPR granularity
+-
+-if G8SR_VGPR_SR_IN_DWX4
+- // the const stride for DWx4 is 4*4 bytes
+- s_and_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0x0000FFFF // reset const stride to 0
+- s_or_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, G8SR_SAVE_BUF_RSRC_WORD1_STRIDE_DWx4 // const stride to 4*4 bytes
+-
+- s_mov_b32 m0, 4 // skip first 4 VGPRs
+- s_cmp_lt_u32 m0, s_save_alloc_size
+- s_cbranch_scc0 L_SAVE_VGPR_LOOP_END // no more vgprs
+-
+- s_set_gpr_idx_on m0, 0x1 // This will change M0
+- s_add_u32 s_save_alloc_size, s_save_alloc_size, 0x1000 // because above inst change m0
+-L_SAVE_VGPR_LOOP:
+- v_mov_b32 v0, v0 // v0 = v[0+m0]
+- v_mov_b32 v1, v1
+- v_mov_b32 v2, v2
+- v_mov_b32 v3, v3
+-
+-
+- buffer_store_dwordx4 v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
+- s_add_u32 m0, m0, 4
+- s_add_u32 s_save_mem_offset, s_save_mem_offset, 256*4
+- s_cmp_lt_u32 m0, s_save_alloc_size
+- s_cbranch_scc1 L_SAVE_VGPR_LOOP //VGPR save is complete?
+- s_set_gpr_idx_off
+-L_SAVE_VGPR_LOOP_END:
+-
+- s_and_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0x0000FFFF // reset const stride to 0
+- s_or_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, S_SAVE_BUF_RSRC_WORD1_STRIDE // reset const stride to 4 bytes
+-else
+- // VGPR store using dw burst
+- s_mov_b32 m0, 0x4 //VGPR initial index value =0
+- s_cmp_lt_u32 m0, s_save_alloc_size
+- s_cbranch_scc0 L_SAVE_VGPR_END
+-
+-
+- s_set_gpr_idx_on m0, 0x1 //M0[7:0] = M0[7:0] and M0[15:12] = 0x1
+- s_add_u32 s_save_alloc_size, s_save_alloc_size, 0x1000 //add 0x1000 since we compare m0 against it later
+-
+- L_SAVE_VGPR_LOOP:
+- v_mov_b32 v0, v0 //v0 = v[0+m0]
+- v_mov_b32 v1, v1 //v0 = v[0+m0]
+- v_mov_b32 v2, v2 //v0 = v[0+m0]
+- v_mov_b32 v3, v3 //v0 = v[0+m0]
+-
+- if(USE_MTBUF_INSTEAD_OF_MUBUF)
+- tbuffer_store_format_x v0, v0, s_save_buf_rsrc0, s_save_mem_offset format:BUF_NUM_FORMAT_FLOAT format: BUF_DATA_FORMAT_32 slc:1 glc:1
+- else
+- buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
+- buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256
+- buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256*2
+- buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256*3
+- end
+-
+- s_add_u32 m0, m0, 4 //next vgpr index
+- s_add_u32 s_save_mem_offset, s_save_mem_offset, 256*4 //every buffer_store_dword does 256 bytes
+- s_cmp_lt_u32 m0, s_save_alloc_size //scc = (m0 < s_save_alloc_size) ? 1 : 0
+- s_cbranch_scc1 L_SAVE_VGPR_LOOP //VGPR save is complete?
+- s_set_gpr_idx_off
+-end
+-
+-L_SAVE_VGPR_END:
+-
+-
+-
+-
+-
+-
+- /* S_PGM_END_SAVED */ //FIXME graphics ONLY
+- if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_SAVE_NORMAL_EXIT))
+- s_and_b32 s_save_pc_hi, s_save_pc_hi, 0x0000ffff //pc[47:32]
+- s_add_u32 s_save_pc_lo, s_save_pc_lo, 4 //pc[31:0]+4
+- s_addc_u32 s_save_pc_hi, s_save_pc_hi, 0x0 //carry bit over
+- s_rfe_b64 s_save_pc_lo //Return to the main shader program
+- else
+- end
+-
+-// Save Done timestamp
+-if G8SR_DEBUG_TIMESTAMP
+- s_memrealtime s_g8sr_ts_save_d
+- // SGPR SR memory offset : size(VGPR)
+- get_vgpr_size_bytes(s_save_mem_offset)
+- s_add_u32 s_save_mem_offset, s_save_mem_offset, G8SR_DEBUG_TS_SAVE_D_OFFSET
+- s_waitcnt lgkmcnt(0) //FIXME, will cause xnack??
+- // Need reset rsrc2??
+- s_mov_b32 m0, s_save_mem_offset
+- s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+- s_buffer_store_dwordx2 s_g8sr_ts_save_d, s_save_buf_rsrc0, m0 glc:1
+-end
+-
+-
+- s_branch L_END_PGM
+-
+-
+-
+-/**************************************************************************/
+-/* restore routine */
+-/**************************************************************************/
+-
+-L_RESTORE:
+- /* Setup Resource Contants */
+- if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_RESTORE_NORMAL))
+- //calculate wd_addr using absolute thread id
+- v_readlane_b32 s_restore_tmp, v9, 0
+- s_lshr_b32 s_restore_tmp, s_restore_tmp, 6
+- s_mul_i32 s_restore_tmp, s_restore_tmp, WAVE_SPACE
+- s_add_i32 s_restore_spi_init_lo, s_restore_tmp, WG_BASE_ADDR_LO
+- s_mov_b32 s_restore_spi_init_hi, WG_BASE_ADDR_HI
+- s_and_b32 s_restore_spi_init_hi, s_restore_spi_init_hi, CTX_RESTORE_CONTROL
+- else
+- end
+-
+-if G8SR_DEBUG_TIMESTAMP
+- s_memrealtime s_g8sr_ts_restore_s
+- s_waitcnt lgkmcnt(0) //FIXME, will cause xnack??
+- // tma_lo/hi are sgpr 110, 111, which will not used for 112 SGPR allocated case...
+- s_mov_b32 s_restore_pc_lo, s_g8sr_ts_restore_s[0]
+- s_mov_b32 s_restore_pc_hi, s_g8sr_ts_restore_s[1] //backup ts to ttmp0/1, sicne exec will be finally restored..
+-end
+-
+-
+-
+- s_mov_b32 s_restore_buf_rsrc0, s_restore_spi_init_lo //base_addr_lo
+- s_and_b32 s_restore_buf_rsrc1, s_restore_spi_init_hi, 0x0000FFFF //base_addr_hi
+- s_or_b32 s_restore_buf_rsrc1, s_restore_buf_rsrc1, S_RESTORE_BUF_RSRC_WORD1_STRIDE
+- s_mov_b32 s_restore_buf_rsrc2, 0 //NUM_RECORDS initial value = 0 (in bytes)
+- s_mov_b32 s_restore_buf_rsrc3, S_RESTORE_BUF_RSRC_WORD3_MISC
+- s_and_b32 s_restore_tmp, s_restore_spi_init_hi, S_RESTORE_SPI_INIT_ATC_MASK
+- s_lshr_b32 s_restore_tmp, s_restore_tmp, (S_RESTORE_SPI_INIT_ATC_SHIFT-SQ_BUF_RSRC_WORD1_ATC_SHIFT) //get ATC bit into position
+- s_or_b32 s_restore_buf_rsrc3, s_restore_buf_rsrc3, s_restore_tmp //or ATC
+- s_and_b32 s_restore_tmp, s_restore_spi_init_hi, S_RESTORE_SPI_INIT_MTYPE_MASK
+- s_lshr_b32 s_restore_tmp, s_restore_tmp, (S_RESTORE_SPI_INIT_MTYPE_SHIFT-SQ_BUF_RSRC_WORD3_MTYPE_SHIFT) //get MTYPE bits into position
+- s_or_b32 s_restore_buf_rsrc3, s_restore_buf_rsrc3, s_restore_tmp //or MTYPE
+-
+- /* global mem offset */
+-// s_mov_b32 s_restore_mem_offset, 0x0 //mem offset initial value = 0
+-
+- /* the first wave in the threadgroup */
+- s_and_b32 s_restore_tmp, s_restore_spi_init_hi, S_RESTORE_SPI_INIT_FIRST_WAVE_MASK
+- s_cbranch_scc0 L_RESTORE_VGPR
+-
+- /* restore LDS */
+- //////////////////////////////
+- L_RESTORE_LDS:
+-
+- s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on //be consistent with SAVE although can be moved ahead
+- s_mov_b32 exec_hi, 0xFFFFFFFF
+-
+- s_getreg_b32 s_restore_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE) //lds_size
+- s_and_b32 s_restore_alloc_size, s_restore_alloc_size, 0xFFFFFFFF //lds_size is zero?
+- s_cbranch_scc0 L_RESTORE_VGPR //no lds used? jump to L_RESTORE_VGPR
+- s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 6 //LDS size in dwords = lds_size * 64dw
+- s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 2 //LDS size in bytes
+- s_mov_b32 s_restore_buf_rsrc2, s_restore_alloc_size //NUM_RECORDS in bytes
+-
+- // LDS at offset: size(VGPR)+SIZE(SGPR)+SIZE(HWREG)
+- //
+- get_vgpr_size_bytes(s_restore_mem_offset)
+- get_sgpr_size_bytes(s_restore_tmp)
+- s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp
+- s_add_u32 s_restore_mem_offset, s_restore_mem_offset, get_hwreg_size_bytes() //FIXME, Check if offset overflow???
+-
+-
+- if (SWIZZLE_EN)
+- s_add_u32 s_restore_buf_rsrc2, s_restore_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+- else
+- s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+- end
+- s_mov_b32 m0, 0x0 //lds_offset initial value = 0
+-
+- L_RESTORE_LDS_LOOP:
+- if (SAVE_LDS)
+- buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset lds:1 // first 64DW
+- buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset lds:1 offset:256 // second 64DW
+- end
+- s_add_u32 m0, m0, 256*2 // 128 DW
+- s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 256*2 //mem offset increased by 128DW
+- s_cmp_lt_u32 m0, s_restore_alloc_size //scc=(m0 < s_restore_alloc_size) ? 1 : 0
+- s_cbranch_scc1 L_RESTORE_LDS_LOOP //LDS restore is complete?
+-
+-
+- /* restore VGPRs */
+- //////////////////////////////
+- L_RESTORE_VGPR:
+- // VGPR SR memory offset : 0
+- s_mov_b32 s_restore_mem_offset, 0x0
+- s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on //be consistent with SAVE although can be moved ahead
+- s_mov_b32 exec_hi, 0xFFFFFFFF
+-
+- s_getreg_b32 s_restore_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE) //vpgr_size
+- s_add_u32 s_restore_alloc_size, s_restore_alloc_size, 1
+- s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 2 //Number of VGPRs = (vgpr_size + 1) * 4 (non-zero value)
+- s_lshl_b32 s_restore_buf_rsrc2, s_restore_alloc_size, 8 //NUM_RECORDS in bytes (64 threads*4)
+- if (SWIZZLE_EN)
+- s_add_u32 s_restore_buf_rsrc2, s_restore_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+- else
+- s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+- end
+-
+-if G8SR_VGPR_SR_IN_DWX4
+- get_vgpr_size_bytes(s_restore_mem_offset)
+- s_sub_u32 s_restore_mem_offset, s_restore_mem_offset, 256*4
+-
+- // the const stride for DWx4 is 4*4 bytes
+- s_and_b32 s_restore_buf_rsrc1, s_restore_buf_rsrc1, 0x0000FFFF // reset const stride to 0
+- s_or_b32 s_restore_buf_rsrc1, s_restore_buf_rsrc1, G8SR_RESTORE_BUF_RSRC_WORD1_STRIDE_DWx4 // const stride to 4*4 bytes
+-
+- s_mov_b32 m0, s_restore_alloc_size
+- s_set_gpr_idx_on m0, 0x8 // Note.. This will change m0
+-
+-L_RESTORE_VGPR_LOOP:
+- buffer_load_dwordx4 v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1
+- s_waitcnt vmcnt(0)
+- s_sub_u32 m0, m0, 4
+- v_mov_b32 v0, v0 // v[0+m0] = v0
+- v_mov_b32 v1, v1
+- v_mov_b32 v2, v2
+- v_mov_b32 v3, v3
+- s_sub_u32 s_restore_mem_offset, s_restore_mem_offset, 256*4
+- s_cmp_eq_u32 m0, 0x8000
+- s_cbranch_scc0 L_RESTORE_VGPR_LOOP
+- s_set_gpr_idx_off
+-
+- s_and_b32 s_restore_buf_rsrc1, s_restore_buf_rsrc1, 0x0000FFFF // reset const stride to 0
+- s_or_b32 s_restore_buf_rsrc1, s_restore_buf_rsrc1, S_RESTORE_BUF_RSRC_WORD1_STRIDE // const stride to 4*4 bytes
+-
+-else
+- // VGPR load using dw burst
+- s_mov_b32 s_restore_mem_offset_save, s_restore_mem_offset // restore start with v1, v0 will be the last
+- s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 256*4
+- s_mov_b32 m0, 4 //VGPR initial index value = 1
+- s_set_gpr_idx_on m0, 0x8 //M0[7:0] = M0[7:0] and M0[15:12] = 0x8
+- s_add_u32 s_restore_alloc_size, s_restore_alloc_size, 0x8000 //add 0x8000 since we compare m0 against it later
+-
+- L_RESTORE_VGPR_LOOP:
+- if(USE_MTBUF_INSTEAD_OF_MUBUF)
+- tbuffer_load_format_x v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset format:BUF_NUM_FORMAT_FLOAT format: BUF_DATA_FORMAT_32 slc:1 glc:1
+- else
+- buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1
+- buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:256
+- buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:256*2
+- buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:256*3
+- end
+- s_waitcnt vmcnt(0) //ensure data ready
+- v_mov_b32 v0, v0 //v[0+m0] = v0
+- v_mov_b32 v1, v1
+- v_mov_b32 v2, v2
+- v_mov_b32 v3, v3
+- s_add_u32 m0, m0, 4 //next vgpr index
+- s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 256*4 //every buffer_load_dword does 256 bytes
+- s_cmp_lt_u32 m0, s_restore_alloc_size //scc = (m0 < s_restore_alloc_size) ? 1 : 0
+- s_cbranch_scc1 L_RESTORE_VGPR_LOOP //VGPR restore (except v0) is complete?
+- s_set_gpr_idx_off
+- /* VGPR restore on v0 */
+- if(USE_MTBUF_INSTEAD_OF_MUBUF)
+- tbuffer_load_format_x v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save format:BUF_NUM_FORMAT_FLOAT format: BUF_DATA_FORMAT_32 slc:1 glc:1
+- else
+- buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1
+- buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1 offset:256
+- buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1 offset:256*2
+- buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1 offset:256*3
+- end
+-
+-end
+-
+- /* restore SGPRs */
+- //////////////////////////////
+-
+- // SGPR SR memory offset : size(VGPR)
+- get_vgpr_size_bytes(s_restore_mem_offset)
+- get_sgpr_size_bytes(s_restore_tmp)
+- s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp
+- s_sub_u32 s_restore_mem_offset, s_restore_mem_offset, 16*4 // restore SGPR from S[n] to S[0], by 16 sgprs group
+- // TODO, change RSRC word to rearrange memory layout for SGPRS
+-
+- s_getreg_b32 s_restore_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE) //spgr_size
+- s_add_u32 s_restore_alloc_size, s_restore_alloc_size, 1
+- s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 4 //Number of SGPRs = (sgpr_size + 1) * 16 (non-zero value)
+-
+- if (SGPR_SAVE_USE_SQC)
+- s_lshl_b32 s_restore_buf_rsrc2, s_restore_alloc_size, 2 //NUM_RECORDS in bytes
+- else
+- s_lshl_b32 s_restore_buf_rsrc2, s_restore_alloc_size, 8 //NUM_RECORDS in bytes (64 threads)
+- end
+- if (SWIZZLE_EN)
+- s_add_u32 s_restore_buf_rsrc2, s_restore_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+- else
+- s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+- end
+-
+- s_mov_b32 m0, s_restore_alloc_size
+-
+- L_RESTORE_SGPR_LOOP:
+- read_16sgpr_from_mem(s0, s_restore_buf_rsrc0, s_restore_mem_offset) //PV: further performance improvement can be made
+- s_waitcnt lgkmcnt(0) //ensure data ready
+-
+- s_sub_u32 m0, m0, 16 // Restore from S[n] to S[0]
+- s_nop 0 // hazard SALU M0=> S_MOVREL
+-
+- s_movreld_b64 s0, s0 //s[0+m0] = s0
+- s_movreld_b64 s2, s2
+- s_movreld_b64 s4, s4
+- s_movreld_b64 s6, s6
+- s_movreld_b64 s8, s8
+- s_movreld_b64 s10, s10
+- s_movreld_b64 s12, s12
+- s_movreld_b64 s14, s14
+-
+- s_cmp_eq_u32 m0, 0 //scc = (m0 < s_restore_alloc_size) ? 1 : 0
+- s_cbranch_scc0 L_RESTORE_SGPR_LOOP //SGPR restore (except s0) is complete?
+-
+- /* restore HW registers */
+- //////////////////////////////
+- L_RESTORE_HWREG:
+-
+-
+-if G8SR_DEBUG_TIMESTAMP
+- s_mov_b32 s_g8sr_ts_restore_s[0], s_restore_pc_lo
+- s_mov_b32 s_g8sr_ts_restore_s[1], s_restore_pc_hi
+-end
+-
+- // HWREG SR memory offset : size(VGPR)+size(SGPR)
+- get_vgpr_size_bytes(s_restore_mem_offset)
+- get_sgpr_size_bytes(s_restore_tmp)
+- s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp
+-
+-
+- s_mov_b32 s_restore_buf_rsrc2, 0x4 //NUM_RECORDS in bytes
+- if (SWIZZLE_EN)
+- s_add_u32 s_restore_buf_rsrc2, s_restore_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+- else
+- s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+- end
+-
+- read_hwreg_from_mem(s_restore_m0, s_restore_buf_rsrc0, s_restore_mem_offset) //M0
+- read_hwreg_from_mem(s_restore_pc_lo, s_restore_buf_rsrc0, s_restore_mem_offset) //PC
+- read_hwreg_from_mem(s_restore_pc_hi, s_restore_buf_rsrc0, s_restore_mem_offset)
+- read_hwreg_from_mem(s_restore_exec_lo, s_restore_buf_rsrc0, s_restore_mem_offset) //EXEC
+- read_hwreg_from_mem(s_restore_exec_hi, s_restore_buf_rsrc0, s_restore_mem_offset)
+- read_hwreg_from_mem(s_restore_status, s_restore_buf_rsrc0, s_restore_mem_offset) //STATUS
+- read_hwreg_from_mem(s_restore_trapsts, s_restore_buf_rsrc0, s_restore_mem_offset) //TRAPSTS
+- read_hwreg_from_mem(xnack_mask_lo, s_restore_buf_rsrc0, s_restore_mem_offset) //XNACK_MASK_LO
+- read_hwreg_from_mem(xnack_mask_hi, s_restore_buf_rsrc0, s_restore_mem_offset) //XNACK_MASK_HI
+- read_hwreg_from_mem(s_restore_mode, s_restore_buf_rsrc0, s_restore_mem_offset) //MODE
+-
+- s_waitcnt lgkmcnt(0) //from now on, it is safe to restore STATUS and IB_STS
+-
+- s_and_b32 s_restore_pc_hi, s_restore_pc_hi, 0x0000ffff //pc[47:32] //Do it here in order not to affect STATUS
+-
+- //for normal save & restore, the saved PC points to the next inst to execute, no adjustment needs to be made, otherwise:
+- if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_RESTORE_NORMAL))
+- s_add_u32 s_restore_pc_lo, s_restore_pc_lo, 8 //pc[31:0]+8 //two back-to-back s_trap are used (first for save and second for restore)
+- s_addc_u32 s_restore_pc_hi, s_restore_pc_hi, 0x0 //carry bit over
+- end
+- if ((EMU_RUN_HACK) && (EMU_RUN_HACK_RESTORE_NORMAL))
+- s_add_u32 s_restore_pc_lo, s_restore_pc_lo, 4 //pc[31:0]+4 // save is hack through s_trap but restore is normal
+- s_addc_u32 s_restore_pc_hi, s_restore_pc_hi, 0x0 //carry bit over
+- end
+-
+- s_mov_b32 m0, s_restore_m0
+- s_mov_b32 exec_lo, s_restore_exec_lo
+- s_mov_b32 exec_hi, s_restore_exec_hi
+-
+- s_and_b32 s_restore_m0, SQ_WAVE_TRAPSTS_PRE_SAVECTX_MASK, s_restore_trapsts
+- s_setreg_b32 hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_PRE_SAVECTX_SHIFT, SQ_WAVE_TRAPSTS_PRE_SAVECTX_SIZE), s_restore_m0
+- s_and_b32 s_restore_m0, SQ_WAVE_TRAPSTS_POST_SAVECTX_MASK, s_restore_trapsts
+- s_lshr_b32 s_restore_m0, s_restore_m0, SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT
+- s_setreg_b32 hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT, SQ_WAVE_TRAPSTS_POST_SAVECTX_SIZE), s_restore_m0
+- //s_setreg_b32 hwreg(HW_REG_TRAPSTS), s_restore_trapsts //don't overwrite SAVECTX bit as it may be set through external SAVECTX during restore
+- s_setreg_b32 hwreg(HW_REG_MODE), s_restore_mode
+- //reuse s_restore_m0 as a temp register
+- s_and_b32 s_restore_m0, s_restore_pc_hi, S_SAVE_PC_HI_RCNT_MASK
+- s_lshr_b32 s_restore_m0, s_restore_m0, S_SAVE_PC_HI_RCNT_SHIFT
+- s_lshl_b32 s_restore_m0, s_restore_m0, SQ_WAVE_IB_STS_RCNT_SHIFT
+- s_mov_b32 s_restore_tmp, 0x0 //IB_STS is zero
+- s_or_b32 s_restore_tmp, s_restore_tmp, s_restore_m0
+- s_and_b32 s_restore_m0, s_restore_pc_hi, S_SAVE_PC_HI_FIRST_REPLAY_MASK
+- s_lshr_b32 s_restore_m0, s_restore_m0, S_SAVE_PC_HI_FIRST_REPLAY_SHIFT
+- s_lshl_b32 s_restore_m0, s_restore_m0, SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT
+- s_or_b32 s_restore_tmp, s_restore_tmp, s_restore_m0
+- s_and_b32 s_restore_m0, s_restore_status, SQ_WAVE_STATUS_INST_ATC_MASK
+- s_lshr_b32 s_restore_m0, s_restore_m0, SQ_WAVE_STATUS_INST_ATC_SHIFT
+- s_setreg_b32 hwreg(HW_REG_IB_STS), s_restore_tmp
+-
+- s_and_b64 exec, exec, exec // Restore STATUS.EXECZ, not writable by s_setreg_b32
+- s_and_b64 vcc, vcc, vcc // Restore STATUS.VCCZ, not writable by s_setreg_b32
+- s_setreg_b32 hwreg(HW_REG_STATUS), s_restore_status // SCC is included, which is changed by previous salu
+-
+- s_barrier //barrier to ensure the readiness of LDS before access attemps from any other wave in the same TG //FIXME not performance-optimal at this time
+-
+-if G8SR_DEBUG_TIMESTAMP
+- s_memrealtime s_g8sr_ts_restore_d
+- s_waitcnt lgkmcnt(0)
+-end
+-
+-// s_rfe_b64 s_restore_pc_lo //Return to the main shader program and resume execution
+- s_rfe_restore_b64 s_restore_pc_lo, s_restore_m0 // s_restore_m0[0] is used to set STATUS.inst_atc
+-
+-
+-/**************************************************************************/
+-/* the END */
+-/**************************************************************************/
+-L_END_PGM:
+- s_endpgm
+-
+-end
+-
+-
+-/**************************************************************************/
+-/* the helper functions */
+-/**************************************************************************/
+-
+-//Only for save hwreg to mem
+-function write_hwreg_to_mem(s, s_rsrc, s_mem_offset)
+- s_mov_b32 exec_lo, m0 //assuming exec_lo is not needed anymore from this point on
+- s_mov_b32 m0, s_mem_offset
+- s_buffer_store_dword s, s_rsrc, m0 glc:1
+- s_add_u32 s_mem_offset, s_mem_offset, 4
+- s_mov_b32 m0, exec_lo
+-end
+-
+-
+-// HWREG are saved before SGPRs, so all HWREG could be use.
+-function write_16sgpr_to_mem(s, s_rsrc, s_mem_offset)
+-
+- s_buffer_store_dwordx4 s[0], s_rsrc, 0 glc:1
+- s_buffer_store_dwordx4 s[4], s_rsrc, 16 glc:1
+- s_buffer_store_dwordx4 s[8], s_rsrc, 32 glc:1
+- s_buffer_store_dwordx4 s[12], s_rsrc, 48 glc:1
+- s_add_u32 s_rsrc[0], s_rsrc[0], 4*16
+- s_addc_u32 s_rsrc[1], s_rsrc[1], 0x0 // +scc
+-end
+-
+-
+-function read_hwreg_from_mem(s, s_rsrc, s_mem_offset)
+- s_buffer_load_dword s, s_rsrc, s_mem_offset glc:1
+- s_add_u32 s_mem_offset, s_mem_offset, 4
+-end
+-
+-function read_16sgpr_from_mem(s, s_rsrc, s_mem_offset)
+- s_buffer_load_dwordx16 s, s_rsrc, s_mem_offset glc:1
+- s_sub_u32 s_mem_offset, s_mem_offset, 4*16
+-end
+-
+-
+-
+-function get_lds_size_bytes(s_lds_size_byte)
+- // SQ LDS granularity is 64DW, while PGM_RSRC2.lds_size is in granularity 128DW
+- s_getreg_b32 s_lds_size_byte, hwreg(HW_REG_LDS_ALLOC, SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT, SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE) // lds_size
+- s_lshl_b32 s_lds_size_byte, s_lds_size_byte, 8 //LDS size in dwords = lds_size * 64 *4Bytes // granularity 64DW
+-end
+-
+-function get_vgpr_size_bytes(s_vgpr_size_byte)
+- s_getreg_b32 s_vgpr_size_byte, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE) //vpgr_size
+- s_add_u32 s_vgpr_size_byte, s_vgpr_size_byte, 1
+- s_lshl_b32 s_vgpr_size_byte, s_vgpr_size_byte, (2+8) //Number of VGPRs = (vgpr_size + 1) * 4 * 64 * 4 (non-zero value) //FIXME for GFX, zero is possible
+-end
+-
+-function get_sgpr_size_bytes(s_sgpr_size_byte)
+- s_getreg_b32 s_sgpr_size_byte, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE) //spgr_size
+- s_add_u32 s_sgpr_size_byte, s_sgpr_size_byte, 1
+- s_lshl_b32 s_sgpr_size_byte, s_sgpr_size_byte, 6 //Number of SGPRs = (sgpr_size + 1) * 16 *4 (non-zero value)
+-end
+-
+-function get_hwreg_size_bytes
+- return 128 //HWREG size 128 bytes
+-end
+-
+-
+-
+-#endif
+-
+-static const uint32_t cwsr_trap_gfx9_hex[] = {
+- 0xbf820001, 0xbf820124,
+- 0xb8f0f802, 0x89708670,
+- 0xb8f1f803, 0x8674ff71,
+- 0x00000400, 0xbf85001d,
+- 0x8674ff71, 0x00000800,
+- 0xbf850003, 0x8674ff71,
+- 0x00000100, 0xbf840003,
+- 0x8770ff70, 0x00002000,
+- 0xbf820010, 0xb8faf812,
+- 0xb8fbf813, 0x8efa887a,
+- 0xc00a1d3d, 0x00000000,
+- 0xbf8cc07f, 0x87737574,
+- 0xbf840002, 0xb970f802,
+- 0xbe801d74, 0xb8f1f803,
+- 0x8671ff71, 0x000001ff,
+- 0xbf850002, 0x806c846c,
+- 0x826d806d, 0x866dff6d,
+- 0x0000ffff, 0xb970f802,
+- 0xbe801f6c, 0xb8f1f803,
+- 0x8671ff71, 0x00000100,
+- 0xbf840006, 0xbef60080,
+- 0xb9760203, 0x866dff6d,
+- 0x0000ffff, 0x80ec886c,
+- 0x82ed806d, 0xbef60080,
+- 0xb9760283, 0xbef20068,
+- 0xbef30069, 0xb8f62407,
+- 0x8e769c76, 0x876d766d,
+- 0xb8f603c7, 0x8e769b76,
+- 0x876d766d, 0xb8f6f807,
+- 0x8676ff76, 0x00007fff,
+- 0xb976f807, 0xbeee007e,
+- 0xbeef007f, 0xbefe0180,
+- 0xbf900004, 0xbf8e0002,
+- 0xbf88fffe, 0xbef4007e,
+- 0x8675ff7f, 0x0000ffff,
+- 0x8775ff75, 0x00040000,
+- 0xbef60080, 0xbef700ff,
+- 0x00807fac, 0x8676ff7f,
+- 0x08000000, 0x8f768376,
+- 0x87777677, 0x8676ff7f,
+- 0x70000000, 0x8f768176,
+- 0x87777677, 0xbefb007c,
+- 0xbefa0080, 0xb8fa2a05,
+- 0x807a817a, 0x8e7a8a7a,
+- 0xb8f61605, 0x80768176,
+- 0x8e768676, 0x807a767a,
+- 0xbef60084, 0xbef600ff,
+- 0x01000000, 0xbefe007c,
+- 0xbefc007a, 0xc0611efa,
+- 0x0000007c, 0x807a847a,
+- 0xbefc007e, 0xbefe007c,
+- 0xbefc007a, 0xc0611b3a,
+- 0x0000007c, 0x807a847a,
+- 0xbefc007e, 0xbefe007c,
+- 0xbefc007a, 0xc0611b7a,
+- 0x0000007c, 0x807a847a,
+- 0xbefc007e, 0xbefe007c,
+- 0xbefc007a, 0xc0611bba,
+- 0x0000007c, 0x807a847a,
+- 0xbefc007e, 0xbefe007c,
+- 0xbefc007a, 0xc0611bfa,
+- 0x0000007c, 0x807a847a,
+- 0xbefc007e, 0xbefe007c,
+- 0xbefc007a, 0xc0611c3a,
+- 0x0000007c, 0x807a847a,
+- 0xbefc007e, 0xb8f1f803,
+- 0xbefe007c, 0xbefc007a,
+- 0xc0611c7a, 0x0000007c,
+- 0x807a847a, 0xbefc007e,
+- 0xbefe007c, 0xbefc007a,
+- 0xc0611cba, 0x0000007c,
+- 0x807a847a, 0xbefc007e,
+- 0xbefe007c, 0xbefc007a,
+- 0xc0611cfa, 0x0000007c,
+- 0x807a847a, 0xbefc007e,
+- 0xb8fbf801, 0xbefe007c,
+- 0xbefc007a, 0xc0611efa,
+- 0x0000007c, 0x807a847a,
+- 0xbefc007e, 0x8676ff7f,
+- 0x04000000, 0xbeef0080,
+- 0x876f6f76, 0xb8fa2a05,
+- 0x807a817a, 0x8e7a8a7a,
+- 0xb8f11605, 0x80718171,
+- 0x8e718471, 0x8e768271,
+- 0xbef600ff, 0x01000000,
+- 0xbef20174, 0x80747a74,
+- 0x82758075, 0xbefc0080,
+- 0xbf800000, 0xbe802b00,
+- 0xbe822b02, 0xbe842b04,
+- 0xbe862b06, 0xbe882b08,
+- 0xbe8a2b0a, 0xbe8c2b0c,
+- 0xbe8e2b0e, 0xc06b003a,
+- 0x00000000, 0xc06b013a,
+- 0x00000010, 0xc06b023a,
+- 0x00000020, 0xc06b033a,
+- 0x00000030, 0x8074c074,
+- 0x82758075, 0x807c907c,
+- 0xbf0a717c, 0xbf85ffeb,
+- 0xbef40172, 0xbefa0080,
+- 0xbefe00c1, 0xbeff00c1,
+- 0xbef600ff, 0x01000000,
+- 0xe0724000, 0x7a1d0000,
+- 0xe0724100, 0x7a1d0100,
+- 0xe0724200, 0x7a1d0200,
+- 0xe0724300, 0x7a1d0300,
+- 0xbefe00c1, 0xbeff00c1,
+- 0xb8f14306, 0x8671c171,
+- 0xbf84002c, 0xbf8a0000,
+- 0x8676ff6f, 0x04000000,
+- 0xbf840028, 0x8e718671,
+- 0x8e718271, 0xbef60071,
+- 0xb8fa2a05, 0x807a817a,
+- 0x8e7a8a7a, 0xb8f61605,
+- 0x80768176, 0x8e768676,
+- 0x807a767a, 0x807aff7a,
+- 0x00000080, 0xbef600ff,
+- 0x01000000, 0xbefc0080,
+- 0xd28c0002, 0x000100c1,
+- 0xd28d0003, 0x000204c1,
+- 0xd1060002, 0x00011103,
+- 0x7e0602ff, 0x00000200,
+- 0xbefc00ff, 0x00010000,
+- 0xbe800077, 0x8677ff77,
+- 0xff7fffff, 0x8777ff77,
+- 0x00058000, 0xd8ec0000,
+- 0x00000002, 0xbf8cc07f,
+- 0xe0765000, 0x7a1d0002,
+- 0x68040702, 0xd0c9006a,
+- 0x0000e302, 0xbf87fff7,
+- 0xbef70000, 0xbefa00ff,
+- 0x00000400, 0xbefe00c1,
+- 0xbeff00c1, 0xb8f12a05,
+- 0x80718171, 0x8e718271,
+- 0x8e768871, 0xbef600ff,
+- 0x01000000, 0xbefc0084,
+- 0xbf0a717c, 0xbf840015,
+- 0xbf11017c, 0x8071ff71,
+- 0x00001000, 0x7e000300,
+- 0x7e020301, 0x7e040302,
+- 0x7e060303, 0xe0724000,
+- 0x7a1d0000, 0xe0724100,
+- 0x7a1d0100, 0xe0724200,
+- 0x7a1d0200, 0xe0724300,
+- 0x7a1d0300, 0x807c847c,
+- 0x807aff7a, 0x00000400,
+- 0xbf0a717c, 0xbf85ffef,
+- 0xbf9c0000, 0xbf8200c5,
+- 0xbef4007e, 0x8675ff7f,
+- 0x0000ffff, 0x8775ff75,
+- 0x00040000, 0xbef60080,
+- 0xbef700ff, 0x00807fac,
+- 0x8672ff7f, 0x08000000,
+- 0x8f728372, 0x87777277,
+- 0x8672ff7f, 0x70000000,
+- 0x8f728172, 0x87777277,
+- 0x8672ff7f, 0x04000000,
+- 0xbf84001e, 0xbefe00c1,
+- 0xbeff00c1, 0xb8ef4306,
+- 0x866fc16f, 0xbf840019,
+- 0x8e6f866f, 0x8e6f826f,
+- 0xbef6006f, 0xb8f82a05,
+- 0x80788178, 0x8e788a78,
+- 0xb8f21605, 0x80728172,
+- 0x8e728672, 0x80787278,
+- 0x8078ff78, 0x00000080,
+- 0xbef600ff, 0x01000000,
+- 0xbefc0080, 0xe0510000,
+- 0x781d0000, 0xe0510100,
+- 0x781d0000, 0x807cff7c,
+- 0x00000200, 0x8078ff78,
+- 0x00000200, 0xbf0a6f7c,
+- 0xbf85fff6, 0xbef80080,
+- 0xbefe00c1, 0xbeff00c1,
+- 0xb8ef2a05, 0x806f816f,
+- 0x8e6f826f, 0x8e76886f,
+- 0xbef600ff, 0x01000000,
+- 0xbef20078, 0x8078ff78,
+- 0x00000400, 0xbefc0084,
+- 0xbf11087c, 0x806fff6f,
+- 0x00008000, 0xe0524000,
+- 0x781d0000, 0xe0524100,
+- 0x781d0100, 0xe0524200,
+- 0x781d0200, 0xe0524300,
+- 0x781d0300, 0xbf8c0f70,
+- 0x7e000300, 0x7e020301,
+- 0x7e040302, 0x7e060303,
+- 0x807c847c, 0x8078ff78,
+- 0x00000400, 0xbf0a6f7c,
+- 0xbf85ffee, 0xbf9c0000,
+- 0xe0524000, 0x721d0000,
+- 0xe0524100, 0x721d0100,
+- 0xe0524200, 0x721d0200,
+- 0xe0524300, 0x721d0300,
+- 0xb8f82a05, 0x80788178,
+- 0x8e788a78, 0xb8f21605,
+- 0x80728172, 0x8e728672,
+- 0x80787278, 0x80f8c078,
+- 0xb8ef1605, 0x806f816f,
+- 0x8e6f846f, 0x8e76826f,
+- 0xbef600ff, 0x01000000,
+- 0xbefc006f, 0xc031003a,
+- 0x00000078, 0x80f8c078,
+- 0xbf8cc07f, 0x80fc907c,
+- 0xbf800000, 0xbe802d00,
+- 0xbe822d02, 0xbe842d04,
+- 0xbe862d06, 0xbe882d08,
+- 0xbe8a2d0a, 0xbe8c2d0c,
+- 0xbe8e2d0e, 0xbf06807c,
+- 0xbf84fff0, 0xb8f82a05,
+- 0x80788178, 0x8e788a78,
+- 0xb8f21605, 0x80728172,
+- 0x8e728672, 0x80787278,
+- 0xbef60084, 0xbef600ff,
+- 0x01000000, 0xc0211bfa,
+- 0x00000078, 0x80788478,
+- 0xc0211b3a, 0x00000078,
+- 0x80788478, 0xc0211b7a,
+- 0x00000078, 0x80788478,
+- 0xc0211eba, 0x00000078,
+- 0x80788478, 0xc0211efa,
+- 0x00000078, 0x80788478,
+- 0xc0211c3a, 0x00000078,
+- 0x80788478, 0xc0211c7a,
+- 0x00000078, 0x80788478,
+- 0xc0211a3a, 0x00000078,
+- 0x80788478, 0xc0211a7a,
+- 0x00000078, 0x80788478,
+- 0xc0211cfa, 0x00000078,
+- 0x80788478, 0xbf8cc07f,
+- 0x866dff6d, 0x0000ffff,
+- 0xbefc006f, 0xbefe007a,
+- 0xbeff007b, 0x866f71ff,
+- 0x000003ff, 0xb96f4803,
+- 0x866f71ff, 0xfffff800,
+- 0x8f6f8b6f, 0xb96fa2c3,
+- 0xb973f801, 0x866fff6d,
+- 0xf0000000, 0x8f6f9c6f,
+- 0x8e6f906f, 0xbef20080,
+- 0x87726f72, 0x866fff6d,
+- 0x08000000, 0x8f6f9b6f,
+- 0x8e6f8f6f, 0x87726f72,
+- 0x866fff70, 0x00800000,
+- 0x8f6f976f, 0xb972f807,
+- 0x86fe7e7e, 0x86ea6a6a,
+- 0xb970f802, 0xbf8a0000,
+- 0x95806f6c, 0xbf810000,
+-};
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+index 207a05e..6316aad 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+@@ -25,7 +25,6 @@
+ #include <linux/err.h>
+ #include <linux/fs.h>
+ #include <linux/sched.h>
+-#include <linux/sched/mm.h>
+ #include <linux/slab.h>
+ #include <linux/uaccess.h>
+ #include <linux/compat.h>
+@@ -34,17 +33,13 @@
+ #include <linux/mm.h>
+ #include <linux/mman.h>
+ #include <asm/processor.h>
+-#include <linux/ptrace.h>
+-
+ #include "kfd_priv.h"
+ #include "kfd_device_queue_manager.h"
+ #include "kfd_dbgmgr.h"
+-#include "kfd_ipc.h"
+
+ static long kfd_ioctl(struct file *, unsigned int, unsigned long);
+ static int kfd_open(struct inode *, struct file *);
+ static int kfd_mmap(struct file *, struct vm_area_struct *);
+-static bool kfd_is_large_bar(struct kfd_dev *dev);
+
+ static const char kfd_dev_name[] = "kfd";
+
+@@ -60,14 +55,6 @@ static int kfd_char_dev_major = -1;
+ static struct class *kfd_class;
+ struct device *kfd_device;
+
+-static char *kfd_devnode(struct device *dev, umode_t *mode)
+-{
+- if (mode && dev->devt == MKDEV(kfd_char_dev_major, 0))
+- *mode = 0666;
+-
+- return NULL;
+-}
+-
+ int kfd_chardev_init(void)
+ {
+ int err = 0;
+@@ -82,8 +69,6 @@ int kfd_chardev_init(void)
+ if (IS_ERR(kfd_class))
+ goto err_class_create;
+
+- kfd_class->devnode = kfd_devnode;
+-
+ kfd_device = device_create(kfd_class, NULL,
+ MKDEV(kfd_char_dev_major, 0),
+ NULL, kfd_dev_name);
+@@ -132,7 +117,7 @@ static int kfd_open(struct inode *inode, struct file *filep)
+ return -EPERM;
+ }
+
+- process = kfd_create_process(filep);
++ process = kfd_create_process(current);
+ if (IS_ERR(process))
+ return PTR_ERR(process);
+
+@@ -157,12 +142,12 @@ static int set_queue_properties_from_user(struct queue_properties *q_properties,
+ struct kfd_ioctl_create_queue_args *args)
+ {
+ if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) {
+- pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
++ pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
+ return -EINVAL;
+ }
+
+ if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
+- pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
++ pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
+ return -EINVAL;
+ }
+
+@@ -170,26 +155,26 @@ static int set_queue_properties_from_user(struct queue_properties *q_properties,
+ (!access_ok(VERIFY_WRITE,
+ (const void __user *) args->ring_base_address,
+ sizeof(uint64_t)))) {
+- pr_err("Can't access ring base address\n");
++ pr_err("kfd: can't access ring base address\n");
+ return -EFAULT;
+ }
+
+ if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) {
+- pr_err("Ring size must be a power of 2 or 0\n");
++ pr_err("kfd: ring size must be a power of 2 or 0\n");
+ return -EINVAL;
+ }
+
+ if (!access_ok(VERIFY_WRITE,
+ (const void __user *) args->read_pointer_address,
+ sizeof(uint32_t))) {
+- pr_err("Can't access read pointer\n");
++ pr_err("kfd: can't access read pointer\n");
+ return -EFAULT;
+ }
+
+ if (!access_ok(VERIFY_WRITE,
+ (const void __user *) args->write_pointer_address,
+ sizeof(uint32_t))) {
+- pr_err("Can't access write pointer\n");
++ pr_err("kfd: can't access write pointer\n");
+ return -EFAULT;
+ }
+
+@@ -197,7 +182,7 @@ static int set_queue_properties_from_user(struct queue_properties *q_properties,
+ !access_ok(VERIFY_WRITE,
+ (const void __user *) args->eop_buffer_address,
+ sizeof(uint32_t))) {
+- pr_debug("Can't access eop buffer");
++ pr_debug("kfd: can't access eop buffer");
+ return -EFAULT;
+ }
+
+@@ -205,7 +190,7 @@ static int set_queue_properties_from_user(struct queue_properties *q_properties,
+ !access_ok(VERIFY_WRITE,
+ (const void __user *) args->ctx_save_restore_address,
+ sizeof(uint32_t))) {
+- pr_debug("Can't access ctx save restore buffer");
++ pr_debug("kfd: can't access ctx save restore buffer");
+ return -EFAULT;
+ }
+
+@@ -221,7 +206,6 @@ static int set_queue_properties_from_user(struct queue_properties *q_properties,
+ q_properties->ctx_save_restore_area_address =
+ args->ctx_save_restore_address;
+ q_properties->ctx_save_restore_area_size = args->ctx_save_restore_size;
+- q_properties->ctl_stack_size = args->ctl_stack_size;
+ if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE ||
+ args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
+ q_properties->type = KFD_QUEUE_TYPE_COMPUTE;
+@@ -235,27 +219,27 @@ static int set_queue_properties_from_user(struct queue_properties *q_properties,
+ else
+ q_properties->format = KFD_QUEUE_FORMAT_PM4;
+
+- pr_debug("Queue Percentage: %d, %d\n",
++ pr_debug("Queue Percentage (%d, %d)\n",
+ q_properties->queue_percent, args->queue_percentage);
+
+- pr_debug("Queue Priority: %d, %d\n",
++ pr_debug("Queue Priority (%d, %d)\n",
+ q_properties->priority, args->queue_priority);
+
+- pr_debug("Queue Address: 0x%llX, 0x%llX\n",
++ pr_debug("Queue Address (0x%llX, 0x%llX)\n",
+ q_properties->queue_address, args->ring_base_address);
+
+- pr_debug("Queue Size: 0x%llX, %u\n",
++ pr_debug("Queue Size (0x%llX, %u)\n",
+ q_properties->queue_size, args->ring_size);
+
+- pr_debug("Queue r/w Pointers: %p, %p\n",
+- q_properties->read_ptr,
+- q_properties->write_ptr);
++ pr_debug("Queue r/w Pointers (0x%llX, 0x%llX)\n",
++ (uint64_t) q_properties->read_ptr,
++ (uint64_t) q_properties->write_ptr);
+
+- pr_debug("Queue Format: %d\n", q_properties->format);
++ pr_debug("Queue Format (%d)\n", q_properties->format);
+
+- pr_debug("Queue EOP: 0x%llX\n", q_properties->eop_ring_buffer_address);
++ pr_debug("Queue EOP (0x%llX)\n", q_properties->eop_ring_buffer_address);
+
+- pr_debug("Queue CTX save area: 0x%llX\n",
++ pr_debug("Queue CTX save arex (0x%llX)\n",
+ q_properties->ctx_save_restore_area_address);
+
+ return 0;
+@@ -273,16 +257,16 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
+
+ memset(&q_properties, 0, sizeof(struct queue_properties));
+
+- pr_debug("Creating queue ioctl\n");
++ pr_debug("kfd: creating queue ioctl\n");
+
+ err = set_queue_properties_from_user(&q_properties, args);
+ if (err)
+ return err;
+
+- pr_debug("Looking for gpu id 0x%x\n", args->gpu_id);
++ pr_debug("kfd: looking for gpu id 0x%x\n", args->gpu_id);
+ dev = kfd_device_by_id(args->gpu_id);
+- if (!dev) {
+- pr_debug("Could not find gpu id 0x%x\n", args->gpu_id);
++ if (dev == NULL) {
++ pr_debug("kfd: gpu id 0x%x was not found\n", args->gpu_id);
+ return -EINVAL;
+ }
+
+@@ -294,11 +278,12 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
+ goto err_bind_process;
+ }
+
+- pr_debug("Creating queue for PASID %d on gpu 0x%x\n",
++ pr_debug("kfd: creating queue for PASID %d on GPU 0x%x\n",
+ p->pasid,
+ dev->id);
+
+- err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, &queue_id);
++ err = pqm_create_queue(&p->pqm, dev, filep, &q_properties,
++ 0, q_properties.type, &queue_id);
+ if (err != 0)
+ goto err_create_queue;
+
+@@ -306,28 +291,20 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
+
+
+ /* Return gpu_id as doorbell offset for mmap usage */
+- args->doorbell_offset = KFD_MMAP_TYPE_DOORBELL;
+- args->doorbell_offset |= KFD_MMAP_GPU_ID(args->gpu_id);
++ args->doorbell_offset = (KFD_MMAP_DOORBELL_MASK | args->gpu_id);
+ args->doorbell_offset <<= PAGE_SHIFT;
+- if (KFD_IS_SOC15(dev->device_info->asic_family))
+- /* On SOC15 ASICs, doorbell allocation must be
+- * per-device, and independent from the per-process
+- * queue_id. Return the doorbell offset within the
+- * doorbell aperture to user mode.
+- */
+- args->doorbell_offset |= q_properties.doorbell_off;
+
+ mutex_unlock(&p->mutex);
+
+- pr_debug("Queue id %d was created successfully\n", args->queue_id);
++ pr_debug("kfd: queue id %d was created successfully\n", args->queue_id);
+
+- pr_debug("Ring buffer address == 0x%016llX\n",
++ pr_debug("ring buffer address == 0x%016llX\n",
+ args->ring_base_address);
+
+- pr_debug("Read ptr address == 0x%016llX\n",
++ pr_debug("read ptr address == 0x%016llX\n",
+ args->read_pointer_address);
+
+- pr_debug("Write ptr address == 0x%016llX\n",
++ pr_debug("write ptr address == 0x%016llX\n",
+ args->write_pointer_address);
+
+ return 0;
+@@ -344,7 +321,7 @@ static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p,
+ int retval;
+ struct kfd_ioctl_destroy_queue_args *args = data;
+
+- pr_debug("Destroying queue id %d for pasid %d\n",
++ pr_debug("kfd: destroying queue id %d for PASID %d\n",
+ args->queue_id,
+ p->pasid);
+
+@@ -364,12 +341,12 @@ static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p,
+ struct queue_properties properties;
+
+ if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) {
+- pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
++ pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
+ return -EINVAL;
+ }
+
+ if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
+- pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
++ pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
+ return -EINVAL;
+ }
+
+@@ -377,12 +354,12 @@ static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p,
+ (!access_ok(VERIFY_WRITE,
+ (const void __user *) args->ring_base_address,
+ sizeof(uint64_t)))) {
+- pr_err("Can't access ring base address\n");
++ pr_err("kfd: can't access ring base address\n");
+ return -EFAULT;
+ }
+
+ if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) {
+- pr_err("Ring size must be a power of 2 or 0\n");
++ pr_err("kfd: ring size must be a power of 2 or 0\n");
+ return -EINVAL;
+ }
+
+@@ -391,7 +368,7 @@ static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p,
+ properties.queue_percent = args->queue_percentage;
+ properties.priority = args->queue_priority;
+
+- pr_debug("Updating queue id %d for pasid %d\n",
++ pr_debug("kfd: updating queue id %d for PASID %d\n",
+ args->queue_id, p->pasid);
+
+ mutex_lock(&p->mutex);
+@@ -403,58 +380,6 @@ static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p,
+ return retval;
+ }
+
+-static int kfd_ioctl_set_cu_mask(struct file *filp, struct kfd_process *p,
+- void *data)
+-{
+- int retval;
+- const int max_num_cus = 1024;
+- struct kfd_ioctl_set_cu_mask_args *args = data;
+- struct queue_properties properties;
+- uint32_t __user *cu_mask_ptr = (uint32_t __user *)args->cu_mask_ptr;
+- size_t cu_mask_size = sizeof(uint32_t) * (args->num_cu_mask / 32);
+-
+- if ((args->num_cu_mask % 32) != 0) {
+- pr_debug("num_cu_mask 0x%x must be a multiple of 32",
+- args->num_cu_mask);
+- return -EINVAL;
+- }
+-
+- properties.cu_mask_count = args->num_cu_mask;
+- if (properties.cu_mask_count == 0) {
+- pr_debug("CU mask cannot be 0");
+- return -EINVAL;
+- }
+-
+- /* To prevent an unreasonably large CU mask size, set an arbitrary
+- * limit of max_num_cus bits. We can then just drop any CU mask bits
+- * past max_num_cus bits and just use the first max_num_cus bits.
+- */
+- if (properties.cu_mask_count > max_num_cus) {
+- pr_debug("CU mask cannot be greater than 1024 bits");
+- properties.cu_mask_count = max_num_cus;
+- cu_mask_size = sizeof(uint32_t) * (max_num_cus/32);
+- }
+-
+- properties.cu_mask = kzalloc(cu_mask_size, GFP_KERNEL);
+- if (!properties.cu_mask)
+- return -ENOMEM;
+-
+- retval = copy_from_user(properties.cu_mask, cu_mask_ptr, cu_mask_size);
+- if (retval) {
+- pr_debug("Could not copy CU mask from userspace");
+- kfree(properties.cu_mask);
+- return -EFAULT;
+- }
+-
+- mutex_lock(&p->mutex);
+-
+- retval = pqm_set_cu_mask(&p->pqm, args->queue_id, &properties);
+-
+- mutex_unlock(&p->mutex);
+-
+- return retval;
+-}
+-
+ static int kfd_ioctl_set_memory_policy(struct file *filep,
+ struct kfd_process *p, void *data)
+ {
+@@ -475,7 +400,7 @@ static int kfd_ioctl_set_memory_policy(struct file *filep,
+ }
+
+ dev = kfd_device_by_id(args->gpu_id);
+- if (!dev)
++ if (dev == NULL)
+ return -EINVAL;
+
+ mutex_lock(&p->mutex);
+@@ -507,38 +432,6 @@ static int kfd_ioctl_set_memory_policy(struct file *filep,
+ return err;
+ }
+
+-static int kfd_ioctl_set_trap_handler(struct file *filep,
+- struct kfd_process *p, void *data)
+-{
+- struct kfd_ioctl_set_trap_handler_args *args = data;
+- struct kfd_dev *dev;
+- int err = 0;
+- struct kfd_process_device *pdd;
+-
+- dev = kfd_device_by_id(args->gpu_id);
+- if (!dev)
+- return -EINVAL;
+-
+- mutex_lock(&p->mutex);
+-
+- pdd = kfd_bind_process_to_device(dev, p);
+- if (IS_ERR(pdd)) {
+- err = -ESRCH;
+- goto out;
+- }
+-
+- if (dev->dqm->ops.set_trap_handler(dev->dqm,
+- &pdd->qpd,
+- args->tba_addr,
+- args->tma_addr))
+- err = -EINVAL;
+-
+-out:
+- mutex_unlock(&p->mutex);
+-
+- return err;
+-}
+-
+ static int kfd_ioctl_dbg_register(struct file *filep,
+ struct kfd_process *p, void *data)
+ {
+@@ -550,11 +443,16 @@ static int kfd_ioctl_dbg_register(struct file *filep,
+ long status = 0;
+
+ dev = kfd_device_by_id(args->gpu_id);
+- if (!dev)
++ if (dev == NULL)
+ return -EINVAL;
+
+- mutex_lock(&p->mutex);
++ if (dev->device_info->asic_family == CHIP_CARRIZO) {
++ pr_debug("kfd_ioctl_dbg_register not supported on CZ\n");
++ return -EINVAL;
++ }
++
+ mutex_lock(kfd_get_dbgmgr_mutex());
++ mutex_lock(&p->mutex);
+
+ /*
+ * make sure that we have pdd, if this the first queue created for
+@@ -562,11 +460,12 @@ static int kfd_ioctl_dbg_register(struct file *filep,
+ */
+ pdd = kfd_bind_process_to_device(dev, p);
+ if (IS_ERR(pdd)) {
+- status = PTR_ERR(pdd);
+- goto out;
++ mutex_unlock(&p->mutex);
++ mutex_unlock(kfd_get_dbgmgr_mutex());
++ return PTR_ERR(pdd);
+ }
+
+- if (!dev->dbgmgr) {
++ if (dev->dbgmgr == NULL) {
+ /* In case of a legal call, we have no dbgmgr yet */
+ create_ok = kfd_dbgmgr_create(&dbgmgr_ptr, dev);
+ if (create_ok) {
+@@ -581,9 +480,8 @@ static int kfd_ioctl_dbg_register(struct file *filep,
+ status = -EINVAL;
+ }
+
+-out:
+- mutex_unlock(kfd_get_dbgmgr_mutex());
+ mutex_unlock(&p->mutex);
++ mutex_unlock(kfd_get_dbgmgr_mutex());
+
+ return status;
+ }
+@@ -596,7 +494,7 @@ static int kfd_ioctl_dbg_unregister(struct file *filep,
+ long status;
+
+ dev = kfd_device_by_id(args->gpu_id);
+- if (!dev)
++ if (dev == NULL)
+ return -EINVAL;
+
+ if (dev->device_info->asic_family == CHIP_CARRIZO) {
+@@ -607,7 +505,7 @@ static int kfd_ioctl_dbg_unregister(struct file *filep,
+ mutex_lock(kfd_get_dbgmgr_mutex());
+
+ status = kfd_dbgmgr_unregister(dev->dbgmgr, p);
+- if (!status) {
++ if (status == 0) {
+ kfd_dbgmgr_destroy(dev->dbgmgr);
+ dev->dbgmgr = NULL;
+ }
+@@ -641,13 +539,21 @@ static int kfd_ioctl_dbg_address_watch(struct file *filep,
+ memset((void *) &aw_info, 0, sizeof(struct dbg_address_watch_info));
+
+ dev = kfd_device_by_id(args->gpu_id);
+- if (!dev)
++ if (dev == NULL)
++ return -EINVAL;
++
++ if (dev->device_info->asic_family == CHIP_CARRIZO) {
++ pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n");
+ return -EINVAL;
++ }
+
+ cmd_from_user = (void __user *) args->content_ptr;
+
+- if (args->buf_size_in_bytes > MAX_ALLOWED_AW_BUFF_SIZE ||
+- (args->buf_size_in_bytes <= sizeof(*args)))
++ /* Validate arguments */
++
++ if ((args->buf_size_in_bytes > MAX_ALLOWED_AW_BUFF_SIZE) ||
++ (args->buf_size_in_bytes <= sizeof(*args) + sizeof(int) * 2) ||
++ (cmd_from_user == NULL))
+ return -EINVAL;
+
+ /* this is the actual buffer to work with */
+@@ -673,9 +579,9 @@ static int kfd_ioctl_dbg_address_watch(struct file *filep,
+ /* skip over the addresses buffer */
+ args_idx += sizeof(aw_info.watch_address) * aw_info.num_watch_points;
+
+- if (args_idx >= args->buf_size_in_bytes) {
+- status = -EINVAL;
+- goto out;
++ if (args_idx >= args->buf_size_in_bytes - sizeof(*args)) {
++ kfree(args_buff);
++ return -EINVAL;
+ }
+
+ watch_mask_value = (uint64_t) args_buff[args_idx];
+@@ -697,9 +603,9 @@ static int kfd_ioctl_dbg_address_watch(struct file *filep,
+ args_idx += sizeof(aw_info.watch_mask);
+ }
+
+- if (args_idx > args->buf_size_in_bytes) {
+- status = -EINVAL;
+- goto out;
++ if (args_idx >= args->buf_size_in_bytes - sizeof(args)) {
++ kfree(args_buff);
++ return -EINVAL;
+ }
+
+ /* Currently HSA Event is not supported for DBG */
+@@ -711,7 +617,6 @@ static int kfd_ioctl_dbg_address_watch(struct file *filep,
+
+ mutex_unlock(kfd_get_dbgmgr_mutex());
+
+-out:
+ kfree(args_buff);
+
+ return status;
+@@ -741,9 +646,14 @@ static int kfd_ioctl_dbg_wave_control(struct file *filep,
+ sizeof(wac_info.trapId);
+
+ dev = kfd_device_by_id(args->gpu_id);
+- if (!dev)
++ if (dev == NULL)
+ return -EINVAL;
+
++ if (dev->device_info->asic_family == CHIP_CARRIZO) {
++ pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n");
++ return -EINVAL;
++ }
++
+ /* input size must match the computed "compact" size */
+ if (args->buf_size_in_bytes != computed_buff_size) {
+ pr_debug("size mismatch, computed : actual %u : %u\n",
+@@ -802,37 +712,22 @@ static int kfd_ioctl_get_clock_counters(struct file *filep,
+ {
+ struct kfd_ioctl_get_clock_counters_args *args = data;
+ struct kfd_dev *dev;
+-#if (defined OS_NAME_RHEL) && (OS_VERSION_MAJOR == 6) \
+- || (defined OS_NAME_RHEL_7_2)
+- struct timespec time;
+-#else
+ struct timespec64 time;
+-#endif
+
+ dev = kfd_device_by_id(args->gpu_id);
+- if (dev)
+- /* Reading GPU clock counter from KGD */
+- args->gpu_clock_counter =
+- dev->kfd2kgd->get_gpu_clock_counter(dev->kgd);
+- else
+- /* Node without GPU resource */
+- args->gpu_clock_counter = 0;
++ if (dev == NULL)
++ return -EINVAL;
++
++ /* Reading GPU clock counter from KGD */
++ args->gpu_clock_counter =
++ dev->kfd2kgd->get_gpu_clock_counter(dev->kgd);
+
+ /* No access to rdtsc. Using raw monotonic time */
+-#if (defined OS_NAME_RHEL) && (OS_VERSION_MAJOR == 6) \
+- || (defined OS_NAME_RHEL_7_2)
+- getrawmonotonic(&time);
+- args->cpu_clock_counter = (uint64_t)timespec_to_ns(&time);
+-
+- get_monotonic_boottime(&time);
+- args->system_clock_counter = (uint64_t)timespec_to_ns(&time);
+-#else
+ getrawmonotonic64(&time);
+ args->cpu_clock_counter = (uint64_t)timespec64_to_ns(&time);
+
+ get_monotonic_boottime64(&time);
+ args->system_clock_counter = (uint64_t)timespec64_to_ns(&time);
+-#endif
+
+ /* Since the counter is in nano-seconds we use 1GHz frequency */
+ args->system_clock_freq = 1000000000;
+@@ -887,104 +782,12 @@ static int kfd_ioctl_get_process_apertures(struct file *filp,
+ "scratch_limit %llX\n", pdd->scratch_limit);
+
+ args->num_of_nodes++;
+-
+- pdd = kfd_get_next_process_device_data(p, pdd);
+- } while (pdd && (args->num_of_nodes < NUM_OF_SUPPORTED_GPUS));
+- }
+-
+- mutex_unlock(&p->mutex);
+-
+- return 0;
+-}
+-
+-static int kfd_ioctl_get_process_apertures_new(struct file *filp,
+- struct kfd_process *p, void *data)
+-{
+- struct kfd_ioctl_get_process_apertures_new_args *args = data;
+- struct kfd_process_device_apertures *pa;
+- struct kfd_process_device *pdd;
+- uint32_t nodes = 0;
+- int ret;
+-
+- dev_dbg(kfd_device, "get apertures for PASID %d", p->pasid);
+-
+- if (args->num_of_nodes == 0) {
+- /* Return number of nodes, so that user space can alloacate
+- * sufficient memory
+- */
+- mutex_lock(&p->mutex);
+-
+- if (!kfd_has_process_device_data(p))
+- goto out_upwrite;
+-
+- /* Run over all pdd of the process */
+- pdd = kfd_get_first_process_device_data(p);
+- do {
+- args->num_of_nodes++;
+- pdd = kfd_get_next_process_device_data(p, pdd);
+- } while (pdd);
+-
+- goto out_upwrite;
+- }
+-
+- /* Fill in process-aperture information for all available
+- * nodes, but not more than args->num_of_nodes as that is
+- * the amount of memory allocated by user
+- */
+- pa = kzalloc((sizeof(struct kfd_process_device_apertures) *
+- args->num_of_nodes), GFP_KERNEL);
+- if (!pa)
+- return -ENOMEM;
+-
+- mutex_lock(&p->mutex);
+-
+- if (!kfd_has_process_device_data(p)) {
+- args->num_of_nodes = 0;
+- kfree(pa);
+- goto out_upwrite;
++ } while ((pdd = kfd_get_next_process_device_data(p, pdd)) != NULL &&
++ (args->num_of_nodes < NUM_OF_SUPPORTED_GPUS));
+ }
+
+- /* Run over all pdd of the process */
+- pdd = kfd_get_first_process_device_data(p);
+- do {
+- pa[nodes].gpu_id = pdd->dev->id;
+- pa[nodes].lds_base = pdd->lds_base;
+- pa[nodes].lds_limit = pdd->lds_limit;
+- pa[nodes].gpuvm_base = pdd->gpuvm_base;
+- pa[nodes].gpuvm_limit = pdd->gpuvm_limit;
+- pa[nodes].scratch_base = pdd->scratch_base;
+- pa[nodes].scratch_limit = pdd->scratch_limit;
+-
+- dev_dbg(kfd_device,
+- "gpu id %u\n", pdd->dev->id);
+- dev_dbg(kfd_device,
+- "lds_base %llX\n", pdd->lds_base);
+- dev_dbg(kfd_device,
+- "lds_limit %llX\n", pdd->lds_limit);
+- dev_dbg(kfd_device,
+- "gpuvm_base %llX\n", pdd->gpuvm_base);
+- dev_dbg(kfd_device,
+- "gpuvm_limit %llX\n", pdd->gpuvm_limit);
+- dev_dbg(kfd_device,
+- "scratch_base %llX\n", pdd->scratch_base);
+- dev_dbg(kfd_device,
+- "scratch_limit %llX\n", pdd->scratch_limit);
+- nodes++;
+-
+- pdd = kfd_get_next_process_device_data(p, pdd);
+- } while (pdd && (nodes < args->num_of_nodes));
+ mutex_unlock(&p->mutex);
+
+- args->num_of_nodes = nodes;
+- ret = copy_to_user(
+- (void __user *)args->kfd_process_device_apertures_ptr,
+- pa,
+- (nodes * sizeof(struct kfd_process_device_apertures)));
+- kfree(pa);
+- return ret ? -EFAULT : 0;
+-
+-out_upwrite:
+- mutex_unlock(&p->mutex);
+ return 0;
+ }
+
+@@ -992,57 +795,15 @@ static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p,
+ void *data)
+ {
+ struct kfd_ioctl_create_event_args *args = data;
+- struct kfd_dev *kfd;
+- struct kfd_process_device *pdd;
+- int err = -EINVAL;
+- void *mem, *kern_addr = NULL;
+-
+- pr_debug("Event page offset 0x%llx\n", args->event_page_offset);
+-
+- if (args->event_page_offset) {
+- kfd = kfd_device_by_id(GET_GPU_ID(args->event_page_offset));
+- if (!kfd) {
+- pr_err("Getting device by id failed in %s\n", __func__);
+- return -EFAULT;
+- }
+- if (!kfd->device_info->is_need_iommu_device) {
+- mutex_lock(&p->mutex);
+- pdd = kfd_bind_process_to_device(kfd, p);
+- if (IS_ERR(pdd)) {
+- err = PTR_ERR(pdd);
+- goto out_upwrite;
+- }
+- mem = kfd_process_device_translate_handle(pdd,
+- GET_IDR_HANDLE(args->event_page_offset));
+- if (!mem) {
+- pr_err("Can't find BO, offset is 0x%llx\n",
+- args->event_page_offset);
+- err = -EFAULT;
+- goto out_upwrite;
+- }
+- mutex_unlock(&p->mutex);
+-
+- /* Map dGPU gtt BO to kernel */
+- kfd->kfd2kgd->map_gtt_bo_to_kernel(kfd->kgd,
+- mem, &kern_addr);
+- }
+- }
++ int err;
+
+- err = kfd_event_create(filp, p,
+- args->event_type,
+- args->auto_reset != 0,
+- args->node_id,
+- &args->event_id,
+- &args->event_trigger_data,
+- &args->event_page_offset,
+- &args->event_slot_index,
+- kern_addr);
++ err = kfd_event_create(filp, p, args->event_type,
++ args->auto_reset != 0, args->node_id,
++ &args->event_id, &args->event_trigger_data,
++ &args->event_page_offset,
++ &args->event_slot_index);
+
+ return err;
+-
+-out_upwrite:
+- mutex_unlock(&p->mutex);
+- return err;
+ }
+
+ static int kfd_ioctl_destroy_event(struct file *filp, struct kfd_process *p,
+@@ -1085,870 +846,9 @@ static int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p,
+
+ return err;
+ }
+-static int kfd_ioctl_alloc_scratch_memory(struct file *filep,
+- struct kfd_process *p, void *data)
+-{
+- struct kfd_ioctl_alloc_memory_of_scratch_args *args = data;
+- struct kfd_process_device *pdd;
+- struct kfd_dev *dev;
+- long err;
+-
+- if (args->size == 0)
+- return -EINVAL;
+-
+- dev = kfd_device_by_id(args->gpu_id);
+- if (!dev)
+- return -EINVAL;
+-
+- mutex_lock(&p->mutex);
+-
+- pdd = kfd_bind_process_to_device(dev, p);
+- if (IS_ERR(pdd)) {
+- err = PTR_ERR(pdd);
+- goto bind_process_to_device_fail;
+- }
+-
+- pdd->sh_hidden_private_base_vmid = args->va_addr;
+- pdd->qpd.sh_hidden_private_base = args->va_addr;
+-
+- mutex_unlock(&p->mutex);
+-
+- if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS &&
+- pdd->qpd.vmid != 0) {
+- err = dev->kfd2kgd->alloc_memory_of_scratch(
+- dev->kgd, args->va_addr, pdd->qpd.vmid);
+- if (err != 0)
+- goto alloc_memory_of_scratch_failed;
+- }
+-
+- return 0;
+-
+-bind_process_to_device_fail:
+- mutex_unlock(&p->mutex);
+-alloc_memory_of_scratch_failed:
+- return -EFAULT;
+-}
+-
+-bool kfd_is_large_bar(struct kfd_dev *dev)
+-{
+- struct kfd_local_mem_info mem_info;
+-
+- if (debug_largebar) {
+- pr_debug("Simulate large-bar allocation on non large-bar machine\n");
+- return true;
+- }
+-
+- if (dev->device_info->is_need_iommu_device)
+- return false;
+-
+- dev->kfd2kgd->get_local_mem_info(dev->kgd, &mem_info);
+- if (mem_info.local_mem_size_private == 0 &&
+- mem_info.local_mem_size_public > 0)
+- return true;
+- return false;
+-}
+-
+-static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep,
+- struct kfd_process *p, void *data)
+-{
+- struct kfd_ioctl_alloc_memory_of_gpu_args *args = data;
+- struct kfd_process_device *pdd;
+- void *mem;
+- struct kfd_dev *dev;
+- int idr_handle;
+- long err;
+- uint64_t offset = args->mmap_offset;
+- uint32_t flags = args->flags;
+- struct vm_area_struct *vma;
+-
+- if (args->size == 0)
+- return -EINVAL;
+-
+- dev = kfd_device_by_id(args->gpu_id);
+- if (!dev)
+- return -EINVAL;
+-
+- if (flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
+- /* Check if the userptr corresponds to another (or third-party)
+- * device local memory. If so treat is as a doorbell. User
+- * space will be oblivious of this and will use this doorbell
+- * BO as a regular userptr BO
+- */
+- vma = find_vma(current->mm, args->mmap_offset);
+- if (vma && (vma->vm_flags & VM_IO)) {
+- unsigned long pfn;
+-
+- follow_pfn(vma, args->mmap_offset, &pfn);
+- flags |= KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL;
+- flags &= ~KFD_IOC_ALLOC_MEM_FLAGS_USERPTR;
+- offset = (pfn << PAGE_SHIFT);
+- }
+- } else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) {
+- if (args->size != kfd_doorbell_process_slice(dev))
+- return -EINVAL;
+- offset = kfd_get_process_doorbells(dev, p);
+- }
+-
+- mutex_lock(&p->mutex);
+-
+- pdd = kfd_bind_process_to_device(dev, p);
+- if (IS_ERR(pdd)) {
+- err = PTR_ERR(pdd);
+- goto err_unlock;
+- }
+-
+- err = dev->kfd2kgd->alloc_memory_of_gpu(
+- dev->kgd, args->va_addr, args->size,
+- pdd->vm, (struct kgd_mem **) &mem, &offset,
+- flags);
+-
+- if (err)
+- goto err_unlock;
+-
+- idr_handle = kfd_process_device_create_obj_handle(pdd, mem,
+- args->va_addr, args->size, NULL);
+- if (idr_handle < 0) {
+- err = -EFAULT;
+- goto err_free;
+- }
+-
+- mutex_unlock(&p->mutex);
+-
+- args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
+- if ((args->flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) != 0 &&
+- !kfd_is_large_bar(dev)) {
+- args->mmap_offset = 0;
+- } else {
+- args->mmap_offset = KFD_MMAP_TYPE_MAP_BO;
+- args->mmap_offset |= KFD_MMAP_GPU_ID(args->gpu_id);
+- args->mmap_offset <<= PAGE_SHIFT;
+- args->mmap_offset |= offset;
+- }
+-
+- return 0;
+-
+-err_free:
+- dev->kfd2kgd->free_memory_of_gpu(dev->kgd,
+- (struct kgd_mem *) mem,
+- pdd->vm);
+-err_unlock:
+- mutex_unlock(&p->mutex);
+- return err;
+-}
+-
+-static int kfd_ioctl_free_memory_of_gpu(struct file *filep,
+- struct kfd_process *p, void *data)
+-{
+- struct kfd_ioctl_free_memory_of_gpu_args *args = data;
+- struct kfd_process_device *pdd;
+- struct kfd_bo *buf_obj;
+- struct kfd_dev *dev;
+- int ret;
+-
+- dev = kfd_device_by_id(GET_GPU_ID(args->handle));
+- if (!dev)
+- return -EINVAL;
+-
+- mutex_lock(&p->mutex);
+-
+- pdd = kfd_get_process_device_data(dev, p);
+- if (!pdd) {
+- pr_err("Process device data doesn't exist\n");
+- ret = -EINVAL;
+- goto err_unlock;
+- }
+-
+- buf_obj = kfd_process_device_find_bo(pdd,
+- GET_IDR_HANDLE(args->handle));
+- if (!buf_obj) {
+- ret = -EINVAL;
+- goto err_unlock;
+- }
+- run_rdma_free_callback(buf_obj);
+-
+- ret = dev->kfd2kgd->free_memory_of_gpu(dev->kgd, buf_obj->mem,
+- pdd->vm);
+-
+- /* If freeing the buffer failed, leave the handle in place for
+- * clean-up during process tear-down.
+- */
+- if (ret == 0)
+- kfd_process_device_remove_obj_handle(
+- pdd, GET_IDR_HANDLE(args->handle));
+-
+-err_unlock:
+- mutex_unlock(&p->mutex);
+- return ret;
+-}
+-
+-static int kfd_ioctl_map_memory_to_gpu(struct file *filep,
+- struct kfd_process *p, void *data)
+-{
+- struct kfd_ioctl_map_memory_to_gpu_args *args = data;
+- struct kfd_process_device *pdd, *peer_pdd;
+- void *mem;
+- struct kfd_dev *dev, *peer;
+- long err = 0;
+- int i, num_dev = 0;
+- uint32_t *devices_arr = NULL;
+-
+- dev = kfd_device_by_id(GET_GPU_ID(args->handle));
+- if (!dev)
+- return -EINVAL;
+-
+- if (args->device_ids_array_size > 0 &&
+- (args->device_ids_array_size < sizeof(uint32_t))) {
+- pr_err("Node IDs array size %u\n",
+- args->device_ids_array_size);
+- return -EFAULT;
+- }
+-
+- if (args->device_ids_array_size > 0) {
+- devices_arr = kmalloc(args->device_ids_array_size, GFP_KERNEL);
+- if (!devices_arr)
+- return -ENOMEM;
+-
+- err = copy_from_user(devices_arr,
+- (void __user *)args->device_ids_array_ptr,
+- args->device_ids_array_size);
+- if (err != 0) {
+- err = -EFAULT;
+- goto copy_from_user_failed;
+- }
+- }
+-
+- mutex_lock(&p->mutex);
+-
+- pdd = kfd_bind_process_to_device(dev, p);
+- if (IS_ERR(pdd)) {
+- err = PTR_ERR(pdd);
+- goto bind_process_to_device_failed;
+- }
+-
+- mem = kfd_process_device_translate_handle(pdd,
+- GET_IDR_HANDLE(args->handle));
+- if (!mem) {
+- err = PTR_ERR(mem);
+- goto get_mem_obj_from_handle_failed;
+- }
+-
+- if (args->device_ids_array_size > 0) {
+- num_dev = args->device_ids_array_size / sizeof(uint32_t);
+- for (i = 0 ; i < num_dev; i++) {
+- peer = kfd_device_by_id(devices_arr[i]);
+- if (!peer) {
+- pr_err("Getting device by id failed for 0x%x\n",
+- devices_arr[i]);
+- err = -EFAULT;
+- goto get_mem_obj_from_handle_failed;
+- }
+-
+- peer_pdd = kfd_bind_process_to_device(peer, p);
+- if (!peer_pdd) {
+- err = -EFAULT;
+- goto get_mem_obj_from_handle_failed;
+- }
+- err = peer->kfd2kgd->map_memory_to_gpu(
+- peer->kgd, (struct kgd_mem *)mem, peer_pdd->vm);
+- if (err != 0)
+- pr_err("Failed to map\n");
+- }
+- } else {
+- err = dev->kfd2kgd->map_memory_to_gpu(
+- dev->kgd, (struct kgd_mem *)mem, pdd->vm);
+- if (err != 0)
+- pr_err("Failed to map\n");
+- }
+-
+- mutex_unlock(&p->mutex);
+-
+- err = dev->kfd2kgd->sync_memory(dev->kgd, (struct kgd_mem *) mem, true);
+- if (err) {
+- pr_debug("Sync memory failed, wait interrupted by user signal\n");
+- goto sync_memory_failed;
+- }
+-
+- /* Flush TLBs after waiting for the page table updates to complete */
+- if (args->device_ids_array_size > 0) {
+- for (i = 0; i < num_dev; i++) {
+- peer = kfd_device_by_id(devices_arr[i]);
+- if (WARN_ON_ONCE(!peer))
+- continue;
+- peer_pdd = kfd_get_process_device_data(dev, p);
+- if (WARN_ON_ONCE(!peer_pdd))
+- continue;
+- kfd_flush_tlb(peer, p->pasid);
+- }
+- } else {
+- kfd_flush_tlb(dev, p->pasid);
+- }
+-
+- if (args->device_ids_array_size > 0 && devices_arr)
+- kfree(devices_arr);
+-
+- return err;
+-
+-bind_process_to_device_failed:
+-get_mem_obj_from_handle_failed:
+- mutex_unlock(&p->mutex);
+-copy_from_user_failed:
+-sync_memory_failed:
+- kfree(devices_arr);
+- return err;
+-}
+-
+-int kfd_unmap_memory_from_gpu(void *mem, struct kfd_process_device *pdd)
+-{
+- int err;
+- struct kfd_dev *dev = pdd->dev;
+-
+- err = dev->kfd2kgd->unmap_memory_to_gpu(
+- dev->kgd, (struct kgd_mem *) mem, pdd->vm);
+-
+- if (err != 0)
+- return err;
+-
+- kfd_flush_tlb(dev, pdd->process->pasid);
+-
+- return 0;
+-}
+-
+-static int kfd_ioctl_unmap_memory_from_gpu(struct file *filep,
+- struct kfd_process *p, void *data)
+-{
+- struct kfd_ioctl_unmap_memory_from_gpu_args *args = data;
+- struct kfd_process_device *pdd, *peer_pdd;
+- void *mem;
+- struct kfd_dev *dev, *peer;
+- long err = 0;
+- uint32_t *devices_arr = NULL, num_dev, i;
+-
+- dev = kfd_device_by_id(GET_GPU_ID(args->handle));
+- if (!dev)
+- return -EINVAL;
+-
+- if (args->device_ids_array_size > 0 &&
+- (args->device_ids_array_size < sizeof(uint32_t))) {
+- pr_err("Node IDs array size %u\n",
+- args->device_ids_array_size);
+- return -EFAULT;
+- }
+-
+- if (args->device_ids_array_size > 0) {
+- devices_arr = kmalloc(args->device_ids_array_size, GFP_KERNEL);
+- if (!devices_arr)
+- return -ENOMEM;
+-
+- err = copy_from_user(devices_arr,
+- (void __user *)args->device_ids_array_ptr,
+- args->device_ids_array_size);
+- if (err != 0) {
+- err = -EFAULT;
+- goto copy_from_user_failed;
+- }
+- }
+-
+- mutex_lock(&p->mutex);
+-
+- pdd = kfd_get_process_device_data(dev, p);
+- if (!pdd) {
+- pr_err("Process device data doesn't exist\n");
+- err = PTR_ERR(pdd);
+- goto bind_process_to_device_failed;
+- }
+-
+- mem = kfd_process_device_translate_handle(pdd,
+- GET_IDR_HANDLE(args->handle));
+- if (!mem) {
+- err = PTR_ERR(mem);
+- goto get_mem_obj_from_handle_failed;
+- }
+-
+- if (args->device_ids_array_size > 0) {
+- num_dev = args->device_ids_array_size / sizeof(uint32_t);
+- for (i = 0 ; i < num_dev; i++) {
+- peer = kfd_device_by_id(devices_arr[i]);
+- if (!peer) {
+- err = -EFAULT;
+- goto get_mem_obj_from_handle_failed;
+- }
+-
+- peer_pdd = kfd_get_process_device_data(peer, p);
+- if (!peer_pdd) {
+- err = -EFAULT;
+- goto get_mem_obj_from_handle_failed;
+- }
+- kfd_unmap_memory_from_gpu(mem, peer_pdd);
+- }
+- kfree(devices_arr);
+- } else
+- kfd_unmap_memory_from_gpu(mem, pdd);
+-
+- mutex_unlock(&p->mutex);
+-
+- return 0;
+-
+-bind_process_to_device_failed:
+-get_mem_obj_from_handle_failed:
+- mutex_unlock(&p->mutex);
+-copy_from_user_failed:
+- kfree(devices_arr);
+- return err;
+-}
+-
+-static int kfd_ioctl_set_process_dgpu_aperture(struct file *filep,
+- struct kfd_process *p, void *data)
+-{
+- struct kfd_ioctl_set_process_dgpu_aperture_args *args = data;
+- struct kfd_dev *dev;
+- struct kfd_process_device *pdd;
+- long err;
+-
+- dev = kfd_device_by_id(args->gpu_id);
+- if (!dev)
+- return -EINVAL;
+-
+- mutex_lock(&p->mutex);
+-
+- pdd = kfd_bind_process_to_device(dev, p);
+- if (IS_ERR(pdd)) {
+- err = PTR_ERR(pdd);
+- goto exit;
+- }
+-
+- err = kfd_set_process_dgpu_aperture(pdd, args->dgpu_base,
+- args->dgpu_limit);
+-
+-exit:
+- mutex_unlock(&p->mutex);
+- return err;
+-}
+-
+-static int kfd_ioctl_get_dmabuf_info(struct file *filep,
+- struct kfd_process *p, void *data)
+-{
+- struct kfd_ioctl_get_dmabuf_info_args *args = data;
+- struct kfd_dev *dev = NULL;
+- struct kgd_dev *dma_buf_kgd;
+- void *metadata_buffer = NULL;
+- uint32_t flags;
+- unsigned int i;
+- int r;
+-
+- /* Find a KFD GPU device that supports the get_dmabuf_info query */
+- for (i = 0; kfd_topology_enum_kfd_devices(i, &dev) == 0; i++)
+- if (dev && dev->kfd2kgd->get_dmabuf_info)
+- break;
+- if (!dev)
+- return -EINVAL;
+-
+- if (args->metadata_ptr) {
+- metadata_buffer = kzalloc(args->metadata_size, GFP_KERNEL);
+- if (!metadata_buffer)
+- return -ENOMEM;
+- }
+-
+- /* Get dmabuf info from KGD */
+- r = dev->kfd2kgd->get_dmabuf_info(dev->kgd, args->dmabuf_fd,
+- &dma_buf_kgd, &args->size,
+- metadata_buffer, args->metadata_size,
+- &args->metadata_size, &flags);
+- if (r)
+- goto exit;
+-
+- /* Reverse-lookup gpu_id from kgd pointer */
+- dev = kfd_device_by_kgd(dma_buf_kgd);
+- if (!dev) {
+- r = -EINVAL;
+- goto exit;
+- }
+- args->gpu_id = dev->id;
+- args->flags = flags;
+-
+- /* Copy metadata buffer to user mode */
+- if (metadata_buffer) {
+- r = copy_to_user((void __user *)args->metadata_ptr,
+- metadata_buffer, args->metadata_size);
+- if (r != 0)
+- r = -EFAULT;
+- }
+-
+-exit:
+- kfree(metadata_buffer);
+-
+- return r;
+-}
+-
+-static int kfd_ioctl_import_dmabuf(struct file *filep,
+- struct kfd_process *p, void *data)
+-{
+- struct kfd_ioctl_import_dmabuf_args *args = data;
+- struct kfd_dev *dev;
+- int r;
+-
+- dev = kfd_device_by_id(args->gpu_id);
+- if (!dev)
+- return -EINVAL;
+-
+- r = kfd_ipc_import_dmabuf(dev, p, args->gpu_id, args->dmabuf_fd,
+- args->va_addr, &args->handle, NULL);
+- if (r)
+- pr_err("Failed to import dmabuf\n");
+-
+- return r;
+-}
+-
+-static int kfd_ioctl_ipc_export_handle(struct file *filep,
+- struct kfd_process *p,
+- void *data)
+-{
+- struct kfd_ioctl_ipc_export_handle_args *args = data;
+- struct kfd_dev *dev;
+- int r;
+-
+- dev = kfd_device_by_id(args->gpu_id);
+- if (!dev)
+- return -EINVAL;
+-
+- r = kfd_ipc_export_as_handle(dev, p, args->handle, args->share_handle);
+- if (r)
+- pr_err("Failed to export IPC handle\n");
+-
+- return r;
+-}
+-
+-static int kfd_ioctl_ipc_import_handle(struct file *filep,
+- struct kfd_process *p,
+- void *data)
+-{
+- struct kfd_ioctl_ipc_import_handle_args *args = data;
+- struct kfd_dev *dev = NULL;
+- int r;
+-
+- dev = kfd_device_by_id(args->gpu_id);
+- if (!dev)
+- return -EINVAL;
+-
+- r = kfd_ipc_import_handle(dev, p, args->gpu_id, args->share_handle,
+- args->va_addr, &args->handle,
+- &args->mmap_offset);
+- if (r)
+- pr_err("Failed to import IPC handle\n");
+-
+- return r;
+-}
+-
+-static int kfd_ioctl_get_tile_config(struct file *filep,
+- struct kfd_process *p, void *data)
+-{
+- struct kfd_ioctl_get_tile_config_args *args = data;
+- struct kfd_dev *dev;
+- struct tile_config config;
+- int err = 0;
+-
+- dev = kfd_device_by_id(args->gpu_id);
+-
+- dev->kfd2kgd->get_tile_config(dev->kgd, &config);
+-
+- args->gb_addr_config = config.gb_addr_config;
+- args->num_banks = config.num_banks;
+- args->num_ranks = config.num_ranks;
+-
+- if (args->num_tile_configs > config.num_tile_configs)
+- args->num_tile_configs = config.num_tile_configs;
+- err = copy_to_user((void __user *)args->tile_config_ptr,
+- config.tile_config_ptr,
+- args->num_tile_configs * sizeof(uint32_t));
+- if (err) {
+- args->num_tile_configs = 0;
+- return -EFAULT;
+- }
+-
+- if (args->num_macro_tile_configs > config.num_macro_tile_configs)
+- args->num_macro_tile_configs =
+- config.num_macro_tile_configs;
+- err = copy_to_user((void __user *)args->macro_tile_config_ptr,
+- config.macro_tile_config_ptr,
+- args->num_macro_tile_configs * sizeof(uint32_t));
+- if (err) {
+- args->num_macro_tile_configs = 0;
+- return -EFAULT;
+- }
+-
+- return 0;
+-}
+-
+-#if defined(BUILD_AS_DKMS)
+-static int kfd_ioctl_cross_memory_copy(struct file *filep,
+- struct kfd_process *local_p, void *data)
+-{
+- return 0;
+-}
+-#else
+-static int kfd_ioctl_cross_memory_copy(struct file *filep,
+- struct kfd_process *local_p, void *data)
+-{
+- struct kfd_ioctl_cross_memory_copy_args *args = data;
+- struct kfd_memory_range *src_array, *dst_array;
+- struct kfd_bo *src_bo, *dst_bo;
+- struct kfd_process *remote_p, *src_p, *dst_p;
+- struct task_struct *remote_task;
+- struct mm_struct *remote_mm;
+- struct pid *remote_pid;
+- struct dma_fence *fence = NULL, *lfence = NULL;
+- uint64_t dst_va_addr;
+- uint64_t copied, total_copied = 0;
+- uint64_t src_offset, dst_offset;
+- int i, j = 0, err = 0;
+-
+- /* Check parameters */
+- if (args->src_mem_range_array == 0 || args->dst_mem_range_array == 0 ||
+- args->src_mem_array_size == 0 || args->dst_mem_array_size == 0)
+- return -EINVAL;
+- args->bytes_copied = 0;
+-
+- /* Allocate space for source and destination arrays */
+- src_array = kmalloc_array((args->src_mem_array_size +
+- args->dst_mem_array_size),
+- sizeof(struct kfd_memory_range),
+- GFP_KERNEL);
+- if (!src_array)
+- return -ENOMEM;
+- dst_array = &src_array[args->src_mem_array_size];
+-
+- if (copy_from_user(src_array, (void __user *)args->src_mem_range_array,
+- args->src_mem_array_size *
+- sizeof(struct kfd_memory_range))) {
+- err = -EFAULT;
+- goto copy_from_user_fail;
+- }
+- if (copy_from_user(dst_array, (void __user *)args->dst_mem_range_array,
+- args->dst_mem_array_size *
+- sizeof(struct kfd_memory_range))) {
+- err = -EFAULT;
+- goto copy_from_user_fail;
+- }
+-
+- /* Get remote process */
+- remote_pid = find_get_pid(args->pid);
+- if (!remote_pid) {
+- pr_err("Cross mem copy failed. Invalid PID %d\n", args->pid);
+- err = -ESRCH;
+- goto copy_from_user_fail;
+- }
+-
+- remote_task = get_pid_task(remote_pid, PIDTYPE_PID);
+- if (!remote_pid) {
+- pr_err("Cross mem copy failed. Invalid PID or task died %d\n",
+- args->pid);
+- err = -ESRCH;
+- goto get_pid_task_fail;
+- }
+-
+- /* Check access permission */
+- remote_mm = mm_access(remote_task, PTRACE_MODE_ATTACH_REALCREDS);
+- if (!remote_mm || IS_ERR(remote_mm)) {
+- err = IS_ERR(remote_mm) ? PTR_ERR(remote_mm) : -ESRCH;
+- if (err == -EACCES) {
+- pr_err("Cross mem copy failed. Permission error\n");
+- err = -EPERM;
+- } else
+- pr_err("Cross mem copy failed. Invalid task %d\n",
+- err);
+- goto mm_access_fail;
+- }
+-
+- remote_p = kfd_get_process(remote_task);
+- if (!remote_p) {
+- pr_err("Cross mem copy failed. Invalid kfd process %d\n",
+- args->pid);
+- err = -EINVAL;
+- goto kfd_process_fail;
+- }
+-
+- if (KFD_IS_CROSS_MEMORY_WRITE(args->flags)) {
+- src_p = local_p;
+- dst_p = remote_p;
+- pr_debug("CMA WRITE: local -> remote\n");
+- } else {
+- src_p = remote_p;
+- dst_p = local_p;
+- pr_debug("CMA READ: remote -> local\n");
+- }
+-
+-
+- /* For each source kfd_range:
+- * - Find the BO. Each range has to be within the same BO.
+- * - Copy this range to single or multiple destination BOs.
+- * - dst_va_addr - will point to next va address into which data will
+- * be copied.
+- * - dst_bo & src_bo - the current destination and source BOs
+- * - src_offset & dst_offset - offset into the respective BOs from
+- * data will be sourced or copied
+- */
+- dst_va_addr = dst_array[0].va_addr;
+- mutex_lock(&dst_p->mutex);
+- dst_bo = kfd_process_find_bo_from_interval(dst_p,
+- dst_va_addr,
+- dst_va_addr + dst_array[0].size - 1);
+- mutex_unlock(&dst_p->mutex);
+- if (!dst_bo) {
+- err = -EFAULT;
+- goto kfd_process_fail;
+- }
+- dst_offset = dst_va_addr - dst_bo->it.start;
+-
+- for (i = 0; i < args->src_mem_array_size; i++) {
+- uint64_t src_va_addr_end = src_array[i].va_addr +
+- src_array[i].size - 1;
+- uint64_t src_size_to_copy = src_array[i].size;
+-
+- mutex_lock(&src_p->mutex);
+- src_bo = kfd_process_find_bo_from_interval(src_p,
+- src_array[i].va_addr,
+- src_va_addr_end);
+- mutex_unlock(&src_p->mutex);
+- if (!src_bo || src_va_addr_end > src_bo->it.last) {
+- pr_err("Cross mem copy failed. Invalid range\n");
+- err = -EFAULT;
+- break;
+- }
+-
+- src_offset = src_array[i].va_addr - src_bo->it.start;
+-
+- /* Copy src_bo to one or multiple dst_bo(s) based on size and
+- * and current copy location.
+- */
+- while (j < args->dst_mem_array_size) {
+- uint64_t copy_size;
+- int64_t space_left;
+-
+- /* Find the current copy_size. This will be smaller of
+- * the following
+- * - space left in the current dest memory range
+- * - data left to copy from source range
+- */
+- space_left = (dst_array[j].va_addr + dst_array[j].size)
+- - dst_va_addr;
+- copy_size = (src_size_to_copy < space_left) ?
+- src_size_to_copy : space_left;
+-
+- /* Check both BOs belong to same device */
+- if (src_bo->dev->kgd != dst_bo->dev->kgd) {
+- pr_err("Cross Memory failed. Not same device\n");
+- err = -EINVAL;
+- break;
+- }
+-
+- /* Store prev fence. Release it when a later fence is
+- * created
+- */
+- lfence = fence;
+- fence = NULL;
+-
+- err = dst_bo->dev->kfd2kgd->copy_mem_to_mem(
+- src_bo->dev->kgd,
+- src_bo->mem, src_offset,
+- dst_bo->mem, dst_offset,
+- copy_size,
+- &fence, &copied);
+-
+- if (err) {
+- pr_err("GPU Cross mem copy failed\n");
+- err = -EFAULT;
+- break;
+- }
+-
+- /* Later fence available. Release old fence */
+- if (fence && lfence) {
+- dma_fence_put(lfence);
+- lfence = NULL;
+- }
+-
+- total_copied += copied;
+- src_size_to_copy -= copied;
+- space_left -= copied;
+- dst_va_addr += copied;
+- dst_offset += copied;
+- src_offset += copied;
+- if (dst_va_addr > dst_bo->it.last + 1) {
+- pr_err("Cross mem copy failed. Memory overflow\n");
+- err = -EFAULT;
+- break;
+- }
+-
+- /* If the cur dest range is full move to next one */
+- if (space_left <= 0) {
+- if (++j >= args->dst_mem_array_size)
+- break;
+-
+- dst_va_addr = dst_array[j].va_addr;
+- dst_bo = kfd_process_find_bo_from_interval(
+- dst_p,
+- dst_va_addr,
+- dst_va_addr +
+- dst_array[j].size - 1);
+- dst_offset = dst_va_addr - dst_bo->it.start;
+- }
+-
+- /* If the cur src range is done, move to next one */
+- if (src_size_to_copy <= 0)
+- break;
+- }
+- if (err)
+- break;
+- }
+-
+- /* Wait for the last fence irrespective of error condition */
+- if (fence) {
+- if (dma_fence_wait_timeout(fence, false, msecs_to_jiffies(1000))
+- < 0)
+- pr_err("Cross mem copy failed. BO timed out\n");
+- dma_fence_put(fence);
+- } else if (lfence) {
+- pr_debug("GPU copy fail. But wait for prev DMA to finish\n");
+- dma_fence_wait_timeout(lfence, true, msecs_to_jiffies(1000));
+- dma_fence_put(lfence);
+- }
+-
+-kfd_process_fail:
+- mmput(remote_mm);
+-mm_access_fail:
+- put_task_struct(remote_task);
+-get_pid_task_fail:
+- put_pid(remote_pid);
+-copy_from_user_fail:
+- kfree(src_array);
+-
+- /* An error could happen after partial copy. In that case this will
+- * reflect partial amount of bytes copied
+- */
+- args->bytes_copied = total_copied;
+- return err;
+-}
+-#endif
+-
+-static int kfd_ioctl_get_queue_wave_state(struct file *filep,
+- struct kfd_process *p, void *data)
+-{
+- struct kfd_ioctl_get_queue_wave_state_args *args = data;
+- int r;
+-
+- mutex_lock(&p->mutex);
+-
+- r = pqm_get_wave_state(&p->pqm, args->queue_id,
+- (void __user *)args->ctl_stack_address,
+- &args->ctl_stack_used_size,
+- &args->save_area_used_size);
+-
+- mutex_unlock(&p->mutex);
+-
+- return r;
+-}
+
+ #define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \
+- [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \
+- .cmd_drv = 0, .name = #ioctl}
++ [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, .cmd_drv = 0, .name = #ioctl}
+
+ /** Ioctl table */
+ static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
+@@ -1999,55 +899,6 @@ static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
+
+ AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_WAVE_CONTROL,
+ kfd_ioctl_dbg_wave_control, 0),
+-
+- AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_MEMORY_OF_GPU,
+- kfd_ioctl_alloc_memory_of_gpu, 0),
+-
+- AMDKFD_IOCTL_DEF(AMDKFD_IOC_FREE_MEMORY_OF_GPU,
+- kfd_ioctl_free_memory_of_gpu, 0),
+-
+- AMDKFD_IOCTL_DEF(AMDKFD_IOC_MAP_MEMORY_TO_GPU,
+- kfd_ioctl_map_memory_to_gpu, 0),
+-
+- AMDKFD_IOCTL_DEF(AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU,
+- kfd_ioctl_unmap_memory_from_gpu, 0),
+-
+- AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_MEMORY_OF_SCRATCH,
+- kfd_ioctl_alloc_scratch_memory, 0),
+-
+- AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_CU_MASK,
+- kfd_ioctl_set_cu_mask, 0),
+-
+- AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_PROCESS_DGPU_APERTURE,
+- kfd_ioctl_set_process_dgpu_aperture, 0),
+-
+- AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_TRAP_HANDLER,
+- kfd_ioctl_set_trap_handler, 0),
+-
+- AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES_NEW,
+- kfd_ioctl_get_process_apertures_new, 0),
+-
+- AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_DMABUF_INFO,
+- kfd_ioctl_get_dmabuf_info, 0),
+-
+- AMDKFD_IOCTL_DEF(AMDKFD_IOC_IMPORT_DMABUF,
+- kfd_ioctl_import_dmabuf, 0),
+-
+- AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_TILE_CONFIG,
+- kfd_ioctl_get_tile_config, 0),
+-
+- AMDKFD_IOCTL_DEF(AMDKFD_IOC_IPC_IMPORT_HANDLE,
+- kfd_ioctl_ipc_import_handle, 0),
+-
+- AMDKFD_IOCTL_DEF(AMDKFD_IOC_IPC_EXPORT_HANDLE,
+- kfd_ioctl_ipc_export_handle, 0),
+-
+- AMDKFD_IOCTL_DEF(AMDKFD_IOC_CROSS_MEMORY_COPY,
+- kfd_ioctl_cross_memory_copy, 0),
+-
+- AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_QUEUE_WAVE_STATE,
+- kfd_ioctl_get_queue_wave_state, 0)
+-
+ };
+
+ #define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls)
+@@ -2143,37 +994,19 @@ static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
+ static int kfd_mmap(struct file *filp, struct vm_area_struct *vma)
+ {
+ struct kfd_process *process;
+- struct kfd_dev *kfd;
+- unsigned long vm_pgoff;
+- int retval;
+
+ process = kfd_get_process(current);
+ if (IS_ERR(process))
+ return PTR_ERR(process);
+
+- vm_pgoff = vma->vm_pgoff;
+- vma->vm_pgoff = KFD_MMAP_OFFSET_VALUE_GET(vma->vm_pgoff);
+-
+- switch (vm_pgoff & KFD_MMAP_TYPE_MASK) {
+- case KFD_MMAP_TYPE_DOORBELL:
+- kfd = kfd_device_by_id(KFD_MMAP_GPU_ID_GET(vm_pgoff));
+- if (!kfd)
+- return -EFAULT;
+- return kfd_doorbell_mmap(kfd, process, vma);
+-
+- case KFD_MMAP_TYPE_EVENTS:
++ if ((vma->vm_pgoff & KFD_MMAP_DOORBELL_MASK) ==
++ KFD_MMAP_DOORBELL_MASK) {
++ vma->vm_pgoff = vma->vm_pgoff ^ KFD_MMAP_DOORBELL_MASK;
++ return kfd_doorbell_mmap(process, vma);
++ } else if ((vma->vm_pgoff & KFD_MMAP_EVENTS_MASK) ==
++ KFD_MMAP_EVENTS_MASK) {
++ vma->vm_pgoff = vma->vm_pgoff ^ KFD_MMAP_EVENTS_MASK;
+ return kfd_event_mmap(process, vma);
+-
+- case KFD_MMAP_TYPE_MAP_BO:
+- kfd = kfd_device_by_id(KFD_MMAP_GPU_ID_GET(vm_pgoff));
+- if (!kfd)
+- return -EFAULT;
+- retval = kfd->kfd2kgd->mmap_bo(kfd->kgd, vma);
+- return retval;
+-
+- case KFD_MMAP_TYPE_RESERVED_MEM:
+- return kfd_reserved_mem_mmap(process, vma);
+-
+ }
+
+ return -EFAULT;
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
+deleted file mode 100644
+index 4e94081..0000000
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
++++ /dev/null
+@@ -1,1304 +0,0 @@
+-#include <linux/kernel.h>
+-#include <linux/acpi.h>
+-#include <linux/mm.h>
+-#if defined(CONFIG_AMD_IOMMU_V2_MODULE) || defined(CONFIG_AMD_IOMMU_V2)
+-#include <linux/amd-iommu.h>
+-#endif
+-#include <linux/pci.h>
+-#include "kfd_crat.h"
+-#include "kfd_priv.h"
+-#include "kfd_topology.h"
+-
+-/* GPU Processor ID base for dGPUs for which VCRAT needs to be created.
+- * GPU processor ID are expressed with Bit[31]=1.
+- * The base is set to 0x8000_0000 + 0x1000 to avoid collision with GPU IDs
+- * used in the CRAT.
+- */
+-static uint32_t gpu_processor_id_low = 0x80001000;
+-
+-/* Return the next available gpu_processor_id and increment it for next GPU
+- * @total_cu_count - Total CUs present in the GPU including ones
+- * masked off
+- */
+-static inline unsigned int get_and_inc_gpu_processor_id(
+- unsigned int total_cu_count)
+-{
+- int current_id = gpu_processor_id_low;
+-
+- gpu_processor_id_low += total_cu_count;
+- return current_id;
+-}
+-
+-/* Static table to describe GPU Cache information */
+-struct kfd_gpu_cache_info {
+- uint32_t cache_size;
+- uint32_t cache_level;
+- uint32_t flags;
+- /* Indicates how many Compute Units share this cache
+- * Value = 1 indicates the cache is not shared
+- */
+- uint32_t num_cu_shared;
+-};
+-
+-static struct kfd_gpu_cache_info kaveri_cache_info[] = {
+- {
+- /* TCP L1 Cache per CU */
+- .cache_size = 16,
+- .cache_level = 1,
+- .flags = (CRAT_CACHE_FLAGS_ENABLED |
+- CRAT_CACHE_FLAGS_DATA_CACHE |
+- CRAT_CACHE_FLAGS_SIMD_CACHE),
+- .num_cu_shared = 1,
+-
+- },
+- {
+- /* Scalar L1 Instruction Cache (in SQC module) per bank */
+- .cache_size = 16,
+- .cache_level = 1,
+- .flags = (CRAT_CACHE_FLAGS_ENABLED |
+- CRAT_CACHE_FLAGS_INST_CACHE |
+- CRAT_CACHE_FLAGS_SIMD_CACHE),
+- .num_cu_shared = 2,
+- },
+- {
+- /* Scalar L1 Data Cache (in SQC module) per bank */
+- .cache_size = 8,
+- .cache_level = 1,
+- .flags = (CRAT_CACHE_FLAGS_ENABLED |
+- CRAT_CACHE_FLAGS_DATA_CACHE |
+- CRAT_CACHE_FLAGS_SIMD_CACHE),
+- .num_cu_shared = 2,
+- },
+-
+- /* TODO: Add L2 Cache information */
+-};
+-
+-
+-static struct kfd_gpu_cache_info carrizo_cache_info[] = {
+- {
+- /* TCP L1 Cache per CU */
+- .cache_size = 16,
+- .cache_level = 1,
+- .flags = (CRAT_CACHE_FLAGS_ENABLED |
+- CRAT_CACHE_FLAGS_DATA_CACHE |
+- CRAT_CACHE_FLAGS_SIMD_CACHE),
+- .num_cu_shared = 1,
+- },
+- {
+- /* Scalar L1 Instruction Cache (in SQC module) per bank */
+- .cache_size = 8,
+- .cache_level = 1,
+- .flags = (CRAT_CACHE_FLAGS_ENABLED |
+- CRAT_CACHE_FLAGS_INST_CACHE |
+- CRAT_CACHE_FLAGS_SIMD_CACHE),
+- .num_cu_shared = 4,
+- },
+- {
+- /* Scalar L1 Data Cache (in SQC module) per bank. */
+- .cache_size = 4,
+- .cache_level = 1,
+- .flags = (CRAT_CACHE_FLAGS_ENABLED |
+- CRAT_CACHE_FLAGS_DATA_CACHE |
+- CRAT_CACHE_FLAGS_SIMD_CACHE),
+- .num_cu_shared = 4,
+- },
+-
+- /* TODO: Add L2 Cache information */
+-};
+-
+-/* NOTE: In future if more information is added to struct kfd_gpu_cache_info
+- * the following ASICs may need a separate table.
+- */
+-#define hawaii_cache_info kaveri_cache_info
+-#define tonga_cache_info carrizo_cache_info
+-#define fiji_cache_info carrizo_cache_info
+-#define polaris10_cache_info carrizo_cache_info
+-#define polaris11_cache_info carrizo_cache_info
+-/* TODO - check & update Vega10 cache details */
+-#define vega10_cache_info carrizo_cache_info
+-#define raven_cache_info carrizo_cache_info
+-
+-static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev,
+- struct crat_subtype_computeunit *cu)
+-{
+- dev->node_props.cpu_cores_count = cu->num_cpu_cores;
+- dev->node_props.cpu_core_id_base = cu->processor_id_low;
+-#if defined(CONFIG_AMD_IOMMU_V2_MODULE) || defined(CONFIG_AMD_IOMMU_V2)
+- if (cu->hsa_capability & CRAT_CU_FLAGS_IOMMU_PRESENT)
+- dev->node_props.capability |= HSA_CAP_ATS_PRESENT;
+-#endif
+-
+- pr_debug("CU CPU: cores=%d id_base=%d\n", cu->num_cpu_cores,
+- cu->processor_id_low);
+-}
+-
+-static void kfd_populated_cu_info_gpu(struct kfd_topology_device *dev,
+- struct crat_subtype_computeunit *cu)
+-{
+- dev->node_props.simd_id_base = cu->processor_id_low;
+- dev->node_props.simd_count = cu->num_simd_cores;
+- dev->node_props.lds_size_in_kb = cu->lds_size_in_kb;
+- dev->node_props.max_waves_per_simd = cu->max_waves_simd;
+- dev->node_props.wave_front_size = cu->wave_front_size;
+- dev->node_props.array_count = cu->array_count;
+- dev->node_props.cu_per_simd_array = cu->num_cu_per_array;
+- dev->node_props.simd_per_cu = cu->num_simd_per_cu;
+- dev->node_props.max_slots_scratch_cu = cu->max_slots_scatch_cu;
+- if (cu->hsa_capability & CRAT_CU_FLAGS_HOT_PLUGGABLE)
+- dev->node_props.capability |= HSA_CAP_HOT_PLUGGABLE;
+- pr_debug("CU GPU: id_base=%d\n", cu->processor_id_low);
+-}
+-
+-/* kfd_parse_subtype_cu - parse compute unit subtypes and attach it to correct
+- * topology device present in the device_list
+- */
+-static int kfd_parse_subtype_cu(struct crat_subtype_computeunit *cu,
+- struct list_head *device_list)
+-{
+- struct kfd_topology_device *dev;
+-
+- pr_debug("Found CU entry in CRAT table with proximity_domain=%d caps=%x\n",
+- cu->proximity_domain, cu->hsa_capability);
+- list_for_each_entry(dev, device_list, list) {
+- if (cu->proximity_domain == dev->proximity_domain) {
+- if (cu->flags & CRAT_CU_FLAGS_CPU_PRESENT)
+- kfd_populated_cu_info_cpu(dev, cu);
+-
+- if (cu->flags & CRAT_CU_FLAGS_GPU_PRESENT)
+- kfd_populated_cu_info_gpu(dev, cu);
+- break;
+- }
+- }
+-
+- return 0;
+-}
+-
+-/* kfd_parse_subtype_mem - parse memory subtypes and attach it to correct
+- * topology device present in the device_list
+- */
+-static int kfd_parse_subtype_mem(struct crat_subtype_memory *mem,
+- struct list_head *device_list)
+-{
+- struct kfd_mem_properties *props;
+- struct kfd_topology_device *dev;
+-
+- pr_debug("Found memory entry in CRAT table with proximity_domain=%d\n",
+- mem->proximity_domain);
+- list_for_each_entry(dev, device_list, list) {
+- if (mem->proximity_domain == dev->proximity_domain) {
+- props = kfd_alloc_struct(props);
+- if (!props)
+- return -ENOMEM;
+-
+- /* We're on GPU node */
+- if (dev->node_props.cpu_cores_count == 0) {
+- /* APU */
+- if (mem->visibility_type == 0)
+- props->heap_type =
+- HSA_MEM_HEAP_TYPE_FB_PRIVATE;
+- /* dGPU */
+- else
+- props->heap_type = mem->visibility_type;
+- } else
+- props->heap_type = HSA_MEM_HEAP_TYPE_SYSTEM;
+-
+- if (mem->flags & CRAT_MEM_FLAGS_HOT_PLUGGABLE)
+- props->flags |= HSA_MEM_FLAGS_HOT_PLUGGABLE;
+- if (mem->flags & CRAT_MEM_FLAGS_NON_VOLATILE)
+- props->flags |= HSA_MEM_FLAGS_NON_VOLATILE;
+-
+- props->size_in_bytes =
+- ((uint64_t)mem->length_high << 32) +
+- mem->length_low;
+- props->width = mem->width;
+-
+- dev->node_props.mem_banks_count++;
+- list_add_tail(&props->list, &dev->mem_props);
+-
+- break;
+- }
+- }
+-
+- return 0;
+-}
+-
+-/* kfd_parse_subtype_cache - parse cache subtypes and attach it to correct
+- * topology device present in the device_list
+- */
+-static int kfd_parse_subtype_cache(struct crat_subtype_cache *cache,
+- struct list_head *device_list)
+-{
+- struct kfd_cache_properties *props;
+- struct kfd_topology_device *dev;
+- uint32_t id;
+- uint32_t total_num_of_cu;
+-
+- id = cache->processor_id_low;
+-
+- list_for_each_entry(dev, device_list, list) {
+- total_num_of_cu = (dev->node_props.array_count *
+- dev->node_props.cu_per_simd_array);
+-
+- /* Cache infomration in CRAT doesn't have proximity_domain
+- * information as it is associated with a CPU core or GPU
+- * Compute Unit. So map the cache using CPU core Id or SIMD
+- * (GPU) ID.
+- * TODO: This works because currently we can safely assume that
+- * Compute Units are parsed before caches are parsed. In
+- * future, remove this dependency
+- */
+- if ((id >= dev->node_props.cpu_core_id_base &&
+- id <= dev->node_props.cpu_core_id_base +
+- dev->node_props.cpu_cores_count) ||
+- (id >= dev->node_props.simd_id_base &&
+- id < dev->node_props.simd_id_base +
+- total_num_of_cu)) {
+- props = kfd_alloc_struct(props);
+- if (!props)
+- return -ENOMEM;
+-
+- props->processor_id_low = id;
+- props->cache_level = cache->cache_level;
+- props->cache_size = cache->cache_size;
+- props->cacheline_size = cache->cache_line_size;
+- props->cachelines_per_tag = cache->lines_per_tag;
+- props->cache_assoc = cache->associativity;
+- props->cache_latency = cache->cache_latency;
+- memcpy(props->sibling_map, cache->sibling_map,
+- sizeof(props->sibling_map));
+-
+- if (cache->flags & CRAT_CACHE_FLAGS_DATA_CACHE)
+- props->cache_type |= HSA_CACHE_TYPE_DATA;
+- if (cache->flags & CRAT_CACHE_FLAGS_INST_CACHE)
+- props->cache_type |= HSA_CACHE_TYPE_INSTRUCTION;
+- if (cache->flags & CRAT_CACHE_FLAGS_CPU_CACHE)
+- props->cache_type |= HSA_CACHE_TYPE_CPU;
+- if (cache->flags & CRAT_CACHE_FLAGS_SIMD_CACHE)
+- props->cache_type |= HSA_CACHE_TYPE_HSACU;
+-
+- dev->cache_count++;
+- dev->node_props.caches_count++;
+- list_add_tail(&props->list, &dev->cache_props);
+-
+- break;
+- }
+- }
+-
+- return 0;
+-}
+-
+-/* kfd_parse_subtype_iolink - parse iolink subtypes and attach it to correct
+- * topology device present in the device_list
+- */
+-static int kfd_parse_subtype_iolink(struct crat_subtype_iolink *iolink,
+- struct list_head *device_list)
+-{
+- struct kfd_iolink_properties *props = NULL, *props2;
+- struct kfd_topology_device *dev, *cpu_dev;
+- uint32_t id_from;
+- uint32_t id_to;
+-
+- id_from = iolink->proximity_domain_from;
+- id_to = iolink->proximity_domain_to;
+-
+- pr_debug("Found IO link entry in CRAT table with id_from=%d\n",
+- id_from);
+- list_for_each_entry(dev, device_list, list) {
+- if (id_from == dev->proximity_domain) {
+- props = kfd_alloc_struct(props);
+- if (!props)
+- return -ENOMEM;
+-
+- props->node_from = id_from;
+- props->node_to = id_to;
+- props->ver_maj = iolink->version_major;
+- props->ver_min = iolink->version_minor;
+- props->iolink_type = iolink->io_interface_type;
+-
+- if (props->iolink_type == CRAT_IOLINK_TYPE_PCIEXPRESS)
+- props->weight = 20;
+- else
+- props->weight = node_distance(id_from, id_to);
+-
+- props->min_latency = iolink->minimum_latency;
+- props->max_latency = iolink->maximum_latency;
+- props->min_bandwidth = iolink->minimum_bandwidth_mbs;
+- props->max_bandwidth = iolink->maximum_bandwidth_mbs;
+- props->rec_transfer_size =
+- iolink->recommended_transfer_size;
+-
+- dev->io_link_count++;
+- dev->node_props.io_links_count++;
+- list_add_tail(&props->list, &dev->io_link_props);
+- break;
+- }
+- }
+-
+- /* CPU topology is created before GPUs are detected, so CPU->GPU
+- * links are not built at that time. If a PCIe type is discovered, it
+- * means a GPU is detected and we are adding GPU->CPU to the topology.
+- * At this time, also add the corresponded CPU->GPU link.
+- */
+- if (props && props->iolink_type == CRAT_IOLINK_TYPE_PCIEXPRESS) {
+- cpu_dev = kfd_topology_device_by_proximity_domain(id_to);
+- if (!cpu_dev)
+- return -ENODEV;
+- /* same everything but the other direction */
+- props2 = kmemdup(props, sizeof(*props2), GFP_KERNEL);
+- props2->node_from = id_to;
+- props2->node_to = id_from;
+- props2->kobj = NULL;
+- cpu_dev->io_link_count++;
+- cpu_dev->node_props.io_links_count++;
+- list_add_tail(&props2->list, &cpu_dev->io_link_props);
+- }
+-
+- return 0;
+-}
+-
+-/* kfd_parse_subtype - parse subtypes and attach it to correct topology device
+- * present in the device_list
+- * @sub_type_hdr - subtype section of crat_image
+- * @device_list - list of topology devices present in this crat_image
+- */
+-static int kfd_parse_subtype(struct crat_subtype_generic *sub_type_hdr,
+- struct list_head *device_list)
+-{
+- struct crat_subtype_computeunit *cu;
+- struct crat_subtype_memory *mem;
+- struct crat_subtype_cache *cache;
+- struct crat_subtype_iolink *iolink;
+- int ret = 0;
+-
+- switch (sub_type_hdr->type) {
+- case CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY:
+- cu = (struct crat_subtype_computeunit *)sub_type_hdr;
+- ret = kfd_parse_subtype_cu(cu, device_list);
+- break;
+- case CRAT_SUBTYPE_MEMORY_AFFINITY:
+- mem = (struct crat_subtype_memory *)sub_type_hdr;
+- ret = kfd_parse_subtype_mem(mem, device_list);
+- break;
+- case CRAT_SUBTYPE_CACHE_AFFINITY:
+- cache = (struct crat_subtype_cache *)sub_type_hdr;
+- ret = kfd_parse_subtype_cache(cache, device_list);
+- break;
+- case CRAT_SUBTYPE_TLB_AFFINITY:
+- /* For now, nothing to do here */
+- pr_debug("Found TLB entry in CRAT table (not processing)\n");
+- break;
+- case CRAT_SUBTYPE_CCOMPUTE_AFFINITY:
+- /* For now, nothing to do here */
+- pr_debug("Found CCOMPUTE entry in CRAT table (not processing)\n");
+- break;
+- case CRAT_SUBTYPE_IOLINK_AFFINITY:
+- iolink = (struct crat_subtype_iolink *)sub_type_hdr;
+- ret = kfd_parse_subtype_iolink(iolink, device_list);
+- break;
+- default:
+- pr_warn("Unknown subtype %d in CRAT\n",
+- sub_type_hdr->type);
+- }
+-
+- return ret;
+-}
+-
+-/* kfd_parse_crat_table - parse CRAT table. For each node present in CRAT
+- * create a kfd_topology_device and add in to device_list. Also parse
+- * CRAT subtypes and attach it to appropriate kfd_topology_device
+- * @crat_image - input image containing CRAT
+- * @device_list - [OUT] list of kfd_topology_device generated after
+- * parsing crat_image
+- * @proximity_domain - Proximity domain of the first device in the table
+- *
+- * Return - 0 if successful else -ve value
+- */
+-int kfd_parse_crat_table(void *crat_image,
+- struct list_head *device_list,
+- uint32_t proximity_domain)
+-{
+- struct kfd_topology_device *top_dev = NULL;
+- struct crat_subtype_generic *sub_type_hdr;
+- uint16_t node_id;
+- int ret = 0;
+- struct crat_header *crat_table = (struct crat_header *)crat_image;
+- uint16_t num_nodes;
+- uint32_t image_len;
+- uint32_t last_header_type, last_header_length;
+-
+- if (!crat_image)
+- return -EINVAL;
+-
+- if (!list_empty(device_list)) {
+- pr_warn("Error device list should be empty\n");
+- return -EINVAL;
+- }
+-
+- num_nodes = crat_table->num_domains;
+- image_len = crat_table->length;
+-
+- pr_info("Parsing CRAT table with %d nodes\n", num_nodes);
+-
+- for (node_id = 0; node_id < num_nodes; node_id++) {
+- top_dev = kfd_create_topology_device(device_list);
+- if (!top_dev)
+- break;
+- top_dev->proximity_domain = proximity_domain++;
+- }
+-
+- if (!top_dev) {
+- ret = -ENOMEM;
+- goto err;
+- }
+-
+- memcpy(top_dev->oem_id, crat_table->oem_id, CRAT_OEMID_LENGTH);
+- memcpy(top_dev->oem_table_id, crat_table->oem_table_id,
+- CRAT_OEMTABLEID_LENGTH);
+- top_dev->oem_revision = crat_table->oem_revision;
+-
+- last_header_type = last_header_length = 0;
+- sub_type_hdr = (struct crat_subtype_generic *)(crat_table+1);
+- while ((char *)sub_type_hdr + sizeof(struct crat_subtype_generic) <
+- ((char *)crat_image) + image_len) {
+- pr_debug("Parsing CRAT subtype header %p enabled: %s type: 0x%x length %d\n",
+- sub_type_hdr,
+- (sub_type_hdr->flags &
+- CRAT_SUBTYPE_FLAGS_ENABLED)
+- ? "true" : "false",
+- sub_type_hdr->type,
+- sub_type_hdr->length);
+-
+- if (sub_type_hdr->length == 0) {
+- pr_err("Parsing wrong CRAT's subtype header last header type: %d last header len %d\n",
+- last_header_type, last_header_type);
+- pr_err("Current header type %d length %d\n",
+- sub_type_hdr->type, sub_type_hdr->length);
+- break;
+- }
+-
+- if (sub_type_hdr->flags & CRAT_SUBTYPE_FLAGS_ENABLED) {
+- ret = kfd_parse_subtype(sub_type_hdr, device_list);
+- if (ret != 0)
+- break;
+- }
+-
+- last_header_type = sub_type_hdr->type;
+- last_header_length = sub_type_hdr->length;
+- sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
+- sub_type_hdr->length);
+- }
+-
+-err:
+- if (ret)
+- kfd_release_topology_device_list(device_list);
+-
+- return ret;
+-}
+-
+-/* Helper function. See kfd_fill_gpu_cache_info for parameter description */
+-static int fill_in_pcache(struct crat_subtype_cache *pcache,
+- struct kfd_gpu_cache_info *pcache_info,
+- struct kfd_cu_info *cu_info,
+- int mem_available,
+- int cu_bitmask,
+- int cache_type, unsigned int cu_processor_id,
+- int cu_block)
+-{
+- unsigned int cu_sibling_map_mask;
+- int first_active_cu;
+-
+- /* First check if enough memory is available */
+- if (mem_available - sizeof(struct crat_subtype_cache) < 0)
+- return -ENOMEM;
+-
+- cu_sibling_map_mask = cu_bitmask;
+- cu_sibling_map_mask >>= cu_block;
+- cu_sibling_map_mask &=
+- ((1 << pcache_info[cache_type].num_cu_shared) - 1);
+- first_active_cu = ffs(cu_sibling_map_mask);
+-
+- /* CU could be inactive. In case of shared cache find the first active
+- * CU. and incase of non-shared cache check if the CU is inactive. If
+- * inactive active skip it
+- */
+- if (first_active_cu) {
+- memset(pcache, 0, sizeof(struct crat_subtype_cache));
+- pcache->type = CRAT_SUBTYPE_CACHE_AFFINITY;
+- pcache->length = sizeof(struct crat_subtype_cache);
+- pcache->flags = pcache_info[cache_type].flags;
+- pcache->processor_id_low = cu_processor_id
+- + (first_active_cu - 1);
+- pcache->cache_level = pcache_info[cache_type].cache_level;
+- pcache->cache_size = pcache_info[cache_type].cache_size;
+-
+- /* Sibling map is w.r.t processor_id_low, so shift out
+- * inactive CU
+- */
+- cu_sibling_map_mask =
+- cu_sibling_map_mask >> (first_active_cu - 1);
+-
+- pcache->sibling_map[0] = (uint8_t)(cu_sibling_map_mask & 0xFF);
+- pcache->sibling_map[1] =
+- (uint8_t)((cu_sibling_map_mask >> 8) & 0xFF);
+- pcache->sibling_map[2] =
+- (uint8_t)((cu_sibling_map_mask >> 16) & 0xFF);
+- pcache->sibling_map[3] =
+- (uint8_t)((cu_sibling_map_mask >> 24) & 0xFF);
+- return 0;
+- }
+- return 1;
+-}
+-
+-/* kfd_fill_gpu_cache_info - Fill GPU cache info using kfd_gpu_cache_info
+- * tables
+- *
+- * @kdev - [IN] GPU device
+- * @gpu_processor_id - [IN] GPU processor ID to which these caches
+- * associate
+- * @available_size - [IN] Amount of memory available in pcache
+- * @cu_info - [IN] Compute Unit info obtained from KGD
+- * @pcache - [OUT] memory into which cache data is to be filled in.
+- * @size_filled - [OUT] amount of data used up in pcache.
+- * @num_of_entries - [OUT] number of caches added
+- */
+-static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev,
+- int gpu_processor_id,
+- int available_size,
+- struct kfd_cu_info *cu_info,
+- struct crat_subtype_cache *pcache,
+- int *size_filled,
+- int *num_of_entries)
+-{
+- struct kfd_gpu_cache_info *pcache_info;
+- int num_of_cache_types = 0;
+- int i, j, k;
+- int ct = 0;
+- int mem_available = available_size;
+- unsigned int cu_processor_id;
+- int ret;
+-
+- switch (kdev->device_info->asic_family) {
+- case CHIP_KAVERI:
+- pcache_info = kaveri_cache_info;
+- num_of_cache_types = ARRAY_SIZE(kaveri_cache_info);
+- break;
+- case CHIP_HAWAII:
+- pcache_info = hawaii_cache_info;
+- num_of_cache_types = ARRAY_SIZE(hawaii_cache_info);
+- break;
+- case CHIP_CARRIZO:
+- pcache_info = carrizo_cache_info;
+- num_of_cache_types = ARRAY_SIZE(carrizo_cache_info);
+- break;
+- case CHIP_TONGA:
+- pcache_info = tonga_cache_info;
+- num_of_cache_types = ARRAY_SIZE(tonga_cache_info);
+- break;
+- case CHIP_FIJI:
+- pcache_info = fiji_cache_info;
+- num_of_cache_types = ARRAY_SIZE(fiji_cache_info);
+- break;
+- case CHIP_POLARIS10:
+- pcache_info = polaris10_cache_info;
+- num_of_cache_types = ARRAY_SIZE(polaris10_cache_info);
+- break;
+- case CHIP_POLARIS11:
+- pcache_info = polaris11_cache_info;
+- num_of_cache_types = ARRAY_SIZE(polaris11_cache_info);
+- break;
+- case CHIP_VEGA10:
+- pcache_info = vega10_cache_info;
+- num_of_cache_types = ARRAY_SIZE(vega10_cache_info);
+- break;
+- case CHIP_RAVEN:
+- pcache_info = raven_cache_info;
+- num_of_cache_types = ARRAY_SIZE(raven_cache_info);
+- break;
+- default:
+- return -EINVAL;
+- }
+-
+- *size_filled = 0;
+- *num_of_entries = 0;
+-
+- /* For each type of cache listed in the kfd_gpu_cache_info table,
+- * go through all available Compute Units.
+- * The [i,j,k] loop will
+- * if kfd_gpu_cache_info.num_cu_shared = 1
+- * will parse through all available CU
+- * If (kfd_gpu_cache_info.num_cu_shared != 1)
+- * then it will consider only one CU from
+- * the shared unit
+- */
+-
+- for (ct = 0; ct < num_of_cache_types; ct++) {
+- cu_processor_id = gpu_processor_id;
+- for (i = 0; i < cu_info->num_shader_engines; i++) {
+- for (j = 0; j < cu_info->num_shader_arrays_per_engine;
+- j++) {
+- for (k = 0; k < cu_info->num_cu_per_sh;
+- k += pcache_info[ct].num_cu_shared) {
+-
+- ret = fill_in_pcache(pcache,
+- pcache_info,
+- cu_info,
+- mem_available,
+- cu_info->cu_bitmap[i][j],
+- ct,
+- cu_processor_id,
+- k);
+-
+- if (ret < 0)
+- break;
+-
+- if (!ret) {
+- pcache++;
+- (*num_of_entries)++;
+- mem_available -=
+- sizeof(*pcache);
+- (*size_filled) +=
+- sizeof(*pcache);
+- }
+-
+- /* Move to next CU block */
+- cu_processor_id +=
+- pcache_info[ct].num_cu_shared;
+- }
+- }
+- }
+- }
+-
+- pr_debug("Added [%d] GPU cache entries\n", *num_of_entries);
+-
+- return 0;
+-}
+-
+-/*
+- * kfd_create_crat_image_acpi - Allocates memory for CRAT image and
+- * copies CRAT from ACPI (if available).
+- * NOTE: Call kfd_destroy_crat_image to free CRAT image memory
+- *
+- * @crat_image: CRAT read from ACPI. If no CRAT in ACPI then
+- * crat_image will be NULL
+- * @size: [OUT] size of crat_image
+- *
+- * Return 0 if successful else return -ve value
+- */
+-#ifdef CONFIG_ACPI
+-int kfd_create_crat_image_acpi(void **crat_image, size_t *size)
+-{
+- struct acpi_table_header *crat_table;
+- acpi_status status;
+- void *pcrat_image;
+-
+- if (!crat_image)
+- return -EINVAL;
+-
+- *crat_image = NULL;
+-
+- /* Fetch the CRAT table from ACPI */
+- status = acpi_get_table(CRAT_SIGNATURE, 0, &crat_table);
+- if (status == AE_NOT_FOUND) {
+- pr_warn("CRAT table not found\n");
+- return -ENODATA;
+- } else if (ACPI_FAILURE(status)) {
+- const char *err = acpi_format_exception(status);
+-
+- pr_err("CRAT table error: %s\n", err);
+- return -EINVAL;
+- }
+-
+- if (ignore_crat) {
+- pr_info("CRAT table disabled by module option\n");
+- return -ENODATA;
+- }
+-
+- pcrat_image = kmalloc(crat_table->length, GFP_KERNEL);
+- if (!pcrat_image) {
+- pr_err("No memory for allocating CRAT image\n");
+- return -ENOMEM;
+- }
+-
+- memcpy(pcrat_image, crat_table, crat_table->length);
+-
+- *crat_image = pcrat_image;
+- *size = crat_table->length;
+-
+- return 0;
+-}
+-#endif
+-
+-/* Memory required to create Virtual CRAT.
+- * Since there is no easy way to predict the amount of memory required, the
+- * following amount are allocated for CPU and GPU Virtual CRAT. This is
+- * expected to cover all known conditions. But to be safe additional check
+- * is put in the code to ensure we don't overwrite.
+- */
+-#define VCRAT_SIZE_FOR_CPU (2 * PAGE_SIZE)
+-#define VCRAT_SIZE_FOR_GPU (3 * PAGE_SIZE)
+-
+-/* kfd_fill_cu_for_cpu - Fill in Compute info for the given CPU NUMA node
+- *
+- * @numa_node_id: CPU NUMA node id
+- * @avail_size: Available size in the memory
+- * @sub_type_hdr: Memory into which compute info will be filled in
+- *
+- * Return 0 if successful else return -ve value
+- */
+-static int kfd_fill_cu_for_cpu(int numa_node_id, int *avail_size,
+- int proximity_domain,
+- struct crat_subtype_computeunit *sub_type_hdr)
+-{
+- const struct cpumask *cpumask;
+-
+- *avail_size -= sizeof(struct crat_subtype_computeunit);
+- if (*avail_size < 0)
+- return -ENOMEM;
+-
+- memset(sub_type_hdr, 0, sizeof(struct crat_subtype_computeunit));
+-
+- /* Fill in subtype header data */
+- sub_type_hdr->type = CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY;
+- sub_type_hdr->length = sizeof(struct crat_subtype_computeunit);
+- sub_type_hdr->flags = CRAT_SUBTYPE_FLAGS_ENABLED;
+-
+- cpumask = cpumask_of_node(numa_node_id);
+-
+- /* Fill in CU data */
+- sub_type_hdr->flags |= CRAT_CU_FLAGS_CPU_PRESENT;
+- sub_type_hdr->proximity_domain = proximity_domain;
+- sub_type_hdr->processor_id_low = kfd_numa_node_to_apic_id(numa_node_id);
+- if (sub_type_hdr->processor_id_low == -1)
+- return -EINVAL;
+-
+- sub_type_hdr->num_cpu_cores = cpumask_weight(cpumask);
+-
+- return 0;
+-}
+-
+-/* kfd_fill_mem_info_for_cpu - Fill in Memory info for the given CPU NUMA node
+- *
+- * @numa_node_id: CPU NUMA node id
+- * @avail_size: Available size in the memory
+- * @sub_type_hdr: Memory into which compute info will be filled in
+- *
+- * Return 0 if successful else return -ve value
+- */
+-static int kfd_fill_mem_info_for_cpu(int numa_node_id, int *avail_size,
+- int proximity_domain,
+- struct crat_subtype_memory *sub_type_hdr)
+-{
+- uint64_t mem_in_bytes = 0;
+- pg_data_t *pgdat;
+- int zone_type;
+-
+- *avail_size -= sizeof(struct crat_subtype_memory);
+- if (*avail_size < 0)
+- return -ENOMEM;
+-
+- memset(sub_type_hdr, 0, sizeof(struct crat_subtype_memory));
+-
+- /* Fill in subtype header data */
+- sub_type_hdr->type = CRAT_SUBTYPE_MEMORY_AFFINITY;
+- sub_type_hdr->length = sizeof(struct crat_subtype_memory);
+- sub_type_hdr->flags = CRAT_SUBTYPE_FLAGS_ENABLED;
+-
+- /* Fill in Memory Subunit data */
+-
+- /* Unlike si_meminfo, si_meminfo_node is not exported. So
+- * the following lines are duplicated from si_meminfo_node
+- * function
+- */
+- pgdat = NODE_DATA(numa_node_id);
+- for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++)
+- mem_in_bytes += pgdat->node_zones[zone_type].managed_pages;
+- mem_in_bytes <<= PAGE_SHIFT;
+-
+- sub_type_hdr->length_low = lower_32_bits(mem_in_bytes);
+- sub_type_hdr->length_high = upper_32_bits(mem_in_bytes);
+- sub_type_hdr->proximity_domain = proximity_domain;
+-
+- return 0;
+-}
+-
+-#ifdef CONFIG_X86_64
+-static int kfd_fill_iolink_info_for_cpu(int numa_node_id, int *avail_size,
+- uint32_t *num_entries,
+- struct crat_subtype_iolink *sub_type_hdr)
+-{
+- int nid;
+- struct cpuinfo_x86 *c = &cpu_data(0);
+- uint8_t link_type;
+-
+- if (c->x86_vendor == X86_VENDOR_AMD)
+- link_type = CRAT_IOLINK_TYPE_HYPERTRANSPORT;
+- else
+- link_type = CRAT_IOLINK_TYPE_QPI_1_1;
+-
+- *num_entries = 0;
+-
+- /* Create IO links from this node to other CPU nodes */
+- for_each_online_node(nid) {
+- if (nid == numa_node_id) /* node itself */
+- continue;
+-
+- *avail_size -= sizeof(struct crat_subtype_iolink);
+- if (*avail_size < 0)
+- return -ENOMEM;
+-
+- memset(sub_type_hdr, 0, sizeof(struct crat_subtype_iolink));
+-
+- /* Fill in subtype header data */
+- sub_type_hdr->type = CRAT_SUBTYPE_IOLINK_AFFINITY;
+- sub_type_hdr->length = sizeof(struct crat_subtype_iolink);
+- sub_type_hdr->flags = CRAT_SUBTYPE_FLAGS_ENABLED;
+-
+- /* Fill in IO link data */
+- sub_type_hdr->proximity_domain_from = numa_node_id;
+- sub_type_hdr->proximity_domain_to = nid;
+- sub_type_hdr->io_interface_type = link_type;
+-
+- (*num_entries)++;
+- sub_type_hdr++;
+- }
+-
+- return 0;
+-}
+-#endif
+-
+-/* kfd_create_vcrat_image_cpu - Create Virtual CRAT for CPU
+- *
+- * @pcrat_image: Fill in VCRAT for CPU
+- * @size: [IN] allocated size of crat_image.
+- * [OUT] actual size of data filled in crat_image
+- */
+-static int kfd_create_vcrat_image_cpu(void *pcrat_image, size_t *size)
+-{
+- struct crat_header *crat_table = (struct crat_header *)pcrat_image;
+- struct crat_subtype_generic *sub_type_hdr;
+- int avail_size = *size;
+- int numa_node_id;
+- int ret = 0;
+-#ifdef CONFIG_ACPI
+- struct acpi_table_header *acpi_table;
+- acpi_status status;
+-#endif
+-#ifdef CONFIG_X86_64
+- uint32_t entries = 0;
+-#endif
+-
+- if (!pcrat_image || avail_size < VCRAT_SIZE_FOR_CPU)
+- return -EINVAL;
+-
+- /* Fill in CRAT Header.
+- * Modify length and total_entries as subunits are added.
+- */
+- avail_size -= sizeof(struct crat_header);
+- if (avail_size < 0)
+- return -ENOMEM;
+-
+- memset(crat_table, 0, sizeof(struct crat_header));
+- memcpy(&crat_table->signature, CRAT_SIGNATURE,
+- sizeof(crat_table->signature));
+- crat_table->length = sizeof(struct crat_header);
+-
+-#ifdef CONFIG_ACPI
+- status = acpi_get_table("DSDT", 0, &acpi_table);
+- if (status == AE_NOT_FOUND)
+- pr_warn("DSDT table not found for OEM information\n");
+- else {
+- crat_table->oem_revision = acpi_table->revision;
+- memcpy(crat_table->oem_id, acpi_table->oem_id,
+- CRAT_OEMID_LENGTH);
+- memcpy(crat_table->oem_table_id, acpi_table->oem_table_id,
+- CRAT_OEMTABLEID_LENGTH);
+- }
+-#else
+- crat_table->oem_revision = 0;
+- memcpy(crat_table->oem_id, "INV", CRAT_OEMID_LENGTH);
+- memcpy(crat_table->oem_table_id, "UNAVAIL", CRAT_OEMTABLEID_LENGTH);
+-#endif
+- crat_table->total_entries = 0;
+- crat_table->num_domains = 0;
+-
+- sub_type_hdr = (struct crat_subtype_generic *)(crat_table+1);
+-
+- for_each_online_node(numa_node_id) {
+- if (kfd_numa_node_to_apic_id(numa_node_id) == -1)
+- continue;
+-
+- /* Fill in Subtype: Compute Unit */
+- ret = kfd_fill_cu_for_cpu(numa_node_id, &avail_size,
+- crat_table->num_domains,
+- (struct crat_subtype_computeunit *)sub_type_hdr);
+- if (ret < 0)
+- return ret;
+- crat_table->length += sub_type_hdr->length;
+- crat_table->total_entries++;
+-
+- sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
+- sub_type_hdr->length);
+-
+- /* Fill in Subtype: Memory */
+- ret = kfd_fill_mem_info_for_cpu(numa_node_id, &avail_size,
+- crat_table->num_domains,
+- (struct crat_subtype_memory *)sub_type_hdr);
+- if (ret < 0)
+- return ret;
+- crat_table->length += sub_type_hdr->length;
+- crat_table->total_entries++;
+-
+- sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
+- sub_type_hdr->length);
+-
+- /* Fill in Subtype: IO Link */
+-#ifdef CONFIG_X86_64
+- ret = kfd_fill_iolink_info_for_cpu(numa_node_id, &avail_size,
+- &entries,
+- (struct crat_subtype_iolink *)sub_type_hdr);
+- if (ret < 0)
+- return ret;
+- crat_table->length += (sub_type_hdr->length * entries);
+- crat_table->total_entries += entries;
+-
+- sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
+- sub_type_hdr->length * entries);
+-#else
+- pr_info("IO link not available for non x86 platforms\n");
+-#endif
+-
+- crat_table->num_domains++;
+- }
+-
+- /* TODO: Add cache Subtype for CPU.
+- * Currently, CPU cache information is available in function
+- * detect_cache_attributes(cpu) defined in the file
+- * ./arch/x86/kernel/cpu/intel_cacheinfo.c. This function is not
+- * exported and to get the same information the code needs to be
+- * duplicated.
+- */
+-
+- *size = crat_table->length;
+- pr_info("Virtual CRAT table created for CPU\n");
+-
+- return 0;
+-}
+-
+-static int kfd_fill_gpu_memory_affinity(int *avail_size,
+- struct kfd_dev *kdev, uint8_t type, uint64_t size,
+- struct crat_subtype_memory *sub_type_hdr,
+- uint32_t proximity_domain,
+- const struct kfd_local_mem_info *local_mem_info)
+-{
+- *avail_size -= sizeof(struct crat_subtype_memory);
+- if (*avail_size < 0)
+- return -ENOMEM;
+-
+- memset((void *)sub_type_hdr, 0, sizeof(struct crat_subtype_memory));
+- sub_type_hdr->type = CRAT_SUBTYPE_MEMORY_AFFINITY;
+- sub_type_hdr->length = sizeof(struct crat_subtype_memory);
+- sub_type_hdr->flags |= CRAT_SUBTYPE_FLAGS_ENABLED;
+-
+- sub_type_hdr->proximity_domain = proximity_domain;
+-
+- pr_debug("Fill gpu memory affinity - type 0x%x size 0x%llx\n",
+- type, size);
+-
+- sub_type_hdr->length_low = lower_32_bits(size);
+- sub_type_hdr->length_high = upper_32_bits(size);
+-
+- sub_type_hdr->width = local_mem_info->vram_width;
+- sub_type_hdr->visibility_type = type;
+-
+- return 0;
+-}
+-
+-/* kfd_fill_gpu_direct_io_link - Fill in direct io link from GPU
+- * to its NUMA node
+- * @avail_size: Available size in the memory
+- * @kdev - [IN] GPU device
+- * @sub_type_hdr: Memory into which io link info will be filled in
+- * @proximity_domain - proximity domain of the GPU node
+- *
+- * Return 0 if successful else return -ve value
+- */
+-static int kfd_fill_gpu_direct_io_link(int *avail_size,
+- struct kfd_dev *kdev,
+- struct crat_subtype_iolink *sub_type_hdr,
+- uint32_t proximity_domain)
+-{
+- *avail_size -= sizeof(struct crat_subtype_iolink);
+- if (*avail_size < 0)
+- return -ENOMEM;
+-
+- memset((void *)sub_type_hdr, 0, sizeof(struct crat_subtype_iolink));
+-
+- /* Fill in subtype header data */
+- sub_type_hdr->type = CRAT_SUBTYPE_IOLINK_AFFINITY;
+- sub_type_hdr->length = sizeof(struct crat_subtype_iolink);
+- sub_type_hdr->flags |= CRAT_SUBTYPE_FLAGS_ENABLED;
+-
+- /* Fill in IOLINK subtype.
+- * TODO: Fill-in other fields of iolink subtype
+- */
+- sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_PCIEXPRESS;
+- sub_type_hdr->proximity_domain_from = proximity_domain;
+-#ifdef CONFIG_NUMA
+- if (kdev->pdev->dev.numa_node == NUMA_NO_NODE)
+- sub_type_hdr->proximity_domain_to = 0;
+- else
+- sub_type_hdr->proximity_domain_to = kdev->pdev->dev.numa_node;
+-#else
+- sub_type_hdr->proximity_domain_to = 0;
+-#endif
+- return 0;
+-}
+-
+-/* kfd_create_vcrat_image_gpu - Create Virtual CRAT for CPU
+- *
+- * @pcrat_image: Fill in VCRAT for GPU
+- * @size: [IN] allocated size of crat_image.
+- * [OUT] actual size of data filled in crat_image
+- */
+-static int kfd_create_vcrat_image_gpu(void *pcrat_image,
+- size_t *size, struct kfd_dev *kdev,
+- uint32_t proximity_domain)
+-{
+- struct crat_header *crat_table = (struct crat_header *)pcrat_image;
+- struct crat_subtype_generic *sub_type_hdr;
+- struct crat_subtype_computeunit *cu;
+- struct kfd_cu_info cu_info;
+- int avail_size = *size;
+- uint32_t total_num_of_cu;
+- int num_of_cache_entries = 0;
+- int cache_mem_filled = 0;
+- int ret = 0;
+-#if defined(CONFIG_AMD_IOMMU_V2_MODULE) || defined(CONFIG_AMD_IOMMU_V2)
+- struct amd_iommu_device_info iommu_info;
+- const u32 required_iommu_flags = AMD_IOMMU_DEVICE_FLAG_ATS_SUP |
+- AMD_IOMMU_DEVICE_FLAG_PRI_SUP |
+- AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
+-#endif
+- struct kfd_local_mem_info local_mem_info;
+-
+- if (!pcrat_image || avail_size < VCRAT_SIZE_FOR_GPU)
+- return -EINVAL;
+-
+- /* Fill the CRAT Header.
+- * Modify length and total_entries as subunits are added.
+- */
+- avail_size -= sizeof(struct crat_header);
+- if (avail_size < 0)
+- return -ENOMEM;
+-
+- memset(crat_table, 0, sizeof(struct crat_header));
+-
+- memcpy(&crat_table->signature, CRAT_SIGNATURE,
+- sizeof(crat_table->signature));
+- /* Change length as we add more subtypes*/
+- crat_table->length = sizeof(struct crat_header);
+- crat_table->num_domains = 1;
+- crat_table->total_entries = 0;
+-
+- /* Fill in Subtype: Compute Unit
+- * First fill in the sub type header and then sub type data
+- */
+- avail_size -= sizeof(struct crat_subtype_computeunit);
+- if (avail_size < 0)
+- return -ENOMEM;
+-
+- sub_type_hdr = (struct crat_subtype_generic *)(crat_table + 1);
+- memset(sub_type_hdr, 0, sizeof(struct crat_subtype_computeunit));
+-
+- sub_type_hdr->type = CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY;
+- sub_type_hdr->length = sizeof(struct crat_subtype_computeunit);
+- sub_type_hdr->flags = CRAT_SUBTYPE_FLAGS_ENABLED;
+-
+- /* Fill CU subtype data */
+- cu = (struct crat_subtype_computeunit *)sub_type_hdr;
+- cu->flags |= CRAT_CU_FLAGS_GPU_PRESENT;
+- cu->proximity_domain = proximity_domain;
+-
+- kdev->kfd2kgd->get_cu_info(kdev->kgd, &cu_info);
+- cu->num_simd_per_cu = cu_info.simd_per_cu;
+- cu->num_simd_cores = cu_info.simd_per_cu * cu_info.cu_active_number;
+- cu->max_waves_simd = cu_info.max_waves_per_simd;
+-
+- cu->wave_front_size = cu_info.wave_front_size;
+- cu->array_count = cu_info.num_shader_arrays_per_engine *
+- cu_info.num_shader_engines;
+- total_num_of_cu = (cu->array_count * cu_info.num_cu_per_sh);
+- cu->processor_id_low = get_and_inc_gpu_processor_id(total_num_of_cu);
+- cu->num_cu_per_array = cu_info.num_cu_per_sh;
+- cu->max_slots_scatch_cu = cu_info.max_scratch_slots_per_cu;
+- cu->num_banks = cu_info.num_shader_engines;
+- cu->lds_size_in_kb = cu_info.lds_size;
+-
+- cu->hsa_capability = 0;
+-
+- /* Check if this node supports IOMMU. During parsing this flag will
+- * translate to HSA_CAP_ATS_PRESENT
+- */
+-#if defined(CONFIG_AMD_IOMMU_V2_MODULE) || defined(CONFIG_AMD_IOMMU_V2)
+- iommu_info.flags = 0;
+- if (amd_iommu_device_info(kdev->pdev, &iommu_info) == 0) {
+- if ((iommu_info.flags & required_iommu_flags) ==
+- required_iommu_flags)
+- cu->hsa_capability |= CRAT_CU_FLAGS_IOMMU_PRESENT;
+- }
+-#endif
+-
+- crat_table->length += sub_type_hdr->length;
+- crat_table->total_entries++;
+-
+- /* Fill in Subtype: Memory. Only on systems with large BAR (no
+- * private FB), report memory as public. On other systems
+- * report the total FB size (public+private) as a single
+- * private heap.
+- */
+- kdev->kfd2kgd->get_local_mem_info(kdev->kgd, &local_mem_info);
+- sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
+- sub_type_hdr->length);
+-
+- if (debug_largebar)
+- local_mem_info.local_mem_size_private = 0;
+-
+- if (local_mem_info.local_mem_size_private == 0)
+- ret = kfd_fill_gpu_memory_affinity(&avail_size,
+- kdev, HSA_MEM_HEAP_TYPE_FB_PUBLIC,
+- local_mem_info.local_mem_size_public,
+- (struct crat_subtype_memory *)sub_type_hdr,
+- proximity_domain,
+- &local_mem_info);
+- else
+- ret = kfd_fill_gpu_memory_affinity(&avail_size,
+- kdev, HSA_MEM_HEAP_TYPE_FB_PRIVATE,
+- local_mem_info.local_mem_size_public +
+- local_mem_info.local_mem_size_private,
+- (struct crat_subtype_memory *)sub_type_hdr,
+- proximity_domain,
+- &local_mem_info);
+- if (ret < 0)
+- return ret;
+-
+- crat_table->length += sizeof(struct crat_subtype_memory);
+- crat_table->total_entries++;
+-
+- /* TODO: Fill in cache information. This information is NOT readily
+- * available in KGD
+- */
+- sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
+- sub_type_hdr->length);
+- ret = kfd_fill_gpu_cache_info(kdev, cu->processor_id_low,
+- avail_size,
+- &cu_info,
+- (struct crat_subtype_cache *)sub_type_hdr,
+- &cache_mem_filled,
+- &num_of_cache_entries);
+-
+- if (ret < 0)
+- return ret;
+-
+- crat_table->length += cache_mem_filled;
+- crat_table->total_entries += num_of_cache_entries;
+- avail_size -= cache_mem_filled;
+-
+- /* Fill in Subtype: IO_LINKS
+- * Only direct links are added here which is Link from GPU to
+- * to its NUMA node. Indirect links are added by userspace.
+- */
+- sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
+- cache_mem_filled);
+- ret = kfd_fill_gpu_direct_io_link(&avail_size, kdev,
+- (struct crat_subtype_iolink *)sub_type_hdr, proximity_domain);
+-
+- if (ret < 0)
+- return ret;
+-
+- crat_table->length += sub_type_hdr->length;
+- crat_table->total_entries++;
+-
+- *size = crat_table->length;
+- pr_info("Virtual CRAT table created for GPU\n");
+-
+- return ret;
+-}
+-
+-/* kfd_create_crat_image_virtual - Allocates memory for CRAT image and
+- * creates a Virtual CRAT (VCRAT) image
+- *
+- * NOTE: Call kfd_destroy_crat_image to free CRAT image memory
+- *
+- * @crat_image: VCRAT image created because ACPI does not have a
+- * CRAT for this device
+- * @size: [OUT] size of virtual crat_image
+- * @flags: COMPUTE_UNIT_CPU - Create VCRAT for CPU device
+- * COMPUTE_UNIT_GPU - Create VCRAT for GPU
+- * (COMPUTE_UNIT_CPU | COMPUTE_UNIT_GPU) - Create VCRAT for APU
+- * -- this option is not currently implemented.
+- * The assumption is that all AMD APUs will have CRAT
+- * @kdev: Valid kfd_device required if flags contain COMPUTE_UNIT_GPU
+- *
+- * Return 0 if successful else return -ve value
+- */
+-int kfd_create_crat_image_virtual(void **crat_image, size_t *size,
+- int flags, struct kfd_dev *kdev, uint32_t proximity_domain)
+-{
+- void *pcrat_image = NULL;
+- int ret = 0;
+-
+- if (!crat_image)
+- return -EINVAL;
+-
+- *crat_image = NULL;
+-
+- /* Allocate one VCRAT_SIZE_FOR_CPU for CPU virtual CRAT image and
+- * VCRAT_SIZE_FOR_GPU for GPU virtual CRAT image. This should cover
+- * all the current conditions. A check is put not to overwrite beyond
+- * allocated size
+- */
+- switch (flags) {
+- case COMPUTE_UNIT_CPU:
+- pcrat_image = kmalloc(VCRAT_SIZE_FOR_CPU, GFP_KERNEL);
+- if (!pcrat_image)
+- return -ENOMEM;
+- *size = VCRAT_SIZE_FOR_CPU;
+- ret = kfd_create_vcrat_image_cpu(pcrat_image, size);
+- break;
+- case COMPUTE_UNIT_GPU:
+- if (!kdev)
+- return -EINVAL;
+- pcrat_image = kmalloc(VCRAT_SIZE_FOR_GPU, GFP_KERNEL);
+- if (!pcrat_image)
+- return -ENOMEM;
+- *size = VCRAT_SIZE_FOR_GPU;
+- ret = kfd_create_vcrat_image_gpu(pcrat_image, size,
+- kdev, proximity_domain);
+- break;
+- case (COMPUTE_UNIT_CPU | COMPUTE_UNIT_GPU):
+- /* TODO: */
+- ret = -EINVAL;
+- pr_err("VCRAT not implemented for APU\n");
+- break;
+- default:
+- ret = -EINVAL;
+- }
+-
+- if (!ret)
+- *crat_image = pcrat_image;
+- else
+- kfree(pcrat_image);
+-
+- return ret;
+-}
+-
+-
+-/* kfd_destroy_crat_image
+- *
+- * @crat_image: [IN] - crat_image from kfd_create_crat_image_xxx(..)
+- *
+- */
+-void kfd_destroy_crat_image(void *crat_image)
+-{
+- kfree(crat_image);
+-}
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.h b/drivers/gpu/drm/amd/amdkfd/kfd_crat.h
+index 00de41f..a374fa3 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.h
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.h
+@@ -24,7 +24,6 @@
+ #define KFD_CRAT_H_INCLUDED
+
+ #include <linux/types.h>
+-#include "kfd_priv.h"
+
+ #pragma pack(1)
+
+@@ -45,10 +44,6 @@
+
+ #define CRAT_OEMID_64BIT_MASK ((1ULL << (CRAT_OEMID_LENGTH * 8)) - 1)
+
+-/* Compute Unit flags */
+-#define COMPUTE_UNIT_CPU (1 << 0) /* Create Virtual CRAT for CPU */
+-#define COMPUTE_UNIT_GPU (1 << 1) /* Create Virtual CRAT for GPU */
+-
+ struct crat_header {
+ uint32_t signature;
+ uint32_t length;
+@@ -110,7 +105,7 @@ struct crat_subtype_computeunit {
+ uint8_t wave_front_size;
+ uint8_t num_banks;
+ uint16_t micro_engine_id;
+- uint8_t array_count;
++ uint8_t num_arrays;
+ uint8_t num_cu_per_array;
+ uint8_t num_simd_per_cu;
+ uint8_t max_slots_scatch_cu;
+@@ -132,14 +127,13 @@ struct crat_subtype_memory {
+ uint8_t length;
+ uint16_t reserved;
+ uint32_t flags;
+- uint32_t proximity_domain;
++ uint32_t promixity_domain;
+ uint32_t base_addr_low;
+ uint32_t base_addr_high;
+ uint32_t length_low;
+ uint32_t length_high;
+ uint32_t width;
+- uint8_t visibility_type; /* for virtual (dGPU) CRAT */
+- uint8_t reserved2[CRAT_MEMORY_RESERVED_LENGTH - 1];
++ uint8_t reserved2[CRAT_MEMORY_RESERVED_LENGTH];
+ };
+
+ /*
+@@ -228,12 +222,9 @@ struct crat_subtype_ccompute {
+ /*
+ * HSA IO Link Affinity structure and definitions
+ */
+-#define CRAT_IOLINK_FLAGS_ENABLED (1 << 0)
+-#define CRAT_IOLINK_FLAGS_NON_COHERENT (1 << 1)
+-#define CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT (1 << 2)
+-#define CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT (1 << 3)
+-#define CRAT_IOLINK_FLAGS_NO_PEER_TO_PEER_DMA (1 << 4)
+-#define CRAT_IOLINK_FLAGS_RESERVED_MASK 0xffffffe0
++#define CRAT_IOLINK_FLAGS_ENABLED 0x00000001
++#define CRAT_IOLINK_FLAGS_COHERENCY 0x00000002
++#define CRAT_IOLINK_FLAGS_RESERVED 0xfffffffc
+
+ /*
+ * IO interface types
+@@ -241,16 +232,8 @@ struct crat_subtype_ccompute {
+ #define CRAT_IOLINK_TYPE_UNDEFINED 0
+ #define CRAT_IOLINK_TYPE_HYPERTRANSPORT 1
+ #define CRAT_IOLINK_TYPE_PCIEXPRESS 2
+-#define CRAT_IOLINK_TYPE_AMBA 3
+-#define CRAT_IOLINK_TYPE_MIPI 4
+-#define CRAT_IOLINK_TYPE_QPI_1_1 5
+-#define CRAT_IOLINK_TYPE_RESERVED1 6
+-#define CRAT_IOLINK_TYPE_RESERVED2 7
+-#define CRAT_IOLINK_TYPE_RAPID_IO 8
+-#define CRAT_IOLINK_TYPE_INFINIBAND 9
+-#define CRAT_IOLINK_TYPE_RESERVED3 10
+-#define CRAT_IOLINK_TYPE_OTHER 11
+-#define CRAT_IOLINK_TYPE_MAX 255
++#define CRAT_IOLINK_TYPE_OTHER 3
++#define CRAT_IOLINK_TYPE_MAX 255
+
+ #define CRAT_IOLINK_RESERVED_LENGTH 24
+
+@@ -308,13 +291,4 @@ struct cdit_header {
+
+ #pragma pack()
+
+-#ifdef CONFIG_ACPI
+-int kfd_create_crat_image_acpi(void **crat_image, size_t *size);
+-#endif
+-void kfd_destroy_crat_image(void *crat_image);
+-int kfd_parse_crat_table(void *crat_image,
+- struct list_head *device_list,
+- uint32_t proximity_domain);
+-int kfd_create_crat_image_virtual(void **crat_image, size_t *size,
+- int flags, struct kfd_dev *kdev, uint32_t proximity_domain);
+ #endif /* KFD_CRAT_H_INCLUDED */
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c b/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c
+index af6d736..d5e19b5 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c
+@@ -29,7 +29,7 @@
+ #include <linux/mutex.h>
+ #include <linux/device.h>
+
+-#include "kfd_pm4_headers_vi.h"
++#include "kfd_pm4_headers.h"
+ #include "kfd_pm4_headers_diq.h"
+ #include "kfd_kernel_queue.h"
+ #include "kfd_priv.h"
+@@ -42,15 +42,16 @@
+
+ static void dbgdev_address_watch_disable_nodiq(struct kfd_dev *dev)
+ {
++ BUG_ON(!dev || !dev->kfd2kgd);
++
+ dev->kfd2kgd->address_watch_disable(dev->kgd);
+ }
+
+ static int dbgdev_diq_submit_ib(struct kfd_dbgdev *dbgdev,
+ unsigned int pasid, uint64_t vmid0_address,
+- uint32_t *packet_buff, size_t size_in_bytes,
+- bool sync)
++ uint32_t *packet_buff, size_t size_in_bytes)
+ {
+- struct pm4_mec_release_mem *rm_packet;
++ struct pm4__release_mem *rm_packet;
+ struct pm4__indirect_buffer_pasid *ib_packet;
+ struct kfd_mem_obj *mem_obj;
+ size_t pq_packets_size_in_bytes;
+@@ -61,14 +62,12 @@ static int dbgdev_diq_submit_ib(struct kfd_dbgdev *dbgdev,
+ unsigned int *ib_packet_buff;
+ int status;
+
+- if (WARN_ON(!size_in_bytes))
+- return -EINVAL;
++ BUG_ON(!dbgdev || !dbgdev->kq || !packet_buff || !size_in_bytes);
+
+ kq = dbgdev->kq;
+
+- pq_packets_size_in_bytes = sizeof(struct pm4__indirect_buffer_pasid);
+- if (sync)
+- pq_packets_size_in_bytes += sizeof(struct pm4_mec_release_mem);
++ pq_packets_size_in_bytes = sizeof(struct pm4__release_mem) +
++ sizeof(struct pm4__indirect_buffer_pasid);
+
+ /*
+ * We acquire a buffer from DIQ
+@@ -78,8 +77,8 @@ static int dbgdev_diq_submit_ib(struct kfd_dbgdev *dbgdev,
+ status = kq->ops.acquire_packet_buffer(kq,
+ pq_packets_size_in_bytes / sizeof(uint32_t),
+ &ib_packet_buff);
+- if (status) {
+- pr_err("acquire_packet_buffer failed\n");
++ if (status != 0) {
++ pr_err("amdkfd: acquire_packet_buffer failed\n");
+ return status;
+ }
+
+@@ -101,11 +100,6 @@ static int dbgdev_diq_submit_ib(struct kfd_dbgdev *dbgdev,
+
+ ib_packet->bitfields5.pasid = pasid;
+
+- if (!sync) {
+- kq->ops.submit_packet(kq);
+- return status;
+- }
+-
+ /*
+ * for now we use release mem for GPU-CPU synchronization
+ * Consider WaitRegMem + WriteData as a better alternative
+@@ -114,15 +108,15 @@ static int dbgdev_diq_submit_ib(struct kfd_dbgdev *dbgdev,
+ * (a) Sync with HW
+ * (b) Sync var is written by CP to mem.
+ */
+- rm_packet = (struct pm4_mec_release_mem *) (ib_packet_buff +
++ rm_packet = (struct pm4__release_mem *) (ib_packet_buff +
+ (sizeof(struct pm4__indirect_buffer_pasid) /
+ sizeof(unsigned int)));
+
+ status = kfd_gtt_sa_allocate(dbgdev->dev, sizeof(uint64_t),
+ &mem_obj);
+
+- if (status) {
+- pr_err("Failed to allocate GART memory\n");
++ if (status != 0) {
++ pr_err("amdkfd: Failed to allocate GART memory\n");
+ kq->ops.rollback_packet(kq);
+ return status;
+ }
+@@ -133,7 +127,7 @@ static int dbgdev_diq_submit_ib(struct kfd_dbgdev *dbgdev,
+
+ rm_packet->header.opcode = IT_RELEASE_MEM;
+ rm_packet->header.type = PM4_TYPE_3;
+- rm_packet->header.count = sizeof(struct pm4_mec_release_mem) /
++ rm_packet->header.count = sizeof(struct pm4__release_mem) /
+ sizeof(unsigned int) - 2;
+
+ rm_packet->bitfields2.event_type = CACHE_FLUSH_AND_INV_TS_EVENT;
+@@ -174,6 +168,8 @@ static int dbgdev_diq_submit_ib(struct kfd_dbgdev *dbgdev,
+
+ static int dbgdev_register_nodiq(struct kfd_dbgdev *dbgdev)
+ {
++ BUG_ON(!dbgdev);
++
+ /*
+ * no action is needed in this case,
+ * just make sure diq will not be used
+@@ -191,12 +187,14 @@ static int dbgdev_register_diq(struct kfd_dbgdev *dbgdev)
+ struct kernel_queue *kq = NULL;
+ int status;
+
+- properties.type = KFD_QUEUE_TYPE_DIQ;
++ BUG_ON(!dbgdev || !dbgdev->pqm || !dbgdev->dev);
++
+ status = pqm_create_queue(dbgdev->pqm, dbgdev->dev, NULL,
+- &properties, &qid);
++ &properties, 0, KFD_QUEUE_TYPE_DIQ,
++ &qid);
+
+ if (status) {
+- pr_err("Failed to create DIQ\n");
++ pr_err("amdkfd: Failed to create DIQ\n");
+ return status;
+ }
+
+@@ -204,8 +202,8 @@ static int dbgdev_register_diq(struct kfd_dbgdev *dbgdev)
+
+ kq = pqm_get_kernel_queue(dbgdev->pqm, qid);
+
+- if (!kq) {
+- pr_err("Error getting DIQ\n");
++ if (kq == NULL) {
++ pr_err("amdkfd: Error getting DIQ\n");
+ pqm_destroy_queue(dbgdev->pqm, qid);
+ return -EFAULT;
+ }
+@@ -217,6 +215,8 @@ static int dbgdev_register_diq(struct kfd_dbgdev *dbgdev)
+
+ static int dbgdev_unregister_nodiq(struct kfd_dbgdev *dbgdev)
+ {
++ BUG_ON(!dbgdev || !dbgdev->dev);
++
+ /* disable watch address */
+ dbgdev_address_watch_disable_nodiq(dbgdev->dev);
+ return 0;
+@@ -227,6 +227,8 @@ static int dbgdev_unregister_diq(struct kfd_dbgdev *dbgdev)
+ /* todo - disable address watch */
+ int status;
+
++ BUG_ON(!dbgdev || !dbgdev->pqm || !dbgdev->kq);
++
+ status = pqm_destroy_queue(dbgdev->pqm,
+ dbgdev->kq->queue->properties.queue_id);
+ dbgdev->kq = NULL;
+@@ -239,17 +241,18 @@ static void dbgdev_address_watch_set_registers(
+ union TCP_WATCH_ADDR_H_BITS *addrHi,
+ union TCP_WATCH_ADDR_L_BITS *addrLo,
+ union TCP_WATCH_CNTL_BITS *cntl,
+- unsigned int index, unsigned int vmid,
+- bool is_apu)
++ unsigned int index, unsigned int vmid)
+ {
+ union ULARGE_INTEGER addr;
+
++ BUG_ON(!adw_info || !addrHi || !addrLo || !cntl);
++
+ addr.quad_part = 0;
+ addrHi->u32All = 0;
+ addrLo->u32All = 0;
+ cntl->u32All = 0;
+
+- if (adw_info->watch_mask)
++ if (adw_info->watch_mask != NULL)
+ cntl->bitfields.mask =
+ (uint32_t) (adw_info->watch_mask[index] &
+ ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK);
+@@ -265,9 +268,9 @@ static void dbgdev_address_watch_set_registers(
+
+ cntl->bitfields.mode = adw_info->watch_mode[index];
+ cntl->bitfields.vmid = (uint32_t) vmid;
+- /* for APU assume it is an ATC address */
+- if (is_apu)
+- cntl->u32All |= ADDRESS_WATCH_REG_CNTL_ATC_BIT;
++ /* for now assume it is an ATC address */
++ cntl->u32All |= ADDRESS_WATCH_REG_CNTL_ATC_BIT;
++
+ pr_debug("\t\t%20s %08x\n", "set reg mask :", cntl->bitfields.mask);
+ pr_debug("\t\t%20s %08x\n", "set reg add high :",
+ addrHi->bitfields.addr);
+@@ -276,7 +279,7 @@ static void dbgdev_address_watch_set_registers(
+ }
+
+ static int dbgdev_address_watch_nodiq(struct kfd_dbgdev *dbgdev,
+- struct dbg_address_watch_info *adw_info)
++ struct dbg_address_watch_info *adw_info)
+ {
+ union TCP_WATCH_ADDR_H_BITS addrHi;
+ union TCP_WATCH_ADDR_L_BITS addrLo;
+@@ -284,11 +287,13 @@ static int dbgdev_address_watch_nodiq(struct kfd_dbgdev *dbgdev,
+ struct kfd_process_device *pdd;
+ unsigned int i;
+
++ BUG_ON(!dbgdev || !dbgdev->dev || !adw_info);
++
+ /* taking the vmid for that process on the safe way using pdd */
+ pdd = kfd_get_process_device_data(dbgdev->dev,
+ adw_info->process);
+ if (!pdd) {
+- pr_err("Failed to get pdd for wave control no DIQ\n");
++ pr_err("amdkfd: Failed to get pdd for wave control no DIQ\n");
+ return -EFAULT;
+ }
+
+@@ -298,19 +303,19 @@ static int dbgdev_address_watch_nodiq(struct kfd_dbgdev *dbgdev,
+
+ if ((adw_info->num_watch_points > MAX_WATCH_ADDRESSES) ||
+ (adw_info->num_watch_points == 0)) {
+- pr_err("num_watch_points is invalid\n");
++ pr_err("amdkfd: num_watch_points is invalid\n");
+ return -EINVAL;
+ }
+
+- if (!adw_info->watch_mode || !adw_info->watch_address) {
+- pr_err("adw_info fields are not valid\n");
++ if ((adw_info->watch_mode == NULL) ||
++ (adw_info->watch_address == NULL)) {
++ pr_err("amdkfd: adw_info fields are not valid\n");
+ return -EINVAL;
+ }
+
+- for (i = 0; i < adw_info->num_watch_points; i++) {
++ for (i = 0 ; i < adw_info->num_watch_points ; i++) {
+ dbgdev_address_watch_set_registers(adw_info, &addrHi, &addrLo,
+- &cntl, i, pdd->qpd.vmid,
+- dbgdev->dev->device_info->is_need_iommu_device);
++ &cntl, i, pdd->qpd.vmid);
+
+ pr_debug("\t\t%30s\n", "* * * * * * * * * * * * * * * * * *");
+ pr_debug("\t\t%20s %08x\n", "register index :", i);
+@@ -343,43 +348,48 @@ static int dbgdev_address_watch_nodiq(struct kfd_dbgdev *dbgdev,
+ }
+
+ static int dbgdev_address_watch_diq(struct kfd_dbgdev *dbgdev,
+- struct dbg_address_watch_info *adw_info)
++ struct dbg_address_watch_info *adw_info)
+ {
+ struct pm4__set_config_reg *packets_vec;
+ union TCP_WATCH_ADDR_H_BITS addrHi;
+ union TCP_WATCH_ADDR_L_BITS addrLo;
+ union TCP_WATCH_CNTL_BITS cntl;
++ struct kfd_mem_obj *mem_obj;
+ unsigned int aw_reg_add_dword;
+ uint32_t *packet_buff_uint;
+- uint64_t packet_buff_gpu_addr;
+ unsigned int i;
+ int status;
+ size_t ib_size = sizeof(struct pm4__set_config_reg) * 4;
+ /* we do not control the vmid in DIQ mode, just a place holder */
+ unsigned int vmid = 0;
+
++ BUG_ON(!dbgdev || !dbgdev->dev || !adw_info);
++
+ addrHi.u32All = 0;
+ addrLo.u32All = 0;
+ cntl.u32All = 0;
+
+ if ((adw_info->num_watch_points > MAX_WATCH_ADDRESSES) ||
+ (adw_info->num_watch_points == 0)) {
+- pr_err("num_watch_points is invalid\n");
++ pr_err("amdkfd: num_watch_points is invalid\n");
+ return -EINVAL;
+ }
+
+- if (!adw_info->watch_mode || !adw_info->watch_address) {
+- pr_err("adw_info fields are not valid\n");
++ if ((NULL == adw_info->watch_mode) ||
++ (NULL == adw_info->watch_address)) {
++ pr_err("amdkfd: adw_info fields are not valid\n");
+ return -EINVAL;
+ }
+
+- status = dbgdev->kq->ops.acquire_inline_ib(dbgdev->kq,
+- ib_size/sizeof(uint32_t),
+- &packet_buff_uint, &packet_buff_gpu_addr);
+- if (status) {
+- pr_err("Failed to allocate IB from DIQ ring\n");
++ status = kfd_gtt_sa_allocate(dbgdev->dev, ib_size, &mem_obj);
++
++ if (status != 0) {
++ pr_err("amdkfd: Failed to allocate GART memory\n");
+ return status;
+ }
++
++ packet_buff_uint = mem_obj->cpu_ptr;
++
+ memset(packet_buff_uint, 0, ib_size);
+
+ packets_vec = (struct pm4__set_config_reg *) (packet_buff_uint);
+@@ -398,9 +408,12 @@ static int dbgdev_address_watch_diq(struct kfd_dbgdev *dbgdev,
+ packets_vec[3].bitfields2.insert_vmid = 1;
+
+ for (i = 0; i < adw_info->num_watch_points; i++) {
+- dbgdev_address_watch_set_registers(adw_info, &addrHi, &addrLo,
+- &cntl, i, vmid,
+- dbgdev->dev->device_info->is_need_iommu_device);
++ dbgdev_address_watch_set_registers(adw_info,
++ &addrHi,
++ &addrLo,
++ &cntl,
++ i,
++ vmid);
+
+ pr_debug("\t\t%30s\n", "* * * * * * * * * * * * * * * * * *");
+ pr_debug("\t\t%20s %08x\n", "register index :", i);
+@@ -429,6 +442,8 @@ static int dbgdev_address_watch_diq(struct kfd_dbgdev *dbgdev,
+ i,
+ ADDRESS_WATCH_REG_CNTL);
+
++ aw_reg_add_dword /= sizeof(uint32_t);
++
+ packets_vec[0].bitfields2.reg_offset =
+ aw_reg_add_dword - AMD_CONFIG_REG_BASE;
+
+@@ -440,6 +455,8 @@ static int dbgdev_address_watch_diq(struct kfd_dbgdev *dbgdev,
+ i,
+ ADDRESS_WATCH_REG_ADDR_HI);
+
++ aw_reg_add_dword /= sizeof(uint32_t);
++
+ packets_vec[1].bitfields2.reg_offset =
+ aw_reg_add_dword - AMD_CONFIG_REG_BASE;
+ packets_vec[1].reg_data[0] = addrHi.u32All;
+@@ -450,6 +467,8 @@ static int dbgdev_address_watch_diq(struct kfd_dbgdev *dbgdev,
+ i,
+ ADDRESS_WATCH_REG_ADDR_LO);
+
++ aw_reg_add_dword /= sizeof(uint32_t);
++
+ packets_vec[2].bitfields2.reg_offset =
+ aw_reg_add_dword - AMD_CONFIG_REG_BASE;
+ packets_vec[2].reg_data[0] = addrLo.u32All;
+@@ -466,6 +485,8 @@ static int dbgdev_address_watch_diq(struct kfd_dbgdev *dbgdev,
+ i,
+ ADDRESS_WATCH_REG_CNTL);
+
++ aw_reg_add_dword /= sizeof(uint32_t);
++
+ packets_vec[3].bitfields2.reg_offset =
+ aw_reg_add_dword - AMD_CONFIG_REG_BASE;
+ packets_vec[3].reg_data[0] = cntl.u32All;
+@@ -473,30 +494,32 @@ static int dbgdev_address_watch_diq(struct kfd_dbgdev *dbgdev,
+ status = dbgdev_diq_submit_ib(
+ dbgdev,
+ adw_info->process->pasid,
+- packet_buff_gpu_addr,
++ mem_obj->gpu_addr,
+ packet_buff_uint,
+- ib_size, true);
++ ib_size);
+
+- if (status) {
+- pr_err("Failed to submit IB to DIQ\n");
+- return status;
++ if (status != 0) {
++ pr_err("amdkfd: Failed to submit IB to DIQ\n");
++ break;
+ }
+ }
+
++ kfd_gtt_sa_free(dbgdev->dev, mem_obj);
+ return status;
+ }
+
+ static int dbgdev_wave_control_set_registers(
+ struct dbg_wave_control_info *wac_info,
+ union SQ_CMD_BITS *in_reg_sq_cmd,
+- union GRBM_GFX_INDEX_BITS *in_reg_gfx_index,
+- unsigned int asic_family)
++ union GRBM_GFX_INDEX_BITS *in_reg_gfx_index)
+ {
+ int status = 0;
+ union SQ_CMD_BITS reg_sq_cmd;
+ union GRBM_GFX_INDEX_BITS reg_gfx_index;
+ struct HsaDbgWaveMsgAMDGen2 *pMsg;
+
++ BUG_ON(!wac_info || !in_reg_sq_cmd || !in_reg_gfx_index);
++
+ reg_sq_cmd.u32All = 0;
+ reg_gfx_index.u32All = 0;
+ pMsg = &wac_info->dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2;
+@@ -548,25 +571,11 @@ static int dbgdev_wave_control_set_registers(
+
+ switch (wac_info->operand) {
+ case HSA_DBG_WAVEOP_HALT:
+- if (asic_family == CHIP_KAVERI) {
+- reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_HALT;
+- pr_debug("Halting KV\n");
+- } else {
+- reg_sq_cmd.bits_sethalt.cmd = SQ_IND_CMD_NEW_SETHALT;
+- reg_sq_cmd.bits_sethalt.data = SQ_IND_CMD_DATA_HALT;
+- pr_debug("Halting CZ\n");
+- }
++ reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_HALT;
+ break;
+
+ case HSA_DBG_WAVEOP_RESUME:
+- if (asic_family == CHIP_KAVERI) {
+- reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_RESUME;
+- pr_debug("Resuming KV\n");
+- } else {
+- reg_sq_cmd.bits_sethalt.cmd = SQ_IND_CMD_NEW_SETHALT;
+- reg_sq_cmd.bits_sethalt.data = SQ_IND_CMD_DATA_RESUME;
+- pr_debug("Resuming CZ\n");
+- }
++ reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_RESUME;
+ break;
+
+ case HSA_DBG_WAVEOP_KILL:
+@@ -606,21 +615,23 @@ static int dbgdev_wave_control_diq(struct kfd_dbgdev *dbgdev,
+ int status;
+ union SQ_CMD_BITS reg_sq_cmd;
+ union GRBM_GFX_INDEX_BITS reg_gfx_index;
++ struct kfd_mem_obj *mem_obj;
+ uint32_t *packet_buff_uint;
+- uint64_t packet_buff_gpu_addr;
+ struct pm4__set_config_reg *packets_vec;
+ size_t ib_size = sizeof(struct pm4__set_config_reg) * 3;
+
++ BUG_ON(!dbgdev || !wac_info);
++
+ reg_sq_cmd.u32All = 0;
+
+ status = dbgdev_wave_control_set_registers(wac_info, &reg_sq_cmd,
+- &reg_gfx_index, dbgdev->dev->device_info->asic_family);
++ &reg_gfx_index);
+ if (status) {
+- pr_err("Failed to set wave control registers\n");
++ pr_err("amdkfd: Failed to set wave control registers\n");
+ return status;
+ }
+
+- /* we do not control the VMID in DIQ, so reset it to a known value */
++ /* we do not control the VMID in DIQ,so reset it to a known value */
+ reg_sq_cmd.bits.vm_id = 0;
+
+ pr_debug("\t\t %30s\n", "* * * * * * * * * * * * * * * * * *");
+@@ -653,13 +664,15 @@ static int dbgdev_wave_control_diq(struct kfd_dbgdev *dbgdev,
+
+ pr_debug("\t\t %30s\n", "* * * * * * * * * * * * * * * * * *");
+
+- status = dbgdev->kq->ops.acquire_inline_ib(dbgdev->kq,
+- ib_size / sizeof(uint32_t),
+- &packet_buff_uint, &packet_buff_gpu_addr);
+- if (status) {
+- pr_err("Failed to allocate IB from DIQ ring\n");
++ status = kfd_gtt_sa_allocate(dbgdev->dev, ib_size, &mem_obj);
++
++ if (status != 0) {
++ pr_err("amdkfd: Failed to allocate GART memory\n");
+ return status;
+ }
++
++ packet_buff_uint = mem_obj->cpu_ptr;
++
+ memset(packet_buff_uint, 0, ib_size);
+
+ packets_vec = (struct pm4__set_config_reg *) packet_buff_uint;
+@@ -702,12 +715,14 @@ static int dbgdev_wave_control_diq(struct kfd_dbgdev *dbgdev,
+ status = dbgdev_diq_submit_ib(
+ dbgdev,
+ wac_info->process->pasid,
+- packet_buff_gpu_addr,
++ mem_obj->gpu_addr,
+ packet_buff_uint,
+- ib_size, false);
++ ib_size);
+
+- if (status)
+- pr_err("Failed to submit IB to DIQ\n");
++ if (status != 0)
++ pr_err("amdkfd: Failed to submit IB to DIQ\n");
++
++ kfd_gtt_sa_free(dbgdev->dev, mem_obj);
+
+ return status;
+ }
+@@ -720,19 +735,21 @@ static int dbgdev_wave_control_nodiq(struct kfd_dbgdev *dbgdev,
+ union GRBM_GFX_INDEX_BITS reg_gfx_index;
+ struct kfd_process_device *pdd;
+
++ BUG_ON(!dbgdev || !dbgdev->dev || !wac_info);
++
+ reg_sq_cmd.u32All = 0;
+
+ /* taking the VMID for that process on the safe way using PDD */
+ pdd = kfd_get_process_device_data(dbgdev->dev, wac_info->process);
+
+ if (!pdd) {
+- pr_err("Failed to get pdd for wave control no DIQ\n");
++ pr_err("amdkfd: Failed to get pdd for wave control no DIQ\n");
+ return -EFAULT;
+ }
+ status = dbgdev_wave_control_set_registers(wac_info, &reg_sq_cmd,
+- &reg_gfx_index, dbgdev->dev->device_info->asic_family);
++ &reg_gfx_index);
+ if (status) {
+- pr_err("Failed to set wave control registers\n");
++ pr_err("amdkfd: Failed to set wave control registers\n");
+ return status;
+ }
+
+@@ -783,8 +800,13 @@ int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p)
+ union GRBM_GFX_INDEX_BITS reg_gfx_index;
+ struct kfd_process_device *pdd;
+ struct dbg_wave_control_info wac_info;
+- int first_vmid_to_scan = dev->vm_info.first_vmid_kfd;
+- int last_vmid_to_scan = dev->vm_info.last_vmid_kfd;
++ int temp;
++ int first_vmid_to_scan = 8;
++ int last_vmid_to_scan = 15;
++
++ first_vmid_to_scan = ffs(dev->shared_resources.compute_vmid_bitmap) - 1;
++ temp = dev->shared_resources.compute_vmid_bitmap >> first_vmid_to_scan;
++ last_vmid_to_scan = first_vmid_to_scan + ffz(temp);
+
+ reg_sq_cmd.u32All = 0;
+ status = 0;
+@@ -796,13 +818,12 @@ int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p)
+
+ /* Scan all registers in the range ATC_VMID8_PASID_MAPPING ..
+ * ATC_VMID15_PASID_MAPPING
+- * to check which VMID the current process is mapped to.
+- */
++ * to check which VMID the current process is mapped to. */
+
+ for (vmid = first_vmid_to_scan; vmid <= last_vmid_to_scan; vmid++) {
+ if (dev->kfd2kgd->get_atc_vmid_pasid_mapping_valid
+ (dev->kgd, vmid)) {
+- if (dev->kfd2kgd->get_atc_vmid_pasid_mapping_pasid
++ if (dev->kfd2kgd->get_atc_vmid_pasid_mapping_valid
+ (dev->kgd, vmid) == p->pasid) {
+ pr_debug("Killing wave fronts of vmid %d and pasid %d\n",
+ vmid, p->pasid);
+@@ -812,7 +833,7 @@ int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p)
+ }
+
+ if (vmid > last_vmid_to_scan) {
+- pr_err("Didn't find vmid for pasid %d\n", p->pasid);
++ pr_err("amdkfd: didn't found vmid for pasid (%d)\n", p->pasid);
+ return -EFAULT;
+ }
+
+@@ -822,7 +843,7 @@ int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p)
+ return -EFAULT;
+
+ status = dbgdev_wave_control_set_registers(&wac_info, &reg_sq_cmd,
+- &reg_gfx_index, dev->device_info->asic_family);
++ &reg_gfx_index);
+ if (status != 0)
+ return -EINVAL;
+
+@@ -839,6 +860,8 @@ int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p)
+ void kfd_dbgdev_init(struct kfd_dbgdev *pdbgdev, struct kfd_dev *pdev,
+ enum DBGDEV_TYPE type)
+ {
++ BUG_ON(!pdbgdev || !pdev);
++
+ pdbgdev->dev = pdev;
+ pdbgdev->kq = NULL;
+ pdbgdev->type = type;
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.h b/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.h
+index 583aaa9..03424c2 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.h
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.h
+@@ -60,24 +60,6 @@ enum {
+ SH_REG_SIZE = SH_REG_END - SH_REG_BASE
+ };
+
+-/* SQ_CMD definitions */
+-
+-enum {
+- SQ_IND_CMD_DATA_RESUME = 0,
+- SQ_IND_CMD_DATA_HALT = 1
+-};
+-
+-enum SQ_IND_CMD_NEW {
+- SQ_IND_CMD_NEW_NULL = 0x00000000,
+- SQ_IND_CMD_NEW_SETHALT = 0x00000001,
+- SQ_IND_CMD_NEW_SAVECTX = 0x00000002,
+- SQ_IND_CMD_NEW_KILL = 0x00000003,
+- SQ_IND_CMD_NEW_DEBUG = 0x00000004,
+- SQ_IND_CMD_NEW_TRAP = 0x00000005,
+- SQ_IND_CMD_NEW_SET_PRIO = 0x00000006
+-
+-};
+-
+ enum SQ_IND_CMD_CMD {
+ SQ_IND_CMD_CMD_NULL = 0x00000000,
+ SQ_IND_CMD_CMD_HALT = 0x00000001,
+@@ -136,20 +118,6 @@ union SQ_CMD_BITS {
+ uint32_t:1;
+ uint32_t vm_id:4;
+ } bitfields, bits;
+- struct {
+- uint32_t cmd:3;
+- uint32_t:1;
+- uint32_t mode:3;
+- uint32_t check_vmid:1;
+- uint32_t data:3;
+- uint32_t:5;
+- uint32_t wave_id:4;
+- uint32_t simd_id:2;
+- uint32_t:2;
+- uint32_t queue_id:3;
+- uint32_t:1;
+- uint32_t vm_id:4;
+- } bitfields_sethalt, bits_sethalt;
+ uint32_t u32All;
+ signed int i32All;
+ float f32All;
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.c b/drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.c
+index 9d4af96..56d6763 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.c
+@@ -33,7 +33,6 @@
+ #include "kfd_pm4_headers_diq.h"
+ #include "kfd_dbgmgr.h"
+ #include "kfd_dbgdev.h"
+-#include "kfd_device_queue_manager.h"
+
+ static DEFINE_MUTEX(kfd_dbgmgr_mutex);
+
+@@ -45,6 +44,8 @@ struct mutex *kfd_get_dbgmgr_mutex(void)
+
+ static void kfd_dbgmgr_uninitialize(struct kfd_dbgmgr *pmgr)
+ {
++ BUG_ON(!pmgr);
++
+ kfree(pmgr->dbgdev);
+
+ pmgr->dbgdev = NULL;
+@@ -54,7 +55,7 @@ static void kfd_dbgmgr_uninitialize(struct kfd_dbgmgr *pmgr)
+
+ void kfd_dbgmgr_destroy(struct kfd_dbgmgr *pmgr)
+ {
+- if (pmgr) {
++ if (pmgr != NULL) {
+ kfd_dbgmgr_uninitialize(pmgr);
+ kfree(pmgr);
+ }
+@@ -65,12 +66,12 @@ bool kfd_dbgmgr_create(struct kfd_dbgmgr **ppmgr, struct kfd_dev *pdev)
+ enum DBGDEV_TYPE type = DBGDEV_TYPE_DIQ;
+ struct kfd_dbgmgr *new_buff;
+
+- if (WARN_ON(!pdev->init_complete))
+- return false;
++ BUG_ON(pdev == NULL);
++ BUG_ON(!pdev->init_complete);
+
+ new_buff = kfd_alloc_struct(new_buff);
+ if (!new_buff) {
+- pr_err("Failed to allocate dbgmgr instance\n");
++ pr_err("amdkfd: Failed to allocate dbgmgr instance\n");
+ return false;
+ }
+
+@@ -78,13 +79,13 @@ bool kfd_dbgmgr_create(struct kfd_dbgmgr **ppmgr, struct kfd_dev *pdev)
+ new_buff->dev = pdev;
+ new_buff->dbgdev = kfd_alloc_struct(new_buff->dbgdev);
+ if (!new_buff->dbgdev) {
+- pr_err("Failed to allocate dbgdev instance\n");
++ pr_err("amdkfd: Failed to allocate dbgdev instance\n");
+ kfree(new_buff);
+ return false;
+ }
+
+ /* get actual type of DBGDevice cpsch or not */
+- if (pdev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS)
++ if (sched_policy == KFD_SCHED_POLICY_NO_HWS)
+ type = DBGDEV_TYPE_NODIQ;
+
+ kfd_dbgdev_init(new_buff->dbgdev, pdev, type);
+@@ -95,6 +96,8 @@ bool kfd_dbgmgr_create(struct kfd_dbgmgr **ppmgr, struct kfd_dev *pdev)
+
+ long kfd_dbgmgr_register(struct kfd_dbgmgr *pmgr, struct kfd_process *p)
+ {
++ BUG_ON(!p || !pmgr || !pmgr->dbgdev);
++
+ if (pmgr->pasid != 0) {
+ pr_debug("H/W debugger is already active using pasid %d\n",
+ pmgr->pasid);
+@@ -115,6 +118,8 @@ long kfd_dbgmgr_register(struct kfd_dbgmgr *pmgr, struct kfd_process *p)
+
+ long kfd_dbgmgr_unregister(struct kfd_dbgmgr *pmgr, struct kfd_process *p)
+ {
++ BUG_ON(!p || !pmgr || !pmgr->dbgdev);
++
+ /* Is the requests coming from the already registered process? */
+ if (pmgr->pasid != p->pasid) {
+ pr_debug("H/W debugger is not registered by calling pasid %d\n",
+@@ -132,6 +137,8 @@ long kfd_dbgmgr_unregister(struct kfd_dbgmgr *pmgr, struct kfd_process *p)
+ long kfd_dbgmgr_wave_control(struct kfd_dbgmgr *pmgr,
+ struct dbg_wave_control_info *wac_info)
+ {
++ BUG_ON(!pmgr || !pmgr->dbgdev || !wac_info);
++
+ /* Is the requests coming from the already registered process? */
+ if (pmgr->pasid != wac_info->process->pasid) {
+ pr_debug("H/W debugger support was not registered for requester pasid %d\n",
+@@ -145,6 +152,9 @@ long kfd_dbgmgr_wave_control(struct kfd_dbgmgr *pmgr,
+ long kfd_dbgmgr_address_watch(struct kfd_dbgmgr *pmgr,
+ struct dbg_address_watch_info *adw_info)
+ {
++ BUG_ON(!pmgr || !pmgr->dbgdev || !adw_info);
++
++
+ /* Is the requests coming from the already registered process? */
+ if (pmgr->pasid != adw_info->process->pasid) {
+ pr_debug("H/W debugger support was not registered for requester pasid %d\n",
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.h b/drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.h
+index a04a1fe..257a745 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.h
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.h
+@@ -30,11 +30,13 @@
+ #pragma pack(push, 4)
+
+ enum HSA_DBG_WAVEOP {
+- HSA_DBG_WAVEOP_HALT = 1, /* Halts a wavefront */
+- HSA_DBG_WAVEOP_RESUME = 2, /* Resumes a wavefront */
+- HSA_DBG_WAVEOP_KILL = 3, /* Kills a wavefront */
+- HSA_DBG_WAVEOP_DEBUG = 4, /* Causes wavefront to enter dbg mode */
+- HSA_DBG_WAVEOP_TRAP = 5, /* Causes wavefront to take a trap */
++ HSA_DBG_WAVEOP_HALT = 1, /* Halts a wavefront */
++ HSA_DBG_WAVEOP_RESUME = 2, /* Resumes a wavefront */
++ HSA_DBG_WAVEOP_KILL = 3, /* Kills a wavefront */
++ HSA_DBG_WAVEOP_DEBUG = 4, /* Causes wavefront to enter
++ debug mode */
++ HSA_DBG_WAVEOP_TRAP = 5, /* Causes wavefront to take
++ a trap */
+ HSA_DBG_NUM_WAVEOP = 5,
+ HSA_DBG_MAX_WAVEOP = 0xFFFFFFFF
+ };
+@@ -79,13 +81,15 @@ struct HsaDbgWaveMsgAMDGen2 {
+ uint32_t UserData:8; /* user data */
+ uint32_t ShaderArray:1; /* Shader array */
+ uint32_t Priv:1; /* Privileged */
+- uint32_t Reserved0:4; /* Reserved, should be 0 */
++ uint32_t Reserved0:4; /* This field is reserved,
++ should be 0 */
+ uint32_t WaveId:4; /* wave id */
+ uint32_t SIMD:2; /* SIMD id */
+ uint32_t HSACU:4; /* Compute unit */
+ uint32_t ShaderEngine:2;/* Shader engine */
+ uint32_t MessageType:2; /* see HSA_DBG_WAVEMSG_TYPE */
+- uint32_t Reserved1:4; /* Reserved, should be 0 */
++ uint32_t Reserved1:4; /* This field is reserved,
++ should be 0 */
+ } ui32;
+ uint32_t Value;
+ };
+@@ -117,23 +121,20 @@ struct HsaDbgWaveMessage {
+ * in the user mode instruction stream. The OS scheduler event is typically
+ * associated and signaled by an interrupt issued by the GPU, but other HSA
+ * system interrupt conditions from other HW (e.g. IOMMUv2) may be surfaced
+- * by the KFD by this mechanism, too.
+- */
++ * by the KFD by this mechanism, too. */
+
+ /* these are the new definitions for events */
+ enum HSA_EVENTTYPE {
+ HSA_EVENTTYPE_SIGNAL = 0, /* user-mode generated GPU signal */
+ HSA_EVENTTYPE_NODECHANGE = 1, /* HSA node change (attach/detach) */
+ HSA_EVENTTYPE_DEVICESTATECHANGE = 2, /* HSA device state change
+- * (start/stop)
+- */
++ (start/stop) */
+ HSA_EVENTTYPE_HW_EXCEPTION = 3, /* GPU shader exception event */
+ HSA_EVENTTYPE_SYSTEM_EVENT = 4, /* GPU SYSCALL with parameter info */
+ HSA_EVENTTYPE_DEBUG_EVENT = 5, /* GPU signal for debugging */
+ HSA_EVENTTYPE_PROFILE_EVENT = 6,/* GPU signal for profiling */
+ HSA_EVENTTYPE_QUEUE_EVENT = 7, /* GPU signal queue idle state
+- * (EOP pm4)
+- */
++ (EOP pm4) */
+ /* ... */
+ HSA_EVENTTYPE_MAXID,
+ HSA_EVENTTYPE_TYPE_SIZE = 0xFFFFFFFF
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c b/drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c
+deleted file mode 100644
+index 232e28f..0000000
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c
++++ /dev/null
+@@ -1,75 +0,0 @@
+-/*
+- * Copyright 2014 Advanced Micro Devices, Inc.
+- *
+- * Permission is hereby granted, free of charge, to any person obtaining a
+- * copy of this software and associated documentation files (the "Software"),
+- * to deal in the Software without restriction, including without limitation
+- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+- * and/or sell copies of the Software, and to permit persons to whom the
+- * Software is furnished to do so, subject to the following conditions:
+- *
+- * The above copyright notice and this permission notice shall be included in
+- * all copies or substantial portions of the Software.
+- *
+- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+- * OTHER DEALINGS IN THE SOFTWARE.
+- */
+-
+-#include <linux/debugfs.h>
+-#include "kfd_priv.h"
+-
+-static struct dentry *debugfs_root;
+-
+-static int kfd_debugfs_open(struct inode *inode, struct file *file)
+-{
+- int (*show)(struct seq_file *, void *) = inode->i_private;
+-
+- return single_open(file, show, NULL);
+-}
+-
+-static const struct file_operations kfd_debugfs_fops = {
+- .owner = THIS_MODULE,
+- .open = kfd_debugfs_open,
+- .read = seq_read,
+- .llseek = seq_lseek,
+- .release = single_release,
+-};
+-
+-void kfd_debugfs_init(void)
+-{
+- struct dentry *ent;
+-
+- debugfs_root = debugfs_create_dir("kfd", NULL);
+- if (!debugfs_root || debugfs_root == ERR_PTR(-ENODEV)) {
+- pr_warn("Failed to create kfd debugfs dir\n");
+- return;
+- }
+-
+- ent = debugfs_create_file("mqds", S_IFREG | 0444, debugfs_root,
+- kfd_debugfs_mqds_by_process,
+- &kfd_debugfs_fops);
+- if (!ent)
+- pr_warn("Failed to create mqds in kfd debugfs\n");
+-
+- ent = debugfs_create_file("hqds", S_IFREG | 0444, debugfs_root,
+- kfd_debugfs_hqds_by_device,
+- &kfd_debugfs_fops);
+- if (!ent)
+- pr_warn("Failed to create hqds in kfd debugfs\n");
+-
+- ent = debugfs_create_file("rls", S_IFREG | 0444, debugfs_root,
+- kfd_debugfs_rls_by_device,
+- &kfd_debugfs_fops);
+- if (!ent)
+- pr_warn("Failed to create rls in kfd debugfs\n");
+-}
+-
+-void kfd_debugfs_fini(void)
+-{
+- debugfs_remove_recursive(debugfs_root);
+-}
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+index 6b3a1fa..3f95f7c 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+@@ -20,209 +20,36 @@
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+-#if defined(CONFIG_AMD_IOMMU_V2_MODULE) || defined(CONFIG_AMD_IOMMU_V2)
+ #include <linux/amd-iommu.h>
+-#endif
++#include <linux/bsearch.h>
+ #include <linux/pci.h>
+ #include <linux/slab.h>
+-#include <linux/highmem.h>
+ #include "kfd_priv.h"
+ #include "kfd_device_queue_manager.h"
+-#include "kfd_pm4_headers_vi.h"
+-#include "cwsr_trap_handler_carrizo.h"
+-#include "cwsr_trap_handler_gfx9.asm"
++#include "kfd_pm4_headers.h"
+
+ #define MQD_SIZE_ALIGNED 768
+-static atomic_t kfd_device_suspended = ATOMIC_INIT(0);
+
+-#if defined(CONFIG_AMD_IOMMU_V2_MODULE) || defined(CONFIG_AMD_IOMMU_V2)
+ static const struct kfd_device_info kaveri_device_info = {
+ .asic_family = CHIP_KAVERI,
+ .max_pasid_bits = 16,
+ /* max num of queues for KV.TODO should be a dynamic value */
+ .max_no_of_hqd = 24,
+- .doorbell_size = 4,
+ .ih_ring_entry_size = 4 * sizeof(uint32_t),
+ .event_interrupt_class = &event_interrupt_class_cik,
+ .num_of_watch_points = 4,
+- .mqd_size_aligned = MQD_SIZE_ALIGNED,
+- .is_need_iommu_device = true,
+- .supports_cwsr = false,
+- .needs_pci_atomics = false,
++ .mqd_size_aligned = MQD_SIZE_ALIGNED
+ };
+-#endif
+
+-static const struct kfd_device_info hawaii_device_info = {
+- .asic_family = CHIP_HAWAII,
+- .max_pasid_bits = 16,
+- /* max num of queues for KV.TODO should be a dynamic value */
+- .max_no_of_hqd = 24,
+- .doorbell_size = 4,
+- .ih_ring_entry_size = 4 * sizeof(uint32_t),
+- .event_interrupt_class = &event_interrupt_class_cik,
+- .num_of_watch_points = 4,
+- .mqd_size_aligned = MQD_SIZE_ALIGNED,
+- .is_need_iommu_device = false,
+- .supports_cwsr = false,
+- .needs_pci_atomics = false,
+-};
+-
+-#if defined(CONFIG_AMD_IOMMU_V2_MODULE) || defined(CONFIG_AMD_IOMMU_V2)
+ static const struct kfd_device_info carrizo_device_info = {
+ .asic_family = CHIP_CARRIZO,
+ .max_pasid_bits = 16,
+ /* max num of queues for CZ.TODO should be a dynamic value */
+ .max_no_of_hqd = 24,
+- .doorbell_size = 4,
+- .ih_ring_entry_size = 4 * sizeof(uint32_t),
+- .event_interrupt_class = &event_interrupt_class_cik,
+- .num_of_watch_points = 4,
+- .mqd_size_aligned = MQD_SIZE_ALIGNED,
+- .is_need_iommu_device = true,
+- .supports_cwsr = true,
+- .needs_pci_atomics = false,
+-};
+-#endif
+-
+-static const struct kfd_device_info tonga_device_info = {
+- .asic_family = CHIP_TONGA,
+- .max_pasid_bits = 16,
+- .max_no_of_hqd = 24,
+- .doorbell_size = 4,
+- .ih_ring_entry_size = 4 * sizeof(uint32_t),
+- .event_interrupt_class = &event_interrupt_class_cik,
+- .num_of_watch_points = 4,
+- .mqd_size_aligned = MQD_SIZE_ALIGNED,
+- .is_need_iommu_device = false,
+- .supports_cwsr = false,
+- .needs_pci_atomics = true,
+-};
+-
+-static const struct kfd_device_info tonga_vf_device_info = {
+- .asic_family = CHIP_TONGA,
+- .max_pasid_bits = 16,
+- .max_no_of_hqd = 24,
+- .doorbell_size = 4,
+- .ih_ring_entry_size = 4 * sizeof(uint32_t),
+- .event_interrupt_class = &event_interrupt_class_cik,
+- .num_of_watch_points = 4,
+- .mqd_size_aligned = MQD_SIZE_ALIGNED,
+- .is_need_iommu_device = false,
+- .supports_cwsr = false,
+- .needs_pci_atomics = false,
+-};
+-
+-static const struct kfd_device_info fiji_device_info = {
+- .asic_family = CHIP_FIJI,
+- .max_pasid_bits = 16,
+- .max_no_of_hqd = 24,
+- .doorbell_size = 4,
+- .ih_ring_entry_size = 4 * sizeof(uint32_t),
+- .event_interrupt_class = &event_interrupt_class_cik,
+- .num_of_watch_points = 4,
+- .mqd_size_aligned = MQD_SIZE_ALIGNED,
+- .is_need_iommu_device = false,
+- .supports_cwsr = true,
+- .needs_pci_atomics = true,
+-};
+-
+-static const struct kfd_device_info fiji_vf_device_info = {
+- .asic_family = CHIP_FIJI,
+- .max_pasid_bits = 16,
+- .max_no_of_hqd = 24,
+- .doorbell_size = 4,
+ .ih_ring_entry_size = 4 * sizeof(uint32_t),
+ .event_interrupt_class = &event_interrupt_class_cik,
+ .num_of_watch_points = 4,
+- .mqd_size_aligned = MQD_SIZE_ALIGNED,
+- .is_need_iommu_device = false,
+- .supports_cwsr = true,
+- .needs_pci_atomics = false,
+-};
+-
+-
+-static const struct kfd_device_info polaris10_device_info = {
+- .asic_family = CHIP_POLARIS10,
+- .max_pasid_bits = 16,
+- .max_no_of_hqd = 24,
+- .doorbell_size = 4,
+- .ih_ring_entry_size = 4 * sizeof(uint32_t),
+- .event_interrupt_class = &event_interrupt_class_cik,
+- .num_of_watch_points = 4,
+- .mqd_size_aligned = MQD_SIZE_ALIGNED,
+- .is_need_iommu_device = false,
+- .supports_cwsr = true,
+- .needs_pci_atomics = true,
+-};
+-
+-static const struct kfd_device_info polaris10_vf_device_info = {
+- .asic_family = CHIP_POLARIS10,
+- .max_pasid_bits = 16,
+- .max_no_of_hqd = 24,
+- .doorbell_size = 4,
+- .ih_ring_entry_size = 4 * sizeof(uint32_t),
+- .event_interrupt_class = &event_interrupt_class_cik,
+- .num_of_watch_points = 4,
+- .mqd_size_aligned = MQD_SIZE_ALIGNED,
+- .is_need_iommu_device = false,
+- .supports_cwsr = true,
+- .needs_pci_atomics = false,
+-};
+-
+-static const struct kfd_device_info polaris11_device_info = {
+- .asic_family = CHIP_POLARIS11,
+- .max_pasid_bits = 16,
+- .max_no_of_hqd = 24,
+- .doorbell_size = 4,
+- .ih_ring_entry_size = 4 * sizeof(uint32_t),
+- .event_interrupt_class = &event_interrupt_class_cik,
+- .num_of_watch_points = 4,
+- .mqd_size_aligned = MQD_SIZE_ALIGNED,
+- .is_need_iommu_device = false,
+- .supports_cwsr = true,
+- .needs_pci_atomics = true,
+-};
+-
+-static const struct kfd_device_info vega10_device_info = {
+- .asic_family = CHIP_VEGA10,
+- .max_pasid_bits = 16,
+- .max_no_of_hqd = 24,
+- .doorbell_size = 8,
+- .ih_ring_entry_size = 8 * sizeof(uint32_t),
+- .event_interrupt_class = &event_interrupt_class_v9,
+- .num_of_watch_points = 4,
+- .mqd_size_aligned = MQD_SIZE_ALIGNED,
+- .is_need_iommu_device = false,
+- .supports_cwsr = true,
+- .needs_pci_atomics = true,
+-};
+-
+-static const struct kfd_device_info vega10_vf_device_info = {
+- .asic_family = CHIP_VEGA10,
+- .max_pasid_bits = 16,
+- .max_no_of_hqd = 24,
+- .doorbell_size = 8,
+- .ih_ring_entry_size = 8 * sizeof(uint32_t),
+- .event_interrupt_class = &event_interrupt_class_v9,
+- .num_of_watch_points = 4,
+- .mqd_size_aligned = MQD_SIZE_ALIGNED,
+- .is_need_iommu_device = false,
+- .supports_cwsr = true,
+- .needs_pci_atomics = false,
+-};
+-
+-static const struct kfd_device_info raven_device_info = {
+- .asic_family = CHIP_RAVEN,
+- .max_pasid_bits = 16,
+- .max_no_of_hqd = 24,
+- .doorbell_size = 8,
+- .ih_ring_entry_size = 8 * sizeof(uint32_t),
+- .event_interrupt_class = &event_interrupt_class_v9,
+- .num_of_watch_points = 4,
+- .mqd_size_aligned = MQD_SIZE_ALIGNED,
+- .is_need_iommu_device = true,
+- .supports_cwsr = true,
+- .needs_pci_atomics = true,
++ .mqd_size_aligned = MQD_SIZE_ALIGNED
+ };
+
+ struct kfd_deviceid {
+@@ -232,7 +59,6 @@ struct kfd_deviceid {
+
+ /* Please keep this sorted by increasing device id. */
+ static const struct kfd_deviceid supported_devices[] = {
+-#if defined(CONFIG_AMD_IOMMU_V2_MODULE) || defined(CONFIG_AMD_IOMMU_V2)
+ { 0x1304, &kaveri_device_info }, /* Kaveri */
+ { 0x1305, &kaveri_device_info }, /* Kaveri */
+ { 0x1306, &kaveri_device_info }, /* Kaveri */
+@@ -255,90 +81,28 @@ static const struct kfd_deviceid supported_devices[] = {
+ { 0x131B, &kaveri_device_info }, /* Kaveri */
+ { 0x131C, &kaveri_device_info }, /* Kaveri */
+ { 0x131D, &kaveri_device_info }, /* Kaveri */
+-#endif
+- { 0x67A0, &hawaii_device_info }, /* Hawaii */
+- { 0x67A1, &hawaii_device_info }, /* Hawaii */
+- { 0x67A2, &hawaii_device_info }, /* Hawaii */
+- { 0x67A8, &hawaii_device_info }, /* Hawaii */
+- { 0x67A9, &hawaii_device_info }, /* Hawaii */
+- { 0x67AA, &hawaii_device_info }, /* Hawaii */
+- { 0x67B0, &hawaii_device_info }, /* Hawaii */
+- { 0x67B1, &hawaii_device_info }, /* Hawaii */
+- { 0x67B8, &hawaii_device_info }, /* Hawaii */
+- { 0x67B9, &hawaii_device_info }, /* Hawaii */
+- { 0x67BA, &hawaii_device_info }, /* Hawaii */
+- { 0x67BE, &hawaii_device_info }, /* Hawaii */
+-#if defined(CONFIG_AMD_IOMMU_V2_MODULE) || defined(CONFIG_AMD_IOMMU_V2)
+ { 0x9870, &carrizo_device_info }, /* Carrizo */
+ { 0x9874, &carrizo_device_info }, /* Carrizo */
+ { 0x9875, &carrizo_device_info }, /* Carrizo */
+ { 0x9876, &carrizo_device_info }, /* Carrizo */
+- { 0x9877, &carrizo_device_info }, /* Carrizo */
+-#endif
+- { 0x6920, &tonga_device_info }, /* Tonga */
+- { 0x6921, &tonga_device_info }, /* Tonga */
+- { 0x6928, &tonga_device_info }, /* Tonga */
+- { 0x6929, &tonga_device_info }, /* Tonga */
+- { 0x692B, &tonga_device_info }, /* Tonga */
+- { 0x692F, &tonga_vf_device_info }, /* Tonga vf */
+- { 0x6938, &tonga_device_info }, /* Tonga */
+- { 0x6939, &tonga_device_info }, /* Tonga */
+- { 0x7300, &fiji_device_info }, /* Fiji */
+- { 0x730F, &fiji_vf_device_info }, /* Fiji vf*/
+- { 0x67C0, &polaris10_device_info }, /* Polaris10 */
+- { 0x67C1, &polaris10_device_info }, /* Polaris10 */
+- { 0x67C2, &polaris10_device_info }, /* Polaris10 */
+- { 0x67C4, &polaris10_device_info }, /* Polaris10 */
+- { 0x67C7, &polaris10_device_info }, /* Polaris10 */
+- { 0x67C8, &polaris10_device_info }, /* Polaris10 */
+- { 0x67C9, &polaris10_device_info }, /* Polaris10 */
+- { 0x67CA, &polaris10_device_info }, /* Polaris10 */
+- { 0x67CC, &polaris10_device_info }, /* Polaris10 */
+- { 0x67CF, &polaris10_device_info }, /* Polaris10 */
+- { 0x67D0, &polaris10_vf_device_info }, /* Polaris10 vf*/
+- { 0x67DF, &polaris10_device_info }, /* Polaris10 */
+- { 0x67E0, &polaris11_device_info }, /* Polaris11 */
+- { 0x67E1, &polaris11_device_info }, /* Polaris11 */
+- { 0x67E3, &polaris11_device_info }, /* Polaris11 */
+- { 0x67E7, &polaris11_device_info }, /* Polaris11 */
+- { 0x67E8, &polaris11_device_info }, /* Polaris11 */
+- { 0x67E9, &polaris11_device_info }, /* Polaris11 */
+- { 0x67EB, &polaris11_device_info }, /* Polaris11 */
+- { 0x67EF, &polaris11_device_info }, /* Polaris11 */
+- { 0x67FF, &polaris11_device_info }, /* Polaris11 */
+- { 0x6860, &vega10_device_info }, /* Vega10 */
+- { 0x6861, &vega10_device_info }, /* Vega10 */
+- { 0x6862, &vega10_device_info }, /* Vega10 */
+- { 0x6863, &vega10_device_info }, /* Vega10 */
+- { 0x6864, &vega10_device_info }, /* Vega10 */
+- { 0x6867, &vega10_device_info }, /* Vega10 */
+- { 0x6868, &vega10_device_info }, /* Vega10 */
+- { 0x686C, &vega10_vf_device_info }, /* Vega10 vf*/
+- { 0x687F, &vega10_device_info }, /* Vega10 */
+-#if defined(CONFIG_AMD_IOMMU_V2_MODULE) || defined(CONFIG_AMD_IOMMU_V2)
+- { 0x15DD, &raven_device_info } /* Raven */
+-#endif
++ { 0x9877, &carrizo_device_info } /* Carrizo */
+ };
+
+ static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size,
+ unsigned int chunk_size);
+ static void kfd_gtt_sa_fini(struct kfd_dev *kfd);
+
+-static int kfd_resume(struct kfd_dev *kfd);
+-
+ static const struct kfd_device_info *lookup_device_info(unsigned short did)
+ {
+ size_t i;
+
+ for (i = 0; i < ARRAY_SIZE(supported_devices); i++) {
+ if (supported_devices[i].did == did) {
+- WARN_ON(!supported_devices[i].device_info);
++ BUG_ON(supported_devices[i].device_info == NULL);
+ return supported_devices[i].device_info;
+ }
+ }
+
+- WARN(1, "device is not added to supported_devices\n");
+-
+ return NULL;
+ }
+
+@@ -350,21 +114,8 @@ struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd,
+ const struct kfd_device_info *device_info =
+ lookup_device_info(pdev->device);
+
+- if (!device_info) {
+- dev_err(kfd_device, "kgd2kfd_probe failed\n");
++ if (!device_info)
+ return NULL;
+- }
+-
+- if (device_info->needs_pci_atomics) {
+- /* Allow BIF to recode atomics to PCIe 3.0 AtomicOps.
+- */
+- if (pci_enable_atomic_ops_to_root(pdev) < 0) {
+- dev_info(kfd_device,
+- "skipped device %x:%x, PCI rejects atomics",
+- pdev->vendor, pdev->device);
+- return NULL;
+- }
+- }
+
+ kfd = kzalloc(sizeof(*kfd), GFP_KERNEL);
+ if (!kfd)
+@@ -383,7 +134,6 @@ struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd,
+ return kfd;
+ }
+
+-#if defined(CONFIG_AMD_IOMMU_V2_MODULE) || defined(CONFIG_AMD_IOMMU_V2)
+ static bool device_iommu_pasid_init(struct kfd_dev *kfd)
+ {
+ const u32 required_iommu_flags = AMD_IOMMU_DEVICE_FLAG_ATS_SUP |
+@@ -402,16 +152,15 @@ static bool device_iommu_pasid_init(struct kfd_dev *kfd)
+ }
+
+ if ((iommu_info.flags & required_iommu_flags) != required_iommu_flags) {
+- dev_err(kfd_device, "error required iommu flags ats %i, pri %i, pasid %i\n",
++ dev_err(kfd_device, "error required iommu flags ats(%i), pri(%i), pasid(%i)\n",
+ (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP) != 0,
+ (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) != 0,
+- (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP)
+- != 0);
++ (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP) != 0);
+ return false;
+ }
+
+ pasid_limit = min_t(unsigned int,
+- (unsigned int)(1 << kfd->device_info->max_pasid_bits),
++ (unsigned int)1 << kfd->device_info->max_pasid_bits,
+ iommu_info.max_pasids);
+ /*
+ * last pasid is used for kernel queues doorbells
+@@ -421,8 +170,15 @@ static bool device_iommu_pasid_init(struct kfd_dev *kfd)
+ pasid_limit,
+ kfd->doorbell_process_limit - 1);
+
++ err = amd_iommu_init_device(kfd->pdev, pasid_limit);
++ if (err < 0) {
++ dev_err(kfd_device, "error initializing iommu device\n");
++ return false;
++ }
++
+ if (!kfd_set_pasid_limit(pasid_limit)) {
+ dev_err(kfd_device, "error setting pasid limit\n");
++ amd_iommu_free_device(kfd->pdev);
+ return false;
+ }
+
+@@ -434,7 +190,7 @@ static void iommu_pasid_shutdown_callback(struct pci_dev *pdev, int pasid)
+ struct kfd_dev *dev = kfd_device_by_pci_dev(pdev);
+
+ if (dev)
+- kfd_process_iommu_unbind_callback(dev, pasid);
++ kfd_unbind_process_from_device(dev, pasid);
+ }
+
+ /*
+@@ -455,108 +211,21 @@ static int iommu_invalid_ppr_cb(struct pci_dev *pdev, int pasid,
+ flags);
+
+ dev = kfd_device_by_pci_dev(pdev);
+- if (!WARN_ON(!dev))
+- kfd_signal_iommu_event(dev, pasid, address,
++ BUG_ON(dev == NULL);
++
++ kfd_signal_iommu_event(dev, pasid, address,
+ flags & PPR_FAULT_WRITE, flags & PPR_FAULT_EXEC);
+
+ return AMD_IOMMU_INV_PRI_RSP_INVALID;
+ }
+-#endif /* CONFIG_AMD_IOMMU_V2 */
+-
+-static int kfd_cwsr_init(struct kfd_dev *kfd)
+-{
+- /*
+- * Initialize the CWSR required memory for TBA and TMA
+- */
+- if (cwsr_enable && kfd->device_info->supports_cwsr) {
+- const uint32_t *cwsr_hex;
+- void *cwsr_addr = NULL;
+- unsigned int size;
+-
+- if (kfd->device_info->asic_family < CHIP_VEGA10) {
+- cwsr_hex = cwsr_trap_carrizo_hex;
+- size = sizeof(cwsr_trap_carrizo_hex);
+- } else {
+- cwsr_hex = cwsr_trap_gfx9_hex;
+- size = sizeof(cwsr_trap_gfx9_hex);
+- }
+-
+- if (size > PAGE_SIZE) {
+- pr_err("Wrong CWSR ISA size.\n");
+- return -EINVAL;
+- }
+- kfd->cwsr_size =
+- ALIGN(size, PAGE_SIZE) + PAGE_SIZE;
+- kfd->cwsr_pages = alloc_pages(GFP_KERNEL | __GFP_HIGHMEM,
+- get_order(kfd->cwsr_size));
+- if (!kfd->cwsr_pages) {
+- pr_err("Failed to allocate CWSR isa memory.\n");
+- return -ENOMEM;
+- }
+- /*Only first page used for cwsr ISA code */
+- cwsr_addr = kmap(kfd->cwsr_pages);
+- memset(cwsr_addr, 0, PAGE_SIZE);
+- memcpy(cwsr_addr, cwsr_hex, size);
+- kunmap(kfd->cwsr_pages);
+- kfd->tma_offset = ALIGN(size, PAGE_SIZE);
+- kfd->cwsr_enabled = true;
+- dev_info(kfd_device,
+- "Reserved %d pages for cwsr.\n",
+- (kfd->cwsr_size >> PAGE_SHIFT));
+- }
+-
+- return 0;
+-}
+-
+-static void kfd_cwsr_fini(struct kfd_dev *kfd)
+-{
+- if (kfd->cwsr_pages)
+- __free_pages(kfd->cwsr_pages, get_order(kfd->cwsr_size));
+-}
+-
+-static void kfd_ib_mem_init(struct kfd_dev *kdev)
+-{
+- /* In certain cases we need to send IB from kernel using the GPU address
+- * space created by user applications.
+- * For example, on GFX v7, we need to flush TC associated to the VMID
+- * before tearing down the VMID. In order to do so, we need an address
+- * valid to the VMID to place the IB while this space was created on
+- * the user's side, not the kernel.
+- * Since kfd_set_process_dgpu_aperture reserves "cwsr_base + cwsr_size"
+- * but CWSR only uses pages above cwsr_base, we'll use one page memory
+- * under cwsr_base for IB submissions
+- */
+- kdev->ib_size = PAGE_SIZE;
+-}
+
+ bool kgd2kfd_device_init(struct kfd_dev *kfd,
+ const struct kgd2kfd_shared_resources *gpu_resources)
+ {
+ unsigned int size;
+- unsigned int vmid_bitmap_kfd, vmid_num_kfd;
+-
+- kfd->mec_fw_version = kfd->kfd2kgd->get_fw_version(kfd->kgd,
+- KGD_ENGINE_MEC1);
+
+ kfd->shared_resources = *gpu_resources;
+
+- vmid_bitmap_kfd = kfd->shared_resources.compute_vmid_bitmap;
+- kfd->vm_info.first_vmid_kfd = ffs(vmid_bitmap_kfd) - 1;
+- kfd->vm_info.last_vmid_kfd = fls(vmid_bitmap_kfd) - 1;
+- vmid_num_kfd = kfd->vm_info.last_vmid_kfd
+- - kfd->vm_info.first_vmid_kfd + 1;
+- kfd->vm_info.vmid_num_kfd = vmid_num_kfd;
+-
+- /* Verify module parameters regarding mapped process number*/
+- if ((hws_max_conc_proc < 0)
+- || (hws_max_conc_proc > vmid_num_kfd)) {
+- dev_err(kfd_device,
+- "hws_max_conc_proc %d must be between 0 and %d, use %d instead\n",
+- hws_max_conc_proc, vmid_num_kfd, vmid_num_kfd);
+- kfd->max_proc_per_quantum = vmid_num_kfd;
+- } else
+- kfd->max_proc_per_quantum = hws_max_conc_proc;
+-
+ /* calculate max size of mqds needed for queues */
+ size = max_num_of_queues_per_device *
+ kfd->device_info->mqd_size_aligned;
+@@ -565,9 +234,9 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
+ * calculate max size of runlist packet.
+ * There can be only 2 packets at once
+ */
+- size += (KFD_MAX_NUM_OF_PROCESSES * sizeof(struct pm4_mes_map_process) +
+- max_num_of_queues_per_device * sizeof(struct pm4_mes_map_queues)
+- + sizeof(struct pm4_mes_runlist)) * 2;
++ size += (KFD_MAX_NUM_OF_PROCESSES * sizeof(struct pm4_map_process) +
++ max_num_of_queues_per_device *
++ sizeof(struct pm4_map_queues) + sizeof(struct pm4_runlist)) * 2;
+
+ /* Add size of HIQ & DIQ */
+ size += KFD_KERNEL_QUEUE_SIZE * 2;
+@@ -578,88 +247,89 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
+ if (kfd->kfd2kgd->init_gtt_mem_allocation(
+ kfd->kgd, size, &kfd->gtt_mem,
+ &kfd->gtt_start_gpu_addr, &kfd->gtt_start_cpu_ptr)){
+- dev_err(kfd_device, "Could not allocate %d bytes\n", size);
++ dev_err(kfd_device,
++ "Could not allocate %d bytes for device (%x:%x)\n",
++ size, kfd->pdev->vendor, kfd->pdev->device);
+ goto out;
+ }
+
+- dev_info(kfd_device, "Allocated %d bytes on gart\n", size);
++ dev_info(kfd_device,
++ "Allocated %d bytes on gart for device(%x:%x)\n",
++ size, kfd->pdev->vendor, kfd->pdev->device);
+
+ /* Initialize GTT sa with 512 byte chunk size */
+ if (kfd_gtt_sa_init(kfd, size, 512) != 0) {
+- dev_err(kfd_device, "Error initializing gtt sub-allocator\n");
++ dev_err(kfd_device,
++ "Error initializing gtt sub-allocator\n");
+ goto kfd_gtt_sa_init_error;
+ }
+
+- if (kfd_doorbell_init(kfd)) {
+- dev_err(kfd_device,
+- "Error initializing doorbell aperture\n");
+- goto kfd_doorbell_error;
+- }
++ kfd_doorbell_init(kfd);
+
+- if (kfd_topology_add_device(kfd)) {
+- dev_err(kfd_device, "Error adding device to topology\n");
++ if (kfd_topology_add_device(kfd) != 0) {
++ dev_err(kfd_device,
++ "Error adding device (%x:%x) to topology\n",
++ kfd->pdev->vendor, kfd->pdev->device);
+ goto kfd_topology_add_device_error;
+ }
+
+ if (kfd_interrupt_init(kfd)) {
+- dev_err(kfd_device, "Error initializing interrupts\n");
++ dev_err(kfd_device,
++ "Error initializing interrupts for device (%x:%x)\n",
++ kfd->pdev->vendor, kfd->pdev->device);
+ goto kfd_interrupt_error;
+ }
+
++ if (!device_iommu_pasid_init(kfd)) {
++ dev_err(kfd_device,
++ "Error initializing iommuv2 for device (%x:%x)\n",
++ kfd->pdev->vendor, kfd->pdev->device);
++ goto device_iommu_pasid_error;
++ }
++ amd_iommu_set_invalidate_ctx_cb(kfd->pdev,
++ iommu_pasid_shutdown_callback);
++ amd_iommu_set_invalid_ppr_cb(kfd->pdev, iommu_invalid_ppr_cb);
++
+ kfd->dqm = device_queue_manager_init(kfd);
+ if (!kfd->dqm) {
+- dev_err(kfd_device, "Error initializing queue manager\n");
++ dev_err(kfd_device,
++ "Error initializing queue manager for device (%x:%x)\n",
++ kfd->pdev->vendor, kfd->pdev->device);
+ goto device_queue_manager_error;
+ }
+
+-#if defined(CONFIG_AMD_IOMMU_V2_MODULE) || defined(CONFIG_AMD_IOMMU_V2)
+- if (kfd->device_info->is_need_iommu_device) {
+- if (!device_iommu_pasid_init(kfd)) {
+- dev_err(kfd_device, "Error initializing iommuv2\n");
+- goto device_iommu_pasid_error;
+- }
+- }
+-#endif
+-
+- if (kfd_cwsr_init(kfd)) {
+- dev_err(kfd_device, "Error initializing cwsr\n");
+- goto device_iommu_pasid_error;
+- }
+-
+- kfd_ib_mem_init(kfd);
+-
+- if (kfd_resume(kfd)) {
+- dev_err(kfd_device, "Error resuming kfd\n");
+- goto kfd_resume_error;
++ if (kfd->dqm->ops.start(kfd->dqm) != 0) {
++ dev_err(kfd_device,
++ "Error starting queuen manager for device (%x:%x)\n",
++ kfd->pdev->vendor, kfd->pdev->device);
++ goto dqm_start_error;
+ }
+
+ kfd->dbgmgr = NULL;
+
+ kfd->init_complete = true;
+- dev_info(kfd_device, "added device %x:%x\n", kfd->pdev->vendor,
++ dev_info(kfd_device, "added device (%x:%x)\n", kfd->pdev->vendor,
+ kfd->pdev->device);
+
+- pr_debug("Starting kfd with the following scheduling policy %d\n",
+- kfd->dqm->sched_policy);
++ pr_debug("kfd: Starting kfd with the following scheduling policy %d\n",
++ sched_policy);
+
+ goto out;
+
+-kfd_resume_error:
+- kfd_cwsr_fini(kfd);
+-device_iommu_pasid_error:
++dqm_start_error:
+ device_queue_manager_uninit(kfd->dqm);
+ device_queue_manager_error:
++ amd_iommu_free_device(kfd->pdev);
++device_iommu_pasid_error:
+ kfd_interrupt_exit(kfd);
+ kfd_interrupt_error:
+ kfd_topology_remove_device(kfd);
+ kfd_topology_add_device_error:
+- kfd_doorbell_fini(kfd);
+-kfd_doorbell_error:
+ kfd_gtt_sa_fini(kfd);
+ kfd_gtt_sa_init_error:
+ kfd->kfd2kgd->free_gtt_mem(kfd->kgd, kfd->gtt_mem);
+ dev_err(kfd_device,
+- "device %x:%x NOT added due to errors\n",
++ "device (%x:%x) NOT added due to errors\n",
+ kfd->pdev->vendor, kfd->pdev->device);
+ out:
+ return kfd->init_complete;
+@@ -668,12 +338,10 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
+ void kgd2kfd_device_exit(struct kfd_dev *kfd)
+ {
+ if (kfd->init_complete) {
+- kgd2kfd_suspend(kfd);
+- kfd_cwsr_fini(kfd);
+ device_queue_manager_uninit(kfd->dqm);
++ amd_iommu_free_device(kfd->pdev);
+ kfd_interrupt_exit(kfd);
+ kfd_topology_remove_device(kfd);
+- kfd_doorbell_fini(kfd);
+ kfd_gtt_sa_fini(kfd);
+ kfd->kfd2kgd->free_gtt_mem(kfd->kgd, kfd->gtt_mem);
+ }
+@@ -683,419 +351,77 @@ void kgd2kfd_device_exit(struct kfd_dev *kfd)
+
+ void kgd2kfd_suspend(struct kfd_dev *kfd)
+ {
+- if (!kfd->init_complete)
+- return;
+-
+- /* For first KFD device suspend all the KFD processes */
+- if (atomic_inc_return(&kfd_device_suspended) == 1)
+- kfd_suspend_all_processes();
+-
+- kfd->dqm->ops.stop(kfd->dqm);
+-
+-#if defined(CONFIG_AMD_IOMMU_V2_MODULE) || defined(CONFIG_AMD_IOMMU_V2)
+- if (!kfd->device_info->is_need_iommu_device)
+- return;
++ BUG_ON(kfd == NULL);
+
+- kfd_unbind_processes_from_device(kfd);
+-
+- amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL);
+- amd_iommu_set_invalid_ppr_cb(kfd->pdev, NULL);
+- amd_iommu_free_device(kfd->pdev);
+-#endif
++ if (kfd->init_complete) {
++ kfd->dqm->ops.stop(kfd->dqm);
++ amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL);
++ amd_iommu_set_invalid_ppr_cb(kfd->pdev, NULL);
++ amd_iommu_free_device(kfd->pdev);
++ }
+ }
+
+ int kgd2kfd_resume(struct kfd_dev *kfd)
+ {
+- int ret;
+-
+- if (!kfd->init_complete)
+- return 0;
+-
+- ret = kfd_resume(kfd);
+- if (ret)
+- return ret;
+-
+- if (atomic_dec_return(&kfd_device_suspended) == 0)
+- ret = kfd_resume_all_processes();
+- WARN(atomic_read(&kfd_device_suspended) < 0,
+- "KFD suspend / resume ref. error\n");
+- return ret;
+-}
++ unsigned int pasid_limit;
++ int err;
+
+-static int kfd_resume(struct kfd_dev *kfd)
+-{
+- int err = 0;
++ BUG_ON(kfd == NULL);
+
+-#if defined(CONFIG_AMD_IOMMU_V2_MODULE) || defined(CONFIG_AMD_IOMMU_V2)
+- if (kfd->device_info->is_need_iommu_device) {
+- unsigned int pasid_limit = kfd_get_pasid_limit();
++ pasid_limit = kfd_get_pasid_limit();
+
++ if (kfd->init_complete) {
+ err = amd_iommu_init_device(kfd->pdev, pasid_limit);
+- if (err) {
+- dev_err(kfd_device, "failed to initialize iommu\n");
++ if (err < 0)
+ return -ENXIO;
+- }
+-
+ amd_iommu_set_invalidate_ctx_cb(kfd->pdev,
+- iommu_pasid_shutdown_callback);
+- amd_iommu_set_invalid_ppr_cb(kfd->pdev,
+- iommu_invalid_ppr_cb);
+-
+- err = kfd_bind_processes_to_device(kfd);
+- if (err) {
+- dev_err(kfd_device,
+- "failed to bind process to device\n");
+- return -ENXIO;
+- }
+- }
+-#endif
+-
+- err = kfd->dqm->ops.start(kfd->dqm);
+- if (err) {
+- dev_err(kfd_device,
+- "Error starting queue manager for device %x:%x\n",
+- kfd->pdev->vendor, kfd->pdev->device);
+- goto dqm_start_error;
++ iommu_pasid_shutdown_callback);
++ amd_iommu_set_invalid_ppr_cb(kfd->pdev, iommu_invalid_ppr_cb);
++ kfd->dqm->ops.start(kfd->dqm);
+ }
+
+- kfd->kfd2kgd->write_config_static_mem(kfd->kgd, true, 1, 3, 0);
+-
+- return err;
+-
+-dqm_start_error:
+-#if defined(CONFIG_AMD_IOMMU_V2_MODULE) || defined(CONFIG_AMD_IOMMU_V2)
+- if (kfd->device_info->is_need_iommu_device)
+- amd_iommu_free_device(kfd->pdev);
+-#endif
+-
+- return err;
++ return 0;
+ }
+
+ /* This is called directly from KGD at ISR. */
+ void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry)
+ {
+- uint32_t patched_ihre[DIV_ROUND_UP(
+- kfd->device_info->ih_ring_entry_size,
+- sizeof(uint32_t))];
+- bool is_patched = false;
+-
+ if (!kfd->init_complete)
+ return;
+
+ spin_lock(&kfd->interrupt_lock);
+
+- if (kfd->interrupts_active && interrupt_is_wanted(kfd, ih_ring_entry,
+- patched_ihre, &is_patched)
+- && enqueue_ih_ring_entry(kfd,
+- is_patched ? patched_ihre : ih_ring_entry))
+- queue_work(kfd->ih_wq, &kfd->interrupt_work);
++ if (kfd->interrupts_active
++ && interrupt_is_wanted(kfd, ih_ring_entry)
++ && enqueue_ih_ring_entry(kfd, ih_ring_entry))
++ schedule_work(&kfd->interrupt_work);
+
+ spin_unlock(&kfd->interrupt_lock);
+ }
+
+-/* quiesce_process_mm -
+- * Quiesce all user queues that belongs to given process p
+- */
+-int quiesce_process_mm(struct kfd_process *p)
+-{
+- struct kfd_process_device *pdd;
+- int r = 0;
+- unsigned int n_evicted = 0;
+-
+- list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
+- r = process_evict_queues(pdd->dev->dqm, &pdd->qpd);
+- if (r != 0) {
+- pr_err("Failed to evict process queues\n");
+- goto fail;
+- }
+- n_evicted++;
+- }
+-
+- return r;
+-
+-fail:
+- /* To keep state consistent, roll back partial eviction by
+- * restoring queues
+- */
+- list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
+- if (n_evicted == 0)
+- break;
+- if (process_restore_queues(pdd->dev->dqm, &pdd->qpd))
+- pr_err("Failed to restore queues\n");
+-
+- n_evicted--;
+- }
+-
+- return r;
+-}
+-
+-/* resume_process_mm -
+- * Resume all user queues that belongs to given process p. The caller must
+- * ensure that process p context is valid.
+- */
+-static int resume_process_mm(struct kfd_process *p)
+-{
+- struct kfd_process_device *pdd;
+- struct mm_struct *mm = (struct mm_struct *)p->mm;
+- int r, ret = 0;
+-
+- list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
+- if (pdd->dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS)
+- down_read(&mm->mmap_sem);
+-
+- r = process_restore_queues(pdd->dev->dqm, &pdd->qpd);
+- if (r != 0) {
+- pr_err("Failed to restore process queues\n");
+- if (ret == 0)
+- ret = r;
+- }
+-
+- if (pdd->dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS)
+- up_read(&mm->mmap_sem);
+- }
+-
+- return ret;
+-}
+-
+-int kgd2kfd_quiesce_mm(struct kfd_dev *kfd, struct mm_struct *mm)
+-{
+- struct kfd_process *p;
+- struct kfd_process_device *pdd;
+- int r;
+-
+- /* Because we are called from arbitrary context (workqueue) as opposed
+- * to process context, kfd_process could attempt to exit while we are
+- * running so the lookup function increments the process ref count.
+- */
+- p = kfd_lookup_process_by_mm(mm);
+- if (!p)
+- return -ENODEV;
+-
+- if (kfd) {
+- r = -ENODEV;
+- pdd = kfd_get_process_device_data(kfd, p);
+- if (pdd)
+- r = process_evict_queues(kfd->dqm, &pdd->qpd);
+- } else {
+- r = quiesce_process_mm(p);
+- }
+-
+- kfd_unref_process(p);
+- return r;
+-}
+-
+-int kgd2kfd_resume_mm(struct kfd_dev *kfd, struct mm_struct *mm)
+-{
+- struct kfd_process *p;
+- struct kfd_process_device *pdd;
+- int r;
+-
+- /* Because we are called from arbitrary context (workqueue) as opposed
+- * to process context, kfd_process could attempt to exit while we are
+- * running so the lookup function increments the process ref count.
+- */
+- p = kfd_lookup_process_by_mm(mm);
+- if (!p)
+- return -ENODEV;
+-
+- if (kfd) {
+- r = -ENODEV;
+- pdd = kfd_get_process_device_data(kfd, p);
+- if (pdd)
+- r = process_restore_queues(kfd->dqm, &pdd->qpd);
+- } else {
+- r = resume_process_mm(p);
+- }
+-
+- kfd_unref_process(p);
+- return r;
+-}
+-
+-
+-void kfd_restore_bo_worker(struct work_struct *work)
+-{
+- struct delayed_work *dwork;
+- struct kfd_process *p;
+- struct kfd_process_device *pdd;
+- int ret = 0;
+-
+- dwork = to_delayed_work(work);
+-
+- /* Process termination destroys this worker thread. So during the
+- * lifetime of this thread, kfd_process p will be valid
+- */
+- p = container_of(dwork, struct kfd_process, restore_work);
+-
+- /* Call restore_process_bos on the first KGD device. This function
+- * takes care of restoring the whole process including other devices.
+- * Restore can fail if enough memory is not available. If so,
+- * reschedule again.
+- */
+- pdd = list_first_entry(&p->per_device_data,
+- struct kfd_process_device,
+- per_device_list);
+-
+- pr_info("Started restoring process of pasid %d\n", p->pasid);
+-
+- /* Setting last_restore_timestamp before successful restoration.
+- * Otherwise this would have to be set by KGD (restore_process_bos)
+- * before KFD BOs are unreserved. If not, the process can be evicted
+- * again before the timestamp is set.
+- * If restore fails, the timestamp will be set again in the next
+- * attempt. This would mean that the minimum GPU quanta would be
+- * PROCESS_ACTIVE_TIME_MS - (time to execute the following two
+- * functions)
+- */
+-
+- p->last_restore_timestamp = get_jiffies_64();
+- ret = pdd->dev->kfd2kgd->restore_process_bos(p->process_info, &p->ef);
+- if (ret) {
+- pr_info("Restore failed, try again after %d ms\n",
+- PROCESS_BACK_OFF_TIME_MS);
+- ret = schedule_delayed_work(&p->restore_work,
+- msecs_to_jiffies(PROCESS_BACK_OFF_TIME_MS));
+- WARN(!ret, "reschedule restore work failed\n");
+- return;
+- }
+-
+- ret = resume_process_mm(p);
+- if (ret)
+- pr_err("Failed to resume user queues\n");
+-
+- pr_info("Finished restoring process of pasid %d\n", p->pasid);
+-}
+-
+-/** kgd2kfd_schedule_evict_and_restore_process - Schedules work queue that will
+- * prepare for safe eviction of KFD BOs that belong to the specified
+- * process.
+- *
+- * @mm: mm_struct that identifies the specified KFD process
+- * @fence: eviction fence attached to KFD process BOs
+- *
+- */
+-int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm,
+- struct dma_fence *fence)
+-{
+- struct kfd_process *p;
+- unsigned long active_time;
+- unsigned long delay_jiffies = msecs_to_jiffies(PROCESS_ACTIVE_TIME_MS);
+-
+- if (!fence)
+- return -EINVAL;
+-
+- if (dma_fence_is_signaled(fence))
+- return 0;
+-
+- p = kfd_lookup_process_by_mm(mm);
+- if (!p)
+- return -ENODEV;
+-
+- if (delayed_work_pending(&p->eviction_work.dwork)) {
+- /* It is possible has TTM has lined up couple of BOs of the same
+- * process to be evicted. Check if the fence is same which
+- * indicates that previous work item scheduled is not completed
+- */
+- if (p->eviction_work.quiesce_fence == fence)
+- goto out;
+- else {
+- WARN(1, "Starting new evict with previous evict is not completed\n");
+- if (cancel_delayed_work_sync(&p->eviction_work.dwork))
+- dma_fence_put(p->eviction_work.quiesce_fence);
+- }
+- }
+-
+- p->eviction_work.quiesce_fence = dma_fence_get(fence);
+-
+- /* Avoid KFD process starvation. Wait for at least
+- * PROCESS_ACTIVE_TIME_MS before evicting the process again
+- */
+- active_time = get_jiffies_64() - p->last_restore_timestamp;
+- if (delay_jiffies > active_time)
+- delay_jiffies -= active_time;
+- else
+- delay_jiffies = 0;
+-
+- /* During process initialization eviction_work.dwork is initialized
+- * to kfd_evict_bo_worker
+- */
+- schedule_delayed_work(&p->eviction_work.dwork, delay_jiffies);
+-out:
+- kfd_unref_process(p);
+- return 0;
+-}
+-
+-void kfd_evict_bo_worker(struct work_struct *work)
+-{
+- int ret;
+- struct kfd_process *p;
+- struct kfd_eviction_work *eviction_work;
+- struct delayed_work *dwork;
+-
+- dwork = to_delayed_work(work);
+- eviction_work = container_of(dwork, struct kfd_eviction_work,
+- dwork);
+-
+- /* Process termination destroys this worker thread. So during the
+- * lifetime of this thread, kfd_process p will be valid
+- */
+- p = container_of(eviction_work, struct kfd_process, eviction_work);
+-
+- /* Narrow window of overlap between restore and evict work item is
+- * possible. Once amdgpu_amdkfd_gpuvm_restore_process_bos unreserves
+- * KFD BOs, it is possible to evicted again. But restore has few more
+- * steps of finish. So lets wait for the restore work to complete
+- */
+- if (delayed_work_pending(&p->restore_work))
+- flush_delayed_work(&p->restore_work);
+-
+- pr_info("Started evicting process of pasid %d\n", p->pasid);
+- ret = quiesce_process_mm(p);
+- if (!ret) {
+- dma_fence_signal(eviction_work->quiesce_fence);
+- WARN_ONCE(eviction_work->quiesce_fence != p->ef,
+- "Eviction fence mismatch\n");
+- dma_fence_put(p->ef);
+- /* TODO: quiesce_fence is same as kfd_process->ef. But
+- * quiesce_fence is also used to avoid starting multiple
+- * eviction work items. This might not be necessary and
+- * one of the variables could be removed
+- */
+- p->ef = NULL;
+- schedule_delayed_work(&p->restore_work,
+- msecs_to_jiffies(PROCESS_RESTORE_TIME_MS));
+- } else
+- pr_err("Failed to quiesce user queues. Cannot evict BOs\n");
+-
+- dma_fence_put(eviction_work->quiesce_fence);
+-
+- pr_info("Finished evicting process of pasid %d\n", p->pasid);
+-
+-}
+-
+ static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size,
+ unsigned int chunk_size)
+ {
+- unsigned int num_of_longs;
++ unsigned int num_of_bits;
+
+- if (WARN_ON(buf_size < chunk_size))
+- return -EINVAL;
+- if (WARN_ON(buf_size == 0))
+- return -EINVAL;
+- if (WARN_ON(chunk_size == 0))
+- return -EINVAL;
++ BUG_ON(!kfd);
++ BUG_ON(!kfd->gtt_mem);
++ BUG_ON(buf_size < chunk_size);
++ BUG_ON(buf_size == 0);
++ BUG_ON(chunk_size == 0);
+
+ kfd->gtt_sa_chunk_size = chunk_size;
+ kfd->gtt_sa_num_of_chunks = buf_size / chunk_size;
+
+- num_of_longs = (kfd->gtt_sa_num_of_chunks + BITS_PER_LONG - 1) /
+- BITS_PER_LONG;
++ num_of_bits = kfd->gtt_sa_num_of_chunks / BITS_PER_BYTE;
++ BUG_ON(num_of_bits == 0);
+
+- kfd->gtt_sa_bitmap = kcalloc(num_of_longs, sizeof(long), GFP_KERNEL);
++ kfd->gtt_sa_bitmap = kzalloc(num_of_bits, GFP_KERNEL);
+
+ if (!kfd->gtt_sa_bitmap)
+ return -ENOMEM;
+
+- pr_debug("gtt_sa_num_of_chunks = %d, gtt_sa_bitmap = %p\n",
++ pr_debug("kfd: gtt_sa_num_of_chunks = %d, gtt_sa_bitmap = %p\n",
+ kfd->gtt_sa_num_of_chunks, kfd->gtt_sa_bitmap);
+
+ mutex_init(&kfd->gtt_sa_lock);
+@@ -1129,17 +455,19 @@ int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size,
+ {
+ unsigned int found, start_search, cur_size;
+
++ BUG_ON(!kfd);
++
+ if (size == 0)
+ return -EINVAL;
+
+ if (size > kfd->gtt_sa_num_of_chunks * kfd->gtt_sa_chunk_size)
+ return -ENOMEM;
+
+- *mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_NOIO);
+- if (!(*mem_obj))
++ *mem_obj = kmalloc(sizeof(struct kfd_mem_obj), GFP_KERNEL);
++ if ((*mem_obj) == NULL)
+ return -ENOMEM;
+
+- pr_debug("Allocated mem_obj = %p for size = %d\n", *mem_obj, size);
++ pr_debug("kfd: allocated mem_obj = %p for size = %d\n", *mem_obj, size);
+
+ start_search = 0;
+
+@@ -1151,7 +479,7 @@ int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size,
+ kfd->gtt_sa_num_of_chunks,
+ start_search);
+
+- pr_debug("Found = %d\n", found);
++ pr_debug("kfd: found = %d\n", found);
+
+ /* If there wasn't any free chunk, bail out */
+ if (found == kfd->gtt_sa_num_of_chunks)
+@@ -1169,12 +497,12 @@ int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size,
+ found,
+ kfd->gtt_sa_chunk_size);
+
+- pr_debug("gpu_addr = %p, cpu_addr = %p\n",
++ pr_debug("kfd: gpu_addr = %p, cpu_addr = %p\n",
+ (uint64_t *) (*mem_obj)->gpu_addr, (*mem_obj)->cpu_ptr);
+
+ /* If we need only one chunk, mark it as allocated and get out */
+ if (size <= kfd->gtt_sa_chunk_size) {
+- pr_debug("Single bit\n");
++ pr_debug("kfd: single bit\n");
+ set_bit(found, kfd->gtt_sa_bitmap);
+ goto kfd_gtt_out;
+ }
+@@ -1209,7 +537,7 @@ int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size,
+
+ } while (cur_size > 0);
+
+- pr_debug("range_start = %d, range_end = %d\n",
++ pr_debug("kfd: range_start = %d, range_end = %d\n",
+ (*mem_obj)->range_start, (*mem_obj)->range_end);
+
+ /* Mark the chunks as allocated */
+@@ -1223,7 +551,7 @@ int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size,
+ return 0;
+
+ kfd_gtt_no_free_chunk:
+- pr_debug("Allocation failed with mem_obj = %p\n", mem_obj);
++ pr_debug("kfd: allocation failed with mem_obj = %p\n", mem_obj);
+ mutex_unlock(&kfd->gtt_sa_lock);
+ kfree(mem_obj);
+ return -ENOMEM;
+@@ -1233,11 +561,13 @@ int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj)
+ {
+ unsigned int bit;
+
++ BUG_ON(!kfd);
++
+ /* Act like kfree when trying to free a NULL object */
+ if (!mem_obj)
+ return 0;
+
+- pr_debug("Free mem_obj = %p, range_start = %d, range_end = %d\n",
++ pr_debug("kfd: free mem_obj = %p, range_start = %d, range_end = %d\n",
+ mem_obj, mem_obj->range_start, mem_obj->range_end);
+
+ mutex_lock(&kfd->gtt_sa_lock);
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+index 1abbaa0..42de22b 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+@@ -44,13 +44,9 @@ static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
+ struct queue *q,
+ struct qcm_process_device *qpd);
+
+-static int execute_queues_cpsch(struct device_queue_manager *dqm,
+- bool static_queues_included);
+-static int unmap_queues_cpsch(struct device_queue_manager *dqm,
+- enum kfd_unmap_queues_filter filter,
+- uint32_t filter_param);
+-
+-static int map_queues_cpsch(struct device_queue_manager *dqm);
++static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock);
++static int destroy_queues_cpsch(struct device_queue_manager *dqm,
++ bool preempt_static_queues, bool lock);
+
+ static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
+ struct queue *q,
+@@ -83,17 +79,20 @@ static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe)
+
+ unsigned int get_queues_num(struct device_queue_manager *dqm)
+ {
++ BUG_ON(!dqm || !dqm->dev);
+ return bitmap_weight(dqm->dev->shared_resources.queue_bitmap,
+ KGD_MAX_QUEUES);
+ }
+
+ unsigned int get_queues_per_pipe(struct device_queue_manager *dqm)
+ {
++ BUG_ON(!dqm || !dqm->dev);
+ return dqm->dev->shared_resources.num_queue_per_pipe;
+ }
+
+ unsigned int get_pipes_per_mec(struct device_queue_manager *dqm)
+ {
++ BUG_ON(!dqm || !dqm->dev);
+ return dqm->dev->shared_resources.num_pipe_per_mec;
+ }
+
+@@ -108,57 +107,6 @@ void program_sh_mem_settings(struct device_queue_manager *dqm,
+ qpd->sh_mem_bases);
+ }
+
+-static int allocate_doorbell(struct qcm_process_device *qpd, struct queue *q)
+-{
+- struct kfd_dev *dev = qpd->dqm->dev;
+-
+- if (!KFD_IS_SOC15(dev->device_info->asic_family)) {
+- /* On pre-SOC15 chips we need to use the queue ID to
+- * preserve the user mode ABI.
+- */
+- q->doorbell_id = q->properties.queue_id;
+- } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
+- /* For SDMA queues on SOC15, use static doorbell
+- * assignments based on the engine and queue.
+- */
+- q->doorbell_id = dev->shared_resources.sdma_doorbell
+- [q->properties.sdma_engine_id]
+- [q->properties.sdma_queue_id];
+- } else {
+- /* For CP queues on SOC15 reserve a free doorbell ID */
+- unsigned int found;
+-
+- found = find_first_zero_bit(qpd->doorbell_bitmap,
+- KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
+- if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) {
+- pr_debug("No doorbells available");
+- return -EBUSY;
+- }
+- set_bit(found, qpd->doorbell_bitmap);
+- q->doorbell_id = found;
+- }
+-
+- q->properties.doorbell_off =
+- kfd_doorbell_id_to_offset(dev, q->process,
+- q->doorbell_id);
+-
+- return 0;
+-}
+-
+-static void deallocate_doorbell(struct qcm_process_device *qpd,
+- struct queue *q)
+-{
+- unsigned int old;
+- struct kfd_dev *dev = qpd->dqm->dev;
+-
+- if (!KFD_IS_SOC15(dev->device_info->asic_family) ||
+- q->properties.type == KFD_QUEUE_TYPE_SDMA)
+- return;
+-
+- old = test_and_clear_bit(q->doorbell_id, qpd->doorbell_bitmap);
+- WARN_ON(!old);
+-}
+-
+ static int allocate_vmid(struct device_queue_manager *dqm,
+ struct qcm_process_device *qpd,
+ struct queue *q)
+@@ -168,59 +116,31 @@ static int allocate_vmid(struct device_queue_manager *dqm,
+ if (dqm->vmid_bitmap == 0)
+ return -ENOMEM;
+
+- bit = ffs(dqm->vmid_bitmap) - 1;
+- dqm->vmid_bitmap &= ~(1 << bit);
++ bit = find_first_bit((unsigned long *)&dqm->vmid_bitmap, CIK_VMID_NUM);
++ clear_bit(bit, (unsigned long *)&dqm->vmid_bitmap);
+
+- allocated_vmid = bit + dqm->dev->vm_info.first_vmid_kfd;
+- pr_debug("vmid allocation %d\n", allocated_vmid);
++ /* Kaveri kfd vmid's starts from vmid 8 */
++ allocated_vmid = bit + KFD_VMID_START_OFFSET;
++ pr_debug("kfd: vmid allocation %d\n", allocated_vmid);
+ qpd->vmid = allocated_vmid;
+ q->properties.vmid = allocated_vmid;
+
+ set_pasid_vmid_mapping(dqm, q->process->pasid, q->properties.vmid);
+ program_sh_mem_settings(dqm, qpd);
+
+- /* qpd->page_table_base is set earlier when register_process()
+- * is called, i.e. when the first queue is created.
+- */
+- dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->kgd,
+- qpd->vmid,
+- qpd->page_table_base);
+- /*invalidate the VM context after pasid and vmid mapping is set up*/
+- kfd_flush_tlb(dqm->dev, qpd->pqm->process->pasid);
+-
+ return 0;
+ }
+
+-static int flush_texture_cache_nocpsch(struct kfd_dev *kdev,
+- struct qcm_process_device *qpd)
+-{
+- uint32_t len;
+-
+- if (!qpd->ib_kaddr)
+- return -ENOMEM;
+-
+- len = qpd->dqm->packets.pmf->release_mem(qpd->ib_base,
+- (uint32_t *)qpd->ib_kaddr);
+-
+- return kdev->kfd2kgd->submit_ib(kdev->kgd, KGD_ENGINE_MEC1, qpd->vmid,
+- qpd->ib_base, (uint32_t *)qpd->ib_kaddr, len);
+-}
+-
+ static void deallocate_vmid(struct device_queue_manager *dqm,
+ struct qcm_process_device *qpd,
+ struct queue *q)
+ {
+- int bit = qpd->vmid - dqm->dev->vm_info.first_vmid_kfd;
+-
+- /* On GFX v7, CP doesn't flush TC at dequeue */
+- if (q->device->device_info->asic_family == CHIP_HAWAII)
+- if (flush_texture_cache_nocpsch(q->device, qpd))
+- pr_err("Failed to flush TC\n");
++ int bit = qpd->vmid - KFD_VMID_START_OFFSET;
+
+ /* Release the vmid mapping */
+ set_pasid_vmid_mapping(dqm, 0, qpd->vmid);
+
+- dqm->vmid_bitmap |= (1 << bit);
++ set_bit(bit, (unsigned long *)&dqm->vmid_bitmap);
+ qpd->vmid = 0;
+ q->properties.vmid = 0;
+ }
+@@ -230,53 +150,47 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
+ struct qcm_process_device *qpd,
+ int *allocated_vmid)
+ {
+- int retval = 0;
++ int retval;
++
++ BUG_ON(!dqm || !q || !qpd || !allocated_vmid);
+
++ pr_debug("kfd: In func %s\n", __func__);
+ print_queue(q);
+
+ mutex_lock(&dqm->lock);
+
+ if (dqm->total_queue_count >= max_num_of_queues_per_device) {
+- pr_warn("Can't create new usermode queue because %d queues were already created\n",
++ pr_warn("amdkfd: Can't create new usermode queue because %d queues were already created\n",
+ dqm->total_queue_count);
+- retval = -EPERM;
+- goto out_unlock;
++ mutex_unlock(&dqm->lock);
++ return -EPERM;
+ }
+
+ if (list_empty(&qpd->queues_list)) {
+ retval = allocate_vmid(dqm, qpd, q);
+- if (retval)
+- goto out_unlock;
++ if (retval != 0) {
++ mutex_unlock(&dqm->lock);
++ return retval;
++ }
+ }
+ *allocated_vmid = qpd->vmid;
+ q->properties.vmid = qpd->vmid;
+- /*
+- * Eviction state logic: we only mark active queues as evicted
+- * to avoid the overhead of restoring inactive queues later
+- */
+- if (qpd->evicted)
+- q->properties.is_evicted = (q->properties.queue_size > 0 &&
+- q->properties.queue_percent > 0 &&
+- q->properties.queue_address != 0);
+-
+- q->properties.tba_addr = qpd->tba_addr;
+- q->properties.tma_addr = qpd->tma_addr;
+
+ if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
+ retval = create_compute_queue_nocpsch(dqm, q, qpd);
+- else if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
++ if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
+ retval = create_sdma_queue_nocpsch(dqm, q, qpd);
+
+- if (retval) {
++ if (retval != 0) {
+ if (list_empty(&qpd->queues_list)) {
+ deallocate_vmid(dqm, qpd, q);
+ *allocated_vmid = 0;
+ }
+- goto out_unlock;
++ mutex_unlock(&dqm->lock);
++ return retval;
+ }
+
+ list_add(&q->list, &qpd->queues_list);
+- qpd->queue_count++;
+ if (q->properties.is_active)
+ dqm->queue_count++;
+
+@@ -291,9 +205,8 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
+ pr_debug("Total of %d queues are accountable so far\n",
+ dqm->total_queue_count);
+
+-out_unlock:
+ mutex_unlock(&dqm->lock);
+- return retval;
++ return 0;
+ }
+
+ static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
+@@ -303,16 +216,19 @@ static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
+
+ set = false;
+
+- for (pipe = dqm->next_pipe_to_allocate, i = 0;
+- i < get_pipes_per_mec(dqm);
++ for (pipe = dqm->next_pipe_to_allocate, i = 0; i < get_pipes_per_mec(dqm);
+ pipe = ((pipe + 1) % get_pipes_per_mec(dqm)), ++i) {
+
+ if (!is_pipe_enabled(dqm, 0, pipe))
+ continue;
+
+ if (dqm->allocated_queues[pipe] != 0) {
+- bit = ffs(dqm->allocated_queues[pipe]) - 1;
+- dqm->allocated_queues[pipe] &= ~(1 << bit);
++ bit = find_first_bit(
++ (unsigned long *)&dqm->allocated_queues[pipe],
++ get_queues_per_pipe(dqm));
++
++ clear_bit(bit,
++ (unsigned long *)&dqm->allocated_queues[pipe]);
+ q->pipe = pipe;
+ q->queue = bit;
+ set = true;
+@@ -323,7 +239,8 @@ static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
+ if (!set)
+ return -EBUSY;
+
+- pr_debug("hqd slot - pipe %d, queue %d\n", q->pipe, q->queue);
++ pr_debug("kfd: DQM %s hqd slot - pipe (%d) queue(%d)\n",
++ __func__, q->pipe, q->queue);
+ /* horizontal hqd allocation */
+ dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_per_mec(dqm);
+
+@@ -333,7 +250,7 @@ static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
+ static inline void deallocate_hqd(struct device_queue_manager *dqm,
+ struct queue *q)
+ {
+- dqm->allocated_queues[q->pipe] |= (1 << q->queue);
++ set_bit(q->queue, (unsigned long *)&dqm->allocated_queues[q->pipe]);
+ }
+
+ static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
+@@ -343,203 +260,138 @@ static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
+ int retval;
+ struct mqd_manager *mqd;
+
++ BUG_ON(!dqm || !q || !qpd);
++
+ mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
+- if (!mqd)
++ if (mqd == NULL)
+ return -ENOMEM;
+
+ retval = allocate_hqd(dqm, q);
+- if (retval)
++ if (retval != 0)
+ return retval;
+
+- retval = allocate_doorbell(qpd, q);
+- if (retval)
+- goto out_deallocate_hqd;
+-
+ retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
+ &q->gart_mqd_addr, &q->properties);
+- if (retval)
+- goto out_deallocate_doorbell;
+-
+- pr_debug("Loading mqd to hqd on pipe %d, queue %d\n",
+- q->pipe, q->queue);
+-
+- dqm->dev->kfd2kgd->alloc_memory_of_scratch(
+- dqm->dev->kgd, qpd->sh_hidden_private_base, qpd->vmid);
++ if (retval != 0) {
++ deallocate_hqd(dqm, q);
++ return retval;
++ }
+
+- if (!q->properties.is_active)
+- return 0;
++ pr_debug("kfd: loading mqd to hqd on pipe (%d) queue (%d)\n",
++ q->pipe,
++ q->queue);
+
+- retval = mqd->load_mqd(mqd, q->mqd, q->pipe, q->queue, &q->properties,
+- q->process->mm);
+- if (retval)
+- goto out_uninit_mqd;
++ retval = mqd->load_mqd(mqd, q->mqd, q->pipe,
++ q->queue, (uint32_t __user *) q->properties.write_ptr);
++ if (retval != 0) {
++ deallocate_hqd(dqm, q);
++ mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
++ return retval;
++ }
+
+ return 0;
+-
+-out_uninit_mqd:
+- mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
+-out_deallocate_doorbell:
+- deallocate_doorbell(qpd, q);
+-out_deallocate_hqd:
+- deallocate_hqd(dqm, q);
+-
+- return retval;
+ }
+
+-/* Access to DQM has to be locked before calling destroy_queue_nocpsch_locked
+- * to avoid asynchronized access
+- */
+-static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
++static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
+ struct qcm_process_device *qpd,
+ struct queue *q)
+ {
+ int retval;
+ struct mqd_manager *mqd;
+
+- mqd = dqm->ops.get_mqd_manager(dqm,
+- get_mqd_type_from_queue_type(q->properties.type));
+- if (!mqd)
+- return -ENOMEM;
++ BUG_ON(!dqm || !q || !q->mqd || !qpd);
++
++ retval = 0;
+
+- deallocate_doorbell(qpd, q);
++ pr_debug("kfd: In Func %s\n", __func__);
+
+- if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
++ mutex_lock(&dqm->lock);
++
++ if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
++ mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
++ if (mqd == NULL) {
++ retval = -ENOMEM;
++ goto out;
++ }
+ deallocate_hqd(dqm, q);
+- else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
++ } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
++ mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA);
++ if (mqd == NULL) {
++ retval = -ENOMEM;
++ goto out;
++ }
+ dqm->sdma_queue_count--;
+ deallocate_sdma_queue(dqm, q->sdma_id);
+ } else {
+- pr_debug("q->properties.type %d is invalid\n",
++ pr_debug("q->properties.type is invalid (%d)\n",
+ q->properties.type);
+ retval = -EINVAL;
++ goto out;
+ }
+- dqm->total_queue_count--;
+
+ retval = mqd->destroy_mqd(mqd, q->mqd,
+ KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
+- KFD_HIQ_TIMEOUT,
++ QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS,
+ q->pipe, q->queue);
+- if (retval == -ETIME)
+- qpd->reset_wavefronts = true;
++
++ if (retval != 0)
++ goto out;
+
+ mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
+
+ list_del(&q->list);
+- if (list_empty(&qpd->queues_list)) {
+- if (qpd->reset_wavefronts) {
+- pr_warn("Resetting wave fronts (nocpsch) on dev %p\n",
+- dqm->dev);
+- /* dbgdev_wave_reset_wavefronts has to be called before
+- * deallocate_vmid(), i.e. when vmid is still in use.
+- */
+- dbgdev_wave_reset_wavefronts(dqm->dev,
+- qpd->pqm->process);
+- qpd->reset_wavefronts = false;
+- }
+-
++ if (list_empty(&qpd->queues_list))
+ deallocate_vmid(dqm, qpd, q);
+- }
+- qpd->queue_count--;
+ if (q->properties.is_active)
+ dqm->queue_count--;
+
+- return retval;
+-}
+-
+-static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
+- struct qcm_process_device *qpd,
+- struct queue *q)
+-{
+- int retval;
++ /*
++ * Unconditionally decrement this counter, regardless of the queue's
++ * type
++ */
++ dqm->total_queue_count--;
++ pr_debug("Total of %d queues are accountable so far\n",
++ dqm->total_queue_count);
+
+- mutex_lock(&dqm->lock);
+- retval = destroy_queue_nocpsch_locked(dqm, qpd, q);
++out:
+ mutex_unlock(&dqm->lock);
+-
+ return retval;
+ }
+
+-static bool is_queue_nocpsch(struct device_queue_manager *dqm, struct queue *q)
+-{
+- return (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS &&
+- (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
+- q->properties.type == KFD_QUEUE_TYPE_SDMA));
+-}
+-
+ static int update_queue(struct device_queue_manager *dqm, struct queue *q)
+ {
+ int retval;
+ struct mqd_manager *mqd;
+- struct kfd_process_device *pdd;
+-
+ bool prev_active = false;
+
+- mutex_lock(&dqm->lock);
++ BUG_ON(!dqm || !q || !q->mqd);
+
+- pdd = kfd_get_process_device_data(q->device, q->process);
+- if (!pdd) {
+- retval = -ENODEV;
+- goto out_unlock;
+- }
++ mutex_lock(&dqm->lock);
+ mqd = dqm->ops.get_mqd_manager(dqm,
+ get_mqd_type_from_queue_type(q->properties.type));
+- if (!mqd) {
+- retval = -ENOMEM;
+- goto out_unlock;
+- }
+- /*
+- * Eviction state logic: we only mark active queues as evicted
+- * to avoid the overhead of restoring inactive queues later
+- */
+- if (pdd->qpd.evicted > 0)
+- q->properties.is_evicted = (q->properties.queue_size > 0 &&
+- q->properties.queue_percent > 0 &&
+- q->properties.queue_address != 0);
+-
+- /* save previous activity state for counters */
+- prev_active = q->properties.is_active;
+-
+- /* HWS mode, unmap first to own mqd */
+- if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) {
+- retval = unmap_queues_cpsch(dqm,
+- KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
+- if (retval) {
+- pr_err("unmap queue failed");
+- goto out_unlock;
+- }
+- } else if (is_queue_nocpsch(dqm, q) && prev_active) {
+- retval = mqd->destroy_mqd(mqd, q->mqd,
+- KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
+- KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
+- if (retval) {
+- pr_err("destroy mqd failed");
+- goto out_unlock;
+- }
++ if (mqd == NULL) {
++ mutex_unlock(&dqm->lock);
++ return -ENOMEM;
+ }
+
+- retval = mqd->update_mqd(mqd, q->mqd, &q->properties);
++ if (q->properties.is_active)
++ prev_active = true;
+
+- if (is_queue_nocpsch(dqm, q)) {
+- if (q->properties.is_active)
+- retval = mqd->load_mqd(mqd, q->mqd, q->pipe, q->queue,
+- &q->properties, q->process->mm);
+- }
+ /*
+ *
+ * check active state vs. the previous state
+ * and modify counter accordingly
+ */
+- if (q->properties.is_active && !prev_active)
++ retval = mqd->update_mqd(mqd, q->mqd, &q->properties);
++ if ((q->properties.is_active) && (!prev_active))
+ dqm->queue_count++;
+- else if (!q->properties.is_active && prev_active)
++ else if ((!q->properties.is_active) && (prev_active))
+ dqm->queue_count--;
+
+- if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS)
+- retval = map_queues_cpsch(dqm);
++ if (sched_policy != KFD_SCHED_POLICY_NO_HWS)
++ retval = execute_queues_cpsch(dqm, false);
+
+-out_unlock:
+ mutex_unlock(&dqm->lock);
+-
+ return retval;
+ }
+
+@@ -548,169 +400,41 @@ static struct mqd_manager *get_mqd_manager_nocpsch(
+ {
+ struct mqd_manager *mqd;
+
+- if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
+- return NULL;
++ BUG_ON(!dqm || type >= KFD_MQD_TYPE_MAX);
+
+- pr_debug("mqd type %d\n", type);
++ pr_debug("kfd: In func %s mqd type %d\n", __func__, type);
+
+ mqd = dqm->mqds[type];
+ if (!mqd) {
+ mqd = mqd_manager_init(type, dqm->dev);
+- if (!mqd)
+- pr_err("mqd manager is NULL");
++ if (mqd == NULL)
++ pr_err("kfd: mqd manager is NULL");
+ dqm->mqds[type] = mqd;
+ }
+
+ return mqd;
+ }
+
+-int process_evict_queues(struct device_queue_manager *dqm,
+- struct qcm_process_device *qpd)
+-{
+- struct queue *q, *next;
+- struct mqd_manager *mqd;
+- struct kfd_process_device *pdd;
+- int retval = 0;
+-
+- mutex_lock(&dqm->lock);
+- if (qpd->evicted++ > 0) /* already evicted, do nothing */
+- goto out;
+-
+- pdd = qpd_to_pdd(qpd);
+- pr_info_ratelimited("Evicting PASID %u queues\n",
+- pdd->process->pasid);
+-
+- /* unactivate all active queues on the qpd */
+- list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
+- mqd = dqm->ops.get_mqd_manager(dqm,
+- get_mqd_type_from_queue_type(q->properties.type));
+- if (!mqd) { /* should not be here */
+- pr_err("Cannot evict queue, mqd is NULL\n");
+- retval = -ENOMEM;
+- goto out;
+- }
+- /* if the queue is not active anyway, it is not evicted */
+- if (q->properties.is_active) {
+- q->properties.is_evicted = true;
+- q->properties.is_active = false;
+- }
+-
+- if (is_queue_nocpsch(dqm, q) &&
+- q->properties.is_evicted)
+- retval = mqd->destroy_mqd(mqd, q->mqd,
+- KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
+- KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
+- if (q->properties.is_evicted)
+- dqm->queue_count--;
+- }
+- if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS)
+- retval = execute_queues_cpsch(dqm, qpd->is_debug);
+-
+-out:
+- mutex_unlock(&dqm->lock);
+- return retval;
+-
+-}
+-
+-int process_restore_queues(struct device_queue_manager *dqm,
+- struct qcm_process_device *qpd)
+-{
+- struct queue *q, *next;
+- struct mqd_manager *mqd;
+- int retval = 0;
+- struct kfd_process_device *pdd;
+- uint32_t pd_base;
+-
+- pdd = qpd_to_pdd(qpd);
+- /* Retrieve PD base */
+- pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm);
+-
+- mutex_lock(&dqm->lock);
+- if (qpd->evicted == 0) /* already restored, do nothing */
+- goto out_unlock;
+-
+- if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
+- qpd->evicted--;
+- goto out_unlock;
+- }
+-
+- pr_info_ratelimited("Restoring PASID %u queues\n",
+- pdd->process->pasid);
+-
+- /* Update PD Base in QPD */
+- qpd->page_table_base = pd_base;
+- pr_debug("Updated PD address to 0x%08x\n", pd_base);
+-
+- if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS &&
+- !list_empty(&qpd->queues_list)) {
+- dqm->dev->kfd2kgd->set_vm_context_page_table_base(
+- dqm->dev->kgd,
+- qpd->vmid,
+- qpd->page_table_base);
+-
+- kfd_flush_tlb(dqm->dev, pdd->process->pasid);
+- }
+-
+- /* activate all active queues on the qpd */
+- list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
+- mqd = dqm->ops.get_mqd_manager(dqm,
+- get_mqd_type_from_queue_type(q->properties.type));
+- if (!mqd) { /* should not be here */
+- pr_err("Cannot restore queue, mqd is NULL\n");
+- retval = -ENOMEM;
+- goto out_unlock;
+- }
+- if (q->properties.is_evicted) {
+- q->properties.is_evicted = false;
+- q->properties.is_active = true;
+-
+- if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS &&
+- (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
+- q->properties.type == KFD_QUEUE_TYPE_SDMA))
+- retval = mqd->load_mqd(mqd, q->mqd, q->pipe,
+- q->queue, &q->properties,
+- q->process->mm);
+- dqm->queue_count++;
+- }
+- }
+- if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS)
+- retval = execute_queues_cpsch(dqm, false);
+-
+- if (retval == 0)
+- qpd->evicted = 0;
+-
+-out_unlock:
+- mutex_unlock(&dqm->lock);
+-
+- return retval;
+-}
+-
+-static int register_process(struct device_queue_manager *dqm,
++static int register_process_nocpsch(struct device_queue_manager *dqm,
+ struct qcm_process_device *qpd)
+ {
+ struct device_process_node *n;
+ int retval;
+- struct kfd_process_device *pdd;
+- uint32_t pd_base;
+
+- n = kzalloc(sizeof(*n), GFP_KERNEL);
++ BUG_ON(!dqm || !qpd);
++
++ pr_debug("kfd: In func %s\n", __func__);
++
++ n = kzalloc(sizeof(struct device_process_node), GFP_KERNEL);
+ if (!n)
+ return -ENOMEM;
+
+ n->qpd = qpd;
+
+- pdd = qpd_to_pdd(qpd);
+- /* Retrieve PD base */
+- pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm);
+-
+ mutex_lock(&dqm->lock);
+ list_add(&n->list, &dqm->queues);
+
+- /* Update PD Base in QPD */
+- qpd->page_table_base = pd_base;
+- pr_debug("Updated PD address to 0x%08x\n", pd_base);
+-
+- retval = dqm->asic_ops.update_qpd(dqm, qpd);
++ retval = dqm->ops_asic_specific.register_process(dqm, qpd);
+
+ dqm->processes_count++;
+
+@@ -719,12 +443,16 @@ static int register_process(struct device_queue_manager *dqm,
+ return retval;
+ }
+
+-static int unregister_process(struct device_queue_manager *dqm,
++static int unregister_process_nocpsch(struct device_queue_manager *dqm,
+ struct qcm_process_device *qpd)
+ {
+ int retval;
+ struct device_process_node *cur, *next;
+
++ BUG_ON(!dqm || !qpd);
++
++ pr_debug("In func %s\n", __func__);
++
+ pr_debug("qpd->queues_list is %s\n",
+ list_empty(&qpd->queues_list) ? "empty" : "not empty");
+
+@@ -765,41 +493,48 @@ static void init_interrupts(struct device_queue_manager *dqm)
+ {
+ unsigned int i;
+
++ BUG_ON(dqm == NULL);
++
+ for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++)
+ if (is_pipe_enabled(dqm, 0, i))
+ dqm->dev->kfd2kgd->init_interrupts(dqm->dev->kgd, i);
+ }
++
+ static int init_scheduler(struct device_queue_manager *dqm)
+ {
+- return 0;
++ int retval = 0;
++
++ BUG_ON(!dqm);
++
++ pr_debug("kfd: In %s\n", __func__);
++
++ return retval;
+ }
+
+ static int initialize_nocpsch(struct device_queue_manager *dqm)
+ {
+- int pipe, queue;
++ int i;
+
+- pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
++ BUG_ON(!dqm);
+
+- dqm->allocated_queues = kcalloc(get_pipes_per_mec(dqm),
+- sizeof(unsigned int), GFP_KERNEL);
+- if (!dqm->allocated_queues)
+- return -ENOMEM;
++ pr_debug("kfd: In func %s num of pipes: %d\n",
++ __func__, get_pipes_per_mec(dqm));
+
+ mutex_init(&dqm->lock);
+ INIT_LIST_HEAD(&dqm->queues);
+ dqm->queue_count = dqm->next_pipe_to_allocate = 0;
+ dqm->sdma_queue_count = 0;
+-
+- for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
+- int pipe_offset = pipe * get_queues_per_pipe(dqm);
+-
+- for (queue = 0; queue < get_queues_per_pipe(dqm); queue++)
+- if (test_bit(pipe_offset + queue,
+- dqm->dev->shared_resources.queue_bitmap))
+- dqm->allocated_queues[pipe] |= 1 << queue;
++ dqm->allocated_queues = kcalloc(get_pipes_per_mec(dqm),
++ sizeof(unsigned int), GFP_KERNEL);
++ if (!dqm->allocated_queues) {
++ mutex_destroy(&dqm->lock);
++ return -ENOMEM;
+ }
+
+- dqm->vmid_bitmap = (1 << dqm->dev->vm_info.vmid_num_kfd) - 1;
++ for (i = 0; i < get_pipes_per_mec(dqm); i++)
++ dqm->allocated_queues[i] = (1 << get_queues_per_pipe(dqm)) - 1;
++
++ dqm->vmid_bitmap = (1 << VMID_PER_DEVICE) - 1;
+ dqm->sdma_bitmap = (1 << CIK_SDMA_QUEUES) - 1;
+
+ init_scheduler(dqm);
+@@ -810,7 +545,9 @@ static void uninitialize_nocpsch(struct device_queue_manager *dqm)
+ {
+ int i;
+
+- WARN_ON(dqm->queue_count > 0 || dqm->processes_count > 0);
++ BUG_ON(!dqm);
++
++ BUG_ON(dqm->queue_count > 0 || dqm->processes_count > 0);
+
+ kfree(dqm->allocated_queues);
+ for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++)
+@@ -822,12 +559,11 @@ static void uninitialize_nocpsch(struct device_queue_manager *dqm)
+ static int start_nocpsch(struct device_queue_manager *dqm)
+ {
+ init_interrupts(dqm);
+- return pm_init(&dqm->packets, dqm, dqm->dev->mec_fw_version);
++ return 0;
+ }
+
+ static int stop_nocpsch(struct device_queue_manager *dqm)
+ {
+- pm_uninit(&dqm->packets);
+ return 0;
+ }
+
+@@ -839,8 +575,10 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm,
+ if (dqm->sdma_bitmap == 0)
+ return -ENOMEM;
+
+- bit = ffs(dqm->sdma_bitmap) - 1;
+- dqm->sdma_bitmap &= ~(1 << bit);
++ bit = find_first_bit((unsigned long *)&dqm->sdma_bitmap,
++ CIK_SDMA_QUEUES);
++
++ clear_bit(bit, (unsigned long *)&dqm->sdma_bitmap);
+ *sdma_queue_id = bit;
+
+ return 0;
+@@ -851,7 +589,7 @@ static void deallocate_sdma_queue(struct device_queue_manager *dqm,
+ {
+ if (sdma_queue_id >= CIK_SDMA_QUEUES)
+ return;
+- dqm->sdma_bitmap |= (1 << sdma_queue_id);
++ set_bit(sdma_queue_id, (unsigned long *)&dqm->sdma_bitmap);
+ }
+
+ static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
+@@ -866,40 +604,33 @@ static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
+ return -ENOMEM;
+
+ retval = allocate_sdma_queue(dqm, &q->sdma_id);
+- if (retval)
++ if (retval != 0)
+ return retval;
+
+- q->properties.sdma_queue_id = q->sdma_id / CIK_SDMA_QUEUES_PER_ENGINE;
+- q->properties.sdma_engine_id = q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;
+-
+- retval = allocate_doorbell(qpd, q);
+- if (retval)
+- goto out_deallocate_sdma_queue;
++ q->properties.sdma_queue_id = q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;
++ q->properties.sdma_engine_id = q->sdma_id / CIK_SDMA_ENGINE_NUM;
+
+- pr_debug("SDMA id is: %d\n", q->sdma_id);
+- pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id);
+- pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id);
++ pr_debug("kfd: sdma id is: %d\n", q->sdma_id);
++ pr_debug(" sdma queue id: %d\n", q->properties.sdma_queue_id);
++ pr_debug(" sdma engine id: %d\n", q->properties.sdma_engine_id);
+
+- dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
++ dqm->ops_asic_specific.init_sdma_vm(dqm, q, qpd);
+ retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
+ &q->gart_mqd_addr, &q->properties);
+- if (retval)
+- goto out_deallocate_doorbell;
++ if (retval != 0) {
++ deallocate_sdma_queue(dqm, q->sdma_id);
++ return retval;
++ }
+
+- retval = mqd->load_mqd(mqd, q->mqd, 0, 0, &q->properties, NULL);
+- if (retval)
+- goto out_uninit_mqd;
++ retval = mqd->load_mqd(mqd, q->mqd, 0,
++ 0, NULL);
++ if (retval != 0) {
++ deallocate_sdma_queue(dqm, q->sdma_id);
++ mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
++ return retval;
++ }
+
+ return 0;
+-
+-out_uninit_mqd:
+- mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
+-out_deallocate_doorbell:
+- deallocate_doorbell(qpd, q);
+-out_deallocate_sdma_queue:
+- deallocate_sdma_queue(dqm, q->sdma_id);
+-
+- return retval;
+ }
+
+ /*
+@@ -911,7 +642,12 @@ static int set_sched_resources(struct device_queue_manager *dqm)
+ int i, mec;
+ struct scheduling_resources res;
+
+- res.vmid_mask = dqm->dev->shared_resources.compute_vmid_bitmap;
++ BUG_ON(!dqm);
++
++ pr_debug("kfd: In func %s\n", __func__);
++
++ res.vmid_mask = (1 << VMID_PER_DEVICE) - 1;
++ res.vmid_mask <<= KFD_VMID_START_OFFSET;
+
+ res.queue_mask = 0;
+ for (i = 0; i < KGD_MAX_QUEUES; ++i) {
+@@ -927,8 +663,7 @@ static int set_sched_resources(struct device_queue_manager *dqm)
+
+ /* This situation may be hit in the future if a new HW
+ * generation exposes more than 64 queues. If so, the
+- * definition of res.queue_mask needs updating
+- */
++ * definition of res.queue_mask needs updating */
+ if (WARN_ON(i >= (sizeof(res.queue_mask)*8))) {
+ pr_err("Invalid queue enabled by amdgpu: %d\n", i);
+ break;
+@@ -939,9 +674,9 @@ static int set_sched_resources(struct device_queue_manager *dqm)
+ res.gws_mask = res.oac_mask = res.gds_heap_base =
+ res.gds_heap_size = 0;
+
+- pr_debug("Scheduling resources:\n"
+- "vmid mask: 0x%8X\n"
+- "queue mask: 0x%8llX\n",
++ pr_debug("kfd: scheduling resources:\n"
++ " vmid mask: 0x%8X\n"
++ " queue mask: 0x%8llX\n",
+ res.vmid_mask, res.queue_mask);
+
+ return pm_send_set_resources(&dqm->packets, &res);
+@@ -951,42 +686,51 @@ static int initialize_cpsch(struct device_queue_manager *dqm)
+ {
+ int retval;
+
+- pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
++ BUG_ON(!dqm);
++
++ pr_debug("kfd: In func %s num of pipes: %d\n",
++ __func__, get_pipes_per_mec(dqm));
+
+ mutex_init(&dqm->lock);
+ INIT_LIST_HEAD(&dqm->queues);
+ dqm->queue_count = dqm->processes_count = 0;
+ dqm->sdma_queue_count = 0;
+ dqm->active_runlist = false;
+- dqm->sdma_bitmap = (1 << CIK_SDMA_QUEUES) - 1;
+- retval = dqm->asic_ops.init_cpsch(dqm);
+- if (retval)
+- mutex_destroy(&dqm->lock);
++ retval = dqm->ops_asic_specific.initialize(dqm);
++ if (retval != 0)
++ goto fail_init_pipelines;
++
++ return 0;
+
++fail_init_pipelines:
++ mutex_destroy(&dqm->lock);
+ return retval;
+ }
+
+ static int start_cpsch(struct device_queue_manager *dqm)
+ {
++ struct device_process_node *node;
+ int retval;
+
++ BUG_ON(!dqm);
++
+ retval = 0;
+
+- retval = pm_init(&dqm->packets, dqm, dqm->dev->mec_fw_version);
+- if (retval)
++ retval = pm_init(&dqm->packets, dqm);
++ if (retval != 0)
+ goto fail_packet_manager_init;
+
+ retval = set_sched_resources(dqm);
+- if (retval)
++ if (retval != 0)
+ goto fail_set_sched_resources;
+
+- pr_debug("Allocating fence memory\n");
++ pr_debug("kfd: allocating fence memory\n");
+
+ /* allocate fence memory on the gart */
+ retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr),
+ &dqm->fence_mem);
+
+- if (retval)
++ if (retval != 0)
+ goto fail_allocate_vidmem;
+
+ dqm->fence_addr = dqm->fence_mem->cpu_ptr;
+@@ -994,9 +738,12 @@ static int start_cpsch(struct device_queue_manager *dqm)
+
+ init_interrupts(dqm);
+
+- mutex_lock(&dqm->lock);
+- execute_queues_cpsch(dqm, false);
+- mutex_unlock(&dqm->lock);
++ list_for_each_entry(node, &dqm->queues, list)
++ if (node->qpd->pqm->process && dqm->dev)
++ kfd_bind_process_to_device(dqm->dev,
++ node->qpd->pqm->process);
++
++ execute_queues_cpsch(dqm, true);
+
+ return 0;
+ fail_allocate_vidmem:
+@@ -1008,12 +755,17 @@ static int start_cpsch(struct device_queue_manager *dqm)
+
+ static int stop_cpsch(struct device_queue_manager *dqm)
+ {
+- mutex_lock(&dqm->lock);
++ struct device_process_node *node;
++ struct kfd_process_device *pdd;
+
+- unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
++ BUG_ON(!dqm);
+
+- mutex_unlock(&dqm->lock);
++ destroy_queues_cpsch(dqm, true, true);
+
++ list_for_each_entry(node, &dqm->queues, list) {
++ pdd = qpd_to_pdd(node->qpd);
++ pdd->bound = false;
++ }
+ kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
+ pm_uninit(&dqm->packets);
+
+@@ -1024,9 +776,13 @@ static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
+ struct kernel_queue *kq,
+ struct qcm_process_device *qpd)
+ {
++ BUG_ON(!dqm || !kq || !qpd);
++
++ pr_debug("kfd: In func %s\n", __func__);
++
+ mutex_lock(&dqm->lock);
+ if (dqm->total_queue_count >= max_num_of_queues_per_device) {
+- pr_warn("Can't create new kernel queue because %d queues were already created\n",
++ pr_warn("amdkfd: Can't create new kernel queue because %d queues were already created\n",
+ dqm->total_queue_count);
+ mutex_unlock(&dqm->lock);
+ return -EPERM;
+@@ -1053,12 +809,17 @@ static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
+ struct kernel_queue *kq,
+ struct qcm_process_device *qpd)
+ {
++ BUG_ON(!dqm || !kq);
++
++ pr_debug("kfd: In %s\n", __func__);
++
+ mutex_lock(&dqm->lock);
+ /* here we actually preempt the DIQ */
++ destroy_queues_cpsch(dqm, true, false);
+ list_del(&kq->list);
+ dqm->queue_count--;
+ qpd->is_debug = false;
+- execute_queues_cpsch(dqm, true);
++ execute_queues_cpsch(dqm, false);
+ /*
+ * Unconditionally decrement this counter, regardless of the queue's
+ * type.
+@@ -1069,12 +830,22 @@ static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
+ mutex_unlock(&dqm->lock);
+ }
+
++static void select_sdma_engine_id(struct queue *q)
++{
++ static int sdma_id;
++
++ q->sdma_id = sdma_id;
++ sdma_id = (sdma_id + 1) % 2;
++}
++
+ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
+ struct qcm_process_device *qpd, int *allocate_vmid)
+ {
+ int retval;
+ struct mqd_manager *mqd;
+
++ BUG_ON(!dqm || !q || !qpd);
++
+ retval = 0;
+
+ if (allocate_vmid)
+@@ -1083,60 +854,37 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
+ mutex_lock(&dqm->lock);
+
+ if (dqm->total_queue_count >= max_num_of_queues_per_device) {
+- pr_warn("Can't create new usermode queue because %d queues were already created\n",
++ pr_warn("amdkfd: Can't create new usermode queue because %d queues were already created\n",
+ dqm->total_queue_count);
+ retval = -EPERM;
+- goto out_unlock;
+- }
+-
+- if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
+- retval = allocate_sdma_queue(dqm, &q->sdma_id);
+- if (retval)
+- goto out_unlock;
+- q->properties.sdma_queue_id =
+- q->sdma_id / CIK_SDMA_QUEUES_PER_ENGINE;
+- q->properties.sdma_engine_id =
+- q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;
++ goto out;
+ }
+
+- retval = allocate_doorbell(qpd, q);
+- if (retval)
+- goto out_deallocate_sdma_queue;
++ if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
++ select_sdma_engine_id(q);
+
+ mqd = dqm->ops.get_mqd_manager(dqm,
+ get_mqd_type_from_queue_type(q->properties.type));
+
+- if (!mqd) {
+- retval = -ENOMEM;
+- goto out_deallocate_doorbell;
++ if (mqd == NULL) {
++ mutex_unlock(&dqm->lock);
++ return -ENOMEM;
+ }
+- /*
+- * Eviction state logic: we only mark active queues as evicted
+- * to avoid the overhead of restoring inactive queues later
+- */
+- if (qpd->evicted)
+- q->properties.is_evicted = (q->properties.queue_size > 0 &&
+- q->properties.queue_percent > 0 &&
+- q->properties.queue_address != 0);
+-
+- dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
+
+- q->properties.tba_addr = qpd->tba_addr;
+- q->properties.tma_addr = qpd->tma_addr;
++ dqm->ops_asic_specific.init_sdma_vm(dqm, q, qpd);
+ retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
+ &q->gart_mqd_addr, &q->properties);
+- if (retval)
+- goto out_deallocate_doorbell;
++ if (retval != 0)
++ goto out;
+
+ list_add(&q->list, &qpd->queues_list);
+- qpd->queue_count++;
+ if (q->properties.is_active) {
+ dqm->queue_count++;
+ retval = execute_queues_cpsch(dqm, false);
+ }
+
+ if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
+- dqm->sdma_queue_count++;
++ dqm->sdma_queue_count++;
+ /*
+ * Unconditionally increment this counter, regardless of the queue's
+ * type or whether the queue is active.
+@@ -1146,31 +894,21 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
+ pr_debug("Total of %d queues are accountable so far\n",
+ dqm->total_queue_count);
+
++out:
+ mutex_unlock(&dqm->lock);
+ return retval;
+-
+-out_deallocate_doorbell:
+- deallocate_doorbell(qpd, q);
+-out_deallocate_sdma_queue:
+- if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
+- deallocate_sdma_queue(dqm, q->sdma_id);
+-out_unlock:
+- mutex_unlock(&dqm->lock);
+-
+- return retval;
+ }
+
+ int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
+ unsigned int fence_value,
+- unsigned long timeout_ms)
++ unsigned long timeout)
+ {
+- unsigned long end_jiffies;
+-
+- end_jiffies = (timeout_ms * HZ / 1000) + jiffies;
++ BUG_ON(!fence_addr);
++ timeout += jiffies;
+
+ while (*fence_addr != fence_value) {
+- if (time_after(jiffies, end_jiffies)) {
+- pr_err("qcm fence wait loop timeout expired\n");
++ if (time_after(jiffies, timeout)) {
++ pr_err("kfd: qcm fence wait loop timeout expired\n");
+ return -ETIME;
+ }
+ schedule();
+@@ -1179,63 +917,46 @@ int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
+ return 0;
+ }
+
+-static int unmap_sdma_queues(struct device_queue_manager *dqm,
+- unsigned int sdma_engine)
++static int destroy_sdma_queues(struct device_queue_manager *dqm,
++ unsigned int sdma_engine)
+ {
+ return pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_SDMA,
+- KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, false,
++ KFD_PREEMPT_TYPE_FILTER_DYNAMIC_QUEUES, 0, false,
+ sdma_engine);
+ }
+
+-/* dqm->lock mutex has to be locked before calling this function */
+-static int map_queues_cpsch(struct device_queue_manager *dqm)
++static int destroy_queues_cpsch(struct device_queue_manager *dqm,
++ bool preempt_static_queues, bool lock)
+ {
+ int retval;
++ enum kfd_preempt_type_filter preempt_type;
++ struct kfd_process_device *pdd;
+
+- if (dqm->queue_count <= 0 || dqm->processes_count <= 0) {
+- retval = 0;
+- return retval;
+- }
+-
+- if (dqm->active_runlist) {
+- retval = 0;
+- return retval;
+- }
+-
+- retval = pm_send_runlist(&dqm->packets, &dqm->queues);
+- if (retval) {
+- pr_err("failed to execute runlist\n");
+- return retval;
+- }
+- dqm->active_runlist = true;
+-
+- return retval;
+-}
+-
+-/* dqm->lock mutex has to be locked before calling this function */
+-static int unmap_queues_cpsch(struct device_queue_manager *dqm,
+- enum kfd_unmap_queues_filter filter,
+- uint32_t filter_param)
+-{
+- int retval;
++ BUG_ON(!dqm);
+
+ retval = 0;
+
++ if (lock)
++ mutex_lock(&dqm->lock);
+ if (!dqm->active_runlist)
+- return retval;
++ goto out;
+
+- pr_debug("Before destroying queues, sdma queue count is : %u\n",
++ pr_debug("kfd: Before destroying queues, sdma queue count is : %u\n",
+ dqm->sdma_queue_count);
+
+ if (dqm->sdma_queue_count > 0) {
+- unmap_sdma_queues(dqm, 0);
+- unmap_sdma_queues(dqm, 1);
++ destroy_sdma_queues(dqm, 0);
++ destroy_sdma_queues(dqm, 1);
+ }
+
++ preempt_type = preempt_static_queues ?
++ KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES :
++ KFD_PREEMPT_TYPE_FILTER_DYNAMIC_QUEUES;
++
+ retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE,
+- filter, filter_param, false, 0);
+- if (retval)
+- return retval;
++ preempt_type, 0, false, 0);
++ if (retval != 0)
++ goto out;
+
+ *dqm->fence_addr = KFD_FENCE_INIT;
+ pm_send_query_status(&dqm->packets, dqm->fence_gpu_addr,
+@@ -1243,36 +964,56 @@ static int unmap_queues_cpsch(struct device_queue_manager *dqm,
+ /* should be timed out */
+ retval = amdkfd_fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED,
+ QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS);
+- if (retval) {
+- pr_err("Unmapping queues failed.\n");
+- return retval;
++ if (retval != 0) {
++ pdd = kfd_get_process_device_data(dqm->dev,
++ kfd_get_process(current));
++ pdd->reset_wavefronts = true;
++ goto out;
+ }
+-
+ pm_release_ib(&dqm->packets);
+ dqm->active_runlist = false;
+
++out:
++ if (lock)
++ mutex_unlock(&dqm->lock);
+ return retval;
+ }
+
+-/* dqm->lock mutex has to be locked before calling this function */
+-static int execute_queues_cpsch(struct device_queue_manager *dqm,
+- bool static_queues_included)
++static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock)
+ {
+ int retval;
+- enum kfd_unmap_queues_filter filter;
+
+- filter = static_queues_included ?
+- KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES :
+- KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES;
++ BUG_ON(!dqm);
+
+- retval = unmap_queues_cpsch(dqm, filter, 0);
+- if (retval) {
+- pr_err("The cp might be in an unrecoverable state due to an unsuccessful queues preemption\n");
+- return retval;
++ if (lock)
++ mutex_lock(&dqm->lock);
++
++ retval = destroy_queues_cpsch(dqm, false, false);
++ if (retval != 0) {
++ pr_err("kfd: the cp might be in an unrecoverable state due to an unsuccessful queues preemption");
++ goto out;
++ }
++
++ if (dqm->queue_count <= 0 || dqm->processes_count <= 0) {
++ retval = 0;
++ goto out;
+ }
+
+- retval = map_queues_cpsch(dqm);
++ if (dqm->active_runlist) {
++ retval = 0;
++ goto out;
++ }
+
++ retval = pm_send_runlist(&dqm->packets, &dqm->queues);
++ if (retval != 0) {
++ pr_err("kfd: failed to execute runlist");
++ goto out;
++ }
++ dqm->active_runlist = true;
++
++out:
++ if (lock)
++ mutex_unlock(&dqm->lock);
+ return retval;
+ }
+
+@@ -1284,6 +1025,8 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,
+ struct mqd_manager *mqd;
+ bool preempt_all_queues;
+
++ BUG_ON(!dqm || !qpd || !q);
++
+ preempt_all_queues = false;
+
+ retval = 0;
+@@ -1308,21 +1051,14 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,
+ goto failed;
+ }
+
+- deallocate_doorbell(qpd, q);
+-
+- if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
++ if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
+ dqm->sdma_queue_count--;
+- deallocate_sdma_queue(dqm, q->sdma_id);
+- }
+
+ list_del(&q->list);
+- qpd->queue_count--;
+ if (q->properties.is_active)
+ dqm->queue_count--;
+
+- retval = execute_queues_cpsch(dqm, false);
+- if (retval == -ETIME)
+- qpd->reset_wavefronts = true;
++ execute_queues_cpsch(dqm, false);
+
+ mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
+
+@@ -1336,7 +1072,7 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,
+
+ mutex_unlock(&dqm->lock);
+
+- return retval;
++ return 0;
+
+ failed:
+ failed_try_destroy_debugged_queue:
+@@ -1360,10 +1096,9 @@ static bool set_cache_memory_policy(struct device_queue_manager *dqm,
+ void __user *alternate_aperture_base,
+ uint64_t alternate_aperture_size)
+ {
+- bool retval = true;
++ bool retval;
+
+- if (!dqm->asic_ops.set_cache_memory_policy)
+- return retval;
++ pr_debug("kfd: In func %s\n", __func__);
+
+ mutex_lock(&dqm->lock);
+
+@@ -1385,17 +1120,20 @@ static bool set_cache_memory_policy(struct device_queue_manager *dqm,
+ uint64_t base = (uintptr_t)alternate_aperture_base;
+ uint64_t limit = base + alternate_aperture_size - 1;
+
+- if (limit <= base || (base & APE1_FIXED_BITS_MASK) != 0 ||
+- (limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT) {
+- retval = false;
++ if (limit <= base)
++ goto out;
++
++ if ((base & APE1_FIXED_BITS_MASK) != 0)
++ goto out;
++
++ if ((limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT)
+ goto out;
+- }
+
+ qpd->sh_mem_ape1_base = base >> 16;
+ qpd->sh_mem_ape1_limit = limit >> 16;
+ }
+
+- retval = dqm->asic_ops.set_cache_memory_policy(
++ retval = dqm->ops_asic_specific.set_cache_memory_policy(
+ dqm,
+ qpd,
+ default_policy,
+@@ -1403,199 +1141,35 @@ static bool set_cache_memory_policy(struct device_queue_manager *dqm,
+ alternate_aperture_base,
+ alternate_aperture_size);
+
+- if ((dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0))
++ if ((sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0))
+ program_sh_mem_settings(dqm, qpd);
+
+- pr_debug("sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n",
++ pr_debug("kfd: sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n",
+ qpd->sh_mem_config, qpd->sh_mem_ape1_base,
+ qpd->sh_mem_ape1_limit);
+
+-out:
+ mutex_unlock(&dqm->lock);
+ return retval;
+-}
+-
+-static int set_trap_handler(struct device_queue_manager *dqm,
+- struct qcm_process_device *qpd,
+- uint64_t tba_addr,
+- uint64_t tma_addr)
+-{
+- uint64_t *tma;
+-
+- if (dqm->dev->cwsr_enabled) {
+- /* Jump from CWSR trap handler to user trap */
+- tma = (uint64_t *)(qpd->cwsr_kaddr + dqm->dev->tma_offset);
+- tma[0] = tba_addr;
+- tma[1] = tma_addr;
+- } else {
+- qpd->tba_addr = tba_addr;
+- qpd->tma_addr = tma_addr;
+- }
+-
+- return 0;
+-}
+-
+-static int process_termination_nocpsch(struct device_queue_manager *dqm,
+- struct qcm_process_device *qpd)
+-{
+- struct queue *q, *next;
+- struct device_process_node *cur, *next_dpn;
+- int retval = 0;
+-
+- mutex_lock(&dqm->lock);
+-
+- /* Clear all user mode queues */
+- list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
+- int ret;
+-
+- ret = destroy_queue_nocpsch_locked(dqm, qpd, q);
+- if (ret)
+- retval = ret;
+- }
+-
+- /* Unregister process */
+- list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
+- if (qpd == cur->qpd) {
+- list_del(&cur->list);
+- kfree(cur);
+- dqm->processes_count--;
+- break;
+- }
+- }
+-
+- mutex_unlock(&dqm->lock);
+- return retval;
+-}
+-
+-static int get_wave_state(struct device_queue_manager *dqm,
+- struct queue *q,
+- void __user *ctl_stack,
+- u32 *ctl_stack_used_size,
+- u32 *save_area_used_size)
+-{
+- struct mqd_manager *mqd;
+- int r;
+-
+- mutex_lock(&dqm->lock);
+-
+- if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE ||
+- q->properties.is_active || !q->device->cwsr_enabled) {
+- r = -EINVAL;
+- goto dqm_unlock;
+- }
+-
+- mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
+- if (!mqd) {
+- r = -ENOMEM;
+- goto dqm_unlock;
+- }
+-
+- if (!mqd->get_wave_state) {
+- r = -EINVAL;
+- goto dqm_unlock;
+- }
+-
+- r = mqd->get_wave_state(mqd, q->mqd, ctl_stack, ctl_stack_used_size,
+- save_area_used_size);
+-
+-dqm_unlock:
+- mutex_unlock(&dqm->lock);
+- return r;
+-}
+-
+-static int process_termination_cpsch(struct device_queue_manager *dqm,
+- struct qcm_process_device *qpd)
+-{
+- int retval;
+- struct queue *q, *next;
+- struct kernel_queue *kq, *kq_next;
+- struct mqd_manager *mqd;
+- struct device_process_node *cur, *next_dpn;
+- bool unmap_static_queues = false;
+-
+- retval = 0;
+-
+- mutex_lock(&dqm->lock);
+-
+- /* Clean all kernel queues */
+- list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) {
+- list_del(&kq->list);
+- dqm->queue_count--;
+- qpd->is_debug = false;
+- dqm->total_queue_count--;
+- unmap_static_queues = true;
+- }
+-
+- /* Clear all user mode queues */
+- list_for_each_entry(q, &qpd->queues_list, list) {
+- if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
+- dqm->sdma_queue_count--;
+- deallocate_sdma_queue(dqm, q->sdma_id);
+- }
+-
+- if (q->properties.is_active)
+- dqm->queue_count--;
+-
+- dqm->total_queue_count--;
+- }
+-
+- /* Unregister process */
+- list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
+- if (qpd == cur->qpd) {
+- list_del(&cur->list);
+- kfree(cur);
+- dqm->processes_count--;
+- break;
+- }
+- }
+-
+- retval = execute_queues_cpsch(dqm, unmap_static_queues);
+- if (retval || qpd->reset_wavefronts) {
+- pr_warn("Resetting wave fronts (cpsch) on dev %p\n", dqm->dev);
+- dbgdev_wave_reset_wavefronts(dqm->dev, qpd->pqm->process);
+- qpd->reset_wavefronts = false;
+- }
+-
+- /* lastly, free mqd resources */
+- list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
+- mqd = dqm->ops.get_mqd_manager(dqm,
+- get_mqd_type_from_queue_type(q->properties.type));
+- if (!mqd) {
+- retval = -ENOMEM;
+- goto out;
+- }
+- list_del(&q->list);
+- qpd->queue_count--;
+- mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
+- }
+
+ out:
+ mutex_unlock(&dqm->lock);
+- return retval;
++ return false;
+ }
+
+ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
+ {
+ struct device_queue_manager *dqm;
+
+- pr_debug("Loading device queue manager\n");
++ BUG_ON(!dev);
++
++ pr_debug("kfd: loading device queue manager\n");
+
+- dqm = kzalloc(sizeof(*dqm), GFP_KERNEL);
++ dqm = kzalloc(sizeof(struct device_queue_manager), GFP_KERNEL);
+ if (!dqm)
+ return NULL;
+
+- switch (dev->device_info->asic_family) {
+- case CHIP_HAWAII:
+- case CHIP_TONGA:
+- dqm->sched_policy = KFD_SCHED_POLICY_NO_HWS;
+- break;
+- default:
+- dqm->sched_policy = sched_policy;
+- break;
+- }
+-
+ dqm->dev = dev;
+- switch (dqm->sched_policy) {
++ switch (sched_policy) {
+ case KFD_SCHED_POLICY_HWS:
+ case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION:
+ /* initialize dqm for cp scheduling */
+@@ -1606,15 +1180,12 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
+ dqm->ops.destroy_queue = destroy_queue_cpsch;
+ dqm->ops.update_queue = update_queue;
+ dqm->ops.get_mqd_manager = get_mqd_manager_nocpsch;
+- dqm->ops.register_process = register_process;
+- dqm->ops.unregister_process = unregister_process;
++ dqm->ops.register_process = register_process_nocpsch;
++ dqm->ops.unregister_process = unregister_process_nocpsch;
+ dqm->ops.uninitialize = uninitialize_nocpsch;
+ dqm->ops.create_kernel_queue = create_kernel_queue_cpsch;
+ dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch;
+ dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
+- dqm->ops.set_trap_handler = set_trap_handler;
+- dqm->ops.process_termination = process_termination_cpsch;
+- dqm->ops.get_wave_state = get_wave_state;
+ break;
+ case KFD_SCHED_POLICY_NO_HWS:
+ /* initialize dqm for no cp scheduling */
+@@ -1624,142 +1195,39 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
+ dqm->ops.destroy_queue = destroy_queue_nocpsch;
+ dqm->ops.update_queue = update_queue;
+ dqm->ops.get_mqd_manager = get_mqd_manager_nocpsch;
+- dqm->ops.register_process = register_process;
+- dqm->ops.unregister_process = unregister_process;
++ dqm->ops.register_process = register_process_nocpsch;
++ dqm->ops.unregister_process = unregister_process_nocpsch;
+ dqm->ops.initialize = initialize_nocpsch;
+ dqm->ops.uninitialize = uninitialize_nocpsch;
+ dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
+- dqm->ops.set_trap_handler = set_trap_handler;
+- dqm->ops.process_termination = process_termination_nocpsch;
+- dqm->ops.get_wave_state = get_wave_state;
+ break;
+ default:
+- WARN(1, "Invalid scheduling policy %d", dqm->sched_policy);
+- goto out_free;
++ BUG();
++ break;
+ }
+
+ switch (dev->device_info->asic_family) {
+ case CHIP_CARRIZO:
+- device_queue_manager_init_vi(&dqm->asic_ops);
++ device_queue_manager_init_vi(&dqm->ops_asic_specific);
+ break;
+
+ case CHIP_KAVERI:
+- device_queue_manager_init_cik(&dqm->asic_ops);
+- break;
+-
+- case CHIP_HAWAII:
+- device_queue_manager_init_cik_hawaii(&dqm->asic_ops);
+- break;
+-
+- case CHIP_TONGA:
+- case CHIP_FIJI:
+- case CHIP_POLARIS10:
+- case CHIP_POLARIS11:
+- device_queue_manager_init_vi_tonga(&dqm->asic_ops);
++ device_queue_manager_init_cik(&dqm->ops_asic_specific);
+ break;
+-
+- case CHIP_VEGA10:
+- case CHIP_RAVEN:
+- device_queue_manager_init_v9_vega10(&dqm->asic_ops);
+- break;
+- default:
+- BUG();
+ }
+
+- if (!dqm->ops.initialize(dqm))
+- return dqm;
++ if (dqm->ops.initialize(dqm) != 0) {
++ kfree(dqm);
++ return NULL;
++ }
+
+-out_free:
+- kfree(dqm);
+- return NULL;
++ return dqm;
+ }
+
+ void device_queue_manager_uninit(struct device_queue_manager *dqm)
+ {
++ BUG_ON(!dqm);
++
+ dqm->ops.uninitialize(dqm);
+ kfree(dqm);
+ }
+-
+-int kfd_process_vm_fault(struct device_queue_manager *dqm,
+- unsigned int pasid)
+-{
+- struct kfd_process_device *pdd;
+- struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
+- int ret = 0;
+-
+- if (!p)
+- return -EINVAL;
+- pdd = kfd_get_process_device_data(dqm->dev, p);
+- if (pdd)
+- ret = process_evict_queues(dqm, &pdd->qpd);
+- kfd_unref_process(p);
+-
+- return ret;
+-}
+-
+-static void seq_reg_dump(struct seq_file *m,
+- uint32_t (*dump)[2], uint32_t n_regs)
+-{
+- uint32_t i, count;
+-
+- for (i = 0, count = 0; i < n_regs; i++) {
+- if (count == 0 ||
+- dump[i-1][0] + sizeof(uint32_t) != dump[i][0]) {
+- seq_printf(m, "%s %08x: %08x",
+- i ? "\n" : "",
+- dump[i][0], dump[i][1]);
+- count = 7;
+- } else {
+- seq_printf(m, " %08x", dump[i][1]);
+- count--;
+- }
+- }
+-
+- seq_puts(m, "\n");
+-}
+-
+-int device_queue_manager_debugfs_hqds(struct seq_file *m, void *data)
+-{
+- struct device_queue_manager *dqm = data;
+- uint32_t (*dump)[2], n_regs;
+- int pipe, queue;
+- int r = 0;
+-
+- for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
+- int pipe_offset = pipe * get_queues_per_pipe(dqm);
+-
+- for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) {
+- if (!test_bit(pipe_offset + queue,
+- dqm->dev->shared_resources.queue_bitmap))
+- continue;
+-
+- r = dqm->dev->kfd2kgd->hqd_dump(
+- dqm->dev->kgd, pipe, queue, &dump, &n_regs);
+- if (r)
+- break;
+-
+- seq_printf(m, " CP Pipe %d, Queue %d\n",
+- pipe, queue);
+- seq_reg_dump(m, dump, n_regs);
+-
+- kfree(dump);
+- }
+- }
+-
+- for (pipe = 0; pipe < CIK_SDMA_ENGINE_NUM; pipe++) {
+- for (queue = 0; queue < CIK_SDMA_QUEUES_PER_ENGINE; queue++) {
+- r = dqm->dev->kfd2kgd->hqd_sdma_dump(
+- dqm->dev->kgd, pipe, queue, &dump, &n_regs);
+- if (r)
+- break;
+-
+- seq_printf(m, " SDMA Engine %d, RLC %d\n",
+- pipe, queue);
+- seq_reg_dump(m, dump, n_regs);
+-
+- kfree(dump);
+- }
+- }
+-
+- return r;
+-}
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
+index 841283a..faf820a 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
+@@ -29,10 +29,10 @@
+ #include "kfd_priv.h"
+ #include "kfd_mqd_manager.h"
+
+-#define KFD_HIQ_TIMEOUT (500)
+-#define KFD_UNMAP_LATENCY_MS (4000)
+-#define QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS (2 * KFD_UNMAP_LATENCY_MS + 1000)
+-
++#define QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS (500)
++#define CIK_VMID_NUM (8)
++#define KFD_VMID_START_OFFSET (8)
++#define VMID_PER_DEVICE CIK_VMID_NUM
+ #define KFD_DQM_FIRST_PIPE (0)
+ #define CIK_SDMA_QUEUES (4)
+ #define CIK_SDMA_QUEUES_PER_ENGINE (2)
+@@ -79,14 +79,6 @@ struct device_process_node {
+ * @set_cache_memory_policy: Sets memory policy (cached/ non cached) for the
+ * memory apertures.
+ *
+- * @set_page_directory_base: Sets the PD base address (GPU local memory)
+- * in all the queues of the relevant process running on the specified device.
+- * It preempts the queues, updates the value and execute the runlist again.
+- *
+- * @process_termination: Clears all process queues belongs to that device.
+- *
+- * @get_wave_state: Retrieves context save state and optionally copies the
+- * control stack, if kept in the MQD, to the given userspace address.
+ */
+
+ struct device_queue_manager_ops {
+@@ -130,26 +122,12 @@ struct device_queue_manager_ops {
+ enum cache_policy alternate_policy,
+ void __user *alternate_aperture_base,
+ uint64_t alternate_aperture_size);
+-
+- int (*set_trap_handler)(struct device_queue_manager *dqm,
+- struct qcm_process_device *qpd,
+- uint64_t tba_addr,
+- uint64_t tma_addr);
+-
+- int (*process_termination)(struct device_queue_manager *dqm,
+- struct qcm_process_device *qpd);
+-
+- int (*get_wave_state)(struct device_queue_manager *dqm,
+- struct queue *q,
+- void __user *ctl_stack,
+- u32 *ctl_stack_used_size,
+- u32 *save_area_used_size);
+ };
+
+ struct device_queue_manager_asic_ops {
+- int (*update_qpd)(struct device_queue_manager *dqm,
++ int (*register_process)(struct device_queue_manager *dqm,
+ struct qcm_process_device *qpd);
+- int (*init_cpsch)(struct device_queue_manager *dqm);
++ int (*initialize)(struct device_queue_manager *dqm);
+ bool (*set_cache_memory_policy)(struct device_queue_manager *dqm,
+ struct qcm_process_device *qpd,
+ enum cache_policy default_policy,
+@@ -175,7 +153,7 @@ struct device_queue_manager_asic_ops {
+
+ struct device_queue_manager {
+ struct device_queue_manager_ops ops;
+- struct device_queue_manager_asic_ops asic_ops;
++ struct device_queue_manager_asic_ops ops_asic_specific;
+
+ struct mqd_manager *mqds[KFD_MQD_TYPE_MAX];
+ struct packet_manager packets;
+@@ -196,37 +174,21 @@ struct device_queue_manager {
+ unsigned int *fence_addr;
+ struct kfd_mem_obj *fence_mem;
+ bool active_runlist;
+- int sched_policy;
+ };
+
+-void device_queue_manager_init_cik(
+- struct device_queue_manager_asic_ops *asic_ops);
+-void device_queue_manager_init_cik_hawaii(
+- struct device_queue_manager_asic_ops *asic_ops);
+-void device_queue_manager_init_vi(
+- struct device_queue_manager_asic_ops *asic_ops);
+-void device_queue_manager_init_vi_tonga(
+- struct device_queue_manager_asic_ops *asic_ops);
+-void device_queue_manager_init_v9_vega10(
+- struct device_queue_manager_asic_ops *asic_ops);
++void device_queue_manager_init_cik(struct device_queue_manager_asic_ops *ops);
++void device_queue_manager_init_vi(struct device_queue_manager_asic_ops *ops);
+ void program_sh_mem_settings(struct device_queue_manager *dqm,
+ struct qcm_process_device *qpd);
+ unsigned int get_queues_num(struct device_queue_manager *dqm);
+ unsigned int get_queues_per_pipe(struct device_queue_manager *dqm);
+ unsigned int get_pipes_per_mec(struct device_queue_manager *dqm);
+
+-int process_evict_queues(struct device_queue_manager *dqm,
+- struct qcm_process_device *qpd);
+-int process_restore_queues(struct device_queue_manager *dqm,
+- struct qcm_process_device *qpd);
+-
+-
+ static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd)
+ {
+ return (pdd->lds_base >> 16) & 0xFF;
+ }
+
+-/* This function is only useful for GFXv7 and v8 */
+ static inline unsigned int
+ get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd)
+ {
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c
+index 8e1eb24..48dc056 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c
+@@ -24,7 +24,6 @@
+ #include "kfd_device_queue_manager.h"
+ #include "cik_regs.h"
+ #include "oss/oss_2_4_sh_mask.h"
+-#include "gca/gfx_7_2_sh_mask.h"
+
+ static bool set_cache_memory_policy_cik(struct device_queue_manager *dqm,
+ struct qcm_process_device *qpd,
+@@ -32,33 +31,18 @@ static bool set_cache_memory_policy_cik(struct device_queue_manager *dqm,
+ enum cache_policy alternate_policy,
+ void __user *alternate_aperture_base,
+ uint64_t alternate_aperture_size);
+-static int update_qpd_cik(struct device_queue_manager *dqm,
+- struct qcm_process_device *qpd);
+-static int update_qpd_cik_hawaii(struct device_queue_manager *dqm,
++static int register_process_cik(struct device_queue_manager *dqm,
+ struct qcm_process_device *qpd);
+ static int initialize_cpsch_cik(struct device_queue_manager *dqm);
+ static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q,
+ struct qcm_process_device *qpd);
+-static void init_sdma_vm_hawaii(struct device_queue_manager *dqm,
+- struct queue *q,
+- struct qcm_process_device *qpd);
+-
+-void device_queue_manager_init_cik(
+- struct device_queue_manager_asic_ops *asic_ops)
+-{
+- asic_ops->set_cache_memory_policy = set_cache_memory_policy_cik;
+- asic_ops->update_qpd = update_qpd_cik;
+- asic_ops->init_cpsch = initialize_cpsch_cik;
+- asic_ops->init_sdma_vm = init_sdma_vm;
+-}
+
+-void device_queue_manager_init_cik_hawaii(
+- struct device_queue_manager_asic_ops *asic_ops)
++void device_queue_manager_init_cik(struct device_queue_manager_asic_ops *ops)
+ {
+- asic_ops->set_cache_memory_policy = set_cache_memory_policy_cik;
+- asic_ops->update_qpd = update_qpd_cik_hawaii;
+- asic_ops->init_cpsch = initialize_cpsch_cik;
+- asic_ops->init_sdma_vm = init_sdma_vm_hawaii;
++ ops->set_cache_memory_policy = set_cache_memory_policy_cik;
++ ops->register_process = register_process_cik;
++ ops->initialize = initialize_cpsch_cik;
++ ops->init_sdma_vm = init_sdma_vm;
+ }
+
+ static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble)
+@@ -81,7 +65,7 @@ static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble)
+ * for LDS/Scratch and GPUVM.
+ */
+
+- WARN_ON((top_address_nybble & 1) || top_address_nybble > 0xE ||
++ BUG_ON((top_address_nybble & 1) || top_address_nybble > 0xE ||
+ top_address_nybble == 0);
+
+ return PRIVATE_BASE(top_address_nybble << 12) |
+@@ -114,12 +98,14 @@ static bool set_cache_memory_policy_cik(struct device_queue_manager *dqm,
+ return true;
+ }
+
+-static int update_qpd_cik(struct device_queue_manager *dqm,
++static int register_process_cik(struct device_queue_manager *dqm,
+ struct qcm_process_device *qpd)
+ {
+ struct kfd_process_device *pdd;
+ unsigned int temp;
+
++ BUG_ON(!dqm || !qpd);
++
+ pdd = qpd_to_pdd(qpd);
+
+ /* check if sh_mem_config register already configured */
+@@ -139,40 +125,9 @@ static int update_qpd_cik(struct device_queue_manager *dqm,
+ } else {
+ temp = get_sh_mem_bases_nybble_64(pdd);
+ qpd->sh_mem_bases = compute_sh_mem_bases_64bit(temp);
+- qpd->sh_mem_config |= 1 << SH_MEM_CONFIG__PRIVATE_ATC__SHIFT;
+ }
+
+- pr_debug("is32bit process: %d sh_mem_bases nybble: 0x%X and register 0x%X\n",
+- qpd->pqm->process->is_32bit_user_mode, temp, qpd->sh_mem_bases);
+-
+- return 0;
+-}
+-
+-static int update_qpd_cik_hawaii(struct device_queue_manager *dqm,
+- struct qcm_process_device *qpd)
+-{
+- struct kfd_process_device *pdd;
+- unsigned int temp;
+-
+- pdd = qpd_to_pdd(qpd);
+-
+- /* check if sh_mem_config register already configured */
+- if (qpd->sh_mem_config == 0) {
+- qpd->sh_mem_config =
+- ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED) |
+- DEFAULT_MTYPE(MTYPE_NONCACHED) |
+- APE1_MTYPE(MTYPE_NONCACHED);
+- qpd->sh_mem_ape1_limit = 0;
+- qpd->sh_mem_ape1_base = 0;
+- }
+-
+- /* On dGPU we're always in GPUVM64 addressing mode with 64-bit
+- * aperture addresses.
+- */
+- temp = get_sh_mem_bases_nybble_64(pdd);
+- qpd->sh_mem_bases = compute_sh_mem_bases_64bit(temp);
+-
+- pr_debug("is32bit process: %d sh_mem_bases nybble: 0x%X and register 0x%X\n",
++ pr_debug("kfd: is32bit process: %d sh_mem_bases nybble: 0x%X and register 0x%X\n",
+ qpd->pqm->process->is_32bit_user_mode, temp, qpd->sh_mem_bases);
+
+ return 0;
+@@ -194,19 +149,6 @@ static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q,
+ q->properties.sdma_vm_addr = value;
+ }
+
+-static void init_sdma_vm_hawaii(struct device_queue_manager *dqm,
+- struct queue *q,
+- struct qcm_process_device *qpd)
+-{
+- /* On dGPU we're always in GPUVM64 addressing mode with 64-bit
+- * aperture addresses.
+- */
+- q->properties.sdma_vm_addr =
+- ((get_sh_mem_bases_nybble_64(qpd_to_pdd(qpd))) <<
+- SDMA0_RLC0_VIRTUAL_ADDR__SHARED_BASE__SHIFT) &
+- SDMA0_RLC0_VIRTUAL_ADDR__SHARED_BASE_MASK;
+-}
+-
+ static int initialize_cpsch_cik(struct device_queue_manager *dqm)
+ {
+ return 0;
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c
+deleted file mode 100644
+index dde5882..0000000
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c
++++ /dev/null
+@@ -1,90 +0,0 @@
+-/*
+- * Copyright 2016 Advanced Micro Devices, Inc.
+- *
+- * Permission is hereby granted, free of charge, to any person obtaining a
+- * copy of this software and associated documentation files (the "Software"),
+- * to deal in the Software without restriction, including without limitation
+- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+- * and/or sell copies of the Software, and to permit persons to whom the
+- * Software is furnished to do so, subject to the following conditions:
+- *
+- * The above copyright notice and this permission notice shall be included in
+- * all copies or substantial portions of the Software.
+- *
+- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+- * OTHER DEALINGS IN THE SOFTWARE.
+- *
+- */
+-
+-#include "kfd_device_queue_manager.h"
+-#include "vega10/vega10_enum.h"
+-#include "vega10/GC/gc_9_0_offset.h"
+-#include "vega10/GC/gc_9_0_sh_mask.h"
+-#include "vega10/SDMA0/sdma0_4_0_sh_mask.h"
+-
+-static int update_qpd_v9(struct device_queue_manager *dqm,
+- struct qcm_process_device *qpd);
+-static int initialize_cpsch_v9(struct device_queue_manager *dqm);
+-static void init_sdma_vm_v9(struct device_queue_manager *dqm, struct queue *q,
+- struct qcm_process_device *qpd);
+-
+-void device_queue_manager_init_v9_vega10(
+- struct device_queue_manager_asic_ops *asic_ops)
+-{
+- asic_ops->update_qpd = update_qpd_v9;
+- asic_ops->init_cpsch = initialize_cpsch_v9;
+- asic_ops->init_sdma_vm = init_sdma_vm_v9;
+-}
+-
+-static uint32_t compute_sh_mem_bases_64bit(struct kfd_process_device *pdd)
+-{
+- uint32_t shared_base = pdd->lds_base >> 48;
+- uint32_t private_base = pdd->scratch_base >> 48;
+-
+- return (shared_base << SH_MEM_BASES__SHARED_BASE__SHIFT) |
+- private_base;
+-}
+-
+-static int update_qpd_v9(struct device_queue_manager *dqm,
+- struct qcm_process_device *qpd)
+-{
+- struct kfd_process_device *pdd;
+-
+- pdd = qpd_to_pdd(qpd);
+-
+- /* check if sh_mem_config register already configured */
+- if (qpd->sh_mem_config == 0) {
+- qpd->sh_mem_config =
+- SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
+- SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
+- if (vega10_noretry)
+- qpd->sh_mem_config |=
+- 1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT;
+-
+- qpd->sh_mem_ape1_limit = 0;
+- qpd->sh_mem_ape1_base = 0;
+- }
+-
+- qpd->sh_mem_bases = compute_sh_mem_bases_64bit(pdd);
+-
+- pr_debug("sh_mem_bases 0x%X\n", qpd->sh_mem_bases);
+-
+- return 0;
+-}
+-
+-static int initialize_cpsch_v9(struct device_queue_manager *dqm)
+-{
+- return 0;
+-}
+-
+-static void init_sdma_vm_v9(struct device_queue_manager *dqm, struct queue *q,
+- struct qcm_process_device *qpd)
+-{
+- /* Not needed on SDMAv4 any more */
+- q->properties.sdma_vm_addr = 0;
+-}
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c
+index ac8d852..7e9cae9 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c
+@@ -33,44 +33,18 @@ static bool set_cache_memory_policy_vi(struct device_queue_manager *dqm,
+ enum cache_policy alternate_policy,
+ void __user *alternate_aperture_base,
+ uint64_t alternate_aperture_size);
+-static int update_qpd_vi(struct device_queue_manager *dqm,
++static int register_process_vi(struct device_queue_manager *dqm,
+ struct qcm_process_device *qpd);
+ static int initialize_cpsch_vi(struct device_queue_manager *dqm);
+ static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q,
+ struct qcm_process_device *qpd);
+
+-/*
+- * Tonga device queue manager functions
+- */
+-static bool set_cache_memory_policy_vi_tonga(struct device_queue_manager *dqm,
+- struct qcm_process_device *qpd,
+- enum cache_policy default_policy,
+- enum cache_policy alternate_policy,
+- void __user *alternate_aperture_base,
+- uint64_t alternate_aperture_size);
+-static int update_qpd_vi_tonga(struct device_queue_manager *dqm,
+- struct qcm_process_device *qpd);
+-static void init_sdma_vm_tonga(struct device_queue_manager *dqm,
+- struct queue *q,
+- struct qcm_process_device *qpd);
+-
+-void device_queue_manager_init_vi_tonga(
+- struct device_queue_manager_asic_ops *asic_ops)
+-{
+- asic_ops->set_cache_memory_policy = set_cache_memory_policy_vi_tonga;
+- asic_ops->update_qpd = update_qpd_vi_tonga;
+- asic_ops->init_cpsch = initialize_cpsch_vi;
+- asic_ops->init_sdma_vm = init_sdma_vm_tonga;
+-}
+-
+-
+-void device_queue_manager_init_vi(
+- struct device_queue_manager_asic_ops *asic_ops)
++void device_queue_manager_init_vi(struct device_queue_manager_asic_ops *ops)
+ {
+- asic_ops->set_cache_memory_policy = set_cache_memory_policy_vi;
+- asic_ops->update_qpd = update_qpd_vi;
+- asic_ops->init_cpsch = initialize_cpsch_vi;
+- asic_ops->init_sdma_vm = init_sdma_vm;
++ ops->set_cache_memory_policy = set_cache_memory_policy_vi;
++ ops->register_process = register_process_vi;
++ ops->initialize = initialize_cpsch_vi;
++ ops->init_sdma_vm = init_sdma_vm;
+ }
+
+ static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble)
+@@ -93,7 +67,7 @@ static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble)
+ * for LDS/Scratch and GPUVM.
+ */
+
+- WARN_ON((top_address_nybble & 1) || top_address_nybble > 0xE ||
++ BUG_ON((top_address_nybble & 1) || top_address_nybble > 0xE ||
+ top_address_nybble == 0);
+
+ return top_address_nybble << 12 |
+@@ -130,39 +104,14 @@ static bool set_cache_memory_policy_vi(struct device_queue_manager *dqm,
+ return true;
+ }
+
+-static bool set_cache_memory_policy_vi_tonga(struct device_queue_manager *dqm,
+- struct qcm_process_device *qpd,
+- enum cache_policy default_policy,
+- enum cache_policy alternate_policy,
+- void __user *alternate_aperture_base,
+- uint64_t alternate_aperture_size)
+-{
+- uint32_t default_mtype;
+- uint32_t ape1_mtype;
+-
+- default_mtype = (default_policy == cache_policy_coherent) ?
+- MTYPE_UC :
+- MTYPE_NC;
+-
+- ape1_mtype = (alternate_policy == cache_policy_coherent) ?
+- MTYPE_UC :
+- MTYPE_NC;
+-
+- qpd->sh_mem_config =
+- SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
+- SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT |
+- default_mtype << SH_MEM_CONFIG__DEFAULT_MTYPE__SHIFT |
+- ape1_mtype << SH_MEM_CONFIG__APE1_MTYPE__SHIFT;
+-
+- return true;
+-}
+-
+-static int update_qpd_vi(struct device_queue_manager *dqm,
++static int register_process_vi(struct device_queue_manager *dqm,
+ struct qcm_process_device *qpd)
+ {
+ struct kfd_process_device *pdd;
+ unsigned int temp;
+
++ BUG_ON(!dqm || !qpd);
++
+ pdd = qpd_to_pdd(qpd);
+
+ /* check if sh_mem_config register already configured */
+@@ -188,50 +137,14 @@ static int update_qpd_vi(struct device_queue_manager *dqm,
+ qpd->sh_mem_bases = compute_sh_mem_bases_64bit(temp);
+ qpd->sh_mem_config |= SH_MEM_ADDRESS_MODE_HSA64 <<
+ SH_MEM_CONFIG__ADDRESS_MODE__SHIFT;
+- qpd->sh_mem_config |= 1 <<
+- SH_MEM_CONFIG__PRIVATE_ATC__SHIFT;
+ }
+
+- pr_debug("is32bit process: %d sh_mem_bases nybble: 0x%X and register 0x%X\n",
++ pr_debug("kfd: is32bit process: %d sh_mem_bases nybble: 0x%X and register 0x%X\n",
+ qpd->pqm->process->is_32bit_user_mode, temp, qpd->sh_mem_bases);
+
+ return 0;
+ }
+
+-static int update_qpd_vi_tonga(struct device_queue_manager *dqm,
+- struct qcm_process_device *qpd)
+-{
+- struct kfd_process_device *pdd;
+- unsigned int temp;
+-
+- pdd = qpd_to_pdd(qpd);
+-
+- /* check if sh_mem_config register already configured */
+- if (qpd->sh_mem_config == 0) {
+- qpd->sh_mem_config =
+- SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
+- SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT |
+- MTYPE_UC <<
+- SH_MEM_CONFIG__DEFAULT_MTYPE__SHIFT |
+- MTYPE_UC <<
+- SH_MEM_CONFIG__APE1_MTYPE__SHIFT;
+-
+- qpd->sh_mem_ape1_limit = 0;
+- qpd->sh_mem_ape1_base = 0;
+- }
+-
+- /* On dGPU we're always in GPUVM64 addressing mode with 64-bit
+- * aperture addresses.
+- */
+- temp = get_sh_mem_bases_nybble_64(pdd);
+- qpd->sh_mem_bases = compute_sh_mem_bases_64bit(temp);
+-
+- pr_debug("sh_mem_bases nybble: 0x%X and register 0x%X\n",
+- temp, qpd->sh_mem_bases);
+-
+- return 0;
+-}
+-
+ static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q,
+ struct qcm_process_device *qpd)
+ {
+@@ -248,20 +161,6 @@ static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q,
+ q->properties.sdma_vm_addr = value;
+ }
+
+-static void init_sdma_vm_tonga(struct device_queue_manager *dqm,
+- struct queue *q,
+- struct qcm_process_device *qpd)
+-{
+- /* On dGPU we're always in GPUVM64 addressing mode with 64-bit
+- * aperture addresses.
+- */
+- q->properties.sdma_vm_addr =
+- ((get_sh_mem_bases_nybble_64(qpd_to_pdd(qpd))) <<
+- SDMA0_RLC0_VIRTUAL_ADDR__SHARED_BASE__SHIFT) &
+- SDMA0_RLC0_VIRTUAL_ADDR__SHARED_BASE_MASK;
+-}
+-
+-
+ static int initialize_cpsch_vi(struct device_queue_manager *dqm)
+ {
+ return 0;
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
+index 008d258..453c5d6 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
+@@ -34,6 +34,7 @@
+ */
+
+ #define KERNEL_DOORBELL_PASID 1
++#define KFD_SIZE_OF_DOORBELL_IN_BYTES 4
+
+ /*
+ * Each device exposes a doorbell aperture, a PCI MMIO aperture that
+@@ -50,15 +51,15 @@
+ */
+
+ /* # of doorbell bytes allocated for each process. */
+-size_t kfd_doorbell_process_slice(struct kfd_dev *kfd)
++static inline size_t doorbell_process_allocation(void)
+ {
+- return roundup(kfd->device_info->doorbell_size *
++ return roundup(KFD_SIZE_OF_DOORBELL_IN_BYTES *
+ KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,
+ PAGE_SIZE);
+ }
+
+ /* Doorbell calculations for device init. */
+-int kfd_doorbell_init(struct kfd_dev *kfd)
++void kfd_doorbell_init(struct kfd_dev *kfd)
+ {
+ size_t doorbell_start_offset;
+ size_t doorbell_aperture_size;
+@@ -72,16 +73,16 @@ int kfd_doorbell_init(struct kfd_dev *kfd)
+
+ doorbell_start_offset =
+ roundup(kfd->shared_resources.doorbell_start_offset,
+- kfd_doorbell_process_slice(kfd));
++ doorbell_process_allocation());
+
+ doorbell_aperture_size =
+ rounddown(kfd->shared_resources.doorbell_aperture_size,
+- kfd_doorbell_process_slice(kfd));
++ doorbell_process_allocation());
+
+ if (doorbell_aperture_size > doorbell_start_offset)
+ doorbell_process_limit =
+ (doorbell_aperture_size - doorbell_start_offset) /
+- kfd_doorbell_process_slice(kfd);
++ doorbell_process_allocation();
+ else
+ doorbell_process_limit = 0;
+
+@@ -92,49 +93,45 @@ int kfd_doorbell_init(struct kfd_dev *kfd)
+ kfd->doorbell_process_limit = doorbell_process_limit - 1;
+
+ kfd->doorbell_kernel_ptr = ioremap(kfd->doorbell_base,
+- kfd_doorbell_process_slice(kfd));
++ doorbell_process_allocation());
+
+- if (!kfd->doorbell_kernel_ptr)
+- return -ENOMEM;
++ BUG_ON(!kfd->doorbell_kernel_ptr);
+
+- pr_debug("Doorbell initialization:\n");
+- pr_debug("doorbell base == 0x%08lX\n",
++ pr_debug("kfd: doorbell initialization:\n");
++ pr_debug("kfd: doorbell base == 0x%08lX\n",
+ (uintptr_t)kfd->doorbell_base);
+
+- pr_debug("doorbell_id_offset == 0x%08lX\n",
++ pr_debug("kfd: doorbell_id_offset == 0x%08lX\n",
+ kfd->doorbell_id_offset);
+
+- pr_debug("doorbell_process_limit == 0x%08lX\n",
++ pr_debug("kfd: doorbell_process_limit == 0x%08lX\n",
+ doorbell_process_limit);
+
+- pr_debug("doorbell_kernel_offset == 0x%08lX\n",
++ pr_debug("kfd: doorbell_kernel_offset == 0x%08lX\n",
+ (uintptr_t)kfd->doorbell_base);
+
+- pr_debug("doorbell aperture size == 0x%08lX\n",
++ pr_debug("kfd: doorbell aperture size == 0x%08lX\n",
+ kfd->shared_resources.doorbell_aperture_size);
+
+- pr_debug("doorbell kernel address == 0x%08lX\n",
++ pr_debug("kfd: doorbell kernel address == 0x%08lX\n",
+ (uintptr_t)kfd->doorbell_kernel_ptr);
+-
+- return 0;
+-}
+-
+-void kfd_doorbell_fini(struct kfd_dev *kfd)
+-{
+- if (kfd->doorbell_kernel_ptr)
+- iounmap(kfd->doorbell_kernel_ptr);
+ }
+
+-int kfd_doorbell_mmap(struct kfd_dev *dev, struct kfd_process *process,
+- struct vm_area_struct *vma)
++int kfd_doorbell_mmap(struct kfd_process *process, struct vm_area_struct *vma)
+ {
+ phys_addr_t address;
++ struct kfd_dev *dev;
+
+ /*
+ * For simplicitly we only allow mapping of the entire doorbell
+ * allocation of a single device & process.
+ */
+- if (vma->vm_end - vma->vm_start != kfd_doorbell_process_slice(dev))
++ if (vma->vm_end - vma->vm_start != doorbell_process_allocation())
++ return -EINVAL;
++
++ /* Find kfd device according to gpu id */
++ dev = kfd_device_by_id(vma->vm_pgoff);
++ if (dev == NULL)
+ return -EINVAL;
+
+ /* Calculate physical address of doorbell */
+@@ -145,29 +142,32 @@ int kfd_doorbell_mmap(struct kfd_dev *dev, struct kfd_process *process,
+
+ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+
+- pr_debug("Mapping doorbell page\n"
++ pr_debug("kfd: mapping doorbell page in %s\n"
+ " target user address == 0x%08llX\n"
+ " physical address == 0x%08llX\n"
+ " vm_flags == 0x%04lX\n"
+ " size == 0x%04lX\n",
++ __func__,
+ (unsigned long long) vma->vm_start, address, vma->vm_flags,
+- kfd_doorbell_process_slice(dev));
++ doorbell_process_allocation());
+
+
+ return io_remap_pfn_range(vma,
+ vma->vm_start,
+ address >> PAGE_SHIFT,
+- kfd_doorbell_process_slice(dev),
++ doorbell_process_allocation(),
+ vma->vm_page_prot);
+ }
+
+
+ /* get kernel iomem pointer for a doorbell */
+-void __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
++u32 __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
+ unsigned int *doorbell_off)
+ {
+ u32 inx;
+
++ BUG_ON(!kfd || !doorbell_off);
++
+ mutex_lock(&kfd->doorbell_mutex);
+ inx = find_first_zero_bit(kfd->doorbell_available_index,
+ KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
+@@ -178,17 +178,14 @@ void __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
+ if (inx >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)
+ return NULL;
+
+- inx *= kfd->device_info->doorbell_size / sizeof(u32);
+-
+ /*
+ * Calculating the kernel doorbell offset using "faked" kernel
+- * pasid that allocated for kernel queues only. Offset is in
+- * dword units regardless of the ASIC-dependent doorbell size.
++ * pasid that allocated for kernel queues only
+ */
+- *doorbell_off = KERNEL_DOORBELL_PASID *
+- (kfd_doorbell_process_slice(kfd) / sizeof(u32)) + inx;
++ *doorbell_off = KERNEL_DOORBELL_PASID * (doorbell_process_allocation() /
++ sizeof(u32)) + inx;
+
+- pr_debug("Get kernel queue doorbell\n"
++ pr_debug("kfd: get kernel queue doorbell\n"
+ " doorbell offset == 0x%08X\n"
+ " kernel address == 0x%08lX\n",
+ *doorbell_off, (uintptr_t)(kfd->doorbell_kernel_ptr + inx));
+@@ -200,6 +197,8 @@ void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr)
+ {
+ unsigned int inx;
+
++ BUG_ON(!kfd || !db_addr);
++
+ inx = (unsigned int)(db_addr - kfd->doorbell_kernel_ptr);
+
+ mutex_lock(&kfd->doorbell_mutex);
+@@ -207,21 +206,11 @@ void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr)
+ mutex_unlock(&kfd->doorbell_mutex);
+ }
+
+-void write_kernel_doorbell(void __iomem *db, u32 value)
++inline void write_kernel_doorbell(u32 __iomem *db, u32 value)
+ {
+ if (db) {
+ writel(value, db);
+- pr_debug("Writing %d to doorbell address 0x%p\n", value, db);
+- }
+-}
+-
+-void write_kernel_doorbell64(void __iomem *db, u64 value)
+-{
+- if (db) {
+- WARN(((unsigned long)db & 7) != 0,
+- "Unaligned 64-bit doorbell");
+- writeq(value, (u64 __iomem *)db);
+- pr_debug("writing %llu to doorbell address 0x%p\n", value, db);
++ pr_debug("writing %d to doorbell address 0x%p\n", value, db);
+ }
+ }
+
+@@ -229,26 +218,25 @@ void write_kernel_doorbell64(void __iomem *db, u64 value)
+ * queue_ids are in the range [0,MAX_PROCESS_QUEUES) and are mapped 1:1
+ * to doorbells with the process's doorbell page
+ */
+-unsigned int kfd_doorbell_id_to_offset(struct kfd_dev *kfd,
++unsigned int kfd_queue_id_to_doorbell(struct kfd_dev *kfd,
+ struct kfd_process *process,
+- unsigned int doorbell_id)
++ unsigned int queue_id)
+ {
+ /*
+ * doorbell_id_offset accounts for doorbells taken by KGD.
+- * pasid * kfd_doorbell_process_slice/sizeof(u32) adjusts to
+- * the process's doorbells. The offset returned is in dword
+- * units regardless of the ASIC-dependent doorbell size.
++ * pasid * doorbell_process_allocation/sizeof(u32) adjusts
++ * to the process's doorbells
+ */
+ return kfd->doorbell_id_offset +
+- process->pasid * (kfd_doorbell_process_slice(kfd)/sizeof(u32)) +
+- doorbell_id * kfd->device_info->doorbell_size / sizeof(u32);
++ process->pasid * (doorbell_process_allocation()/sizeof(u32)) +
++ queue_id;
+ }
+
+ uint64_t kfd_get_number_elems(struct kfd_dev *kfd)
+ {
+ uint64_t num_of_elems = (kfd->shared_resources.doorbell_aperture_size -
+ kfd->shared_resources.doorbell_start_offset) /
+- kfd_doorbell_process_slice(kfd) + 1;
++ doorbell_process_allocation() + 1;
+
+ return num_of_elems;
+
+@@ -258,5 +246,5 @@ phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev,
+ struct kfd_process *process)
+ {
+ return dev->doorbell_base +
+- process->pasid * kfd_doorbell_process_slice(dev);
++ process->pasid * doorbell_process_allocation();
+ }
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
+index 7eacf42..d1ce83d 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
+@@ -23,9 +23,9 @@
+ #include <linux/mm_types.h>
+ #include <linux/slab.h>
+ #include <linux/types.h>
+-#include <linux/uaccess.h>
+-#include <linux/sched/mm.h>
+ #include <linux/sched/signal.h>
++#include <linux/uaccess.h>
++#include <linux/mm.h>
+ #include <linux/mman.h>
+ #include <linux/memory.h>
+ #include "kfd_priv.h"
+@@ -52,9 +52,6 @@ struct kfd_event_waiter {
+ uint32_t input_index;
+ };
+
+-#define SLOTS_PER_PAGE KFD_SIGNAL_EVENT_LIMIT
+-#define SLOT_BITMAP_LONGS BITS_TO_LONGS(SLOTS_PER_PAGE)
+-
+ /*
+ * Over-complicated pooled allocator for event notification slots.
+ *
+@@ -68,19 +65,24 @@ struct kfd_event_waiter {
+ struct signal_page {
+ struct list_head event_pages; /* kfd_process.signal_event_pages */
+ uint64_t *kernel_address;
+- uint64_t handle;
+ uint64_t __user *user_address;
+ uint32_t page_index; /* Index into the mmap aperture. */
+ unsigned int free_slots;
+- unsigned long used_slot_bitmap[SLOT_BITMAP_LONGS];
++ unsigned long used_slot_bitmap[0];
+ };
+
++#define SLOTS_PER_PAGE KFD_SIGNAL_EVENT_LIMIT
++#define SLOT_BITMAP_SIZE BITS_TO_LONGS(SLOTS_PER_PAGE)
++#define BITS_PER_PAGE (ilog2(SLOTS_PER_PAGE)+1)
++#define SIGNAL_PAGE_SIZE (sizeof(struct signal_page) + \
++ SLOT_BITMAP_SIZE * sizeof(long))
++
+ /*
+ * For signal events, the event ID is used as the interrupt user data.
+ * For SQ s_sendmsg interrupts, this is limited to 8 bits.
+ */
+
+-#define INTERRUPT_DATA_BITS 12
++#define INTERRUPT_DATA_BITS 8
+ #define SIGNAL_EVENT_ID_SLOT_SHIFT 0
+
+ static uint64_t *page_slots(struct signal_page *page)
+@@ -108,7 +110,7 @@ static bool allocate_free_slot(struct kfd_process *process,
+ *out_page = page;
+ *out_slot_index = slot;
+
+- pr_debug("Allocated event signal slot in page %p, slot %d\n",
++ pr_debug("allocated event signal slot in page %p, slot %d\n",
+ page, slot);
+
+ return true;
+@@ -129,7 +131,7 @@ static bool allocate_signal_page(struct file *devkfd, struct kfd_process *p)
+ void *backing_store;
+ struct signal_page *page;
+
+- page = kzalloc(sizeof(*page), GFP_KERNEL);
++ page = kzalloc(SIGNAL_PAGE_SIZE, GFP_KERNEL);
+ if (!page)
+ goto fail_alloc_signal_page;
+
+@@ -153,9 +155,9 @@ static bool allocate_signal_page(struct file *devkfd, struct kfd_process *p)
+ struct signal_page,
+ event_pages)->page_index + 1;
+
+- pr_debug("Allocated new event signal page at %p, for process %p\n",
++ pr_debug("allocated new event signal page at %p, for process %p\n",
+ page, p);
+- pr_debug("Page index is %d\n", page->page_index);
++ pr_debug("page index is %d\n", page->page_index);
+
+ list_add(&page->event_pages, &p->signal_event_pages);
+
+@@ -184,53 +186,6 @@ static bool allocate_event_notification_slot(struct file *devkfd,
+ return ret;
+ }
+
+-static bool allocate_signal_page_dgpu(struct kfd_process *p,
+- uint64_t *kernel_address, uint64_t handle)
+-{
+- struct signal_page *my_page;
+-
+- my_page = kzalloc(sizeof(*my_page), GFP_KERNEL);
+- if (!my_page)
+- return false;
+-
+- /* prevent user-mode info leaks */
+- memset(kernel_address, (uint8_t) UNSIGNALED_EVENT_SLOT,
+- KFD_SIGNAL_EVENT_LIMIT * 8);
+-
+- my_page->kernel_address = kernel_address;
+- my_page->handle = handle;
+- my_page->user_address = NULL;
+- my_page->free_slots = SLOTS_PER_PAGE;
+- if (list_empty(&p->signal_event_pages))
+- my_page->page_index = 0;
+- else
+- my_page->page_index = list_tail_entry(&p->signal_event_pages,
+- struct signal_page,
+- event_pages)->page_index + 1;
+-
+- pr_debug("Allocated new event signal page at %p, for process %p\n",
+- my_page, p);
+- pr_debug("Page index is %d\n", my_page->page_index);
+-
+- list_add(&my_page->event_pages, &p->signal_event_pages);
+-
+- return true;
+-}
+-
+-void kfd_free_signal_page_dgpu(struct kfd_process *p, uint64_t handle)
+-{
+- struct signal_page *page, *tmp;
+-
+- list_for_each_entry_safe(page, tmp, &p->signal_event_pages,
+- event_pages) {
+- if (page->handle == handle) {
+- list_del(&page->event_pages);
+- kfree(page);
+- break;
+- }
+- }
+-}
+-
+ /* Assumes that the process's event_mutex is locked. */
+ static void release_event_notification_slot(struct signal_page *page,
+ size_t slot_index)
+@@ -239,8 +194,7 @@ static void release_event_notification_slot(struct signal_page *page,
+ page->free_slots++;
+
+ /* We don't free signal pages, they are retained by the process
+- * and reused until it exits.
+- */
++ * and reused until it exits. */
+ }
+
+ static struct signal_page *lookup_signal_page_by_index(struct kfd_process *p,
+@@ -292,7 +246,7 @@ static u32 make_nonsignal_event_id(struct kfd_process *p)
+
+ for (id = p->next_nonsignal_event_id;
+ id < KFD_LAST_NONSIGNAL_EVENT_ID &&
+- lookup_event_by_id(p, id);
++ lookup_event_by_id(p, id) != NULL;
+ id++)
+ ;
+
+@@ -311,7 +265,7 @@ static u32 make_nonsignal_event_id(struct kfd_process *p)
+
+ for (id = KFD_FIRST_NONSIGNAL_EVENT_ID;
+ id < KFD_LAST_NONSIGNAL_EVENT_ID &&
+- lookup_event_by_id(p, id);
++ lookup_event_by_id(p, id) != NULL;
+ id++)
+ ;
+
+@@ -337,16 +291,13 @@ static int create_signal_event(struct file *devkfd,
+ struct kfd_event *ev)
+ {
+ if (p->signal_event_count == KFD_SIGNAL_EVENT_LIMIT) {
+- if (!p->signal_event_limit_reached) {
+- pr_warn("Signal event wasn't created because limit was reached\n");
+- p->signal_event_limit_reached = true;
+- }
++ pr_warn("amdkfd: Signal event wasn't created because limit was reached\n");
+ return -ENOMEM;
+ }
+
+ if (!allocate_event_notification_slot(devkfd, p, &ev->signal_page,
+- &ev->signal_slot_index)) {
+- pr_warn("Signal event wasn't created because out of kernel memory\n");
++ &ev->signal_slot_index)) {
++ pr_warn("amdkfd: Signal event wasn't created because out of kernel memory\n");
+ return -ENOMEM;
+ }
+
+@@ -358,7 +309,11 @@ static int create_signal_event(struct file *devkfd,
+ ev->event_id = make_signal_event_id(ev->signal_page,
+ ev->signal_slot_index);
+
+- pr_debug("Signal event number %zu created with id %d, address %p\n",
++ pr_debug("signal event number %zu created with id %d, address %p\n",
++ p->signal_event_count, ev->event_id,
++ ev->user_signal_address);
++
++ pr_debug("signal event number %zu created with id %d, address %p\n",
+ p->signal_event_count, ev->event_id,
+ ev->user_signal_address);
+
+@@ -390,7 +345,7 @@ void kfd_event_init_process(struct kfd_process *p)
+
+ static void destroy_event(struct kfd_process *p, struct kfd_event *ev)
+ {
+- if (ev->signal_page) {
++ if (ev->signal_page != NULL) {
+ release_event_notification_slot(ev->signal_page,
+ ev->signal_slot_index);
+ p->signal_event_count--;
+@@ -426,9 +381,8 @@ static void shutdown_signal_pages(struct kfd_process *p)
+
+ list_for_each_entry_safe(page, tmp, &p->signal_event_pages,
+ event_pages) {
+- if (page->user_address)
+- free_pages((unsigned long)page->kernel_address,
+- get_order(KFD_SIGNAL_EVENT_LIMIT * 8));
++ free_pages((unsigned long)page->kernel_address,
++ get_order(KFD_SIGNAL_EVENT_LIMIT * 8));
+ kfree(page);
+ }
+ }
+@@ -453,8 +407,7 @@ static bool event_can_be_cpu_signaled(const struct kfd_event *ev)
+ int kfd_event_create(struct file *devkfd, struct kfd_process *p,
+ uint32_t event_type, bool auto_reset, uint32_t node_id,
+ uint32_t *event_id, uint32_t *event_trigger_data,
+- uint64_t *event_page_offset, uint32_t *event_slot_index,
+- void *kern_addr)
++ uint64_t *event_page_offset, uint32_t *event_slot_index)
+ {
+ int ret = 0;
+ struct kfd_event *ev = kzalloc(sizeof(*ev), GFP_KERNEL);
+@@ -468,20 +421,17 @@ int kfd_event_create(struct file *devkfd, struct kfd_process *p,
+
+ INIT_LIST_HEAD(&ev->waiters);
+
+- mutex_lock(&p->event_mutex);
+-
+- if (kern_addr && list_empty(&p->signal_event_pages))
+- allocate_signal_page_dgpu(p, kern_addr, *event_page_offset);
+-
+ *event_page_offset = 0;
+
++ mutex_lock(&p->event_mutex);
++
+ switch (event_type) {
+ case KFD_EVENT_TYPE_SIGNAL:
+ case KFD_EVENT_TYPE_DEBUG:
+ ret = create_signal_event(devkfd, p, ev);
+ if (!ret) {
+ *event_page_offset = (ev->signal_page->page_index |
+- KFD_MMAP_TYPE_EVENTS);
++ KFD_MMAP_EVENTS_MASK);
+ *event_page_offset <<= PAGE_SHIFT;
+ *event_slot_index = ev->signal_slot_index;
+ }
+@@ -614,7 +564,7 @@ void kfd_signal_event_interrupt(unsigned int pasid, uint32_t partial_id,
+ /*
+ * Because we are called from arbitrary context (workqueue) as opposed
+ * to process context, kfd_process could attempt to exit while we are
+- * running so the lookup function increments the process ref count.
++ * running so the lookup function returns a locked process.
+ */
+ struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
+
+@@ -634,7 +584,7 @@ void kfd_signal_event_interrupt(unsigned int pasid, uint32_t partial_id,
+ * search faster.
+ */
+ struct signal_page *page;
+- unsigned int i;
++ unsigned i;
+
+ list_for_each_entry(page, &p->signal_event_pages, event_pages)
+ for (i = 0; i < SLOTS_PER_PAGE; i++)
+@@ -646,7 +596,7 @@ void kfd_signal_event_interrupt(unsigned int pasid, uint32_t partial_id,
+ }
+
+ mutex_unlock(&p->event_mutex);
+- kfd_unref_process(p);
++ mutex_unlock(&p->mutex);
+ }
+
+ static struct kfd_event_waiter *alloc_event_waiters(uint32_t num_events)
+@@ -667,7 +617,7 @@ static struct kfd_event_waiter *alloc_event_waiters(uint32_t num_events)
+ return event_waiters;
+ }
+
+-static int init_event_waiter_get_status(struct kfd_process *p,
++static int init_event_waiter(struct kfd_process *p,
+ struct kfd_event_waiter *waiter,
+ uint32_t event_id,
+ uint32_t input_index)
+@@ -682,18 +632,9 @@ static int init_event_waiter_get_status(struct kfd_process *p,
+ waiter->activated = ev->signaled;
+ ev->signaled = ev->signaled && !ev->auto_reset;
+
+- return 0;
+-}
++ list_add(&waiter->waiters, &ev->waiters);
+
+-static void init_event_waiter_add_to_waitlist(struct kfd_event_waiter *waiter)
+-{
+- struct kfd_event *ev = waiter->event;
+-
+- /* Only add to the wait list if we actually need to
+- * wait on this event.
+- */
+- if (!waiter->activated)
+- list_add(&waiter->waiters, &ev->waiters);
++ return 0;
+ }
+
+ static bool test_event_condition(bool all, uint32_t num_events,
+@@ -788,11 +729,6 @@ int kfd_wait_on_events(struct kfd_process *p,
+
+ mutex_lock(&p->event_mutex);
+
+- /* Set to something unreasonable - this is really
+- * just a bool for now.
+- */
+- *wait_result = KFD_WAIT_TIMEOUT;
+-
+ event_waiters = alloc_event_waiters(num_events);
+ if (!event_waiters) {
+ ret = -ENOMEM;
+@@ -808,34 +744,14 @@ int kfd_wait_on_events(struct kfd_process *p,
+ goto fail;
+ }
+
+- ret = init_event_waiter_get_status(p, &event_waiters[i],
++ ret = init_event_waiter(p, &event_waiters[i],
+ event_data.event_id, i);
+ if (ret)
+ goto fail;
+ }
+
+- /* Check condition once. */
+- if (test_event_condition(all, num_events, event_waiters)) {
+- if (copy_signaled_event_data(num_events,
+- event_waiters, events))
+- *wait_result = KFD_WAIT_COMPLETE;
+- else
+- *wait_result = KFD_WAIT_ERROR;
+- free_waiters(num_events, event_waiters);
+- } else {
+- /* Add to wait lists if we need to wait. */
+- for (i = 0; i < num_events; i++)
+- init_event_waiter_add_to_waitlist(&event_waiters[i]);
+- }
+-
+ mutex_unlock(&p->event_mutex);
+
+- /* Return if all waits were already satisfied. */
+- if (*wait_result != KFD_WAIT_TIMEOUT) {
+- __set_current_state(TASK_RUNNING);
+- return ret;
+- }
+-
+ while (true) {
+ if (fatal_signal_pending(current)) {
+ ret = -EINTR;
+@@ -855,17 +771,6 @@ int kfd_wait_on_events(struct kfd_process *p,
+ break;
+ }
+
+- /* Set task state to interruptible sleep before
+- * checking wake-up conditions. A concurrent wake-up
+- * will put the task back into runnable state. In that
+- * case schedule_timeout will not put the task to
+- * sleep and we'll get a chance to re-check the
+- * updated conditions almost immediately. Otherwise,
+- * this race condition would lead to a soft hang or a
+- * very long sleep.
+- */
+- set_current_state(TASK_INTERRUPTIBLE);
+-
+ if (test_event_condition(all, num_events, event_waiters)) {
+ if (copy_signaled_event_data(num_events,
+ event_waiters, events))
+@@ -880,7 +785,7 @@ int kfd_wait_on_events(struct kfd_process *p,
+ break;
+ }
+
+- timeout = schedule_timeout(timeout);
++ timeout = schedule_timeout_interruptible(timeout);
+ }
+ __set_current_state(TASK_RUNNING);
+
+@@ -911,7 +816,7 @@ int kfd_event_mmap(struct kfd_process *p, struct vm_area_struct *vma)
+ /* check required size is logical */
+ if (get_order(KFD_SIGNAL_EVENT_LIMIT * 8) !=
+ get_order(vma->vm_end - vma->vm_start)) {
+- pr_err("Event page mmap requested illegal size\n");
++ pr_err("amdkfd: event page mmap requested illegal size\n");
+ return -EINVAL;
+ }
+
+@@ -920,7 +825,7 @@ int kfd_event_mmap(struct kfd_process *p, struct vm_area_struct *vma)
+ page = lookup_signal_page_by_index(p, page_index);
+ if (!page) {
+ /* Probably KFD bug, but mmap is user-accessible. */
+- pr_debug("Signal page could not be found for page_index %u\n",
++ pr_debug("signal page could not be found for page_index %u\n",
+ page_index);
+ return -EINVAL;
+ }
+@@ -931,7 +836,7 @@ int kfd_event_mmap(struct kfd_process *p, struct vm_area_struct *vma)
+ vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE
+ | VM_DONTDUMP | VM_PFNMAP;
+
+- pr_debug("Mapping signal page\n");
++ pr_debug("mapping signal page\n");
+ pr_debug(" start user address == 0x%08lx\n", vma->vm_start);
+ pr_debug(" end user address == 0x%08lx\n", vma->vm_end);
+ pr_debug(" pfn == 0x%016lX\n", pfn);
+@@ -971,13 +876,6 @@ static void lookup_events_by_type_and_signal(struct kfd_process *p,
+ ev->memory_exception_data = *ev_data;
+ }
+
+- if (type == KFD_EVENT_TYPE_MEMORY) {
+- dev_warn(kfd_device,
+- "Sending SIGSEGV to HSA Process with PID %d ",
+- p->lead_thread->pid);
+- send_sig(SIGSEGV, p->lead_thread, 0);
+- }
+-
+ /* Send SIGTERM no event of type "type" has been found*/
+ if (send_signal) {
+ if (send_sigterm) {
+@@ -993,7 +891,6 @@ static void lookup_events_by_type_and_signal(struct kfd_process *p,
+ }
+ }
+
+-#if defined(CONFIG_AMD_IOMMU_V2_MODULE) || defined(CONFIG_AMD_IOMMU_V2)
+ void kfd_signal_iommu_event(struct kfd_dev *dev, unsigned int pasid,
+ unsigned long address, bool is_write_requested,
+ bool is_execute_requested)
+@@ -1004,27 +901,17 @@ void kfd_signal_iommu_event(struct kfd_dev *dev, unsigned int pasid,
+ /*
+ * Because we are called from arbitrary context (workqueue) as opposed
+ * to process context, kfd_process could attempt to exit while we are
+- * running so the lookup function increments the process ref count.
++ * running so the lookup function returns a locked process.
+ */
+ struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
+- struct mm_struct *mm;
+
+ if (!p)
+ return; /* Presumably process exited. */
+
+- /* Take a safe reference to the mm_struct, which may otherwise
+- * disappear even while the kfd_process is still referenced.
+- */
+- mm = get_task_mm(p->lead_thread);
+- if (!mm) {
+- kfd_unref_process(p);
+- return; /* Process is exiting */
+- }
+-
+ memset(&memory_exception_data, 0, sizeof(memory_exception_data));
+
+- down_read(&mm->mmap_sem);
+- vma = find_vma(mm, address);
++ down_read(&p->mm->mmap_sem);
++ vma = find_vma(p->mm, address);
+
+ memory_exception_data.gpu_id = dev->id;
+ memory_exception_data.va = address;
+@@ -1050,8 +937,7 @@ void kfd_signal_iommu_event(struct kfd_dev *dev, unsigned int pasid,
+ }
+ }
+
+- up_read(&mm->mmap_sem);
+- mmput(mm);
++ up_read(&p->mm->mmap_sem);
+
+ mutex_lock(&p->event_mutex);
+
+@@ -1060,17 +946,15 @@ void kfd_signal_iommu_event(struct kfd_dev *dev, unsigned int pasid,
+ &memory_exception_data);
+
+ mutex_unlock(&p->event_mutex);
+-
+- kfd_unref_process(p);
++ mutex_unlock(&p->mutex);
+ }
+-#endif /* CONFIG_AMD_IOMMU_V2_MODULE */
+
+ void kfd_signal_hw_exception_event(unsigned int pasid)
+ {
+ /*
+ * Because we are called from arbitrary context (workqueue) as opposed
+ * to process context, kfd_process could attempt to exit while we are
+- * running so the lookup function increments the process ref count.
++ * running so the lookup function returns a locked process.
+ */
+ struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
+
+@@ -1083,42 +967,5 @@ void kfd_signal_hw_exception_event(unsigned int pasid)
+ lookup_events_by_type_and_signal(p, KFD_EVENT_TYPE_HW_EXCEPTION, NULL);
+
+ mutex_unlock(&p->event_mutex);
+- kfd_unref_process(p);
+-}
+-
+-void kfd_signal_vm_fault_event(struct kfd_dev *dev, unsigned int pasid,
+- struct kfd_vm_fault_info *info)
+-{
+- struct kfd_event *ev;
+- int bkt;
+- struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
+- struct kfd_hsa_memory_exception_data memory_exception_data;
+-
+- if (!p)
+- return; /* Presumably process exited. */
+- memset(&memory_exception_data, 0, sizeof(memory_exception_data));
+- memory_exception_data.gpu_id = dev->id;
+- memory_exception_data.failure.imprecise = true;
+- /* Set failure reason */
+- if (info) {
+- memory_exception_data.va = (info->page_addr) << PAGE_SHIFT;
+- memory_exception_data.failure.NotPresent =
+- info->prot_valid ? 1 : 0;
+- memory_exception_data.failure.NoExecute =
+- info->prot_exec ? 1 : 0;
+- memory_exception_data.failure.ReadOnly =
+- info->prot_write ? 1 : 0;
+- memory_exception_data.failure.imprecise = 0;
+- }
+- mutex_lock(&p->event_mutex);
+-
+- hash_for_each(p->events, bkt, ev, events) {
+- if (ev->type == KFD_EVENT_TYPE_MEMORY) {
+- ev->memory_exception_data = memory_exception_data;
+- set_event(ev);
+- }
+- }
+-
+- mutex_unlock(&p->event_mutex);
+- kfd_unref_process(p);
++ mutex_unlock(&p->mutex);
+ }
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c b/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
+index 09595a9..2b65510 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
+@@ -275,80 +275,24 @@
+ * for FLAT_* / S_LOAD operations.
+ */
+
+-#define MAKE_GPUVM_APP_BASE_VI(gpu_num) \
++#define MAKE_GPUVM_APP_BASE(gpu_num) \
+ (((uint64_t)(gpu_num) << 61) + 0x1000000000000L)
+
+-#define MAKE_GPUVM_APP_LIMIT(base, size) \
+- (((uint64_t)(base) & 0xFFFFFF0000000000UL) + (size) - 1)
++#define MAKE_GPUVM_APP_LIMIT(base) \
++ (((uint64_t)(base) & \
++ 0xFFFFFF0000000000UL) | 0xFFFFFFFFFFL)
+
+-#define MAKE_SCRATCH_APP_BASE_VI() \
+- (((uint64_t)(0x1UL) << 61) + 0x100000000L)
++#define MAKE_SCRATCH_APP_BASE(gpu_num) \
++ (((uint64_t)(gpu_num) << 61) + 0x100000000L)
+
+ #define MAKE_SCRATCH_APP_LIMIT(base) \
+ (((uint64_t)base & 0xFFFFFFFF00000000UL) | 0xFFFFFFFF)
+
+-#define MAKE_LDS_APP_BASE_VI() \
+- (((uint64_t)(0x1UL) << 61) + 0x0)
+-
++#define MAKE_LDS_APP_BASE(gpu_num) \
++ (((uint64_t)(gpu_num) << 61) + 0x0)
+ #define MAKE_LDS_APP_LIMIT(base) \
+ (((uint64_t)(base) & 0xFFFFFFFF00000000UL) | 0xFFFFFFFF)
+
+-/* On GFXv9 the LDS and scratch apertures are programmed independently
+- * using the high 16 bits of the 64-bit virtual address. They must be
+- * in the hole, which will be the case as long as the high 16 bits are
+- * not 0.
+- *
+- * The aperture sizes are still 4GB implicitly.
+- *
+- * A GPUVM aperture is not applicable on GFXv9.
+- */
+-#define MAKE_LDS_APP_BASE_V9() ((uint64_t)(0x1UL) << 48)
+-#define MAKE_SCRATCH_APP_BASE_V9() ((uint64_t)(0x2UL) << 48)
+-
+-/* Some VM address space reserved for kernel use (CWSR trap handlers
+- * and kernel IBs)
+- */
+-#define DGPU_VM_BASE_DEFAULT 0x100000
+-#define DGPU_IB_BASE_DEFAULT (DGPU_VM_BASE_DEFAULT - PAGE_SIZE)
+-
+-int kfd_set_process_dgpu_aperture(struct kfd_process_device *pdd,
+- uint64_t base, uint64_t limit)
+-{
+- if (base < (pdd->qpd.cwsr_base + pdd->dev->cwsr_size)) {
+- pr_err("Set dgpu vm base 0x%llx failed.\n", base);
+- return -EINVAL;
+- }
+- pdd->dgpu_base = base;
+- pdd->dgpu_limit = limit;
+- return 0;
+-}
+-
+-void kfd_init_apertures_vi(struct kfd_process_device *pdd, uint8_t id)
+-{
+- /*
+- * node id couldn't be 0 - the three MSB bits of
+- * aperture shoudn't be 0
+- */
+- pdd->lds_base = MAKE_LDS_APP_BASE_VI();
+- pdd->lds_limit = MAKE_LDS_APP_LIMIT(pdd->lds_base);
+-
+- pdd->gpuvm_base = MAKE_GPUVM_APP_BASE_VI(id + 1);
+- pdd->gpuvm_limit = MAKE_GPUVM_APP_LIMIT(
+- pdd->gpuvm_base, pdd->dev->shared_resources.gpuvm_size);
+-
+- pdd->scratch_base = MAKE_SCRATCH_APP_BASE_VI();
+- pdd->scratch_limit = MAKE_SCRATCH_APP_LIMIT(pdd->scratch_base);
+-}
+-
+-void kfd_init_apertures_v9(struct kfd_process_device *pdd, uint8_t id)
+-{
+- pdd->lds_base = MAKE_LDS_APP_BASE_V9();
+- pdd->lds_limit = MAKE_LDS_APP_LIMIT(pdd->lds_base);
+-
+- pdd->scratch_base = MAKE_SCRATCH_APP_BASE_V9();
+- pdd->scratch_limit = MAKE_SCRATCH_APP_LIMIT(pdd->scratch_base);
+-}
+-
+ int kfd_init_apertures(struct kfd_process *process)
+ {
+ uint8_t id = 0;
+@@ -356,14 +300,11 @@ int kfd_init_apertures(struct kfd_process *process)
+ struct kfd_process_device *pdd;
+
+ /*Iterating over all devices*/
+- while (kfd_topology_enum_kfd_devices(id, &dev) == 0) {
+- if (!dev) {
+- id++; /* Skip non GPU devices */
+- continue;
+- }
++ while ((dev = kfd_topology_enum_kfd_devices(id)) != NULL &&
++ id < NUM_OF_SUPPORTED_GPUS) {
+
+ pdd = kfd_create_process_device_data(dev, process);
+- if (!pdd) {
++ if (pdd == NULL) {
+ pr_err("Failed to create process device data\n");
+ return -1;
+ }
+@@ -377,29 +318,23 @@ int kfd_init_apertures(struct kfd_process *process)
+ pdd->gpuvm_base = pdd->gpuvm_limit = 0;
+ pdd->scratch_base = pdd->scratch_limit = 0;
+ } else {
+- switch (dev->device_info->asic_family) {
+- case CHIP_KAVERI:
+- case CHIP_HAWAII:
+- case CHIP_CARRIZO:
+- case CHIP_TONGA:
+- case CHIP_FIJI:
+- case CHIP_POLARIS10:
+- case CHIP_POLARIS11:
+- kfd_init_apertures_vi(pdd, id);
+- break;
+- case CHIP_VEGA10:
+- case CHIP_RAVEN:
+- kfd_init_apertures_v9(pdd, id);
+- break;
+- default:
+- pr_err("Unknown chip in kfd_init_apertures\n");
+- return -1;
+- }
++ /*
++ * node id couldn't be 0 - the three MSB bits of
++ * aperture shoudn't be 0
++ */
++ pdd->lds_base = MAKE_LDS_APP_BASE(id + 1);
++
++ pdd->lds_limit = MAKE_LDS_APP_LIMIT(pdd->lds_base);
++
++ pdd->gpuvm_base = MAKE_GPUVM_APP_BASE(id + 1);
++
++ pdd->gpuvm_limit =
++ MAKE_GPUVM_APP_LIMIT(pdd->gpuvm_base);
+
+- if (!dev->device_info->is_need_iommu_device) {
+- pdd->qpd.cwsr_base = DGPU_VM_BASE_DEFAULT;
+- pdd->qpd.ib_base = DGPU_IB_BASE_DEFAULT;
+- }
++ pdd->scratch_base = MAKE_SCRATCH_APP_BASE(id + 1);
++
++ pdd->scratch_limit =
++ MAKE_SCRATCH_APP_LIMIT(pdd->scratch_base);
+ }
+
+ dev_dbg(kfd_device, "node id %u\n", id);
+@@ -417,9 +352,4 @@ int kfd_init_apertures(struct kfd_process *process)
+ return 0;
+ }
+
+-void kfd_flush_tlb(struct kfd_dev *dev, uint32_t pasid)
+-{
+- const struct kfd2kgd_calls *f2g = dev->kfd2kgd;
+
+- f2g->invalidate_tlbs(dev->kgd, pasid);
+-}
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
+deleted file mode 100644
+index b2c6b52..0000000
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
++++ /dev/null
+@@ -1,133 +0,0 @@
+-/*
+- * Copyright 2016 Advanced Micro Devices, Inc.
+- *
+- * Permission is hereby granted, free of charge, to any person obtaining a
+- * copy of this software and associated documentation files (the "Software"),
+- * to deal in the Software without restriction, including without limitation
+- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+- * and/or sell copies of the Software, and to permit persons to whom the
+- * Software is furnished to do so, subject to the following conditions:
+- *
+- * The above copyright notice and this permission notice shall be included in
+- * all copies or substantial portions of the Software.
+- *
+- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+- * OTHER DEALINGS IN THE SOFTWARE.
+- */
+-
+-#include "kfd_priv.h"
+-#include "kfd_events.h"
+-#include "soc15_int.h"
+-
+-
+-static uint32_t kfd_get_pasid_from_vmid(struct kfd_dev *dev, uint8_t vmid)
+-{
+- uint32_t pasid = 0;
+- const struct kfd2kgd_calls *f2g = dev->kfd2kgd;
+-
+- if (f2g->get_atc_vmid_pasid_mapping_valid(dev->kgd, vmid))
+- pasid = f2g->get_atc_vmid_pasid_mapping_pasid(dev->kgd, vmid);
+-
+- return pasid;
+-}
+-
+-static bool event_interrupt_isr_v9(struct kfd_dev *dev,
+- const uint32_t *ih_ring_entry,
+- uint32_t *patched_ihre,
+- bool *patched_flag)
+-{
+- uint16_t source_id, client_id, pasid, vmid;
+- bool result = false;
+-
+- source_id = SOC15_SOURCE_ID_FROM_IH_ENTRY(ih_ring_entry);
+- client_id = SOC15_CLIENT_ID_FROM_IH_ENTRY(ih_ring_entry);
+- pasid = SOC15_PASID_FROM_IH_ENTRY(ih_ring_entry);
+- vmid = SOC15_VMID_FROM_IH_ENTRY(ih_ring_entry);
+-
+- if (pasid) {
+- const uint32_t *data = ih_ring_entry;
+-
+- pr_debug("client id 0x%x, source id %d, pasid 0x%x. raw data:\n",
+- client_id, source_id, pasid);
+- pr_debug("%8X, %8X, %8X, %8X, %8X, %8X, %8X, %8X.\n",
+- data[0], data[1], data[2], data[3],
+- data[4], data[5], data[6], data[7]);
+- }
+-
+- if ((vmid >= dev->vm_info.first_vmid_kfd &&
+- vmid <= dev->vm_info.last_vmid_kfd) &&
+- (source_id == SOC15_INTSRC_CP_END_OF_PIPE ||
+- source_id == SOC15_INTSRC_SDMA_TRAP ||
+- source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG ||
+- source_id == SOC15_INTSRC_CP_BAD_OPCODE ||
+- client_id == SOC15_IH_CLIENTID_VMC ||
+- client_id == SOC15_IH_CLIENTID_UTCL2)) {
+-
+- /*
+- * KFD want to handle this INT, but MEC firmware did
+- * not send pasid. Try to get it from vmid mapping
+- * and patch the ih entry. It's a temp workaround.
+- */
+- WARN_ONCE((!pasid), "Fix me.\n");
+- if (!pasid) {
+- uint32_t temp = le32_to_cpu(ih_ring_entry[3]);
+-
+- pasid = kfd_get_pasid_from_vmid(dev, vmid);
+- memcpy(patched_ihre, ih_ring_entry,
+- dev->device_info->ih_ring_entry_size);
+- patched_ihre[3] = cpu_to_le32(temp | pasid);
+- *patched_flag = true;
+- }
+- result = pasid ? true : false;
+- }
+-
+- /* Do not process in ISR, just request it to be forwarded to WQ. */
+- return result;
+-
+-}
+-
+-static void event_interrupt_wq_v9(struct kfd_dev *dev,
+- const uint32_t *ih_ring_entry)
+-{
+- uint16_t source_id, client_id, pasid, vmid;
+-
+- source_id = SOC15_SOURCE_ID_FROM_IH_ENTRY(ih_ring_entry);
+- client_id = SOC15_CLIENT_ID_FROM_IH_ENTRY(ih_ring_entry);
+- pasid = SOC15_PASID_FROM_IH_ENTRY(ih_ring_entry);
+- vmid = SOC15_VMID_FROM_IH_ENTRY(ih_ring_entry);
+-
+- if (source_id == SOC15_INTSRC_CP_END_OF_PIPE)
+- kfd_signal_event_interrupt(pasid, 0, 0);
+- else if (source_id == SOC15_INTSRC_SDMA_TRAP)
+- kfd_signal_event_interrupt(pasid, 0, 0);
+- else if (source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG)
+- kfd_signal_event_interrupt(pasid, 0, 0); /*todo */
+- else if (source_id == SOC15_INTSRC_CP_BAD_OPCODE)
+- kfd_signal_hw_exception_event(pasid);
+- else if (client_id == SOC15_IH_CLIENTID_VMC ||
+- client_id == SOC15_IH_CLIENTID_UTCL2) {
+- struct kfd_vm_fault_info info = {0};
+- uint16_t ring_id = SOC15_RING_ID_FROM_IH_ENTRY(ih_ring_entry);
+-
+- info.vmid = vmid;
+- info.mc_id = client_id;
+- info.page_addr = ih_ring_entry[4] |
+- (uint64_t)(ih_ring_entry[5] & 0xf) << 32;
+- info.prot_valid = ring_id & 0x08;
+- info.prot_read = ring_id & 0x10;
+- info.prot_write = ring_id & 0x20;
+-
+- kfd_process_vm_fault(dev->dqm, pasid);
+- kfd_signal_vm_fault_event(dev, pasid, &info);
+- }
+-}
+-
+-const struct kfd_event_interrupt_class event_interrupt_class_v9 = {
+- .interrupt_isr = event_interrupt_isr_v9,
+- .interrupt_wq = event_interrupt_wq_v9,
+-};
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c b/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c
+index 47dcf4a..7f134aa 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c
+@@ -44,24 +44,24 @@
+ #include <linux/device.h>
+ #include "kfd_priv.h"
+
+-#define KFD_IH_NUM_ENTRIES 8192
++#define KFD_INTERRUPT_RING_SIZE 1024
+
+ static void interrupt_wq(struct work_struct *);
+
+ int kfd_interrupt_init(struct kfd_dev *kfd)
+ {
+- int r;
+-
+- r = kfifo_alloc(&kfd->ih_fifo,
+- KFD_IH_NUM_ENTRIES *
+- kfd->device_info->ih_ring_entry_size,
+- GFP_KERNEL);
+- if (r) {
+- dev_err(kfd_chardev(), "Failed to allocate IH fifo\n");
+- return r;
+- }
++ void *interrupt_ring = kmalloc_array(KFD_INTERRUPT_RING_SIZE,
++ kfd->device_info->ih_ring_entry_size,
++ GFP_KERNEL);
++ if (!interrupt_ring)
++ return -ENOMEM;
++
++ kfd->interrupt_ring = interrupt_ring;
++ kfd->interrupt_ring_size =
++ KFD_INTERRUPT_RING_SIZE * kfd->device_info->ih_ring_entry_size;
++ atomic_set(&kfd->interrupt_ring_wptr, 0);
++ atomic_set(&kfd->interrupt_ring_rptr, 0);
+
+- kfd->ih_wq = alloc_workqueue("KFD IH", WQ_HIGHPRI, 1);
+ spin_lock_init(&kfd->interrupt_lock);
+
+ INIT_WORK(&kfd->interrupt_work, interrupt_wq);
+@@ -92,47 +92,74 @@ void kfd_interrupt_exit(struct kfd_dev *kfd)
+ spin_unlock_irqrestore(&kfd->interrupt_lock, flags);
+
+ /*
+- * flush_work ensures that there are no outstanding
++ * Flush_scheduled_work ensures that there are no outstanding
+ * work-queue items that will access interrupt_ring. New work items
+ * can't be created because we stopped interrupt handling above.
+ */
+- flush_workqueue(kfd->ih_wq);
++ flush_scheduled_work();
+
+- kfifo_free(&kfd->ih_fifo);
++ kfree(kfd->interrupt_ring);
+ }
+
+ /*
+- * Assumption: single reader/writer. This function is not re-entrant
++ * This assumes that it can't be called concurrently with itself
++ * but only with dequeue_ih_ring_entry.
+ */
+ bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry)
+ {
+- int count;
++ unsigned int rptr = atomic_read(&kfd->interrupt_ring_rptr);
++ unsigned int wptr = atomic_read(&kfd->interrupt_ring_wptr);
+
+- count = kfifo_in(&kfd->ih_fifo, ih_ring_entry,
+- kfd->device_info->ih_ring_entry_size);
+- if (count != kfd->device_info->ih_ring_entry_size) {
++ if ((rptr - wptr) % kfd->interrupt_ring_size ==
++ kfd->device_info->ih_ring_entry_size) {
++ /* This is very bad, the system is likely to hang. */
+ dev_err_ratelimited(kfd_chardev(),
+- "Interrupt ring overflow, dropping interrupt %d\n",
+- count);
++ "Interrupt ring overflow, dropping interrupt.\n");
+ return false;
+ }
+
++ memcpy(kfd->interrupt_ring + wptr, ih_ring_entry,
++ kfd->device_info->ih_ring_entry_size);
++
++ wptr = (wptr + kfd->device_info->ih_ring_entry_size) %
++ kfd->interrupt_ring_size;
++ smp_wmb(); /* Ensure memcpy'd data is visible before wptr update. */
++ atomic_set(&kfd->interrupt_ring_wptr, wptr);
++
+ return true;
+ }
+
+ /*
+- * Assumption: single reader/writer. This function is not re-entrant
++ * This assumes that it can't be called concurrently with itself
++ * but only with enqueue_ih_ring_entry.
+ */
+ static bool dequeue_ih_ring_entry(struct kfd_dev *kfd, void *ih_ring_entry)
+ {
+- int count;
++ /*
++ * Assume that wait queues have an implicit barrier, i.e. anything that
++ * happened in the ISR before it queued work is visible.
++ */
++
++ unsigned int wptr = atomic_read(&kfd->interrupt_ring_wptr);
++ unsigned int rptr = atomic_read(&kfd->interrupt_ring_rptr);
+
+- count = kfifo_out(&kfd->ih_fifo, ih_ring_entry,
+- kfd->device_info->ih_ring_entry_size);
++ if (rptr == wptr)
++ return false;
+
+- WARN_ON(count && count != kfd->device_info->ih_ring_entry_size);
++ memcpy(ih_ring_entry, kfd->interrupt_ring + rptr,
++ kfd->device_info->ih_ring_entry_size);
+
+- return count == kfd->device_info->ih_ring_entry_size;
++ rptr = (rptr + kfd->device_info->ih_ring_entry_size) %
++ kfd->interrupt_ring_size;
++
++ /*
++ * Ensure the rptr write update is not visible until
++ * memcpy has finished reading.
++ */
++ smp_mb();
++ atomic_set(&kfd->interrupt_ring_rptr, rptr);
++
++ return true;
+ }
+
+ static void interrupt_wq(struct work_struct *work)
+@@ -149,15 +176,13 @@ static void interrupt_wq(struct work_struct *work)
+ ih_ring_entry);
+ }
+
+-bool interrupt_is_wanted(struct kfd_dev *dev,
+- const uint32_t *ih_ring_entry,
+- uint32_t *patched_ihre, bool *flag)
++bool interrupt_is_wanted(struct kfd_dev *dev, const uint32_t *ih_ring_entry)
+ {
+ /* integer and bitwise OR so there is no boolean short-circuiting */
+- unsigned int wanted = 0;
++ unsigned wanted = 0;
+
+ wanted |= dev->device_info->event_interrupt_class->interrupt_isr(dev,
+- ih_ring_entry, patched_ihre, flag);
++ ih_ring_entry);
+
+ return wanted != 0;
+ }
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_ipc.c b/drivers/gpu/drm/amd/amdkfd/kfd_ipc.c
+deleted file mode 100644
+index e67eb9f..0000000
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_ipc.c
++++ /dev/null
+@@ -1,275 +0,0 @@
+-/*
+- * Copyright 2014 Advanced Micro Devices, Inc.
+- *
+- * Permission is hereby granted, free of charge, to any person obtaining a
+- * copy of this software and associated documentation files (the "Software"),
+- * to deal in the Software without restriction, including without limitation
+- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+- * and/or sell copies of the Software, and to permit persons to whom the
+- * Software is furnished to do so, subject to the following conditions:
+- *
+- * The above copyright notice and this permission notice shall be included in
+- * all copies or substantial portions of the Software.
+- *
+- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+- * OTHER DEALINGS IN THE SOFTWARE.
+- */
+-
+-#include <linux/dma-buf.h>
+-#include <linux/slab.h>
+-#include <linux/random.h>
+-
+-#include "kfd_ipc.h"
+-#include "kfd_priv.h"
+-
+-#define KFD_IPC_HASH_TABLE_SIZE_SHIFT 4
+-#define KFD_IPC_HASH_TABLE_SIZE_MASK ((1 << KFD_IPC_HASH_TABLE_SIZE_SHIFT) - 1)
+-
+-static struct kfd_ipc_handles {
+- DECLARE_HASHTABLE(handles, KFD_IPC_HASH_TABLE_SIZE_SHIFT);
+- struct mutex lock;
+-} kfd_ipc_handles;
+-
+-/* Since, handles are random numbers, it can be used directly as hashing key.
+- * The least 4 bits of the handle are used as key. However, during import all
+- * 128 bits of the handle are checked to prevent handle snooping.
+- */
+-#define HANDLE_TO_KEY(sh) ((*(uint64_t *)sh) & KFD_IPC_HASH_TABLE_SIZE_MASK)
+-
+-static int ipc_store_insert(void *val, void *sh, struct kfd_ipc_obj **ipc_obj)
+-{
+- struct kfd_ipc_obj *obj;
+-
+- obj = kmalloc(sizeof(*obj), GFP_KERNEL);
+- if (!obj)
+- return -ENOMEM;
+-
+- /* The initial ref belongs to the allocator process.
+- * The IPC object store itself does not hold a ref since
+- * there is no specific moment in time where that ref should
+- * be dropped, except "when there are no more userspace processes
+- * holding a ref to the object". Therefore the removal from IPC
+- * storage happens at ipc_obj release time.
+- */
+- kref_init(&obj->ref);
+- obj->data = val;
+- get_random_bytes(obj->share_handle, sizeof(obj->share_handle));
+-
+- memcpy(sh, obj->share_handle, sizeof(obj->share_handle));
+-
+- mutex_lock(&kfd_ipc_handles.lock);
+- hlist_add_head(&obj->node,
+- &kfd_ipc_handles.handles[HANDLE_TO_KEY(obj->share_handle)]);
+- mutex_unlock(&kfd_ipc_handles.lock);
+-
+- if (ipc_obj)
+- *ipc_obj = obj;
+-
+- return 0;
+-}
+-
+-static void ipc_obj_release(struct kref *r)
+-{
+- struct kfd_ipc_obj *obj;
+-
+- obj = container_of(r, struct kfd_ipc_obj, ref);
+-
+- mutex_lock(&kfd_ipc_handles.lock);
+- hash_del(&obj->node);
+- mutex_unlock(&kfd_ipc_handles.lock);
+-
+- dma_buf_put(obj->data);
+- kfree(obj);
+-}
+-
+-void ipc_obj_get(struct kfd_ipc_obj *obj)
+-{
+- kref_get(&obj->ref);
+-}
+-
+-void ipc_obj_put(struct kfd_ipc_obj **obj)
+-{
+- kref_put(&(*obj)->ref, ipc_obj_release);
+- *obj = NULL;
+-}
+-
+-int kfd_ipc_init(void)
+-{
+- mutex_init(&kfd_ipc_handles.lock);
+- hash_init(kfd_ipc_handles.handles);
+- return 0;
+-}
+-
+-static int kfd_import_dmabuf_create_kfd_bo(struct kfd_dev *dev,
+- struct kfd_process *p,
+- uint32_t gpu_id, struct dma_buf *dmabuf,
+- uint64_t va_addr, uint64_t *handle,
+- uint64_t *mmap_offset,
+- struct kfd_ipc_obj *ipc_obj)
+-{
+- int r;
+- void *mem;
+- uint64_t size;
+- int idr_handle;
+- struct kfd_process_device *pdd = NULL;
+- uint64_t kfd_mmap_flags = KFD_MMAP_TYPE_MAP_BO |
+- KFD_MMAP_GPU_ID(gpu_id);
+-
+- if (!handle)
+- return -EINVAL;
+-
+- if (!dev || !dev->kfd2kgd->import_dmabuf)
+- return -EINVAL;
+-
+- mutex_lock(&p->mutex);
+-
+- pdd = kfd_bind_process_to_device(dev, p);
+- if (IS_ERR(pdd)) {
+- r = PTR_ERR(pdd);
+- goto err_unlock;
+- }
+-
+- r = dev->kfd2kgd->import_dmabuf(dev->kgd, dmabuf,
+- va_addr, pdd->vm,
+- (struct kgd_mem **)&mem, &size,
+- mmap_offset);
+- if (r)
+- goto err_unlock;
+-
+- idr_handle = kfd_process_device_create_obj_handle(pdd, mem,
+- va_addr, size,
+- ipc_obj);
+- if (idr_handle < 0) {
+- r = -EFAULT;
+- goto err_free;
+- }
+-
+- mutex_unlock(&p->mutex);
+-
+- *handle = MAKE_HANDLE(gpu_id, idr_handle);
+- if (mmap_offset)
+- *mmap_offset = (kfd_mmap_flags << PAGE_SHIFT) | *mmap_offset;
+-
+- return 0;
+-
+-err_free:
+- dev->kfd2kgd->free_memory_of_gpu(dev->kgd,
+- (struct kgd_mem *)mem,
+- pdd->vm);
+-err_unlock:
+- mutex_unlock(&p->mutex);
+- return r;
+-}
+-
+-int kfd_ipc_import_dmabuf(struct kfd_dev *dev,
+- struct kfd_process *p,
+- uint32_t gpu_id, int dmabuf_fd,
+- uint64_t va_addr, uint64_t *handle,
+- uint64_t *mmap_offset)
+-{
+- int r;
+- struct dma_buf *dmabuf = dma_buf_get(dmabuf_fd);
+-
+- if (!dmabuf)
+- return -EINVAL;
+-
+- r = kfd_import_dmabuf_create_kfd_bo(dev, p, gpu_id, dmabuf,
+- va_addr, handle, mmap_offset,
+- NULL);
+- dma_buf_put(dmabuf);
+- return r;
+-}
+-
+-int kfd_ipc_import_handle(struct kfd_dev *dev, struct kfd_process *p,
+- uint32_t gpu_id, uint32_t *share_handle,
+- uint64_t va_addr, uint64_t *handle,
+- uint64_t *mmap_offset)
+-{
+- int r;
+- struct kfd_ipc_obj *entry, *found = NULL;
+-
+- mutex_lock(&kfd_ipc_handles.lock);
+- /* Convert the user provided handle to hash key and search only in that
+- * bucket
+- */
+- hlist_for_each_entry(entry,
+- &kfd_ipc_handles.handles[HANDLE_TO_KEY(share_handle)], node) {
+- if (!memcmp(entry->share_handle, share_handle,
+- sizeof(entry->share_handle))) {
+- found = entry;
+- break;
+- }
+- }
+- mutex_unlock(&kfd_ipc_handles.lock);
+-
+- if (!found)
+- return -EINVAL;
+- ipc_obj_get(found);
+-
+- pr_debug("Found ipc_dma_buf: %p\n", found->data);
+-
+- r = kfd_import_dmabuf_create_kfd_bo(dev, p, gpu_id, found->data,
+- va_addr, handle, mmap_offset,
+- found);
+- if (r)
+- goto error_unref;
+-
+- return r;
+-
+-error_unref:
+- ipc_obj_put(&found);
+- return r;
+-}
+-
+-int kfd_ipc_export_as_handle(struct kfd_dev *dev, struct kfd_process *p,
+- uint64_t handle, uint32_t *ipc_handle)
+-{
+- struct kfd_process_device *pdd = NULL;
+- struct kfd_ipc_obj *obj;
+- struct kfd_bo *kfd_bo = NULL;
+- struct dma_buf *dmabuf;
+- int r;
+-
+- if (!dev || !ipc_handle)
+- return -EINVAL;
+-
+- mutex_lock(&p->mutex);
+- pdd = kfd_bind_process_to_device(dev, p);
+- if (IS_ERR(pdd)) {
+- mutex_unlock(&p->mutex);
+- pr_err("Failed to get pdd\n");
+- return PTR_ERR(pdd);
+- }
+-
+- kfd_bo = kfd_process_device_find_bo(pdd, GET_IDR_HANDLE(handle));
+- mutex_unlock(&p->mutex);
+-
+- if (!kfd_bo) {
+- pr_err("Failed to get bo");
+- return -EINVAL;
+- }
+- if (kfd_bo->kfd_ipc_obj) {
+- memcpy(ipc_handle, kfd_bo->kfd_ipc_obj->share_handle,
+- sizeof(kfd_bo->kfd_ipc_obj->share_handle));
+- return 0;
+- }
+-
+- r = dev->kfd2kgd->export_dmabuf(dev->kgd, pdd->vm,
+- (struct kgd_mem *)kfd_bo->mem,
+- &dmabuf);
+- if (r)
+- return r;
+-
+- r = ipc_store_insert(dmabuf, ipc_handle, &obj);
+- if (r)
+- return r;
+-
+- kfd_bo->kfd_ipc_obj = obj;
+-
+- return r;
+-}
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_ipc.h b/drivers/gpu/drm/amd/amdkfd/kfd_ipc.h
+deleted file mode 100644
+index 9ee8627..0000000
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_ipc.h
++++ /dev/null
+@@ -1,51 +0,0 @@
+-/*
+- * Copyright 2014 Advanced Micro Devices, Inc.
+- *
+- * Permission is hereby granted, free of charge, to any person obtaining a
+- * copy of this software and associated documentation files (the "Software"),
+- * to deal in the Software without restriction, including without limitation
+- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+- * and/or sell copies of the Software, and to permit persons to whom the
+- * Software is furnished to do so, subject to the following conditions:
+- *
+- * The above copyright notice and this permission notice shall be included in
+- * all copies or substantial portions of the Software.
+- *
+- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+- * OTHER DEALINGS IN THE SOFTWARE.
+- *
+- */
+-
+-#ifndef KFD_IPC_H_
+-#define KFD_IPC_H_
+-
+-#include <linux/types.h>
+-#include "kfd_priv.h"
+-
+-struct kfd_ipc_obj {
+- struct hlist_node node;
+- struct kref ref;
+- void *data;
+- uint32_t share_handle[4];
+-};
+-
+-int kfd_ipc_import_handle(struct kfd_dev *dev, struct kfd_process *p,
+- uint32_t gpu_id, uint32_t *share_handle,
+- uint64_t va_addr, uint64_t *handle,
+- uint64_t *mmap_offset);
+-int kfd_ipc_import_dmabuf(struct kfd_dev *kfd, struct kfd_process *p,
+- uint32_t gpu_id, int dmabuf_fd,
+- uint64_t va_addr, uint64_t *handle,
+- uint64_t *mmap_offset);
+-int kfd_ipc_export_as_handle(struct kfd_dev *dev, struct kfd_process *p,
+- uint64_t handle, uint32_t *ipc_handle);
+-
+-void ipc_obj_get(struct kfd_ipc_obj *obj);
+-void ipc_obj_put(struct kfd_ipc_obj **obj);
+-
+-#endif /* KFD_IPC_H_ */
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
+index 903ef25..d135cd0 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
+@@ -41,8 +41,11 @@ static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev,
+ int retval;
+ union PM4_MES_TYPE_3_HEADER nop;
+
+- pr_debug("Initializing queue type %d size %d\n", KFD_QUEUE_TYPE_HIQ,
+- queue_size);
++ BUG_ON(!kq || !dev);
++ BUG_ON(type != KFD_QUEUE_TYPE_DIQ && type != KFD_QUEUE_TYPE_HIQ);
++
++ pr_debug("amdkfd: In func %s initializing queue type %d size %d\n",
++ __func__, KFD_QUEUE_TYPE_HIQ, queue_size);
+
+ memset(&prop, 0, sizeof(prop));
+ memset(&nop, 0, sizeof(nop));
+@@ -60,23 +63,23 @@ static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev,
+ KFD_MQD_TYPE_HIQ);
+ break;
+ default:
+- pr_err("Invalid queue type %d\n", type);
+- return false;
++ BUG();
++ break;
+ }
+
+- if (!kq->mqd)
++ if (kq->mqd == NULL)
+ return false;
+
+ prop.doorbell_ptr = kfd_get_kernel_doorbell(dev, &prop.doorbell_off);
+
+- if (!prop.doorbell_ptr) {
+- pr_err("Failed to initialize doorbell");
++ if (prop.doorbell_ptr == NULL) {
++ pr_err("amdkfd: error init doorbell");
+ goto err_get_kernel_doorbell;
+ }
+
+ retval = kfd_gtt_sa_allocate(dev, queue_size, &kq->pq);
+ if (retval != 0) {
+- pr_err("Failed to init pq queues size %d\n", queue_size);
++ pr_err("amdkfd: error init pq queues size (%d)\n", queue_size);
+ goto err_pq_allocate_vidmem;
+ }
+
+@@ -84,7 +87,7 @@ static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev,
+ kq->pq_gpu_addr = kq->pq->gpu_addr;
+
+ retval = kq->ops_asic_specific.initialize(kq, dev, type, queue_size);
+- if (!retval)
++ if (retval == false)
+ goto err_eop_allocate_vidmem;
+
+ retval = kfd_gtt_sa_allocate(dev, sizeof(*kq->rptr_kernel),
+@@ -96,7 +99,7 @@ static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev,
+ kq->rptr_kernel = kq->rptr_mem->cpu_ptr;
+ kq->rptr_gpu_addr = kq->rptr_mem->gpu_addr;
+
+- retval = kfd_gtt_sa_allocate(dev, dev->device_info->doorbell_size,
++ retval = kfd_gtt_sa_allocate(dev, sizeof(*kq->wptr_kernel),
+ &kq->wptr_mem);
+
+ if (retval != 0)
+@@ -120,7 +123,6 @@ static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev,
+ prop.write_ptr = (uint32_t *) kq->wptr_gpu_addr;
+ prop.eop_ring_buffer_address = kq->eop_gpu_addr;
+ prop.eop_ring_buffer_size = PAGE_SIZE;
+- prop.cu_mask = NULL;
+
+ if (init_queue(&kq->queue, &prop) != 0)
+ goto err_init_queue;
+@@ -137,12 +139,11 @@ static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev,
+
+ /* assign HIQ to HQD */
+ if (type == KFD_QUEUE_TYPE_HIQ) {
+- pr_debug("Assigning hiq to hqd\n");
++ pr_debug("assigning hiq to hqd\n");
+ kq->queue->pipe = KFD_CIK_HIQ_PIPE;
+ kq->queue->queue = KFD_CIK_HIQ_QUEUE;
+ kq->mqd->load_mqd(kq->mqd, kq->queue->mqd, kq->queue->pipe,
+- kq->queue->queue, &kq->queue->properties,
+- NULL);
++ kq->queue->queue, NULL);
+ } else {
+ /* allocate fence for DIQ */
+
+@@ -179,10 +180,12 @@ static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev,
+
+ static void uninitialize(struct kernel_queue *kq)
+ {
++ BUG_ON(!kq);
++
+ if (kq->queue->properties.type == KFD_QUEUE_TYPE_HIQ)
+ kq->mqd->destroy_mqd(kq->mqd,
+- kq->queue->mqd,
+- KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
++ NULL,
++ false,
+ QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS,
+ kq->queue->pipe,
+ kq->queue->queue);
+@@ -206,17 +209,12 @@ static int acquire_packet_buffer(struct kernel_queue *kq,
+ size_t available_size;
+ size_t queue_size_dwords;
+ uint32_t wptr, rptr;
+- uint64_t wptr64;
+ unsigned int *queue_address;
+
+- /* When rptr == wptr, the buffer is empty.
+- * When rptr == wptr + 1, the buffer is full.
+- * It is always rptr that advances to the position of wptr, rather than
+- * the opposite. So we can only use up to queue_size_dwords - 1 dwords.
+- */
++ BUG_ON(!kq || !buffer_ptr);
++
+ rptr = *kq->rptr_kernel;
+- wptr = kq->pending_wptr;
+- wptr64 = kq->pending_wptr64;
++ wptr = *kq->wptr_kernel;
+ queue_address = (unsigned int *)kq->pq_kernel_addr;
+ queue_size_dwords = kq->queue->properties.queue_size / sizeof(uint32_t);
+
+@@ -224,72 +222,28 @@ static int acquire_packet_buffer(struct kernel_queue *kq,
+ pr_debug("wptr: %d\n", wptr);
+ pr_debug("queue_address 0x%p\n", queue_address);
+
+- available_size = (rptr + queue_size_dwords - 1 - wptr) %
++ available_size = (rptr - 1 - wptr + queue_size_dwords) %
+ queue_size_dwords;
+
+- if (packet_size_in_dwords > available_size) {
++ if (packet_size_in_dwords >= queue_size_dwords ||
++ packet_size_in_dwords >= available_size) {
+ /*
+ * make sure calling functions know
+ * acquire_packet_buffer() failed
+ */
+- goto err_no_space;
++ *buffer_ptr = NULL;
++ return -ENOMEM;
+ }
+
+ if (wptr + packet_size_in_dwords >= queue_size_dwords) {
+- /* make sure after rolling back to position 0, there is
+- * still enough space.
+- */
+- if (packet_size_in_dwords >= rptr)
+- goto err_no_space;
+-
+- /* fill nops, roll back and start at position 0 */
+ while (wptr > 0) {
+ queue_address[wptr] = kq->nop_packet;
+ wptr = (wptr + 1) % queue_size_dwords;
+- wptr64++;
+ }
+ }
+
+ *buffer_ptr = &queue_address[wptr];
+ kq->pending_wptr = wptr + packet_size_in_dwords;
+- kq->pending_wptr64 = wptr64 + packet_size_in_dwords;
+-
+- return 0;
+-
+-err_no_space:
+- *buffer_ptr = NULL;
+- return -ENOMEM;
+-}
+-
+-static int acquire_inline_ib(struct kernel_queue *kq,
+- size_t size_in_dwords,
+- unsigned int **buffer_ptr,
+- uint64_t *gpu_addr)
+-{
+- int ret;
+- unsigned int *buf;
+- union PM4_MES_TYPE_3_HEADER nop;
+-
+- if (size_in_dwords >= (1 << 14))
+- return -EINVAL;
+-
+- /* Allocate size_in_dwords on the ring, plus an extra dword
+- * for a NOP packet header
+- */
+- ret = acquire_packet_buffer(kq, size_in_dwords + 1, &buf);
+- if (ret)
+- return ret;
+-
+- /* Build a NOP packet that contains the IB as "payload". */
+- nop.u32all = 0;
+- nop.opcode = IT_NOP;
+- nop.count = size_in_dwords - 1;
+- nop.type = PM4_TYPE_3;
+-
+- *buf = nop.u32all;
+- *buffer_ptr = buf + 1;
+- *gpu_addr = kq->pq_gpu_addr + ((unsigned long)*buffer_ptr -
+- (unsigned long)kq->pq_kernel_addr);
+
+ return 0;
+ }
+@@ -298,7 +252,11 @@ static void submit_packet(struct kernel_queue *kq)
+ {
+ #ifdef DEBUG
+ int i;
++#endif
++
++ BUG_ON(!kq);
+
++#ifdef DEBUG
+ for (i = *kq->wptr_kernel; i < kq->pending_wptr; i++) {
+ pr_debug("0x%2X ", kq->pq_kernel_addr[i]);
+ if (i % 15 == 0)
+@@ -307,11 +265,14 @@ static void submit_packet(struct kernel_queue *kq)
+ pr_debug("\n");
+ #endif
+
+- kq->ops_asic_specific.submit_packet(kq);
++ *kq->wptr_kernel = kq->pending_wptr;
++ write_kernel_doorbell(kq->queue->properties.doorbell_ptr,
++ kq->pending_wptr);
+ }
+
+ static void rollback_packet(struct kernel_queue *kq)
+ {
++ BUG_ON(!kq);
+ kq->pending_wptr = *kq->queue->properties.write_ptr;
+ }
+
+@@ -320,41 +281,30 @@ struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
+ {
+ struct kernel_queue *kq;
+
+- kq = kzalloc(sizeof(*kq), GFP_KERNEL);
++ BUG_ON(!dev);
++
++ kq = kzalloc(sizeof(struct kernel_queue), GFP_KERNEL);
+ if (!kq)
+ return NULL;
+
+ kq->ops.initialize = initialize;
+ kq->ops.uninitialize = uninitialize;
+ kq->ops.acquire_packet_buffer = acquire_packet_buffer;
+- kq->ops.acquire_inline_ib = acquire_inline_ib;
+ kq->ops.submit_packet = submit_packet;
+ kq->ops.rollback_packet = rollback_packet;
+
+ switch (dev->device_info->asic_family) {
+ case CHIP_CARRIZO:
+- case CHIP_TONGA:
+- case CHIP_FIJI:
+- case CHIP_POLARIS10:
+- case CHIP_POLARIS11:
+ kernel_queue_init_vi(&kq->ops_asic_specific);
+ break;
+
+ case CHIP_KAVERI:
+- case CHIP_HAWAII:
+ kernel_queue_init_cik(&kq->ops_asic_specific);
+ break;
+-
+- case CHIP_VEGA10:
+- case CHIP_RAVEN:
+- kernel_queue_init_v9(&kq->ops_asic_specific);
+- break;
+- default:
+- BUG();
+ }
+
+ if (!kq->ops.initialize(kq, dev, type, KFD_KERNEL_QUEUE_SIZE)) {
+- pr_err("Failed to init kernel queue\n");
++ pr_err("amdkfd: failed to init kernel queue\n");
+ kfree(kq);
+ return NULL;
+ }
+@@ -363,37 +313,32 @@ struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
+
+ void kernel_queue_uninit(struct kernel_queue *kq)
+ {
++ BUG_ON(!kq);
++
+ kq->ops.uninitialize(kq);
+ kfree(kq);
+ }
+
+-/* FIXME: Can this test be removed? */
+ static __attribute__((unused)) void test_kq(struct kfd_dev *dev)
+ {
+ struct kernel_queue *kq;
+ uint32_t *buffer, i;
+ int retval;
+
+- pr_err("Starting kernel queue test\n");
++ BUG_ON(!dev);
++
++ pr_err("amdkfd: starting kernel queue test\n");
+
+ kq = kernel_queue_init(dev, KFD_QUEUE_TYPE_HIQ);
+- if (unlikely(!kq)) {
+- pr_err(" Failed to initialize HIQ\n");
+- pr_err("Kernel queue test failed\n");
+- return;
+- }
++ BUG_ON(!kq);
+
+ retval = kq->ops.acquire_packet_buffer(kq, 5, &buffer);
+- if (unlikely(retval != 0)) {
+- pr_err(" Failed to acquire packet buffer\n");
+- pr_err("Kernel queue test failed\n");
+- return;
+- }
++ BUG_ON(retval != 0);
+ for (i = 0; i < 5; i++)
+ buffer[i] = kq->nop_packet;
+ kq->ops.submit_packet(kq);
+
+- pr_err("Ending kernel queue test\n");
++ pr_err("amdkfd: ending kernel queue test\n");
+ }
+
+
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.h b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.h
+index 82c94a6..5940531 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.h
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.h
+@@ -42,12 +42,6 @@
+ * pending write pointer to that location so subsequent calls to
+ * acquire_packet_buffer will get a correct write pointer
+ *
+- * @acquire_inline_ib: Returns a pointer to the location in the kernel
+- * queue ring buffer where the calling function can write an inline IB. It is
+- * Guaranteed that there is enough space for that IB. It also updates the
+- * pending write pointer to that location so subsequent calls to
+- * acquire_packet_buffer will get a correct write pointer
+- *
+ * @submit_packet: Update the write pointer and doorbell of a kernel queue.
+ *
+ * @sync_with_hw: Wait until the write pointer and the read pointer of a kernel
+@@ -65,10 +59,6 @@ struct kernel_queue_ops {
+ int (*acquire_packet_buffer)(struct kernel_queue *kq,
+ size_t packet_size_in_dwords,
+ unsigned int **buffer_ptr);
+- int (*acquire_inline_ib)(struct kernel_queue *kq,
+- size_t packet_size_in_dwords,
+- unsigned int **buffer_ptr,
+- uint64_t *gpu_addr);
+
+ void (*submit_packet)(struct kernel_queue *kq);
+ void (*rollback_packet)(struct kernel_queue *kq);
+@@ -82,7 +72,6 @@ struct kernel_queue {
+ struct kfd_dev *dev;
+ struct mqd_manager *mqd;
+ struct queue *queue;
+- uint64_t pending_wptr64;
+ uint32_t pending_wptr;
+ unsigned int nop_packet;
+
+@@ -90,10 +79,7 @@ struct kernel_queue {
+ uint32_t *rptr_kernel;
+ uint64_t rptr_gpu_addr;
+ struct kfd_mem_obj *wptr_mem;
+- union {
+- uint64_t *wptr64_kernel;
+- uint32_t *wptr_kernel;
+- };
++ uint32_t *wptr_kernel;
+ uint64_t wptr_gpu_addr;
+ struct kfd_mem_obj *pq;
+ uint64_t pq_gpu_addr;
+@@ -111,6 +97,5 @@ struct kernel_queue {
+
+ void kernel_queue_init_cik(struct kernel_queue_ops *ops);
+ void kernel_queue_init_vi(struct kernel_queue_ops *ops);
+-void kernel_queue_init_v9(struct kernel_queue_ops *ops);
+
+ #endif /* KFD_KERNEL_QUEUE_H_ */
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_cik.c
+index 2808422..a90eb44 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_cik.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_cik.c
+@@ -22,19 +22,15 @@
+ */
+
+ #include "kfd_kernel_queue.h"
+-#include "kfd_pm4_headers.h"
+-#include "kfd_pm4_opcodes.h"
+
+ static bool initialize_cik(struct kernel_queue *kq, struct kfd_dev *dev,
+ enum kfd_queue_type type, unsigned int queue_size);
+ static void uninitialize_cik(struct kernel_queue *kq);
+-static void submit_packet_cik(struct kernel_queue *kq);
+
+ void kernel_queue_init_cik(struct kernel_queue_ops *ops)
+ {
+ ops->initialize = initialize_cik;
+ ops->uninitialize = uninitialize_cik;
+- ops->submit_packet = submit_packet_cik;
+ }
+
+ static bool initialize_cik(struct kernel_queue *kq, struct kfd_dev *dev,
+@@ -46,127 +42,3 @@ static bool initialize_cik(struct kernel_queue *kq, struct kfd_dev *dev,
+ static void uninitialize_cik(struct kernel_queue *kq)
+ {
+ }
+-
+-static void submit_packet_cik(struct kernel_queue *kq)
+-{
+- *kq->wptr_kernel = kq->pending_wptr;
+- write_kernel_doorbell(kq->queue->properties.doorbell_ptr,
+- kq->pending_wptr);
+-}
+-
+-static int pm_map_process_cik(struct packet_manager *pm, uint32_t *buffer,
+- struct qcm_process_device *qpd)
+-{
+- struct pm4_map_process *packet;
+-
+- packet = (struct pm4_map_process *)buffer;
+-
+- memset(buffer, 0, sizeof(struct pm4_map_process));
+-
+- packet->header.u32all = pm_build_pm4_header(IT_MAP_PROCESS,
+- sizeof(struct pm4_map_process));
+- packet->bitfields2.diq_enable = (qpd->is_debug) ? 1 : 0;
+- packet->bitfields2.process_quantum = 1;
+- packet->bitfields2.pasid = qpd->pqm->process->pasid;
+- packet->bitfields3.page_table_base = qpd->page_table_base;
+- packet->bitfields10.gds_size = qpd->gds_size;
+- packet->bitfields10.num_gws = qpd->num_gws;
+- packet->bitfields10.num_oac = qpd->num_oac;
+- packet->bitfields10.num_queues = (qpd->is_debug) ? 0 : qpd->queue_count;
+-
+- packet->sh_mem_config = qpd->sh_mem_config;
+- packet->sh_mem_bases = qpd->sh_mem_bases;
+- packet->sh_mem_ape1_base = qpd->sh_mem_ape1_base;
+- packet->sh_mem_ape1_limit = qpd->sh_mem_ape1_limit;
+-
+- packet->gds_addr_lo = lower_32_bits(qpd->gds_context_area);
+- packet->gds_addr_hi = upper_32_bits(qpd->gds_context_area);
+-
+- return 0;
+-}
+-
+-static int pm_map_process_scratch_cik(struct packet_manager *pm,
+- uint32_t *buffer, struct qcm_process_device *qpd)
+-{
+- struct pm4_map_process_scratch_kv *packet;
+-
+- packet = (struct pm4_map_process_scratch_kv *)buffer;
+-
+- memset(buffer, 0, sizeof(struct pm4_map_process_scratch_kv));
+-
+- packet->header.u32all = pm_build_pm4_header(IT_MAP_PROCESS,
+- sizeof(struct pm4_map_process_scratch_kv));
+- packet->bitfields2.diq_enable = (qpd->is_debug) ? 1 : 0;
+- packet->bitfields2.process_quantum = 1;
+- packet->bitfields2.pasid = qpd->pqm->process->pasid;
+- packet->bitfields3.page_table_base = qpd->page_table_base;
+- packet->bitfields14.gds_size = qpd->gds_size;
+- packet->bitfields14.num_gws = qpd->num_gws;
+- packet->bitfields14.num_oac = qpd->num_oac;
+- packet->bitfields14.num_queues = (qpd->is_debug) ? 0 : qpd->queue_count;
+-
+- packet->sh_mem_config = qpd->sh_mem_config;
+- packet->sh_mem_bases = qpd->sh_mem_bases;
+- packet->sh_mem_ape1_base = qpd->sh_mem_ape1_base;
+- packet->sh_mem_ape1_limit = qpd->sh_mem_ape1_limit;
+-
+- packet->sh_hidden_private_base_vmid = qpd->sh_hidden_private_base;
+-
+- packet->gds_addr_lo = lower_32_bits(qpd->gds_context_area);
+- packet->gds_addr_hi = upper_32_bits(qpd->gds_context_area);
+-
+- return 0;
+-}
+-
+-static uint32_t pm_get_map_process_packet_size_cik(void)
+-{
+- return sizeof(struct pm4_map_process);
+-}
+-static uint32_t pm_get_map_process_scratch_packet_size_cik(void)
+-{
+- return sizeof(struct pm4_map_process_scratch_kv);
+-}
+-
+-
+-static struct packet_manager_funcs kfd_cik_pm_funcs = {
+- .map_process = pm_map_process_cik,
+- .runlist = pm_runlist_vi,
+- .set_resources = pm_set_resources_vi,
+- .map_queues = pm_map_queues_vi,
+- .unmap_queues = pm_unmap_queues_vi,
+- .query_status = pm_query_status_vi,
+- .release_mem = pm_release_mem_vi,
+- .get_map_process_packet_size = pm_get_map_process_packet_size_cik,
+- .get_runlist_packet_size = pm_get_runlist_packet_size_vi,
+- .get_set_resources_packet_size = pm_get_set_resources_packet_size_vi,
+- .get_map_queues_packet_size = pm_get_map_queues_packet_size_vi,
+- .get_unmap_queues_packet_size = pm_get_unmap_queues_packet_size_vi,
+- .get_query_status_packet_size = pm_get_query_status_packet_size_vi,
+- .get_release_mem_packet_size = pm_get_release_mem_packet_size_vi,
+-};
+-
+-static struct packet_manager_funcs kfd_cik_scratch_pm_funcs = {
+- .map_process = pm_map_process_scratch_cik,
+- .runlist = pm_runlist_vi,
+- .set_resources = pm_set_resources_vi,
+- .map_queues = pm_map_queues_vi,
+- .unmap_queues = pm_unmap_queues_vi,
+- .query_status = pm_query_status_vi,
+- .release_mem = pm_release_mem_vi,
+- .get_map_process_packet_size =
+- pm_get_map_process_scratch_packet_size_cik,
+- .get_runlist_packet_size = pm_get_runlist_packet_size_vi,
+- .get_set_resources_packet_size = pm_get_set_resources_packet_size_vi,
+- .get_map_queues_packet_size = pm_get_map_queues_packet_size_vi,
+- .get_unmap_queues_packet_size = pm_get_unmap_queues_packet_size_vi,
+- .get_query_status_packet_size = pm_get_query_status_packet_size_vi,
+- .get_release_mem_packet_size = pm_get_release_mem_packet_size_vi,
+-};
+-
+-void kfd_pm_func_init_cik(struct packet_manager *pm, uint16_t fw_ver)
+-{
+- if (fw_ver >= KFD_SCRATCH_KV_FW_VER)
+- pm->pmf = &kfd_cik_scratch_pm_funcs;
+- else
+- pm->pmf = &kfd_cik_pm_funcs;
+-}
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_v9.c
+deleted file mode 100644
+index 5fe4f60..0000000
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_v9.c
++++ /dev/null
+@@ -1,377 +0,0 @@
+-/*
+- * Copyright 2016 Advanced Micro Devices, Inc.
+- *
+- * Permission is hereby granted, free of charge, to any person obtaining a
+- * copy of this software and associated documentation files (the "Software"),
+- * to deal in the Software without restriction, including without limitation
+- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+- * and/or sell copies of the Software, and to permit persons to whom the
+- * Software is furnished to do so, subject to the following conditions:
+- *
+- * The above copyright notice and this permission notice shall be included in
+- * all copies or substantial portions of the Software.
+- *
+- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+- * OTHER DEALINGS IN THE SOFTWARE.
+- *
+- */
+-
+-#include "kfd_kernel_queue.h"
+-#include "kfd_device_queue_manager.h"
+-#include "kfd_pm4_headers_ai.h"
+-#include "kfd_pm4_opcodes.h"
+-
+-static bool initialize_v9(struct kernel_queue *kq, struct kfd_dev *dev,
+- enum kfd_queue_type type, unsigned int queue_size);
+-static void uninitialize_v9(struct kernel_queue *kq);
+-static void submit_packet_v9(struct kernel_queue *kq);
+-
+-void kernel_queue_init_v9(struct kernel_queue_ops *ops)
+-{
+- ops->initialize = initialize_v9;
+- ops->uninitialize = uninitialize_v9;
+- ops->submit_packet = submit_packet_v9;
+-}
+-
+-static bool initialize_v9(struct kernel_queue *kq, struct kfd_dev *dev,
+- enum kfd_queue_type type, unsigned int queue_size)
+-{
+- int retval;
+-
+- retval = kfd_gtt_sa_allocate(dev, PAGE_SIZE, &kq->eop_mem);
+- if (retval != 0)
+- return false;
+-
+- kq->eop_gpu_addr = kq->eop_mem->gpu_addr;
+- kq->eop_kernel_addr = kq->eop_mem->cpu_ptr;
+-
+- memset(kq->eop_kernel_addr, 0, PAGE_SIZE);
+-
+- return true;
+-}
+-
+-static void uninitialize_v9(struct kernel_queue *kq)
+-{
+- kfd_gtt_sa_free(kq->dev, kq->eop_mem);
+-}
+-
+-static void submit_packet_v9(struct kernel_queue *kq)
+-{
+- *kq->wptr64_kernel = kq->pending_wptr64;
+- write_kernel_doorbell64(kq->queue->properties.doorbell_ptr,
+- kq->pending_wptr64);
+-}
+-
+-static int pm_map_process_v9(struct packet_manager *pm,
+- uint32_t *buffer, struct qcm_process_device *qpd)
+-{
+- struct pm4_mes_map_process *packet;
+- uint64_t vm_page_table_base_addr =
+- (uint64_t)(qpd->page_table_base) << 12;
+-
+- packet = (struct pm4_mes_map_process *)buffer;
+- memset(buffer, 0, sizeof(struct pm4_mes_map_process));
+-
+- packet->header.u32All = pm_build_pm4_header(IT_MAP_PROCESS,
+- sizeof(struct pm4_mes_map_process));
+- packet->bitfields2.diq_enable = (qpd->is_debug) ? 1 : 0;
+- packet->bitfields2.process_quantum = 1;
+- packet->bitfields2.pasid = qpd->pqm->process->pasid;
+- packet->bitfields14.gds_size = qpd->gds_size;
+- packet->bitfields14.num_gws = qpd->num_gws;
+- packet->bitfields14.num_oac = qpd->num_oac;
+- packet->bitfields14.sdma_enable = 1;
+- packet->bitfields14.num_queues = (qpd->is_debug) ? 0 : qpd->queue_count;
+-
+- packet->sh_mem_config = qpd->sh_mem_config;
+- packet->sh_mem_bases = qpd->sh_mem_bases;
+- packet->sq_shader_tba_lo = lower_32_bits(qpd->tba_addr >> 8);
+- packet->sq_shader_tba_hi = upper_32_bits(qpd->tba_addr >> 8);
+- packet->sq_shader_tma_lo = lower_32_bits(qpd->tma_addr >> 8);
+- packet->sq_shader_tma_hi = upper_32_bits(qpd->tma_addr >> 8);
+-
+- packet->gds_addr_lo = lower_32_bits(qpd->gds_context_area);
+- packet->gds_addr_hi = upper_32_bits(qpd->gds_context_area);
+-
+- packet->vm_context_page_table_base_addr_lo32 =
+- lower_32_bits(vm_page_table_base_addr);
+- packet->vm_context_page_table_base_addr_hi32 =
+- upper_32_bits(vm_page_table_base_addr);
+-
+- return 0;
+-}
+-
+-static int pm_runlist_v9(struct packet_manager *pm, uint32_t *buffer,
+- uint64_t ib, size_t ib_size_in_dwords, bool chain)
+-{
+- struct pm4_mes_runlist *packet;
+-
+- int concurrent_proc_cnt = 0;
+- struct kfd_dev *kfd = pm->dqm->dev;
+-
+- /* Determine the number of processes to map together to HW:
+- * it can not exceed the number of VMIDs available to the
+- * scheduler, and it is determined by the smaller of the number
+- * of processes in the runlist and kfd module parameter
+- * hws_max_conc_proc.
+- * Note: the arbitration between the number of VMIDs and
+- * hws_max_conc_proc has been done in
+- * kgd2kfd_device_init().
+- */
+- concurrent_proc_cnt = min(pm->dqm->processes_count,
+- kfd->max_proc_per_quantum);
+-
+-
+- packet = (struct pm4_mes_runlist *)buffer;
+-
+- memset(buffer, 0, sizeof(struct pm4_mes_runlist));
+- packet->header.u32All = pm_build_pm4_header(IT_RUN_LIST,
+- sizeof(struct pm4_mes_runlist));
+-
+- packet->bitfields4.ib_size = ib_size_in_dwords;
+- packet->bitfields4.chain = chain ? 1 : 0;
+- packet->bitfields4.offload_polling = 0;
+- packet->bitfields4.valid = 1;
+- packet->bitfields4.process_cnt = concurrent_proc_cnt;
+- packet->ordinal2 = lower_32_bits(ib);
+- packet->ib_base_hi = upper_32_bits(ib);
+-
+- return 0;
+-}
+-
+-static int pm_map_queues_v9(struct packet_manager *pm, uint32_t *buffer,
+- struct queue *q, bool is_static)
+-{
+- struct pm4_mes_map_queues *packet;
+- bool use_static = is_static;
+-
+- packet = (struct pm4_mes_map_queues *)buffer;
+- memset(buffer, 0, sizeof(struct pm4_mes_map_queues));
+-
+- packet->header.u32All = pm_build_pm4_header(IT_MAP_QUEUES,
+- sizeof(struct pm4_mes_map_queues));
+- packet->bitfields2.alloc_format =
+- alloc_format__mes_map_queues__one_per_pipe_vi;
+- packet->bitfields2.num_queues = 1;
+- packet->bitfields2.queue_sel =
+- queue_sel__mes_map_queues__map_to_hws_determined_queue_slots_vi;
+-
+- packet->bitfields2.engine_sel =
+- engine_sel__mes_map_queues__compute_vi;
+- packet->bitfields2.queue_type =
+- queue_type__mes_map_queues__normal_compute_vi;
+-
+- switch (q->properties.type) {
+- case KFD_QUEUE_TYPE_COMPUTE:
+- if (use_static)
+- packet->bitfields2.queue_type =
+- queue_type__mes_map_queues__normal_latency_static_queue_vi;
+- break;
+- case KFD_QUEUE_TYPE_DIQ:
+- packet->bitfields2.queue_type =
+- queue_type__mes_map_queues__debug_interface_queue_vi;
+- break;
+- case KFD_QUEUE_TYPE_SDMA:
+- packet->bitfields2.engine_sel = q->properties.sdma_engine_id +
+- engine_sel__mes_map_queues__sdma0_vi;
+- use_static = false; /* no static queues under SDMA */
+- break;
+- default:
+- WARN(1, "queue type %d", q->properties.type);
+- return -EINVAL;
+- }
+- packet->bitfields3.doorbell_offset =
+- q->properties.doorbell_off;
+-
+- packet->mqd_addr_lo =
+- lower_32_bits(q->gart_mqd_addr);
+-
+- packet->mqd_addr_hi =
+- upper_32_bits(q->gart_mqd_addr);
+-
+- packet->wptr_addr_lo =
+- lower_32_bits((uint64_t)q->properties.write_ptr);
+-
+- packet->wptr_addr_hi =
+- upper_32_bits((uint64_t)q->properties.write_ptr);
+-
+- return 0;
+-}
+-
+-static int pm_unmap_queues_v9(struct packet_manager *pm, uint32_t *buffer,
+- enum kfd_queue_type type,
+- enum kfd_unmap_queues_filter filter,
+- uint32_t filter_param, bool reset,
+- unsigned int sdma_engine)
+-{
+- struct pm4_mes_unmap_queues *packet;
+-
+- packet = (struct pm4_mes_unmap_queues *)buffer;
+- memset(buffer, 0, sizeof(struct pm4_mes_unmap_queues));
+-
+- packet->header.u32All = pm_build_pm4_header(IT_UNMAP_QUEUES,
+- sizeof(struct pm4_mes_unmap_queues));
+- switch (type) {
+- case KFD_QUEUE_TYPE_COMPUTE:
+- case KFD_QUEUE_TYPE_DIQ:
+- packet->bitfields2.engine_sel =
+- engine_sel__mes_unmap_queues__compute;
+- break;
+- case KFD_QUEUE_TYPE_SDMA:
+- packet->bitfields2.engine_sel =
+- engine_sel__mes_unmap_queues__sdma0 + sdma_engine;
+- break;
+- default:
+- WARN(1, "queue type %d", type);
+- return -EINVAL;
+- }
+-
+- if (reset)
+- packet->bitfields2.action =
+- action__mes_unmap_queues__reset_queues;
+- else
+- packet->bitfields2.action =
+- action__mes_unmap_queues__preempt_queues;
+-
+- switch (filter) {
+- case KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE:
+- packet->bitfields2.queue_sel =
+- queue_sel__mes_unmap_queues__perform_request_on_specified_queues;
+- packet->bitfields2.num_queues = 1;
+- packet->bitfields3b.doorbell_offset0 = filter_param;
+- break;
+- case KFD_UNMAP_QUEUES_FILTER_BY_PASID:
+- packet->bitfields2.queue_sel =
+- queue_sel__mes_unmap_queues__perform_request_on_pasid_queues;
+- packet->bitfields3a.pasid = filter_param;
+- break;
+- case KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES:
+- packet->bitfields2.queue_sel =
+- queue_sel__mes_unmap_queues__unmap_all_queues;
+- break;
+- case KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES:
+- /* in this case, we do not preempt static queues */
+- packet->bitfields2.queue_sel =
+- queue_sel__mes_unmap_queues__unmap_all_non_static_queues;
+- break;
+- default:
+- WARN(1, "filter %d", filter);
+- return -EINVAL;
+- }
+-
+- return 0;
+-
+-}
+-
+-static int pm_query_status_v9(struct packet_manager *pm, uint32_t *buffer,
+- uint64_t fence_address, uint32_t fence_value)
+-{
+- struct pm4_mes_query_status *packet;
+-
+- packet = (struct pm4_mes_query_status *)buffer;
+- memset(buffer, 0, sizeof(struct pm4_mes_query_status));
+-
+-
+- packet->header.u32All = pm_build_pm4_header(IT_QUERY_STATUS,
+- sizeof(struct pm4_mes_query_status));
+-
+- packet->bitfields2.context_id = 0;
+- packet->bitfields2.interrupt_sel =
+- interrupt_sel__mes_query_status__completion_status;
+- packet->bitfields2.command =
+- command__mes_query_status__fence_only_after_write_ack;
+-
+- packet->addr_hi = upper_32_bits((uint64_t)fence_address);
+- packet->addr_lo = lower_32_bits((uint64_t)fence_address);
+- packet->data_hi = upper_32_bits((uint64_t)fence_value);
+- packet->data_lo = lower_32_bits((uint64_t)fence_value);
+-
+- return 0;
+-}
+-
+-
+-static uint32_t pm_release_mem_v9(uint64_t gpu_addr, uint32_t *buffer)
+-{
+- struct pm4_mec_release_mem *packet;
+-
+- packet = (struct pm4_mec_release_mem *)buffer;
+- memset(buffer, 0, sizeof(struct pm4_mec_release_mem));
+-
+- packet->header.u32All = pm_build_pm4_header(IT_RELEASE_MEM,
+- sizeof(struct pm4_mec_release_mem));
+-
+- packet->bitfields2.event_type = CACHE_FLUSH_AND_INV_TS_EVENT;
+- packet->bitfields2.event_index = event_index__mec_release_mem__end_of_pipe;
+- packet->bitfields2.tcl1_action_ena = 1;
+- packet->bitfields2.tc_action_ena = 1;
+- packet->bitfields2.cache_policy = cache_policy__mec_release_mem__lru;
+-
+- packet->bitfields3.data_sel = data_sel__mec_release_mem__send_32_bit_low;
+- packet->bitfields3.int_sel =
+- int_sel__mec_release_mem__send_interrupt_after_write_confirm;
+-
+- packet->bitfields4.address_lo_32b = (gpu_addr & 0xffffffff) >> 2;
+- packet->address_hi = upper_32_bits(gpu_addr);
+-
+- packet->data_lo = 0;
+-
+- return sizeof(struct pm4_mec_release_mem) / sizeof(unsigned int);
+-}
+-
+-static uint32_t pm_get_map_process_packet_size_v9(void)
+-{
+- return sizeof(struct pm4_mes_map_process);
+-}
+-
+-static uint32_t pm_get_runlist_packet_size_v9(void)
+-{
+- return sizeof(struct pm4_mes_runlist);
+-}
+-
+-static uint32_t pm_get_map_queues_packet_size_v9(void)
+-{
+- return sizeof(struct pm4_mes_map_queues);
+-}
+-
+-static uint32_t pm_get_unmap_queues_packet_size_v9(void)
+-{
+- return sizeof(struct pm4_mes_unmap_queues);
+-}
+-
+-static uint32_t pm_get_query_status_packet_size_v9(void)
+-{
+- return sizeof(struct pm4_mes_query_status);
+-}
+-
+-static uint32_t pm_get_release_mem_packet_size_v9(void)
+-{
+- return sizeof(struct pm4_mec_release_mem);
+-}
+-
+-static struct packet_manager_funcs kfd_v9_pm_funcs = {
+- .map_process = pm_map_process_v9,
+- .runlist = pm_runlist_v9,
+- .set_resources = pm_set_resources_vi,
+- .map_queues = pm_map_queues_v9,
+- .unmap_queues = pm_unmap_queues_v9,
+- .query_status = pm_query_status_v9,
+- .release_mem = pm_release_mem_v9,
+- .get_map_process_packet_size = pm_get_map_process_packet_size_v9,
+- .get_runlist_packet_size = pm_get_runlist_packet_size_v9,
+- .get_set_resources_packet_size = pm_get_set_resources_packet_size_vi,
+- .get_map_queues_packet_size = pm_get_map_queues_packet_size_v9,
+- .get_unmap_queues_packet_size = pm_get_unmap_queues_packet_size_v9,
+- .get_query_status_packet_size = pm_get_query_status_packet_size_v9,
+- .get_release_mem_packet_size = pm_get_release_mem_packet_size_v9,
+-};
+-
+-void kfd_pm_func_init_v9(struct packet_manager *pm, uint16_t fw_ver)
+-{
+- pm->pmf = &kfd_v9_pm_funcs;
+-}
+-
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_vi.c
+index ecf4a33..f1d4828 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_vi.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_vi.c
+@@ -22,20 +22,15 @@
+ */
+
+ #include "kfd_kernel_queue.h"
+-#include "kfd_device_queue_manager.h"
+-#include "kfd_pm4_headers_vi.h"
+-#include "kfd_pm4_opcodes.h"
+
+ static bool initialize_vi(struct kernel_queue *kq, struct kfd_dev *dev,
+ enum kfd_queue_type type, unsigned int queue_size);
+ static void uninitialize_vi(struct kernel_queue *kq);
+-static void submit_packet_vi(struct kernel_queue *kq);
+
+ void kernel_queue_init_vi(struct kernel_queue_ops *ops)
+ {
+ ops->initialize = initialize_vi;
+ ops->uninitialize = uninitialize_vi;
+- ops->submit_packet = submit_packet_vi;
+ }
+
+ static bool initialize_vi(struct kernel_queue *kq, struct kfd_dev *dev,
+@@ -59,359 +54,3 @@ static void uninitialize_vi(struct kernel_queue *kq)
+ {
+ kfd_gtt_sa_free(kq->dev, kq->eop_mem);
+ }
+-
+-static void submit_packet_vi(struct kernel_queue *kq)
+-{
+- *kq->wptr_kernel = kq->pending_wptr;
+- write_kernel_doorbell(kq->queue->properties.doorbell_ptr,
+- kq->pending_wptr);
+-}
+-
+-static int pm_map_process_vi(struct packet_manager *pm,
+- uint32_t *buffer, struct qcm_process_device *qpd)
+-{
+- struct pm4_mes_map_process *packet;
+-
+- packet = (struct pm4_mes_map_process *)buffer;
+- memset(buffer, 0, sizeof(struct pm4_mes_map_process));
+-
+- packet->header.u32All = pm_build_pm4_header(IT_MAP_PROCESS,
+- sizeof(struct pm4_mes_map_process));
+- packet->bitfields2.diq_enable = (qpd->is_debug) ? 1 : 0;
+- packet->bitfields2.process_quantum = 1;
+- packet->bitfields2.pasid = qpd->pqm->process->pasid;
+- packet->bitfields3.page_table_base = qpd->page_table_base;
+- packet->bitfields10.gds_size = qpd->gds_size;
+- packet->bitfields10.num_gws = qpd->num_gws;
+- packet->bitfields10.num_oac = qpd->num_oac;
+- packet->bitfields10.num_queues = (qpd->is_debug) ? 0 : qpd->queue_count;
+-
+- packet->sh_mem_config = qpd->sh_mem_config;
+- packet->sh_mem_bases = qpd->sh_mem_bases;
+- packet->sh_mem_ape1_base = qpd->sh_mem_ape1_base;
+- packet->sh_mem_ape1_limit = qpd->sh_mem_ape1_limit;
+-
+- packet->sh_hidden_private_base_vmid = qpd->sh_hidden_private_base;
+-
+- packet->gds_addr_lo = lower_32_bits(qpd->gds_context_area);
+- packet->gds_addr_hi = upper_32_bits(qpd->gds_context_area);
+-
+- return 0;
+-}
+-
+-
+-unsigned int pm_build_pm4_header(unsigned int opcode, size_t packet_size)
+-{
+- union PM4_MES_TYPE_3_HEADER header;
+-
+- header.u32All = 0;
+- header.opcode = opcode;
+- header.count = packet_size/sizeof(uint32_t) - 2;
+- header.type = PM4_TYPE_3;
+-
+- return header.u32All;
+-}
+-
+-int pm_runlist_vi(struct packet_manager *pm, uint32_t *buffer,
+- uint64_t ib, size_t ib_size_in_dwords, bool chain)
+-{
+- struct pm4_mes_runlist *packet;
+-
+- int concurrent_proc_cnt = 0;
+- struct kfd_dev *kfd = pm->dqm->dev;
+-
+- /* Determine the number of processes to map together to HW:
+- * it can not exceed the number of VMIDs available to the
+- * scheduler, and it is determined by the smaller of the number
+- * of processes in the runlist and kfd module parameter
+- * hws_max_conc_proc.
+- * Note: the arbitration between the number of VMIDs and
+- * hws_max_conc_proc has been done in
+- * kgd2kfd_device_init().
+- */
+- concurrent_proc_cnt = min(pm->dqm->processes_count,
+- kfd->max_proc_per_quantum);
+-
+-
+- packet = (struct pm4_mes_runlist *)buffer;
+-
+- memset(buffer, 0, sizeof(struct pm4_mes_runlist));
+- packet->header.u32All = pm_build_pm4_header(IT_RUN_LIST,
+- sizeof(struct pm4_mes_runlist));
+-
+- packet->bitfields4.ib_size = ib_size_in_dwords;
+- packet->bitfields4.chain = chain ? 1 : 0;
+- packet->bitfields4.offload_polling = 0;
+- packet->bitfields4.valid = 1;
+- packet->bitfields4.process_cnt = concurrent_proc_cnt;
+- packet->ordinal2 = lower_32_bits(ib);
+- packet->bitfields3.ib_base_hi = upper_32_bits(ib);
+-
+- return 0;
+-}
+-
+-int pm_map_queues_vi(struct packet_manager *pm, uint32_t *buffer,
+- struct queue *q, bool is_static)
+-{
+- struct pm4_mes_map_queues *packet;
+- bool use_static = is_static;
+-
+- packet = (struct pm4_mes_map_queues *)buffer;
+- memset(buffer, 0, sizeof(struct pm4_mes_map_queues));
+-
+- packet->header.u32All = pm_build_pm4_header(IT_MAP_QUEUES,
+- sizeof(struct pm4_mes_map_queues));
+- packet->bitfields2.alloc_format =
+- alloc_format__mes_map_queues__one_per_pipe_vi;
+- packet->bitfields2.num_queues = 1;
+- packet->bitfields2.queue_sel =
+- queue_sel__mes_map_queues__map_to_hws_determined_queue_slots_vi;
+-
+- packet->bitfields2.engine_sel =
+- engine_sel__mes_map_queues__compute_vi;
+- packet->bitfields2.queue_type =
+- queue_type__mes_map_queues__normal_compute_vi;
+-
+- switch (q->properties.type) {
+- case KFD_QUEUE_TYPE_COMPUTE:
+- if (use_static)
+- packet->bitfields2.queue_type =
+- queue_type__mes_map_queues__normal_latency_static_queue_vi;
+- break;
+- case KFD_QUEUE_TYPE_DIQ:
+- packet->bitfields2.queue_type =
+- queue_type__mes_map_queues__debug_interface_queue_vi;
+- break;
+- case KFD_QUEUE_TYPE_SDMA:
+- packet->bitfields2.engine_sel = q->properties.sdma_engine_id +
+- engine_sel__mes_map_queues__sdma0_vi;
+- use_static = false; /* no static queues under SDMA */
+- break;
+- default:
+- WARN(1, "queue type %d", q->properties.type);
+- return -EINVAL;
+- }
+- packet->bitfields3.doorbell_offset =
+- q->properties.doorbell_off;
+-
+- packet->mqd_addr_lo =
+- lower_32_bits(q->gart_mqd_addr);
+-
+- packet->mqd_addr_hi =
+- upper_32_bits(q->gart_mqd_addr);
+-
+- packet->wptr_addr_lo =
+- lower_32_bits((uint64_t)q->properties.write_ptr);
+-
+- packet->wptr_addr_hi =
+- upper_32_bits((uint64_t)q->properties.write_ptr);
+-
+- return 0;
+-}
+-
+-int pm_set_resources_vi(struct packet_manager *pm, uint32_t *buffer,
+- struct scheduling_resources *res)
+-{
+- struct pm4_mes_set_resources *packet;
+-
+- packet = (struct pm4_mes_set_resources *)buffer;
+- memset(buffer, 0, sizeof(struct pm4_mes_set_resources));
+-
+- packet->header.u32All = pm_build_pm4_header(IT_SET_RESOURCES,
+- sizeof(struct pm4_mes_set_resources));
+-
+- packet->bitfields2.queue_type =
+- queue_type__mes_set_resources__hsa_interface_queue_hiq;
+- packet->bitfields2.vmid_mask = res->vmid_mask;
+- packet->bitfields2.unmap_latency = KFD_UNMAP_LATENCY_MS / 100;
+- packet->bitfields7.oac_mask = res->oac_mask;
+- packet->bitfields8.gds_heap_base = res->gds_heap_base;
+- packet->bitfields8.gds_heap_size = res->gds_heap_size;
+-
+- packet->gws_mask_lo = lower_32_bits(res->gws_mask);
+- packet->gws_mask_hi = upper_32_bits(res->gws_mask);
+-
+- packet->queue_mask_lo = lower_32_bits(res->queue_mask);
+- packet->queue_mask_hi = upper_32_bits(res->queue_mask);
+-
+- return 0;
+-}
+-
+-int pm_unmap_queues_vi(struct packet_manager *pm, uint32_t *buffer,
+- enum kfd_queue_type type,
+- enum kfd_unmap_queues_filter filter,
+- uint32_t filter_param, bool reset,
+- unsigned int sdma_engine)
+-{
+- struct pm4_mes_unmap_queues *packet;
+-
+- packet = (struct pm4_mes_unmap_queues *)buffer;
+- memset(buffer, 0, sizeof(struct pm4_mes_unmap_queues));
+-
+- packet->header.u32All = pm_build_pm4_header(IT_UNMAP_QUEUES,
+- sizeof(struct pm4_mes_unmap_queues));
+- switch (type) {
+- case KFD_QUEUE_TYPE_COMPUTE:
+- case KFD_QUEUE_TYPE_DIQ:
+- packet->bitfields2.engine_sel =
+- engine_sel__mes_unmap_queues__compute;
+- break;
+- case KFD_QUEUE_TYPE_SDMA:
+- packet->bitfields2.engine_sel =
+- engine_sel__mes_unmap_queues__sdma0 + sdma_engine;
+- break;
+- default:
+- WARN(1, "queue type %d", type);
+- return -EINVAL;
+- }
+-
+- if (reset)
+- packet->bitfields2.action =
+- action__mes_unmap_queues__reset_queues;
+- else
+- packet->bitfields2.action =
+- action__mes_unmap_queues__preempt_queues;
+-
+- switch (filter) {
+- case KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE:
+- packet->bitfields2.queue_sel =
+- queue_sel__mes_unmap_queues__perform_request_on_specified_queues;
+- packet->bitfields2.num_queues = 1;
+- packet->bitfields3b.doorbell_offset0 = filter_param;
+- break;
+- case KFD_UNMAP_QUEUES_FILTER_BY_PASID:
+- packet->bitfields2.queue_sel =
+- queue_sel__mes_unmap_queues__perform_request_on_pasid_queues;
+- packet->bitfields3a.pasid = filter_param;
+- break;
+- case KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES:
+- packet->bitfields2.queue_sel =
+- queue_sel__mes_unmap_queues__unmap_all_queues;
+- break;
+- case KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES:
+- /* in this case, we do not preempt static queues */
+- packet->bitfields2.queue_sel =
+- queue_sel__mes_unmap_queues__unmap_all_non_static_queues;
+- break;
+- default:
+- WARN(1, "filter %d", filter);
+- return -EINVAL;
+- }
+-
+- return 0;
+-
+-}
+-
+-int pm_query_status_vi(struct packet_manager *pm, uint32_t *buffer,
+- uint64_t fence_address, uint32_t fence_value)
+-{
+- struct pm4_mes_query_status *packet;
+-
+- packet = (struct pm4_mes_query_status *)buffer;
+- memset(buffer, 0, sizeof(struct pm4_mes_query_status));
+-
+-
+- packet->header.u32All = pm_build_pm4_header(IT_QUERY_STATUS,
+- sizeof(struct pm4_mes_query_status));
+-
+- packet->bitfields2.context_id = 0;
+- packet->bitfields2.interrupt_sel =
+- interrupt_sel__mes_query_status__completion_status;
+- packet->bitfields2.command =
+- command__mes_query_status__fence_only_after_write_ack;
+-
+- packet->addr_hi = upper_32_bits((uint64_t)fence_address);
+- packet->addr_lo = lower_32_bits((uint64_t)fence_address);
+- packet->data_hi = upper_32_bits((uint64_t)fence_value);
+- packet->data_lo = lower_32_bits((uint64_t)fence_value);
+-
+- return 0;
+-}
+-
+-
+-uint32_t pm_release_mem_vi(uint64_t gpu_addr, uint32_t *buffer)
+-{
+- struct pm4_mec_release_mem *packet;
+-
+- packet = (struct pm4_mec_release_mem *)buffer;
+- memset(buffer, 0, sizeof(struct pm4_mec_release_mem));
+-
+- packet->header.u32All = pm_build_pm4_header(IT_RELEASE_MEM,
+- sizeof(struct pm4_mec_release_mem));
+-
+- packet->bitfields2.event_type = CACHE_FLUSH_AND_INV_TS_EVENT;
+- packet->bitfields2.event_index = event_index___release_mem__end_of_pipe;
+- packet->bitfields2.tcl1_action_ena = 1;
+- packet->bitfields2.tc_action_ena = 1;
+- packet->bitfields2.cache_policy = cache_policy___release_mem__lru;
+- packet->bitfields2.atc = 0;
+-
+- packet->bitfields3.data_sel = data_sel___release_mem__send_32_bit_low;
+- packet->bitfields3.int_sel =
+- int_sel___release_mem__send_interrupt_after_write_confirm;
+-
+- packet->bitfields4.address_lo_32b = (gpu_addr & 0xffffffff) >> 2;
+- packet->address_hi = upper_32_bits(gpu_addr);
+-
+- packet->data_lo = 0;
+-
+- return sizeof(struct pm4_mec_release_mem) / sizeof(unsigned int);
+-}
+-
+-uint32_t pm_get_map_process_packet_size_vi(void)
+-{
+- return sizeof(struct pm4_mes_map_process);
+-}
+-
+-uint32_t pm_get_runlist_packet_size_vi(void)
+-{
+- return sizeof(struct pm4_mes_runlist);
+-}
+-
+-uint32_t pm_get_set_resources_packet_size_vi(void)
+-{
+- return sizeof(struct pm4_mes_set_resources);
+-}
+-
+-uint32_t pm_get_map_queues_packet_size_vi(void)
+-{
+- return sizeof(struct pm4_mes_map_queues);
+-}
+-
+-uint32_t pm_get_unmap_queues_packet_size_vi(void)
+-{
+- return sizeof(struct pm4_mes_unmap_queues);
+-}
+-
+-uint32_t pm_get_query_status_packet_size_vi(void)
+-{
+- return sizeof(struct pm4_mes_query_status);
+-}
+-
+-uint32_t pm_get_release_mem_packet_size_vi(void)
+-{
+- return sizeof(struct pm4_mec_release_mem);
+-}
+-
+-
+-static struct packet_manager_funcs kfd_vi_pm_funcs = {
+- .map_process = pm_map_process_vi,
+- .runlist = pm_runlist_vi,
+- .set_resources = pm_set_resources_vi,
+- .map_queues = pm_map_queues_vi,
+- .unmap_queues = pm_unmap_queues_vi,
+- .query_status = pm_query_status_vi,
+- .release_mem = pm_release_mem_vi,
+- .get_map_process_packet_size = pm_get_map_process_packet_size_vi,
+- .get_runlist_packet_size = pm_get_runlist_packet_size_vi,
+- .get_set_resources_packet_size = pm_get_set_resources_packet_size_vi,
+- .get_map_queues_packet_size = pm_get_map_queues_packet_size_vi,
+- .get_unmap_queues_packet_size = pm_get_unmap_queues_packet_size_vi,
+- .get_query_status_packet_size = pm_get_query_status_packet_size_vi,
+- .get_release_mem_packet_size = pm_get_release_mem_packet_size_vi,
+-};
+-
+-void kfd_pm_func_init_vi(struct packet_manager *pm, uint16_t fw_ver)
+-{
+- pm->pmf = &kfd_vi_pm_funcs;
+-}
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_module.c b/drivers/gpu/drm/amd/amdkfd/kfd_module.c
+index ba4d5de..850a562 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_module.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_module.c
+@@ -29,10 +29,10 @@
+ #define KFD_DRIVER_AUTHOR "AMD Inc. and others"
+
+ #define KFD_DRIVER_DESC "Standalone HSA driver for AMD's GPUs"
+-#define KFD_DRIVER_DATE "20160408"
+-#define KFD_DRIVER_MAJOR 2
+-#define KFD_DRIVER_MINOR 0
+-#define KFD_DRIVER_PATCHLEVEL 0
++#define KFD_DRIVER_DATE "20150421"
++#define KFD_DRIVER_MAJOR 0
++#define KFD_DRIVER_MINOR 7
++#define KFD_DRIVER_PATCHLEVEL 2
+
+ static const struct kgd2kfd_calls kgd2kfd = {
+ .exit = kgd2kfd_exit,
+@@ -42,10 +42,6 @@ static const struct kgd2kfd_calls kgd2kfd = {
+ .interrupt = kgd2kfd_interrupt,
+ .suspend = kgd2kfd_suspend,
+ .resume = kgd2kfd_resume,
+- .quiesce_mm = kgd2kfd_quiesce_mm,
+- .resume_mm = kgd2kfd_resume_mm,
+- .schedule_evict_and_restore_process =
+- kgd2kfd_schedule_evict_and_restore_process,
+ };
+
+ int sched_policy = KFD_SCHED_POLICY_HWS;
+@@ -53,15 +49,6 @@ module_param(sched_policy, int, 0444);
+ MODULE_PARM_DESC(sched_policy,
+ "Scheduling policy (0 = HWS (Default), 1 = HWS without over-subscription, 2 = Non-HWS (Used for debugging only)");
+
+-int hws_max_conc_proc = 8;
+-module_param(hws_max_conc_proc, int, 0444);
+-MODULE_PARM_DESC(hws_max_conc_proc,
+- "Max # processes HWS can execute concurrently when sched_policy=0 (0 = no concurrency, #VMIDs for KFD = Maximum(default))");
+-
+-int cwsr_enable = 1;
+-module_param(cwsr_enable, int, 0444);
+-MODULE_PARM_DESC(cwsr_enable, "CWSR enable (0 = Off, 1 = On (Default))");
+-
+ int max_num_of_queues_per_device = KFD_MAX_NUM_OF_QUEUES_PER_DEVICE_DEFAULT;
+ module_param(max_num_of_queues_per_device, int, 0444);
+ MODULE_PARM_DESC(max_num_of_queues_per_device,
+@@ -74,28 +61,7 @@ MODULE_PARM_DESC(send_sigterm,
+
+ static int amdkfd_init_completed;
+
+-int debug_largebar;
+-module_param(debug_largebar, int, 0444);
+-MODULE_PARM_DESC(debug_largebar,
+- "Debug large-bar flag used to simulate large-bar capability on non-large bar machine (0 = disable, 1 = enable)");
+-
+-int ignore_crat;
+-module_param(ignore_crat, int, 0444);
+-MODULE_PARM_DESC(ignore_crat,
+- "Ignore CRAT table during KFD initialization (0 = use CRAT (default), 1 = ignore CRAT)");
+-
+-int vega10_noretry;
+-module_param_named(noretry, vega10_noretry, int, 0644);
+-MODULE_PARM_DESC(noretry,
+- "Set sh_mem_config.retry_disable on Vega10 (0 = retry enabled (default), 1 = retry disabled)");
+-
+-int priv_cp_queues;
+-module_param(priv_cp_queues, int, 0644);
+-MODULE_PARM_DESC(priv_cp_queues,
+- "Enable privileged mode for CP queues (0 = off (default), 1 = on)");
+-
+-int kgd2kfd_init(unsigned int interface_version,
+- const struct kgd2kfd_calls **g2f)
++int kgd2kfd_init(unsigned interface_version, const struct kgd2kfd_calls **g2f)
+ {
+ if (!amdkfd_init_completed)
+ return -EPROBE_DEFER;
+@@ -124,7 +90,7 @@ static int __init kfd_module_init(void)
+ /* Verify module parameters */
+ if ((sched_policy < KFD_SCHED_POLICY_HWS) ||
+ (sched_policy > KFD_SCHED_POLICY_NO_HWS)) {
+- pr_err("sched_policy has invalid value\n");
++ pr_err("kfd: sched_policy has invalid value\n");
+ return -1;
+ }
+
+@@ -132,13 +98,13 @@ static int __init kfd_module_init(void)
+ if ((max_num_of_queues_per_device < 1) ||
+ (max_num_of_queues_per_device >
+ KFD_MAX_NUM_OF_QUEUES_PER_DEVICE)) {
+- pr_err("max_num_of_queues_per_device must be between 1 to KFD_MAX_NUM_OF_QUEUES_PER_DEVICE\n");
++ pr_err("kfd: max_num_of_queues_per_device must be between 1 to KFD_MAX_NUM_OF_QUEUES_PER_DEVICE\n");
+ return -1;
+ }
+
+ err = kfd_pasid_init();
+ if (err < 0)
+- return err;
++ goto err_pasid;
+
+ err = kfd_chardev_init();
+ if (err < 0)
+@@ -148,16 +114,8 @@ static int __init kfd_module_init(void)
+ if (err < 0)
+ goto err_topology;
+
+- err = kfd_ipc_init();
+- if (err < 0)
+- goto err_topology;
+-
+ kfd_process_create_wq();
+
+- kfd_init_peer_direct();
+-
+- kfd_debugfs_init();
+-
+ amdkfd_init_completed = 1;
+
+ dev_info(kfd_device, "Initialized module\n");
+@@ -168,6 +126,7 @@ static int __init kfd_module_init(void)
+ kfd_chardev_exit();
+ err_ioctl:
+ kfd_pasid_exit();
++err_pasid:
+ return err;
+ }
+
+@@ -175,8 +134,6 @@ static void __exit kfd_module_exit(void)
+ {
+ amdkfd_init_completed = 0;
+
+- kfd_debugfs_fini();
+- kfd_close_peer_direct();
+ kfd_process_destroy_wq();
+ kfd_topology_shutdown();
+ kfd_chardev_exit();
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
+index 9eb2d54..b1ef136 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
+@@ -23,68 +23,14 @@
+
+ #include "kfd_priv.h"
+
+-/* Mapping queue priority to pipe priority, indexed by queue priority */
+-int pipe_priority_map[] = {
+- KFD_PIPE_PRIORITY_CS_LOW,
+- KFD_PIPE_PRIORITY_CS_LOW,
+- KFD_PIPE_PRIORITY_CS_LOW,
+- KFD_PIPE_PRIORITY_CS_LOW,
+- KFD_PIPE_PRIORITY_CS_LOW,
+- KFD_PIPE_PRIORITY_CS_LOW,
+- KFD_PIPE_PRIORITY_CS_LOW,
+- KFD_PIPE_PRIORITY_CS_MEDIUM,
+- KFD_PIPE_PRIORITY_CS_MEDIUM,
+- KFD_PIPE_PRIORITY_CS_MEDIUM,
+- KFD_PIPE_PRIORITY_CS_MEDIUM,
+- KFD_PIPE_PRIORITY_CS_HIGH,
+- KFD_PIPE_PRIORITY_CS_HIGH,
+- KFD_PIPE_PRIORITY_CS_HIGH,
+- KFD_PIPE_PRIORITY_CS_HIGH,
+- KFD_PIPE_PRIORITY_CS_HIGH
+-};
+-
+-/* Mapping queue priority to SPI priority, indexed by queue priority
+- * SPI priority 2 and 3 are reserved for trap handler context save
+- */
+-int spi_priority_map[] = {
+- KFD_SPI_PRIORITY_EXTRA_LOW,
+- KFD_SPI_PRIORITY_EXTRA_LOW,
+- KFD_SPI_PRIORITY_EXTRA_LOW,
+- KFD_SPI_PRIORITY_EXTRA_LOW,
+- KFD_SPI_PRIORITY_EXTRA_LOW,
+- KFD_SPI_PRIORITY_EXTRA_LOW,
+- KFD_SPI_PRIORITY_EXTRA_LOW,
+- KFD_SPI_PRIORITY_EXTRA_LOW,
+- KFD_SPI_PRIORITY_LOW,
+- KFD_SPI_PRIORITY_LOW,
+- KFD_SPI_PRIORITY_LOW,
+- KFD_SPI_PRIORITY_LOW,
+- KFD_SPI_PRIORITY_LOW,
+- KFD_SPI_PRIORITY_LOW,
+- KFD_SPI_PRIORITY_LOW,
+- KFD_SPI_PRIORITY_LOW
+-};
+-
+ struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type,
+ struct kfd_dev *dev)
+ {
+ switch (dev->device_info->asic_family) {
+ case CHIP_KAVERI:
+ return mqd_manager_init_cik(type, dev);
+- case CHIP_HAWAII:
+- return mqd_manager_init_cik_hawaii(type, dev);
+ case CHIP_CARRIZO:
+ return mqd_manager_init_vi(type, dev);
+- case CHIP_TONGA:
+- case CHIP_FIJI:
+- case CHIP_POLARIS10:
+- case CHIP_POLARIS11:
+- return mqd_manager_init_vi_tonga(type, dev);
+- case CHIP_VEGA10:
+- case CHIP_RAVEN:
+- return mqd_manager_init_v9(type, dev);
+- default:
+- BUG();
+ }
+
+ return NULL;
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
+index dcaeda8..213a71e 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
+@@ -43,9 +43,6 @@
+ *
+ * @is_occupied: Checks if the relevant HQD slot is occupied.
+ *
+- * @get_wave_state: Retrieves context save state and optionally copies the
+- * control stack, if kept in the MQD, to the given userspace address.
+- *
+ * @mqd_mutex: Mqd manager mutex.
+ *
+ * @dev: The kfd device structure coupled with this module.
+@@ -62,8 +59,7 @@
+ * per KFD_MQD_TYPE for each device.
+ *
+ */
+-extern int pipe_priority_map[];
+-extern int spi_priority_map[];
++
+ struct mqd_manager {
+ int (*init_mqd)(struct mqd_manager *mm, void **mqd,
+ struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
+@@ -71,8 +67,7 @@ struct mqd_manager {
+
+ int (*load_mqd)(struct mqd_manager *mm, void *mqd,
+ uint32_t pipe_id, uint32_t queue_id,
+- struct queue_properties *p,
+- struct mm_struct *mms);
++ uint32_t __user *wptr);
+
+ int (*update_mqd)(struct mqd_manager *mm, void *mqd,
+ struct queue_properties *q);
+@@ -89,15 +84,6 @@ struct mqd_manager {
+ uint64_t queue_address, uint32_t pipe_id,
+ uint32_t queue_id);
+
+- int (*get_wave_state)(struct mqd_manager *mm, void *mqd,
+- void __user *ctl_stack,
+- u32 *ctl_stack_used_size,
+- u32 *save_area_used_size);
+-
+-#if defined(CONFIG_DEBUG_FS)
+- int (*debugfs_show_mqd)(struct seq_file *m, void *data);
+-#endif
+-
+ struct mutex mqd_mutex;
+ struct kfd_dev *dev;
+ };
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
+index 5724d33..6acc431 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
+@@ -30,80 +30,12 @@
+ #include "cik_regs.h"
+ #include "cik_structs.h"
+ #include "oss/oss_2_4_sh_mask.h"
+-#include "gca/gfx_7_2_sh_mask.h"
+
+ static inline struct cik_mqd *get_mqd(void *mqd)
+ {
+ return (struct cik_mqd *)mqd;
+ }
+
+-static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
+-{
+- return (struct cik_sdma_rlc_registers *)mqd;
+-}
+-
+-static void update_cu_mask(struct mqd_manager *mm, void *mqd,
+- struct queue_properties *q)
+-{
+- struct cik_mqd *m;
+- struct kfd_cu_info cu_info;
+- uint32_t se_mask[4] = {0}; /* 4 is the max # of SEs */
+- uint32_t cu_mask_count = q->cu_mask_count;
+- const uint32_t *cu_mask = q->cu_mask;
+- int se, cu_per_sh, cu_index, i;
+-
+- if (cu_mask_count == 0)
+- return;
+-
+- m = get_mqd(mqd);
+- m->compute_static_thread_mgmt_se0 = 0;
+- m->compute_static_thread_mgmt_se1 = 0;
+- m->compute_static_thread_mgmt_se2 = 0;
+- m->compute_static_thread_mgmt_se3 = 0;
+-
+- mm->dev->kfd2kgd->get_cu_info(mm->dev->kgd, &cu_info);
+-
+- /* If # CU mask bits > # CUs, set it to the # of CUs */
+- if (cu_mask_count > cu_info.cu_active_number)
+- cu_mask_count = cu_info.cu_active_number;
+-
+- cu_index = 0;
+- for (se = 0; se < cu_info.num_shader_engines; se++) {
+- cu_per_sh = 0;
+-
+- /* Get the number of CUs on this Shader Engine */
+- for (i = 0; i < 4; i++)
+- cu_per_sh += hweight32(cu_info.cu_bitmap[se][i]);
+-
+- se_mask[se] = cu_mask[cu_index / 32] >> (cu_index % 32);
+- if ((cu_per_sh + (cu_index % 32)) > 32)
+- se_mask[se] |= cu_mask[(cu_index / 32) + 1]
+- << (32 - (cu_index % 32));
+- se_mask[se] &= (1 << cu_per_sh) - 1;
+- cu_index += cu_per_sh;
+- }
+- m->compute_static_thread_mgmt_se0 = se_mask[0];
+- m->compute_static_thread_mgmt_se1 = se_mask[1];
+- m->compute_static_thread_mgmt_se2 = se_mask[2];
+- m->compute_static_thread_mgmt_se3 = se_mask[3];
+-
+- pr_debug("Update cu mask to %#x %#x %#x %#x\n",
+- m->compute_static_thread_mgmt_se0,
+- m->compute_static_thread_mgmt_se1,
+- m->compute_static_thread_mgmt_se2,
+- m->compute_static_thread_mgmt_se3);
+-}
+-
+-static void set_priority(struct cik_mqd *m, struct queue_properties *q)
+-{
+- m->cp_hqd_pipe_priority = pipe_priority_map[q->priority];
+- m->cp_hqd_queue_priority = q->priority;
+- m->compute_pgm_rsrc1 = (m->compute_pgm_rsrc1 &
+- (~COMPUTE_PGM_RSRC1__PRIORITY_MASK)) |
+- (spi_priority_map[q->priority] <<
+- COMPUTE_PGM_RSRC1__PRIORITY__SHIFT);
+-}
+-
+ static int init_mqd(struct mqd_manager *mm, void **mqd,
+ struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
+ struct queue_properties *q)
+@@ -112,6 +44,10 @@ static int init_mqd(struct mqd_manager *mm, void **mqd,
+ struct cik_mqd *m;
+ int retval;
+
++ BUG_ON(!mm || !q || !mqd);
++
++ pr_debug("kfd: In func %s\n", __func__);
++
+ retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct cik_mqd),
+ mqd_mem_obj);
+
+@@ -142,6 +78,10 @@ static int init_mqd(struct mqd_manager *mm, void **mqd,
+ m->cp_mqd_base_addr_lo = lower_32_bits(addr);
+ m->cp_mqd_base_addr_hi = upper_32_bits(addr);
+
++ m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE | IB_ATC_EN;
++ /* Although WinKFD writes this, I suspect it should not be necessary */
++ m->cp_hqd_ib_control = IB_ATC_EN | DEFAULT_MIN_IB_AVAIL_SIZE;
++
+ m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS |
+ QUANTUM_DURATION(10);
+
+@@ -154,17 +94,14 @@ static int init_mqd(struct mqd_manager *mm, void **mqd,
+ * 1 = CS_MEDIUM (typically between HP3D and GFX
+ * 2 = CS_HIGH (typically above HP3D)
+ */
+- set_priority(m, q);
++ m->cp_hqd_pipe_priority = 1;
++ m->cp_hqd_queue_priority = 15;
+
+ if (q->format == KFD_QUEUE_FORMAT_AQL)
+ m->cp_hqd_iq_rptr = AQL_ENABLE;
+
+- if (priv_cp_queues)
+- m->cp_hqd_pq_control |=
+- 1 << CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT;
+-
+ *mqd = m;
+- if (gart_addr)
++ if (gart_addr != NULL)
+ *gart_addr = addr;
+ retval = mm->update_mqd(mm, m, q);
+
+@@ -178,6 +115,8 @@ static int init_mqd_sdma(struct mqd_manager *mm, void **mqd,
+ int retval;
+ struct cik_sdma_rlc_registers *m;
+
++ BUG_ON(!mm || !mqd || !mqd_mem_obj);
++
+ retval = kfd_gtt_sa_allocate(mm->dev,
+ sizeof(struct cik_sdma_rlc_registers),
+ mqd_mem_obj);
+@@ -190,7 +129,7 @@ static int init_mqd_sdma(struct mqd_manager *mm, void **mqd,
+ memset(m, 0, sizeof(struct cik_sdma_rlc_registers));
+
+ *mqd = m;
+- if (gart_addr)
++ if (gart_addr != NULL)
+ *gart_addr = (*mqd_mem_obj)->gpu_addr;
+
+ retval = mm->update_mqd(mm, m, q);
+@@ -201,50 +140,43 @@ static int init_mqd_sdma(struct mqd_manager *mm, void **mqd,
+ static void uninit_mqd(struct mqd_manager *mm, void *mqd,
+ struct kfd_mem_obj *mqd_mem_obj)
+ {
++ BUG_ON(!mm || !mqd);
+ kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
+ }
+
+ static void uninit_mqd_sdma(struct mqd_manager *mm, void *mqd,
+ struct kfd_mem_obj *mqd_mem_obj)
+ {
++ BUG_ON(!mm || !mqd);
+ kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
+ }
+
+ static int load_mqd(struct mqd_manager *mm, void *mqd, uint32_t pipe_id,
+- uint32_t queue_id, struct queue_properties *p,
+- struct mm_struct *mms)
++ uint32_t queue_id, uint32_t __user *wptr)
+ {
+- /* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */
+- uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0);
+- uint32_t wptr_mask = (uint32_t)((p->queue_size / sizeof(uint32_t)) - 1);
+-
+- return mm->dev->kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id,
+- (uint32_t __user *)p->write_ptr,
+- wptr_shift, wptr_mask, mms);
++ return mm->dev->kfd2kgd->hqd_load
++ (mm->dev->kgd, mqd, pipe_id, queue_id, wptr);
+ }
+
+ static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
+- uint32_t pipe_id, uint32_t queue_id,
+- struct queue_properties *p, struct mm_struct *mms)
++ uint32_t pipe_id, uint32_t queue_id,
++ uint32_t __user *wptr)
+ {
+- return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd,
+- (uint32_t __user *)p->write_ptr,
+- mms);
++ return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd);
+ }
+
+-static int __update_mqd(struct mqd_manager *mm, void *mqd,
+- struct queue_properties *q, unsigned int atc_bit)
++static int update_mqd(struct mqd_manager *mm, void *mqd,
++ struct queue_properties *q)
+ {
+ struct cik_mqd *m;
+
++ BUG_ON(!mm || !q || !mqd);
++
++ pr_debug("kfd: In func %s\n", __func__);
++
+ m = get_mqd(mqd);
+ m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE |
+- DEFAULT_MIN_AVAIL_SIZE;
+- m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE;
+- if (atc_bit) {
+- m->cp_hqd_pq_control |= PQ_ATC_EN;
+- m->cp_hqd_ib_control |= IB_ATC_EN;
+- }
++ DEFAULT_MIN_AVAIL_SIZE | PQ_ATC_EN;
+
+ /*
+ * Calculating queue size which is log base 2 of actual queue size -1
+@@ -256,47 +188,37 @@ static int __update_mqd(struct mqd_manager *mm, void *mqd,
+ m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
+ m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
+ m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
+- m->cp_hqd_pq_doorbell_control = DOORBELL_OFFSET(q->doorbell_off);
++ m->cp_hqd_pq_doorbell_control = DOORBELL_EN |
++ DOORBELL_OFFSET(q->doorbell_off);
+
+ m->cp_hqd_vmid = q->vmid;
+
+- if (q->format == KFD_QUEUE_FORMAT_AQL)
++ if (q->format == KFD_QUEUE_FORMAT_AQL) {
+ m->cp_hqd_pq_control |= NO_UPDATE_RPTR;
++ }
+
+- update_cu_mask(mm, mqd, q);
+- set_priority(m, q);
+-
++ m->cp_hqd_active = 0;
+ q->is_active = false;
+ if (q->queue_size > 0 &&
+ q->queue_address != 0 &&
+- q->queue_percent > 0 &&
+- !q->is_evicted) {
++ q->queue_percent > 0) {
++ m->cp_hqd_active = 1;
+ q->is_active = true;
+ }
+
+ return 0;
+ }
+
+-static int update_mqd(struct mqd_manager *mm, void *mqd,
+- struct queue_properties *q)
+-{
+- return __update_mqd(mm, mqd, q, 1);
+-}
+-
+-static int update_mqd_hawaii(struct mqd_manager *mm, void *mqd,
+- struct queue_properties *q)
+-{
+- return __update_mqd(mm, mqd, q, 0);
+-}
+-
+ static int update_mqd_sdma(struct mqd_manager *mm, void *mqd,
+ struct queue_properties *q)
+ {
+ struct cik_sdma_rlc_registers *m;
+
++ BUG_ON(!mm || !mqd || !q);
++
+ m = get_sdma_mqd(mqd);
+- m->sdma_rlc_rb_cntl = (ffs(q->queue_size / sizeof(unsigned int)) - 1)
+- << SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT |
++ m->sdma_rlc_rb_cntl = ffs(q->queue_size / sizeof(unsigned int)) <<
++ SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT |
+ q->vmid << SDMA0_RLC0_RB_CNTL__RB_VMID__SHIFT |
+ 1 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
+ 6 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT;
+@@ -305,8 +227,9 @@ static int update_mqd_sdma(struct mqd_manager *mm, void *mqd,
+ m->sdma_rlc_rb_base_hi = upper_32_bits(q->queue_address >> 8);
+ m->sdma_rlc_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
+ m->sdma_rlc_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
+- m->sdma_rlc_doorbell =
+- q->doorbell_off << SDMA0_RLC0_DOORBELL__OFFSET__SHIFT;
++ m->sdma_rlc_doorbell = q->doorbell_off <<
++ SDMA0_RLC0_DOORBELL__OFFSET__SHIFT |
++ 1 << SDMA0_RLC0_DOORBELL__ENABLE__SHIFT;
+
+ m->sdma_rlc_virtual_addr = q->sdma_vm_addr;
+
+@@ -316,8 +239,10 @@ static int update_mqd_sdma(struct mqd_manager *mm, void *mqd,
+ q->is_active = false;
+ if (q->queue_size > 0 &&
+ q->queue_address != 0 &&
+- q->queue_percent > 0 &&
+- !q->is_evicted) {
++ q->queue_percent > 0) {
++ m->sdma_rlc_rb_cntl |=
++ 1 << SDMA0_RLC0_RB_CNTL__RB_ENABLE__SHIFT;
++
+ q->is_active = true;
+ }
+
+@@ -329,7 +254,7 @@ static int destroy_mqd(struct mqd_manager *mm, void *mqd,
+ unsigned int timeout, uint32_t pipe_id,
+ uint32_t queue_id)
+ {
+- return mm->dev->kfd2kgd->hqd_destroy(mm->dev->kgd, mqd, type, timeout,
++ return mm->dev->kfd2kgd->hqd_destroy(mm->dev->kgd, type, timeout,
+ pipe_id, queue_id);
+ }
+
+@@ -376,6 +301,10 @@ static int init_mqd_hiq(struct mqd_manager *mm, void **mqd,
+ struct cik_mqd *m;
+ int retval;
+
++ BUG_ON(!mm || !q || !mqd || !mqd_mem_obj);
++
++ pr_debug("kfd: In func %s\n", __func__);
++
+ retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct cik_mqd),
+ mqd_mem_obj);
+
+@@ -414,7 +343,8 @@ static int init_mqd_hiq(struct mqd_manager *mm, void **mqd,
+ * 1 = CS_MEDIUM (typically between HP3D and GFX
+ * 2 = CS_HIGH (typically above HP3D)
+ */
+- set_priority(m, q);
++ m->cp_hqd_pipe_priority = 1;
++ m->cp_hqd_queue_priority = 15;
+
+ *mqd = m;
+ if (gart_addr)
+@@ -429,6 +359,10 @@ static int update_mqd_hiq(struct mqd_manager *mm, void *mqd,
+ {
+ struct cik_mqd *m;
+
++ BUG_ON(!mm || !q || !mqd);
++
++ pr_debug("kfd: In func %s\n", __func__);
++
+ m = get_mqd(mqd);
+ m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE |
+ DEFAULT_MIN_AVAIL_SIZE |
+@@ -445,50 +379,45 @@ static int update_mqd_hiq(struct mqd_manager *mm, void *mqd,
+ m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
+ m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
+ m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
+- m->cp_hqd_pq_doorbell_control = DOORBELL_OFFSET(q->doorbell_off);
++ m->cp_hqd_pq_doorbell_control = DOORBELL_EN |
++ DOORBELL_OFFSET(q->doorbell_off);
+
+ m->cp_hqd_vmid = q->vmid;
+
++ m->cp_hqd_active = 0;
+ q->is_active = false;
+ if (q->queue_size > 0 &&
+ q->queue_address != 0 &&
+- q->queue_percent > 0 &&
+- !q->is_evicted) {
++ q->queue_percent > 0) {
++ m->cp_hqd_active = 1;
+ q->is_active = true;
+ }
+
+- set_priority(m, q);
+ return 0;
+ }
+
+-#if defined(CONFIG_DEBUG_FS)
+-
+-static int debugfs_show_mqd(struct seq_file *m, void *data)
++struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
+ {
+- seq_hex_dump(m, " ", DUMP_PREFIX_OFFSET, 32, 4,
+- data, sizeof(struct cik_mqd), false);
+- return 0;
+-}
++ struct cik_sdma_rlc_registers *m;
+
+-static int debugfs_show_mqd_sdma(struct seq_file *m, void *data)
+-{
+- seq_hex_dump(m, " ", DUMP_PREFIX_OFFSET, 32, 4,
+- data, sizeof(struct cik_sdma_rlc_registers), false);
+- return 0;
+-}
++ BUG_ON(!mqd);
+
+-#endif
++ m = (struct cik_sdma_rlc_registers *)mqd;
+
++ return m;
++}
+
+ struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
+ struct kfd_dev *dev)
+ {
+ struct mqd_manager *mqd;
+
+- if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
+- return NULL;
++ BUG_ON(!dev);
++ BUG_ON(type >= KFD_MQD_TYPE_MAX);
+
+- mqd = kzalloc(sizeof(*mqd), GFP_NOIO);
++ pr_debug("kfd: In func %s\n", __func__);
++
++ mqd = kzalloc(sizeof(struct mqd_manager), GFP_KERNEL);
+ if (!mqd)
+ return NULL;
+
+@@ -503,9 +432,6 @@ struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
+ mqd->update_mqd = update_mqd;
+ mqd->destroy_mqd = destroy_mqd;
+ mqd->is_occupied = is_occupied;
+-#if defined(CONFIG_DEBUG_FS)
+- mqd->debugfs_show_mqd = debugfs_show_mqd;
+-#endif
+ break;
+ case KFD_MQD_TYPE_HIQ:
+ mqd->init_mqd = init_mqd_hiq;
+@@ -514,9 +440,6 @@ struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
+ mqd->update_mqd = update_mqd_hiq;
+ mqd->destroy_mqd = destroy_mqd;
+ mqd->is_occupied = is_occupied;
+-#if defined(CONFIG_DEBUG_FS)
+- mqd->debugfs_show_mqd = debugfs_show_mqd;
+-#endif
+ break;
+ case KFD_MQD_TYPE_SDMA:
+ mqd->init_mqd = init_mqd_sdma;
+@@ -525,9 +448,6 @@ struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
+ mqd->update_mqd = update_mqd_sdma;
+ mqd->destroy_mqd = destroy_mqd_sdma;
+ mqd->is_occupied = is_occupied_sdma;
+-#if defined(CONFIG_DEBUG_FS)
+- mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;
+-#endif
+ break;
+ default:
+ kfree(mqd);
+@@ -537,15 +457,3 @@ struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
+ return mqd;
+ }
+
+-struct mqd_manager *mqd_manager_init_cik_hawaii(enum KFD_MQD_TYPE type,
+- struct kfd_dev *dev)
+-{
+- struct mqd_manager *mqd;
+-
+- mqd = mqd_manager_init_cik(type, dev);
+- if (!mqd)
+- return NULL;
+- if ((type == KFD_MQD_TYPE_CP) || (type == KFD_MQD_TYPE_COMPUTE))
+- mqd->update_mqd = update_mqd_hawaii;
+- return mqd;
+-}
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
+deleted file mode 100644
+index 6c302d2..0000000
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
++++ /dev/null
+@@ -1,528 +0,0 @@
+-/*
+- * Copyright 2016 Advanced Micro Devices, Inc.
+- *
+- * Permission is hereby granted, free of charge, to any person obtaining a
+- * copy of this software and associated documentation files (the "Software"),
+- * to deal in the Software without restriction, including without limitation
+- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+- * and/or sell copies of the Software, and to permit persons to whom the
+- * Software is furnished to do so, subject to the following conditions:
+- *
+- * The above copyright notice and this permission notice shall be included in
+- * all copies or substantial portions of the Software.
+- *
+- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+- * OTHER DEALINGS IN THE SOFTWARE.
+- *
+- */
+-
+-#include <linux/printk.h>
+-#include <linux/slab.h>
+-#include <linux/uaccess.h>
+-#include "kfd_priv.h"
+-#include "kfd_mqd_manager.h"
+-#include "v9_structs.h"
+-#include "vega10/GC/gc_9_0_offset.h"
+-#include "vega10/GC/gc_9_0_sh_mask.h"
+-#include "vega10/SDMA0/sdma0_4_0_sh_mask.h"
+-
+-static inline struct v9_mqd *get_mqd(void *mqd)
+-{
+- return (struct v9_mqd *)mqd;
+-}
+-
+-static inline struct v9_sdma_mqd *get_sdma_mqd(void *mqd)
+-{
+- return (struct v9_sdma_mqd *)mqd;
+-}
+-
+-static void update_cu_mask(struct mqd_manager *mm, void *mqd,
+- struct queue_properties *q)
+-{
+- struct v9_mqd *m;
+- struct kfd_cu_info cu_info;
+- uint32_t se_mask[4] = {0}; /* 4 is the max # of SEs */
+- uint32_t cu_mask_count = q->cu_mask_count;
+- const uint32_t *cu_mask = q->cu_mask;
+- int se, cu_per_sh, cu_index, i;
+-
+- if (cu_mask_count == 0)
+- return;
+-
+- m = get_mqd(mqd);
+- m->compute_static_thread_mgmt_se0 = 0;
+- m->compute_static_thread_mgmt_se1 = 0;
+- m->compute_static_thread_mgmt_se2 = 0;
+- m->compute_static_thread_mgmt_se3 = 0;
+-
+- mm->dev->kfd2kgd->get_cu_info(mm->dev->kgd, &cu_info);
+-
+- /* If # CU mask bits > # CUs, set it to the # of CUs */
+- if (cu_mask_count > cu_info.cu_active_number)
+- cu_mask_count = cu_info.cu_active_number;
+-
+- cu_index = 0;
+- for (se = 0; se < cu_info.num_shader_engines; se++) {
+- cu_per_sh = 0;
+-
+- /* Get the number of CUs on this Shader Engine */
+- for (i = 0; i < 4; i++)
+- cu_per_sh += hweight32(cu_info.cu_bitmap[se][i]);
+-
+- se_mask[se] = cu_mask[cu_index / 32] >> (cu_index % 32);
+- if ((cu_per_sh + (cu_index % 32)) > 32)
+- se_mask[se] |= cu_mask[(cu_index / 32) + 1]
+- << (32 - (cu_index % 32));
+- se_mask[se] &= (1 << cu_per_sh) - 1;
+- cu_index += cu_per_sh;
+- }
+- m->compute_static_thread_mgmt_se0 = se_mask[0];
+- m->compute_static_thread_mgmt_se1 = se_mask[1];
+- m->compute_static_thread_mgmt_se2 = se_mask[2];
+- m->compute_static_thread_mgmt_se3 = se_mask[3];
+-
+- pr_debug("update cu mask to %#x %#x %#x %#x\n",
+- m->compute_static_thread_mgmt_se0,
+- m->compute_static_thread_mgmt_se1,
+- m->compute_static_thread_mgmt_se2,
+- m->compute_static_thread_mgmt_se3);
+-}
+-
+-static int init_mqd(struct mqd_manager *mm, void **mqd,
+- struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
+- struct queue_properties *q)
+-{
+- int retval;
+- uint64_t addr;
+- struct v9_mqd *m;
+- struct kfd_dev *kfd = mm->dev;
+-
+- /* From V9, for CWSR, the control stack is located on the next page
+- * boundary after the mqd, we will use the gtt allocation function
+- * instead of sub-allocation function.
+- */
+- if (kfd->cwsr_enabled && (q->type == KFD_QUEUE_TYPE_COMPUTE)) {
+- *mqd_mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_NOIO);
+- retval = kfd->kfd2kgd->init_gtt_mem_allocation(kfd->kgd,
+- ALIGN(q->ctl_stack_size, PAGE_SIZE) +
+- ALIGN(sizeof(struct v9_mqd), PAGE_SIZE),
+- &((*mqd_mem_obj)->gtt_mem),
+- &((*mqd_mem_obj)->gpu_addr),
+- (void *)&((*mqd_mem_obj)->cpu_ptr));
+- } else
+- retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct v9_mqd),
+- mqd_mem_obj);
+- if (retval != 0)
+- return -ENOMEM;
+-
+- m = (struct v9_mqd *) (*mqd_mem_obj)->cpu_ptr;
+- addr = (*mqd_mem_obj)->gpu_addr;
+-
+- memset(m, 0, sizeof(struct v9_mqd));
+-
+- m->header = 0xC0310800;
+- m->compute_pipelinestat_enable = 1;
+- m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF;
+- m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF;
+- m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF;
+- m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF;
+-
+- m->cp_hqd_persistent_state = CP_HQD_PERSISTENT_STATE__PRELOAD_REQ_MASK |
+- 0x53 << CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE__SHIFT;
+-
+- m->cp_mqd_control = 1 << CP_MQD_CONTROL__PRIV_STATE__SHIFT;
+-
+- m->cp_mqd_base_addr_lo = lower_32_bits(addr);
+- m->cp_mqd_base_addr_hi = upper_32_bits(addr);
+-
+- m->cp_hqd_quantum = 1 << CP_HQD_QUANTUM__QUANTUM_EN__SHIFT |
+- 1 << CP_HQD_QUANTUM__QUANTUM_SCALE__SHIFT |
+- 10 << CP_HQD_QUANTUM__QUANTUM_DURATION__SHIFT;
+-
+- m->cp_hqd_pipe_priority = 1;
+- m->cp_hqd_queue_priority = 15;
+-
+- if (q->format == KFD_QUEUE_FORMAT_AQL) {
+- m->cp_hqd_aql_control =
+- 1 << CP_HQD_AQL_CONTROL__CONTROL0__SHIFT;
+- }
+-
+- if (q->tba_addr) {
+- m->compute_pgm_rsrc2 |=
+- (1 << COMPUTE_PGM_RSRC2__TRAP_PRESENT__SHIFT);
+- }
+-
+- if (mm->dev->cwsr_enabled) {
+- m->cp_hqd_persistent_state |=
+- (1 << CP_HQD_PERSISTENT_STATE__QSWITCH_MODE__SHIFT);
+- m->cp_hqd_ctx_save_base_addr_lo =
+- lower_32_bits(q->ctx_save_restore_area_address);
+- m->cp_hqd_ctx_save_base_addr_hi =
+- upper_32_bits(q->ctx_save_restore_area_address);
+- m->cp_hqd_ctx_save_size = q->ctx_save_restore_area_size;
+- m->cp_hqd_cntl_stack_size = q->ctl_stack_size;
+- m->cp_hqd_cntl_stack_offset = q->ctl_stack_size;
+- m->cp_hqd_wg_state_offset = q->ctl_stack_size;
+- }
+-
+- if (priv_cp_queues)
+- m->cp_hqd_pq_control |=
+- 1 << CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT;
+-
+- *mqd = m;
+- if (gart_addr)
+- *gart_addr = addr;
+- retval = mm->update_mqd(mm, m, q);
+-
+- return retval;
+-}
+-
+-static int load_mqd(struct mqd_manager *mm, void *mqd,
+- uint32_t pipe_id, uint32_t queue_id,
+- struct queue_properties *p, struct mm_struct *mms)
+-{
+- /* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */
+- uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0);
+-
+- return mm->dev->kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id,
+- (uint32_t __user *)p->write_ptr,
+- wptr_shift, 0, mms);
+-}
+-
+-static int update_mqd(struct mqd_manager *mm, void *mqd,
+- struct queue_properties *q)
+-{
+- struct v9_mqd *m;
+-
+- m = get_mqd(mqd);
+-
+- m->cp_hqd_pq_control = 5 << CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE__SHIFT;
+- m->cp_hqd_pq_control |=
+- ffs(q->queue_size / sizeof(unsigned int)) - 1 - 1;
+- pr_debug("cp_hqd_pq_control 0x%x\n", m->cp_hqd_pq_control);
+-
+- m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
+- m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
+-
+- m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
+- m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
+- m->cp_hqd_pq_wptr_poll_addr_lo = lower_32_bits((uint64_t)q->write_ptr);
+- m->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits((uint64_t)q->write_ptr);
+-
+- m->cp_hqd_pq_doorbell_control =
+- q->doorbell_off <<
+- CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
+- pr_debug("cp_hqd_pq_doorbell_control 0x%x\n",
+- m->cp_hqd_pq_doorbell_control);
+-
+- m->cp_hqd_ib_control = 3 << CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE__SHIFT;
+-
+- /*
+- * HW does not clamp this field correctly. Maximum EOP queue size
+- * is constrained by per-SE EOP done signal count, which is 8-bit.
+- * Limit is 0xFF EOP entries (= 0x7F8 dwords). CP will not submit
+- * more than (EOP entry count - 1) so a queue size of 0x800 dwords
+- * is safe, giving a maximum field value of 0xA.
+- */
+- m->cp_hqd_eop_control = min(0xA,
+- ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1);
+- m->cp_hqd_eop_base_addr_lo =
+- lower_32_bits(q->eop_ring_buffer_address >> 8);
+- m->cp_hqd_eop_base_addr_hi =
+- upper_32_bits(q->eop_ring_buffer_address >> 8);
+-
+- m->cp_hqd_iq_timer = 0;
+-
+- m->cp_hqd_vmid = q->vmid;
+-
+- if (q->format == KFD_QUEUE_FORMAT_AQL) {
+- m->cp_hqd_pq_control |= CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK |
+- 2 << CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT |
+- 1 << CP_HQD_PQ_CONTROL__QUEUE_FULL_EN__SHIFT |
+- 1 << CP_HQD_PQ_CONTROL__WPP_CLAMP_EN__SHIFT;
+- m->cp_hqd_pq_doorbell_control |=
+- 1 << CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_BIF_DROP__SHIFT;
+- }
+- if (mm->dev->cwsr_enabled)
+- m->cp_hqd_ctx_save_control = 0;
+-
+- update_cu_mask(mm, mqd, q);
+-
+- q->is_active = false;
+- if (q->queue_size > 0 &&
+- q->queue_address != 0 &&
+- q->queue_percent > 0 &&
+- !q->is_evicted) {
+- q->is_active = true;
+- }
+-
+- return 0;
+-}
+-
+-
+-static int destroy_mqd(struct mqd_manager *mm, void *mqd,
+- enum kfd_preempt_type type,
+- unsigned int timeout, uint32_t pipe_id,
+- uint32_t queue_id)
+-{
+- return mm->dev->kfd2kgd->hqd_destroy
+- (mm->dev->kgd, mqd, type, timeout,
+- pipe_id, queue_id);
+-}
+-
+-static void uninit_mqd(struct mqd_manager *mm, void *mqd,
+- struct kfd_mem_obj *mqd_mem_obj)
+-{
+- struct kfd_dev *kfd = mm->dev;
+-
+- if (mqd_mem_obj->gtt_mem) {
+- kfd->kfd2kgd->free_gtt_mem(kfd->kgd, mqd_mem_obj->gtt_mem);
+- kfree(mqd_mem_obj);
+- } else {
+- kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
+- }
+-}
+-
+-static bool is_occupied(struct mqd_manager *mm, void *mqd,
+- uint64_t queue_address, uint32_t pipe_id,
+- uint32_t queue_id)
+-{
+- return mm->dev->kfd2kgd->hqd_is_occupied(
+- mm->dev->kgd, queue_address,
+- pipe_id, queue_id);
+-}
+-
+-static int get_wave_state(struct mqd_manager *mm, void *mqd,
+- void __user *ctl_stack,
+- u32 *ctl_stack_used_size,
+- u32 *save_area_used_size)
+-{
+- struct v9_mqd *m;
+-
+- /* Control stack is located one page after MQD. */
+- void *mqd_ctl_stack = (void *)((uintptr_t)mqd + PAGE_SIZE);
+-
+- m = get_mqd(mqd);
+-
+- *ctl_stack_used_size = m->cp_hqd_cntl_stack_size -
+- m->cp_hqd_cntl_stack_offset;
+- *save_area_used_size = m->cp_hqd_wg_state_offset -
+- m->cp_hqd_cntl_stack_size;
+-
+- if (copy_to_user(ctl_stack, mqd_ctl_stack, m->cp_hqd_cntl_stack_size))
+- return -EFAULT;
+-
+- return 0;
+-}
+-
+-static int init_mqd_hiq(struct mqd_manager *mm, void **mqd,
+- struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
+- struct queue_properties *q)
+-{
+- struct v9_mqd *m;
+- int retval = init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q);
+-
+- if (retval != 0)
+- return retval;
+-
+- m = get_mqd(*mqd);
+-
+- m->cp_hqd_pq_control |= 1 << CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT |
+- 1 << CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT;
+-
+- return retval;
+-}
+-
+-static int update_mqd_hiq(struct mqd_manager *mm, void *mqd,
+- struct queue_properties *q)
+-{
+- struct v9_mqd *m;
+- int retval = update_mqd(mm, mqd, q);
+-
+- if (retval != 0)
+- return retval;
+-
+- /* TODO: what's the point? update_mqd already does this. */
+- m = get_mqd(mqd);
+- m->cp_hqd_vmid = q->vmid;
+- return retval;
+-}
+-
+-static int init_mqd_sdma(struct mqd_manager *mm, void **mqd,
+- struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
+- struct queue_properties *q)
+-{
+- int retval;
+- struct v9_sdma_mqd *m;
+-
+-
+- retval = kfd_gtt_sa_allocate(mm->dev,
+- sizeof(struct v9_sdma_mqd),
+- mqd_mem_obj);
+-
+- if (retval != 0)
+- return -ENOMEM;
+-
+- m = (struct v9_sdma_mqd *) (*mqd_mem_obj)->cpu_ptr;
+-
+- memset(m, 0, sizeof(struct v9_sdma_mqd));
+-
+- *mqd = m;
+- if (gart_addr)
+- *gart_addr = (*mqd_mem_obj)->gpu_addr;
+-
+- retval = mm->update_mqd(mm, m, q);
+-
+- return retval;
+-}
+-
+-static void uninit_mqd_sdma(struct mqd_manager *mm, void *mqd,
+- struct kfd_mem_obj *mqd_mem_obj)
+-{
+- kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
+-}
+-
+-static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
+- uint32_t pipe_id, uint32_t queue_id,
+- struct queue_properties *p, struct mm_struct *mms)
+-{
+- return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd,
+- (uint32_t __user *)p->write_ptr,
+- mms);
+-}
+-
+-#define SDMA_RLC_DUMMY_DEFAULT 0xf
+-
+-static int update_mqd_sdma(struct mqd_manager *mm, void *mqd,
+- struct queue_properties *q)
+-{
+- struct v9_sdma_mqd *m;
+-
+- m = get_sdma_mqd(mqd);
+- m->sdmax_rlcx_rb_cntl = (ffs(q->queue_size / sizeof(unsigned int)) - 1)
+- << SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT |
+- q->vmid << SDMA0_RLC0_RB_CNTL__RB_VMID__SHIFT |
+- 1 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
+- 6 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT;
+-
+- m->sdmax_rlcx_rb_base = lower_32_bits(q->queue_address >> 8);
+- m->sdmax_rlcx_rb_base_hi = upper_32_bits(q->queue_address >> 8);
+- m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
+- m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
+- m->sdmax_rlcx_doorbell_offset =
+- q->doorbell_off << SDMA0_RLC0_DOORBELL_OFFSET__OFFSET__SHIFT;
+-
+- m->sdma_engine_id = q->sdma_engine_id;
+- m->sdma_queue_id = q->sdma_queue_id;
+- m->sdmax_rlcx_dummy_reg = SDMA_RLC_DUMMY_DEFAULT;
+-
+- q->is_active = false;
+- if (q->queue_size > 0 &&
+- q->queue_address != 0 &&
+- q->queue_percent > 0 &&
+- !q->is_evicted) {
+- q->is_active = true;
+- }
+-
+- return 0;
+-}
+-
+-/*
+- * * preempt type here is ignored because there is only one way
+- * * to preempt sdma queue
+- */
+-static int destroy_mqd_sdma(struct mqd_manager *mm, void *mqd,
+- enum kfd_preempt_type type,
+- unsigned int timeout, uint32_t pipe_id,
+- uint32_t queue_id)
+-{
+- return mm->dev->kfd2kgd->hqd_sdma_destroy(mm->dev->kgd, mqd, timeout);
+-}
+-
+-static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
+- uint64_t queue_address, uint32_t pipe_id,
+- uint32_t queue_id)
+-{
+- return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd);
+-}
+-
+-#if defined(CONFIG_DEBUG_FS)
+-
+-static int debugfs_show_mqd(struct seq_file *m, void *data)
+-{
+- seq_hex_dump(m, " ", DUMP_PREFIX_OFFSET, 32, 4,
+- data, sizeof(struct v9_mqd), false);
+- return 0;
+-}
+-
+-static int debugfs_show_mqd_sdma(struct seq_file *m, void *data)
+-{
+- seq_hex_dump(m, " ", DUMP_PREFIX_OFFSET, 32, 4,
+- data, sizeof(struct v9_sdma_mqd), false);
+- return 0;
+-}
+-
+-#endif
+-
+-struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
+- struct kfd_dev *dev)
+-{
+- struct mqd_manager *mqd;
+-
+- if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
+- return NULL;
+-
+- mqd = kzalloc(sizeof(*mqd), GFP_NOIO);
+- if (!mqd)
+- return NULL;
+-
+- mqd->dev = dev;
+-
+- switch (type) {
+- case KFD_MQD_TYPE_CP:
+- case KFD_MQD_TYPE_COMPUTE:
+- mqd->init_mqd = init_mqd;
+- mqd->uninit_mqd = uninit_mqd;
+- mqd->load_mqd = load_mqd;
+- mqd->update_mqd = update_mqd;
+- mqd->destroy_mqd = destroy_mqd;
+- mqd->is_occupied = is_occupied;
+- mqd->get_wave_state = get_wave_state;
+-#if defined(CONFIG_DEBUG_FS)
+- mqd->debugfs_show_mqd = debugfs_show_mqd;
+-#endif
+- break;
+- case KFD_MQD_TYPE_HIQ:
+- mqd->init_mqd = init_mqd_hiq;
+- mqd->uninit_mqd = uninit_mqd;
+- mqd->load_mqd = load_mqd;
+- mqd->update_mqd = update_mqd_hiq;
+- mqd->destroy_mqd = destroy_mqd;
+- mqd->is_occupied = is_occupied;
+-#if defined(CONFIG_DEBUG_FS)
+- mqd->debugfs_show_mqd = debugfs_show_mqd;
+-#endif
+- break;
+- case KFD_MQD_TYPE_SDMA:
+- mqd->init_mqd = init_mqd_sdma;
+- mqd->uninit_mqd = uninit_mqd_sdma;
+- mqd->load_mqd = load_mqd_sdma;
+- mqd->update_mqd = update_mqd_sdma;
+- mqd->destroy_mqd = destroy_mqd_sdma;
+- mqd->is_occupied = is_occupied_sdma;
+-#if defined(CONFIG_DEBUG_FS)
+- mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;
+-#endif
+- break;
+- default:
+- kfree(mqd);
+- return NULL;
+- }
+-
+- return mqd;
+-}
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
+index 5c26e5a..a9b9882 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
+@@ -30,7 +30,6 @@
+ #include "vi_structs.h"
+ #include "gca/gfx_8_0_sh_mask.h"
+ #include "gca/gfx_8_0_enum.h"
+-#include "oss/oss_3_0_sh_mask.h"
+
+ #define CP_MQD_CONTROL__PRIV_STATE__SHIFT 0x8
+
+@@ -39,73 +38,6 @@ static inline struct vi_mqd *get_mqd(void *mqd)
+ return (struct vi_mqd *)mqd;
+ }
+
+-static inline struct vi_sdma_mqd *get_sdma_mqd(void *mqd)
+-{
+- return (struct vi_sdma_mqd *)mqd;
+-}
+-
+-static void update_cu_mask(struct mqd_manager *mm, void *mqd,
+- struct queue_properties *q)
+-{
+- struct vi_mqd *m;
+- struct kfd_cu_info cu_info;
+- uint32_t se_mask[4] = {0}; /* 4 is the max # of SEs */
+- uint32_t cu_mask_count = q->cu_mask_count;
+- const uint32_t *cu_mask = q->cu_mask;
+- int se, cu_per_sh, cu_index, i;
+-
+- if (cu_mask_count == 0)
+- return;
+-
+- m = get_mqd(mqd);
+- m->compute_static_thread_mgmt_se0 = 0;
+- m->compute_static_thread_mgmt_se1 = 0;
+- m->compute_static_thread_mgmt_se2 = 0;
+- m->compute_static_thread_mgmt_se3 = 0;
+-
+- mm->dev->kfd2kgd->get_cu_info(mm->dev->kgd, &cu_info);
+-
+- /* If # CU mask bits > # CUs, set it to the # of CUs */
+- if (cu_mask_count > cu_info.cu_active_number)
+- cu_mask_count = cu_info.cu_active_number;
+-
+- cu_index = 0;
+- for (se = 0; se < cu_info.num_shader_engines; se++) {
+- cu_per_sh = 0;
+-
+- /* Get the number of CUs on this Shader Engine */
+- for (i = 0; i < 4; i++)
+- cu_per_sh += hweight32(cu_info.cu_bitmap[se][i]);
+-
+- se_mask[se] = cu_mask[cu_index / 32] >> (cu_index % 32);
+- if ((cu_per_sh + (cu_index % 32)) > 32)
+- se_mask[se] |= cu_mask[(cu_index / 32) + 1]
+- << (32 - (cu_index % 32));
+- se_mask[se] &= (1 << cu_per_sh) - 1;
+- cu_index += cu_per_sh;
+- }
+- m->compute_static_thread_mgmt_se0 = se_mask[0];
+- m->compute_static_thread_mgmt_se1 = se_mask[1];
+- m->compute_static_thread_mgmt_se2 = se_mask[2];
+- m->compute_static_thread_mgmt_se3 = se_mask[3];
+-
+- pr_debug("Update cu mask to %#x %#x %#x %#x\n",
+- m->compute_static_thread_mgmt_se0,
+- m->compute_static_thread_mgmt_se1,
+- m->compute_static_thread_mgmt_se2,
+- m->compute_static_thread_mgmt_se3);
+-}
+-
+-static void set_priority(struct vi_mqd *m, struct queue_properties *q)
+-{
+- m->cp_hqd_pipe_priority = pipe_priority_map[q->priority];
+- m->cp_hqd_queue_priority = q->priority;
+- m->compute_pgm_rsrc1 = (m->compute_pgm_rsrc1 &
+- (~COMPUTE_PGM_RSRC1__PRIORITY_MASK)) |
+- (spi_priority_map[q->priority] <<
+- COMPUTE_PGM_RSRC1__PRIORITY__SHIFT);
+-}
+-
+ static int init_mqd(struct mqd_manager *mm, void **mqd,
+ struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
+ struct queue_properties *q)
+@@ -144,40 +76,16 @@ static int init_mqd(struct mqd_manager *mm, void **mqd,
+ 1 << CP_HQD_QUANTUM__QUANTUM_SCALE__SHIFT |
+ 10 << CP_HQD_QUANTUM__QUANTUM_DURATION__SHIFT;
+
+- set_priority(m, q);
++ m->cp_hqd_pipe_priority = 1;
++ m->cp_hqd_queue_priority = 15;
++
+ m->cp_hqd_eop_rptr = 1 << CP_HQD_EOP_RPTR__INIT_FETCHER__SHIFT;
+
+ if (q->format == KFD_QUEUE_FORMAT_AQL)
+ m->cp_hqd_iq_rptr = 1;
+
+- if (q->tba_addr) {
+- m->compute_tba_lo = lower_32_bits(q->tba_addr >> 8);
+- m->compute_tba_hi = upper_32_bits(q->tba_addr >> 8);
+- m->compute_tma_lo = lower_32_bits(q->tma_addr >> 8);
+- m->compute_tma_hi = upper_32_bits(q->tma_addr >> 8);
+- m->compute_pgm_rsrc2 |=
+- (1 << COMPUTE_PGM_RSRC2__TRAP_PRESENT__SHIFT);
+- }
+-
+- if (mm->dev->cwsr_enabled) {
+- m->cp_hqd_persistent_state |=
+- (1 << CP_HQD_PERSISTENT_STATE__QSWITCH_MODE__SHIFT);
+- m->cp_hqd_ctx_save_base_addr_lo =
+- lower_32_bits(q->ctx_save_restore_area_address);
+- m->cp_hqd_ctx_save_base_addr_hi =
+- upper_32_bits(q->ctx_save_restore_area_address);
+- m->cp_hqd_ctx_save_size = q->ctx_save_restore_area_size;
+- m->cp_hqd_cntl_stack_size = q->ctl_stack_size;
+- m->cp_hqd_cntl_stack_offset = q->ctl_stack_size;
+- m->cp_hqd_wg_state_offset = q->ctl_stack_size;
+- }
+-
+- if (priv_cp_queues)
+- m->cp_hqd_pq_control |=
+- 1 << CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT;
+-
+ *mqd = m;
+- if (gart_addr)
++ if (gart_addr != NULL)
+ *gart_addr = addr;
+ retval = mm->update_mqd(mm, m, q);
+
+@@ -186,15 +94,10 @@ static int init_mqd(struct mqd_manager *mm, void **mqd,
+
+ static int load_mqd(struct mqd_manager *mm, void *mqd,
+ uint32_t pipe_id, uint32_t queue_id,
+- struct queue_properties *p, struct mm_struct *mms)
++ uint32_t __user *wptr)
+ {
+- /* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */
+- uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0);
+- uint32_t wptr_mask = (uint32_t)((p->queue_size / sizeof(uint32_t)) - 1);
+-
+- return mm->dev->kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id,
+- (uint32_t __user *)p->write_ptr,
+- wptr_shift, wptr_mask, mms);
++ return mm->dev->kfd2kgd->hqd_load
++ (mm->dev->kgd, mqd, pipe_id, queue_id, wptr);
+ }
+
+ static int __update_mqd(struct mqd_manager *mm, void *mqd,
+@@ -203,6 +106,10 @@ static int __update_mqd(struct mqd_manager *mm, void *mqd,
+ {
+ struct vi_mqd *m;
+
++ BUG_ON(!mm || !q || !mqd);
++
++ pr_debug("kfd: In func %s\n", __func__);
++
+ m = get_mqd(mqd);
+
+ m->cp_hqd_pq_control = 5 << CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE__SHIFT |
+@@ -210,20 +117,19 @@ static int __update_mqd(struct mqd_manager *mm, void *mqd,
+ mtype << CP_HQD_PQ_CONTROL__MTYPE__SHIFT;
+ m->cp_hqd_pq_control |=
+ ffs(q->queue_size / sizeof(unsigned int)) - 1 - 1;
+- pr_debug("cp_hqd_pq_control 0x%x\n", m->cp_hqd_pq_control);
++ pr_debug("kfd: cp_hqd_pq_control 0x%x\n", m->cp_hqd_pq_control);
+
+ m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
+ m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
+
+ m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
+ m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
+- m->cp_hqd_pq_wptr_poll_addr_lo = lower_32_bits((uint64_t)q->write_ptr);
+- m->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits((uint64_t)q->write_ptr);
+
+ m->cp_hqd_pq_doorbell_control =
++ 1 << CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_EN__SHIFT |
+ q->doorbell_off <<
+ CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
+- pr_debug("cp_hqd_pq_doorbell_control 0x%x\n",
++ pr_debug("kfd: cp_hqd_pq_doorbell_control 0x%x\n",
+ m->cp_hqd_pq_doorbell_control);
+
+ m->cp_hqd_eop_control = atc_bit << CP_HQD_EOP_CONTROL__EOP_ATC__SHIFT |
+@@ -233,15 +139,8 @@ static int __update_mqd(struct mqd_manager *mm, void *mqd,
+ 3 << CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE__SHIFT |
+ mtype << CP_HQD_IB_CONTROL__MTYPE__SHIFT;
+
+- /*
+- * HW does not clamp this field correctly. Maximum EOP queue size
+- * is constrained by per-SE EOP done signal count, which is 8-bit.
+- * Limit is 0xFF EOP entries (= 0x7F8 dwords). CP will not submit
+- * more than (EOP entry count - 1) so a queue size of 0x800 dwords
+- * is safe, giving a maximum field value of 0xA.
+- */
+- m->cp_hqd_eop_control |= min(0xA,
+- ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1);
++ m->cp_hqd_eop_control |=
++ ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1;
+ m->cp_hqd_eop_base_addr_lo =
+ lower_32_bits(q->eop_ring_buffer_address >> 8);
+ m->cp_hqd_eop_base_addr_hi =
+@@ -256,19 +155,13 @@ static int __update_mqd(struct mqd_manager *mm, void *mqd,
+ m->cp_hqd_pq_control |= CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK |
+ 2 << CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT;
+ }
+- if (mm->dev->cwsr_enabled)
+- m->cp_hqd_ctx_save_control =
+- atc_bit << CP_HQD_CTX_SAVE_CONTROL__ATC__SHIFT |
+- mtype << CP_HQD_CTX_SAVE_CONTROL__MTYPE__SHIFT;
+-
+- update_cu_mask(mm, mqd, q);
+- set_priority(m, q);
+
++ m->cp_hqd_active = 0;
+ q->is_active = false;
+ if (q->queue_size > 0 &&
+ q->queue_address != 0 &&
+- q->queue_percent > 0 &&
+- !q->is_evicted) {
++ q->queue_percent > 0) {
++ m->cp_hqd_active = 1;
+ q->is_active = true;
+ }
+
+@@ -282,25 +175,20 @@ static int update_mqd(struct mqd_manager *mm, void *mqd,
+ return __update_mqd(mm, mqd, q, MTYPE_CC, 1);
+ }
+
+-static int update_mqd_tonga(struct mqd_manager *mm, void *mqd,
+- struct queue_properties *q)
+-{
+- return __update_mqd(mm, mqd, q, MTYPE_UC, 0);
+-}
+-
+ static int destroy_mqd(struct mqd_manager *mm, void *mqd,
+ enum kfd_preempt_type type,
+ unsigned int timeout, uint32_t pipe_id,
+ uint32_t queue_id)
+ {
+ return mm->dev->kfd2kgd->hqd_destroy
+- (mm->dev->kgd, mqd, type, timeout,
++ (mm->dev->kgd, type, timeout,
+ pipe_id, queue_id);
+ }
+
+ static void uninit_mqd(struct mqd_manager *mm, void *mqd,
+ struct kfd_mem_obj *mqd_mem_obj)
+ {
++ BUG_ON(!mm || !mqd);
+ kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
+ }
+
+@@ -313,28 +201,6 @@ static bool is_occupied(struct mqd_manager *mm, void *mqd,
+ pipe_id, queue_id);
+ }
+
+-static int get_wave_state(struct mqd_manager *mm, void *mqd,
+- void __user *ctl_stack,
+- u32 *ctl_stack_used_size,
+- u32 *save_area_used_size)
+-{
+- struct vi_mqd *m;
+-
+- m = get_mqd(mqd);
+-
+- *ctl_stack_used_size = m->cp_hqd_cntl_stack_size -
+- m->cp_hqd_cntl_stack_offset;
+- *save_area_used_size = m->cp_hqd_wg_state_offset -
+- m->cp_hqd_cntl_stack_size;
+-
+- /* Control stack is not copied to user mode for GFXv8 because
+- * it's part of the context save area that is already
+- * accessible to user mode
+- */
+-
+- return 0;
+-}
+-
+ static int init_mqd_hiq(struct mqd_manager *mm, void **mqd,
+ struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
+ struct queue_properties *q)
+@@ -367,130 +233,17 @@ static int update_mqd_hiq(struct mqd_manager *mm, void *mqd,
+ return retval;
+ }
+
+-static int init_mqd_sdma(struct mqd_manager *mm, void **mqd,
+- struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
+- struct queue_properties *q)
+-{
+- int retval;
+- struct vi_sdma_mqd *m;
+-
+-
+- retval = kfd_gtt_sa_allocate(mm->dev,
+- sizeof(struct vi_sdma_mqd),
+- mqd_mem_obj);
+-
+- if (retval != 0)
+- return -ENOMEM;
+-
+- m = (struct vi_sdma_mqd *) (*mqd_mem_obj)->cpu_ptr;
+-
+- memset(m, 0, sizeof(struct vi_sdma_mqd));
+-
+- *mqd = m;
+- if (gart_addr)
+- *gart_addr = (*mqd_mem_obj)->gpu_addr;
+-
+- retval = mm->update_mqd(mm, m, q);
+-
+- return retval;
+-}
+-
+-static void uninit_mqd_sdma(struct mqd_manager *mm, void *mqd,
+- struct kfd_mem_obj *mqd_mem_obj)
+-{
+- kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
+-}
+-
+-static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
+- uint32_t pipe_id, uint32_t queue_id,
+- struct queue_properties *p, struct mm_struct *mms)
+-{
+- return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd,
+- (uint32_t __user *)p->write_ptr,
+- mms);
+-}
+-
+-static int update_mqd_sdma(struct mqd_manager *mm, void *mqd,
+- struct queue_properties *q)
+-{
+- struct vi_sdma_mqd *m;
+-
+- m = get_sdma_mqd(mqd);
+- m->sdmax_rlcx_rb_cntl = (ffs(q->queue_size / sizeof(unsigned int)) - 1)
+- << SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT |
+- q->vmid << SDMA0_RLC0_RB_CNTL__RB_VMID__SHIFT |
+- 1 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
+- 6 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT;
+-
+- m->sdmax_rlcx_rb_base = lower_32_bits(q->queue_address >> 8);
+- m->sdmax_rlcx_rb_base_hi = upper_32_bits(q->queue_address >> 8);
+- m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
+- m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
+- m->sdmax_rlcx_doorbell =
+- q->doorbell_off << SDMA0_RLC0_DOORBELL__OFFSET__SHIFT;
+-
+- m->sdmax_rlcx_virtual_addr = q->sdma_vm_addr;
+-
+- m->sdma_engine_id = q->sdma_engine_id;
+- m->sdma_queue_id = q->sdma_queue_id;
+-
+- q->is_active = false;
+- if (q->queue_size > 0 &&
+- q->queue_address != 0 &&
+- q->queue_percent > 0 &&
+- !q->is_evicted) {
+- q->is_active = true;
+- }
+-
+- return 0;
+-}
+-
+-/*
+- * * preempt type here is ignored because there is only one way
+- * * to preempt sdma queue
+- */
+-static int destroy_mqd_sdma(struct mqd_manager *mm, void *mqd,
+- enum kfd_preempt_type type,
+- unsigned int timeout, uint32_t pipe_id,
+- uint32_t queue_id)
+-{
+- return mm->dev->kfd2kgd->hqd_sdma_destroy(mm->dev->kgd, mqd, timeout);
+-}
+-
+-static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
+- uint64_t queue_address, uint32_t pipe_id,
+- uint32_t queue_id)
+-{
+- return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd);
+-}
+-
+-#if defined(CONFIG_DEBUG_FS)
+-
+-static int debugfs_show_mqd(struct seq_file *m, void *data)
+-{
+- seq_hex_dump(m, " ", DUMP_PREFIX_OFFSET, 32, 4,
+- data, sizeof(struct vi_mqd), false);
+- return 0;
+-}
+-
+-static int debugfs_show_mqd_sdma(struct seq_file *m, void *data)
+-{
+- seq_hex_dump(m, " ", DUMP_PREFIX_OFFSET, 32, 4,
+- data, sizeof(struct vi_sdma_mqd), false);
+- return 0;
+-}
+-
+-#endif
+-
+ struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
+ struct kfd_dev *dev)
+ {
+ struct mqd_manager *mqd;
+
+- if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
+- return NULL;
++ BUG_ON(!dev);
++ BUG_ON(type >= KFD_MQD_TYPE_MAX);
++
++ pr_debug("kfd: In func %s\n", __func__);
+
+- mqd = kzalloc(sizeof(*mqd), GFP_NOIO);
++ mqd = kzalloc(sizeof(struct mqd_manager), GFP_KERNEL);
+ if (!mqd)
+ return NULL;
+
+@@ -505,10 +258,6 @@ struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
+ mqd->update_mqd = update_mqd;
+ mqd->destroy_mqd = destroy_mqd;
+ mqd->is_occupied = is_occupied;
+- mqd->get_wave_state = get_wave_state;
+-#if defined(CONFIG_DEBUG_FS)
+- mqd->debugfs_show_mqd = debugfs_show_mqd;
+-#endif
+ break;
+ case KFD_MQD_TYPE_HIQ:
+ mqd->init_mqd = init_mqd_hiq;
+@@ -517,20 +266,8 @@ struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
+ mqd->update_mqd = update_mqd_hiq;
+ mqd->destroy_mqd = destroy_mqd;
+ mqd->is_occupied = is_occupied;
+-#if defined(CONFIG_DEBUG_FS)
+- mqd->debugfs_show_mqd = debugfs_show_mqd;
+-#endif
+ break;
+ case KFD_MQD_TYPE_SDMA:
+- mqd->init_mqd = init_mqd_sdma;
+- mqd->uninit_mqd = uninit_mqd_sdma;
+- mqd->load_mqd = load_mqd_sdma;
+- mqd->update_mqd = update_mqd_sdma;
+- mqd->destroy_mqd = destroy_mqd_sdma;
+- mqd->is_occupied = is_occupied_sdma;
+-#if defined(CONFIG_DEBUG_FS)
+- mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;
+-#endif
+ break;
+ default:
+ kfree(mqd);
+@@ -539,17 +276,3 @@ struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
+
+ return mqd;
+ }
+-
+-struct mqd_manager *mqd_manager_init_vi_tonga(enum KFD_MQD_TYPE type,
+- struct kfd_dev *dev)
+-{
+- struct mqd_manager *mqd;
+-
+- mqd = mqd_manager_init_vi(type, dev);
+- if (!mqd)
+- return NULL;
+- if ((type == KFD_MQD_TYPE_CP) || (type == KFD_MQD_TYPE_COMPUTE))
+- mqd->update_mqd = update_mqd_tonga;
+- return mqd;
+-}
+-
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
+index 7cca7b4..7e92921 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
+@@ -26,6 +26,8 @@
+ #include "kfd_device_queue_manager.h"
+ #include "kfd_kernel_queue.h"
+ #include "kfd_priv.h"
++#include "kfd_pm4_headers.h"
++#include "kfd_pm4_headers_vi.h"
+ #include "kfd_pm4_opcodes.h"
+
+ static inline void inc_wptr(unsigned int *wptr, unsigned int increment_bytes,
+@@ -33,45 +35,47 @@ static inline void inc_wptr(unsigned int *wptr, unsigned int increment_bytes,
+ {
+ unsigned int temp = *wptr + increment_bytes / sizeof(uint32_t);
+
+- WARN((temp * sizeof(uint32_t)) > buffer_size_bytes,
+- "Runlist IB overflow");
++ BUG_ON((temp * sizeof(uint32_t)) > buffer_size_bytes);
+ *wptr = temp;
+ }
+
++static unsigned int build_pm4_header(unsigned int opcode, size_t packet_size)
++{
++ union PM4_MES_TYPE_3_HEADER header;
++
++ header.u32all = 0;
++ header.opcode = opcode;
++ header.count = packet_size/sizeof(uint32_t) - 2;
++ header.type = PM4_TYPE_3;
++
++ return header.u32all;
++}
++
+ static void pm_calc_rlib_size(struct packet_manager *pm,
+ unsigned int *rlib_size,
+ bool *over_subscription)
+ {
+- unsigned int process_count, queue_count, compute_queue_count;
++ unsigned int process_count, queue_count;
+ unsigned int map_queue_size;
+- unsigned int max_proc_per_quantum = 1;
+
+- struct kfd_dev *dev = pm->dqm->dev;
++ BUG_ON(!pm || !rlib_size || !over_subscription);
+
+ process_count = pm->dqm->processes_count;
+ queue_count = pm->dqm->queue_count;
+- compute_queue_count = queue_count - pm->dqm->sdma_queue_count;
+-
+- /* check if there is over subscription
+- * Note: the arbitration between the number of VMIDs and
+- * hws_max_conc_proc has been done in
+- * kgd2kfd_device_init().
+- */
+
++ /* check if there is over subscription*/
+ *over_subscription = false;
+-
+- if (dev->max_proc_per_quantum > 1)
+- max_proc_per_quantum = dev->max_proc_per_quantum;
+-
+- if ((process_count > max_proc_per_quantum) ||
+- compute_queue_count > get_queues_num(pm->dqm)) {
++ if ((process_count > 1) || queue_count > get_queues_num(pm->dqm)) {
+ *over_subscription = true;
+- pr_debug("Over subscribed runlist\n");
++ pr_debug("kfd: over subscribed runlist\n");
+ }
+
+- map_queue_size = pm->pmf->get_map_queues_packet_size();
++ map_queue_size =
++ (pm->dqm->dev->device_info->asic_family == CHIP_CARRIZO) ?
++ sizeof(struct pm4_mes_map_queues) :
++ sizeof(struct pm4_map_queues);
+ /* calculate run list ib allocation size */
+- *rlib_size = process_count * pm->pmf->get_map_process_packet_size() +
++ *rlib_size = process_count * sizeof(struct pm4_map_process) +
+ queue_count * map_queue_size;
+
+ /*
+@@ -79,9 +83,9 @@ static void pm_calc_rlib_size(struct packet_manager *pm,
+ * when over subscription
+ */
+ if (*over_subscription)
+- *rlib_size += pm->pmf->get_runlist_packet_size();
++ *rlib_size += sizeof(struct pm4_runlist);
+
+- pr_debug("runlist ib size %d\n", *rlib_size);
++ pr_debug("kfd: runlist ib size %d\n", *rlib_size);
+ }
+
+ static int pm_allocate_runlist_ib(struct packet_manager *pm,
+@@ -92,19 +96,18 @@ static int pm_allocate_runlist_ib(struct packet_manager *pm,
+ {
+ int retval;
+
+- if (WARN_ON(pm->allocated))
+- return -EINVAL;
++ BUG_ON(!pm);
++ BUG_ON(pm->allocated);
++ BUG_ON(is_over_subscription == NULL);
+
+ pm_calc_rlib_size(pm, rl_buffer_size, is_over_subscription);
+
+- mutex_lock(&pm->lock);
+-
+ retval = kfd_gtt_sa_allocate(pm->dqm->dev, *rl_buffer_size,
+ &pm->ib_buffer_obj);
+
+- if (retval) {
+- pr_err("Failed to allocate runlist IB\n");
+- goto out;
++ if (retval != 0) {
++ pr_err("kfd: failed to allocate runlist IB\n");
++ return retval;
+ }
+
+ *(void **)rl_buffer = pm->ib_buffer_obj->cpu_ptr;
+@@ -112,12 +115,198 @@ static int pm_allocate_runlist_ib(struct packet_manager *pm,
+
+ memset(*rl_buffer, 0, *rl_buffer_size);
+ pm->allocated = true;
+-
+-out:
+- mutex_unlock(&pm->lock);
+ return retval;
+ }
+
++static int pm_create_runlist(struct packet_manager *pm, uint32_t *buffer,
++ uint64_t ib, size_t ib_size_in_dwords, bool chain)
++{
++ struct pm4_runlist *packet;
++
++ BUG_ON(!pm || !buffer || !ib);
++
++ packet = (struct pm4_runlist *)buffer;
++
++ memset(buffer, 0, sizeof(struct pm4_runlist));
++ packet->header.u32all = build_pm4_header(IT_RUN_LIST,
++ sizeof(struct pm4_runlist));
++
++ packet->bitfields4.ib_size = ib_size_in_dwords;
++ packet->bitfields4.chain = chain ? 1 : 0;
++ packet->bitfields4.offload_polling = 0;
++ packet->bitfields4.valid = 1;
++ packet->ordinal2 = lower_32_bits(ib);
++ packet->bitfields3.ib_base_hi = upper_32_bits(ib);
++
++ return 0;
++}
++
++static int pm_create_map_process(struct packet_manager *pm, uint32_t *buffer,
++ struct qcm_process_device *qpd)
++{
++ struct pm4_map_process *packet;
++ struct queue *cur;
++ uint32_t num_queues;
++
++ BUG_ON(!pm || !buffer || !qpd);
++
++ packet = (struct pm4_map_process *)buffer;
++
++ pr_debug("kfd: In func %s\n", __func__);
++
++ memset(buffer, 0, sizeof(struct pm4_map_process));
++
++ packet->header.u32all = build_pm4_header(IT_MAP_PROCESS,
++ sizeof(struct pm4_map_process));
++ packet->bitfields2.diq_enable = (qpd->is_debug) ? 1 : 0;
++ packet->bitfields2.process_quantum = 1;
++ packet->bitfields2.pasid = qpd->pqm->process->pasid;
++ packet->bitfields3.page_table_base = qpd->page_table_base;
++ packet->bitfields10.gds_size = qpd->gds_size;
++ packet->bitfields10.num_gws = qpd->num_gws;
++ packet->bitfields10.num_oac = qpd->num_oac;
++ num_queues = 0;
++ list_for_each_entry(cur, &qpd->queues_list, list)
++ num_queues++;
++ packet->bitfields10.num_queues = (qpd->is_debug) ? 0 : num_queues;
++
++ packet->sh_mem_config = qpd->sh_mem_config;
++ packet->sh_mem_bases = qpd->sh_mem_bases;
++ packet->sh_mem_ape1_base = qpd->sh_mem_ape1_base;
++ packet->sh_mem_ape1_limit = qpd->sh_mem_ape1_limit;
++
++ packet->gds_addr_lo = lower_32_bits(qpd->gds_context_area);
++ packet->gds_addr_hi = upper_32_bits(qpd->gds_context_area);
++
++ return 0;
++}
++
++static int pm_create_map_queue_vi(struct packet_manager *pm, uint32_t *buffer,
++ struct queue *q, bool is_static)
++{
++ struct pm4_mes_map_queues *packet;
++ bool use_static = is_static;
++
++ BUG_ON(!pm || !buffer || !q);
++
++ pr_debug("kfd: In func %s\n", __func__);
++
++ packet = (struct pm4_mes_map_queues *)buffer;
++ memset(buffer, 0, sizeof(struct pm4_map_queues));
++
++ packet->header.u32all = build_pm4_header(IT_MAP_QUEUES,
++ sizeof(struct pm4_map_queues));
++ packet->bitfields2.alloc_format =
++ alloc_format__mes_map_queues__one_per_pipe_vi;
++ packet->bitfields2.num_queues = 1;
++ packet->bitfields2.queue_sel =
++ queue_sel__mes_map_queues__map_to_hws_determined_queue_slots_vi;
++
++ packet->bitfields2.engine_sel =
++ engine_sel__mes_map_queues__compute_vi;
++ packet->bitfields2.queue_type =
++ queue_type__mes_map_queues__normal_compute_vi;
++
++ switch (q->properties.type) {
++ case KFD_QUEUE_TYPE_COMPUTE:
++ if (use_static)
++ packet->bitfields2.queue_type =
++ queue_type__mes_map_queues__normal_latency_static_queue_vi;
++ break;
++ case KFD_QUEUE_TYPE_DIQ:
++ packet->bitfields2.queue_type =
++ queue_type__mes_map_queues__debug_interface_queue_vi;
++ break;
++ case KFD_QUEUE_TYPE_SDMA:
++ packet->bitfields2.engine_sel =
++ engine_sel__mes_map_queues__sdma0_vi;
++ use_static = false; /* no static queues under SDMA */
++ break;
++ default:
++ pr_err("kfd: in %s queue type %d\n", __func__,
++ q->properties.type);
++ BUG();
++ break;
++ }
++ packet->bitfields3.doorbell_offset =
++ q->properties.doorbell_off;
++
++ packet->mqd_addr_lo =
++ lower_32_bits(q->gart_mqd_addr);
++
++ packet->mqd_addr_hi =
++ upper_32_bits(q->gart_mqd_addr);
++
++ packet->wptr_addr_lo =
++ lower_32_bits((uint64_t)q->properties.write_ptr);
++
++ packet->wptr_addr_hi =
++ upper_32_bits((uint64_t)q->properties.write_ptr);
++
++ return 0;
++}
++
++static int pm_create_map_queue(struct packet_manager *pm, uint32_t *buffer,
++ struct queue *q, bool is_static)
++{
++ struct pm4_map_queues *packet;
++ bool use_static = is_static;
++
++ BUG_ON(!pm || !buffer || !q);
++
++ pr_debug("kfd: In func %s\n", __func__);
++
++ packet = (struct pm4_map_queues *)buffer;
++ memset(buffer, 0, sizeof(struct pm4_map_queues));
++
++ packet->header.u32all = build_pm4_header(IT_MAP_QUEUES,
++ sizeof(struct pm4_map_queues));
++ packet->bitfields2.alloc_format =
++ alloc_format__mes_map_queues__one_per_pipe;
++ packet->bitfields2.num_queues = 1;
++ packet->bitfields2.queue_sel =
++ queue_sel__mes_map_queues__map_to_hws_determined_queue_slots;
++
++ packet->bitfields2.vidmem = (q->properties.is_interop) ?
++ vidmem__mes_map_queues__uses_video_memory :
++ vidmem__mes_map_queues__uses_no_video_memory;
++
++ switch (q->properties.type) {
++ case KFD_QUEUE_TYPE_COMPUTE:
++ case KFD_QUEUE_TYPE_DIQ:
++ packet->bitfields2.engine_sel =
++ engine_sel__mes_map_queues__compute;
++ break;
++ case KFD_QUEUE_TYPE_SDMA:
++ packet->bitfields2.engine_sel =
++ engine_sel__mes_map_queues__sdma0;
++ use_static = false; /* no static queues under SDMA */
++ break;
++ default:
++ BUG();
++ break;
++ }
++
++ packet->mes_map_queues_ordinals[0].bitfields3.doorbell_offset =
++ q->properties.doorbell_off;
++
++ packet->mes_map_queues_ordinals[0].bitfields3.is_static =
++ (use_static) ? 1 : 0;
++
++ packet->mes_map_queues_ordinals[0].mqd_addr_lo =
++ lower_32_bits(q->gart_mqd_addr);
++
++ packet->mes_map_queues_ordinals[0].mqd_addr_hi =
++ upper_32_bits(q->gart_mqd_addr);
++
++ packet->mes_map_queues_ordinals[0].wptr_addr_lo =
++ lower_32_bits((uint64_t)q->properties.write_ptr);
++
++ packet->mes_map_queues_ordinals[0].wptr_addr_hi =
++ upper_32_bits((uint64_t)q->properties.write_ptr);
++
++ return 0;
++}
+
+ static int pm_create_runlist_ib(struct packet_manager *pm,
+ struct list_head *queues,
+@@ -133,17 +322,19 @@ static int pm_create_runlist_ib(struct packet_manager *pm,
+ struct kernel_queue *kq;
+ bool is_over_subscription;
+
++ BUG_ON(!pm || !queues || !rl_size_bytes || !rl_gpu_addr);
++
+ rl_wptr = retval = proccesses_mapped = 0;
+
+ retval = pm_allocate_runlist_ib(pm, &rl_buffer, rl_gpu_addr,
+ &alloc_size_bytes, &is_over_subscription);
+- if (retval)
++ if (retval != 0)
+ return retval;
+
+ *rl_size_bytes = alloc_size_bytes;
+- pm->ib_size_bytes = alloc_size_bytes;
+
+- pr_debug("Building runlist ib process count: %d queues count %d\n",
++ pr_debug("kfd: In func %s\n", __func__);
++ pr_debug("kfd: building runlist ib process count: %d queues count %d\n",
+ pm->dqm->processes_count, pm->dqm->queue_count);
+
+ /* build the run list ib packet */
+@@ -151,35 +342,42 @@ static int pm_create_runlist_ib(struct packet_manager *pm,
+ qpd = cur->qpd;
+ /* build map process packet */
+ if (proccesses_mapped >= pm->dqm->processes_count) {
+- pr_debug("Not enough space left in runlist IB\n");
++ pr_debug("kfd: not enough space left in runlist IB\n");
+ pm_release_ib(pm);
+ return -ENOMEM;
+ }
+
+- retval = pm->pmf->map_process(pm, &rl_buffer[rl_wptr], qpd);
+- if (retval)
++ retval = pm_create_map_process(pm, &rl_buffer[rl_wptr], qpd);
++ if (retval != 0)
+ return retval;
+
+ proccesses_mapped++;
+- inc_wptr(&rl_wptr, pm->pmf->get_map_process_packet_size(),
++ inc_wptr(&rl_wptr, sizeof(struct pm4_map_process),
+ alloc_size_bytes);
+
+ list_for_each_entry(kq, &qpd->priv_queue_list, list) {
+ if (!kq->queue->properties.is_active)
+ continue;
+
+- pr_debug("static_queue, mapping kernel q %d, is debug status %d\n",
++ pr_debug("kfd: static_queue, mapping kernel q %d, is debug status %d\n",
+ kq->queue->queue, qpd->is_debug);
+
+- retval = pm->pmf->map_queues(pm,
++ if (pm->dqm->dev->device_info->asic_family ==
++ CHIP_CARRIZO)
++ retval = pm_create_map_queue_vi(pm,
++ &rl_buffer[rl_wptr],
++ kq->queue,
++ qpd->is_debug);
++ else
++ retval = pm_create_map_queue(pm,
+ &rl_buffer[rl_wptr],
+ kq->queue,
+ qpd->is_debug);
+- if (retval)
++ if (retval != 0)
+ return retval;
+
+ inc_wptr(&rl_wptr,
+- pm->pmf->get_map_queues_packet_size(),
++ sizeof(struct pm4_map_queues),
+ alloc_size_bytes);
+ }
+
+@@ -187,74 +385,63 @@ static int pm_create_runlist_ib(struct packet_manager *pm,
+ if (!q->properties.is_active)
+ continue;
+
+- pr_debug("static_queue, mapping user queue %d, is debug status %d\n",
++ pr_debug("kfd: static_queue, mapping user queue %d, is debug status %d\n",
+ q->queue, qpd->is_debug);
+
+- retval = pm->pmf->map_queues(pm,
++ if (pm->dqm->dev->device_info->asic_family ==
++ CHIP_CARRIZO)
++ retval = pm_create_map_queue_vi(pm,
+ &rl_buffer[rl_wptr],
+ q,
+ qpd->is_debug);
+- if (retval)
++ else
++ retval = pm_create_map_queue(pm,
++ &rl_buffer[rl_wptr],
++ q,
++ qpd->is_debug);
++
++ if (retval != 0)
+ return retval;
+
+ inc_wptr(&rl_wptr,
+- pm->pmf->get_map_queues_packet_size(),
++ sizeof(struct pm4_map_queues),
+ alloc_size_bytes);
+ }
+ }
+
+- pr_debug("Finished map process and queues to runlist\n");
++ pr_debug("kfd: finished map process and queues to runlist\n");
+
+ if (is_over_subscription)
+- retval = pm->pmf->runlist(pm, &rl_buffer[rl_wptr],
+- *rl_gpu_addr,
+- alloc_size_bytes / sizeof(uint32_t),
+- true);
++ pm_create_runlist(pm, &rl_buffer[rl_wptr], *rl_gpu_addr,
++ alloc_size_bytes / sizeof(uint32_t), true);
+
+ for (i = 0; i < alloc_size_bytes / sizeof(uint32_t); i++)
+ pr_debug("0x%2X ", rl_buffer[i]);
+ pr_debug("\n");
+
+- return retval;
++ return 0;
+ }
+
+-int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm,
+- uint16_t fw_ver)
++int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm)
+ {
++ BUG_ON(!dqm);
++
+ pm->dqm = dqm;
+ mutex_init(&pm->lock);
+ pm->priv_queue = kernel_queue_init(dqm->dev, KFD_QUEUE_TYPE_HIQ);
+- if (!pm->priv_queue) {
++ if (pm->priv_queue == NULL) {
+ mutex_destroy(&pm->lock);
+ return -ENOMEM;
+ }
+ pm->allocated = false;
+
+- switch (pm->dqm->dev->device_info->asic_family) {
+- case CHIP_KAVERI:
+- case CHIP_HAWAII:
+- kfd_pm_func_init_cik(pm, fw_ver);
+- break;
+- case CHIP_CARRIZO:
+- case CHIP_TONGA:
+- case CHIP_FIJI:
+- case CHIP_POLARIS10:
+- case CHIP_POLARIS11:
+- kfd_pm_func_init_vi(pm, fw_ver);
+- break;
+- case CHIP_VEGA10:
+- case CHIP_RAVEN:
+- kfd_pm_func_init_v9(pm, fw_ver);
+- break;
+- default:
+- BUG();
+- }
+-
+ return 0;
+ }
+
+ void pm_uninit(struct packet_manager *pm)
+ {
++ BUG_ON(!pm);
++
+ mutex_destroy(&pm->lock);
+ kernel_queue_uninit(pm->priv_queue);
+ }
+@@ -262,30 +449,45 @@ void pm_uninit(struct packet_manager *pm)
+ int pm_send_set_resources(struct packet_manager *pm,
+ struct scheduling_resources *res)
+ {
+- uint32_t *buffer, size;
+- int retval = 0;
++ struct pm4_set_resources *packet;
++
++ BUG_ON(!pm || !res);
++
++ pr_debug("kfd: In func %s\n", __func__);
+
+- size = pm->pmf->get_set_resources_packet_size();
+ mutex_lock(&pm->lock);
+ pm->priv_queue->ops.acquire_packet_buffer(pm->priv_queue,
+- size / sizeof(uint32_t),
+- (unsigned int **)&buffer);
+- if (!buffer) {
+- pr_err("Failed to allocate buffer on kernel queue\n");
+- retval = -ENOMEM;
+- goto out;
++ sizeof(*packet) / sizeof(uint32_t),
++ (unsigned int **)&packet);
++ if (packet == NULL) {
++ mutex_unlock(&pm->lock);
++ pr_err("kfd: failed to allocate buffer on kernel queue\n");
++ return -ENOMEM;
+ }
+
+- retval = pm->pmf->set_resources(pm, buffer, res);
+- if (!retval)
+- pm->priv_queue->ops.submit_packet(pm->priv_queue);
+- else
+- pm->priv_queue->ops.rollback_packet(pm->priv_queue);
++ memset(packet, 0, sizeof(struct pm4_set_resources));
++ packet->header.u32all = build_pm4_header(IT_SET_RESOURCES,
++ sizeof(struct pm4_set_resources));
++
++ packet->bitfields2.queue_type =
++ queue_type__mes_set_resources__hsa_interface_queue_hiq;
++ packet->bitfields2.vmid_mask = res->vmid_mask;
++ packet->bitfields2.unmap_latency = KFD_UNMAP_LATENCY;
++ packet->bitfields7.oac_mask = res->oac_mask;
++ packet->bitfields8.gds_heap_base = res->gds_heap_base;
++ packet->bitfields8.gds_heap_size = res->gds_heap_size;
++
++ packet->gws_mask_lo = lower_32_bits(res->gws_mask);
++ packet->gws_mask_hi = upper_32_bits(res->gws_mask);
++
++ packet->queue_mask_lo = lower_32_bits(res->queue_mask);
++ packet->queue_mask_hi = upper_32_bits(res->queue_mask);
++
++ pm->priv_queue->ops.submit_packet(pm->priv_queue);
+
+-out:
+ mutex_unlock(&pm->lock);
+
+- return retval;
++ return 0;
+ }
+
+ int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues)
+@@ -295,25 +497,26 @@ int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues)
+ size_t rl_ib_size, packet_size_dwords;
+ int retval;
+
++ BUG_ON(!pm || !dqm_queues);
++
+ retval = pm_create_runlist_ib(pm, dqm_queues, &rl_gpu_ib_addr,
+ &rl_ib_size);
+- if (retval)
++ if (retval != 0)
+ goto fail_create_runlist_ib;
+
+- pr_debug("runlist IB address: 0x%llX\n", rl_gpu_ib_addr);
++ pr_debug("kfd: runlist IB address: 0x%llX\n", rl_gpu_ib_addr);
+
+- packet_size_dwords = pm->pmf->get_runlist_packet_size() /
+- sizeof(uint32_t);
++ packet_size_dwords = sizeof(struct pm4_runlist) / sizeof(uint32_t);
+ mutex_lock(&pm->lock);
+
+ retval = pm->priv_queue->ops.acquire_packet_buffer(pm->priv_queue,
+ packet_size_dwords, &rl_buffer);
+- if (retval)
++ if (retval != 0)
+ goto fail_acquire_packet_buffer;
+
+- retval = pm->pmf->runlist(pm, rl_buffer, rl_gpu_ib_addr,
+- rl_ib_size / sizeof(uint32_t), false);
+- if (retval)
++ retval = pm_create_runlist(pm, rl_buffer, rl_gpu_ib_addr,
++ rl_ib_size / sizeof(uint32_t), false);
++ if (retval != 0)
+ goto fail_create_runlist;
+
+ pm->priv_queue->ops.submit_packet(pm->priv_queue);
+@@ -327,72 +530,138 @@ int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues)
+ fail_acquire_packet_buffer:
+ mutex_unlock(&pm->lock);
+ fail_create_runlist_ib:
+- pm_release_ib(pm);
++ if (pm->allocated)
++ pm_release_ib(pm);
+ return retval;
+ }
+
+ int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address,
+ uint32_t fence_value)
+ {
+- uint32_t *buffer, size;
+- int retval = 0;
++ int retval;
++ struct pm4_query_status *packet;
+
+- if (WARN_ON(!fence_address))
+- return -EFAULT;
++ BUG_ON(!pm || !fence_address);
+
+- size = pm->pmf->get_query_status_packet_size();
+ mutex_lock(&pm->lock);
+- pm->priv_queue->ops.acquire_packet_buffer(pm->priv_queue,
+- size / sizeof(uint32_t), (unsigned int **)&buffer);
+- if (!buffer) {
+- pr_err("Failed to allocate buffer on kernel queue\n");
+- retval = -ENOMEM;
+- goto out;
+- }
++ retval = pm->priv_queue->ops.acquire_packet_buffer(
++ pm->priv_queue,
++ sizeof(struct pm4_query_status) / sizeof(uint32_t),
++ (unsigned int **)&packet);
++ if (retval != 0)
++ goto fail_acquire_packet_buffer;
+
+- retval = pm->pmf->query_status(pm, buffer, fence_address, fence_value);
+- if (!retval)
+- pm->priv_queue->ops.submit_packet(pm->priv_queue);
+- else
+- pm->priv_queue->ops.rollback_packet(pm->priv_queue);
++ packet->header.u32all = build_pm4_header(IT_QUERY_STATUS,
++ sizeof(struct pm4_query_status));
++
++ packet->bitfields2.context_id = 0;
++ packet->bitfields2.interrupt_sel =
++ interrupt_sel__mes_query_status__completion_status;
++ packet->bitfields2.command =
++ command__mes_query_status__fence_only_after_write_ack;
++
++ packet->addr_hi = upper_32_bits((uint64_t)fence_address);
++ packet->addr_lo = lower_32_bits((uint64_t)fence_address);
++ packet->data_hi = upper_32_bits((uint64_t)fence_value);
++ packet->data_lo = lower_32_bits((uint64_t)fence_value);
++
++ pm->priv_queue->ops.submit_packet(pm->priv_queue);
++ mutex_unlock(&pm->lock);
++
++ return 0;
+
+-out:
++fail_acquire_packet_buffer:
+ mutex_unlock(&pm->lock);
+ return retval;
+ }
+
+ int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,
+- enum kfd_unmap_queues_filter filter,
++ enum kfd_preempt_type_filter mode,
+ uint32_t filter_param, bool reset,
+ unsigned int sdma_engine)
+ {
+- uint32_t *buffer, size;
+- int retval = 0;
++ int retval;
++ uint32_t *buffer;
++ struct pm4_unmap_queues *packet;
++
++ BUG_ON(!pm);
+
+- size = pm->pmf->get_unmap_queues_packet_size();
+ mutex_lock(&pm->lock);
+- pm->priv_queue->ops.acquire_packet_buffer(pm->priv_queue,
+- size / sizeof(uint32_t), (unsigned int **)&buffer);
+- if (!buffer) {
+- pr_err("Failed to allocate buffer on kernel queue\n");
+- retval = -ENOMEM;
+- goto out;
++ retval = pm->priv_queue->ops.acquire_packet_buffer(
++ pm->priv_queue,
++ sizeof(struct pm4_unmap_queues) / sizeof(uint32_t),
++ &buffer);
++ if (retval != 0)
++ goto err_acquire_packet_buffer;
++
++ packet = (struct pm4_unmap_queues *)buffer;
++ memset(buffer, 0, sizeof(struct pm4_unmap_queues));
++ pr_debug("kfd: static_queue: unmapping queues: mode is %d , reset is %d , type is %d\n",
++ mode, reset, type);
++ packet->header.u32all = build_pm4_header(IT_UNMAP_QUEUES,
++ sizeof(struct pm4_unmap_queues));
++ switch (type) {
++ case KFD_QUEUE_TYPE_COMPUTE:
++ case KFD_QUEUE_TYPE_DIQ:
++ packet->bitfields2.engine_sel =
++ engine_sel__mes_unmap_queues__compute;
++ break;
++ case KFD_QUEUE_TYPE_SDMA:
++ packet->bitfields2.engine_sel =
++ engine_sel__mes_unmap_queues__sdma0 + sdma_engine;
++ break;
++ default:
++ BUG();
++ break;
+ }
+
+- retval = pm->pmf->unmap_queues(pm, buffer, type, filter, filter_param,
+- reset, sdma_engine);
+- if (!retval)
+- pm->priv_queue->ops.submit_packet(pm->priv_queue);
++ if (reset)
++ packet->bitfields2.action =
++ action__mes_unmap_queues__reset_queues;
+ else
+- pm->priv_queue->ops.rollback_packet(pm->priv_queue);
++ packet->bitfields2.action =
++ action__mes_unmap_queues__preempt_queues;
++
++ switch (mode) {
++ case KFD_PREEMPT_TYPE_FILTER_SINGLE_QUEUE:
++ packet->bitfields2.queue_sel =
++ queue_sel__mes_unmap_queues__perform_request_on_specified_queues;
++ packet->bitfields2.num_queues = 1;
++ packet->bitfields3b.doorbell_offset0 = filter_param;
++ break;
++ case KFD_PREEMPT_TYPE_FILTER_BY_PASID:
++ packet->bitfields2.queue_sel =
++ queue_sel__mes_unmap_queues__perform_request_on_pasid_queues;
++ packet->bitfields3a.pasid = filter_param;
++ break;
++ case KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES:
++ packet->bitfields2.queue_sel =
++ queue_sel__mes_unmap_queues__perform_request_on_all_active_queues;
++ break;
++ case KFD_PREEMPT_TYPE_FILTER_DYNAMIC_QUEUES:
++ /* in this case, we do not preempt static queues */
++ packet->bitfields2.queue_sel =
++ queue_sel__mes_unmap_queues__perform_request_on_dynamic_queues_only;
++ break;
++ default:
++ BUG();
++ break;
++ }
++
++ pm->priv_queue->ops.submit_packet(pm->priv_queue);
+
+-out:
++ mutex_unlock(&pm->lock);
++ return 0;
++
++err_acquire_packet_buffer:
+ mutex_unlock(&pm->lock);
+ return retval;
+ }
+
+ void pm_release_ib(struct packet_manager *pm)
+ {
++ BUG_ON(!pm);
++
+ mutex_lock(&pm->lock);
+ if (pm->allocated) {
+ kfd_gtt_sa_free(pm->dqm->dev, pm->ib_buffer_obj);
+@@ -400,18 +669,3 @@ void pm_release_ib(struct packet_manager *pm)
+ }
+ mutex_unlock(&pm->lock);
+ }
+-
+-int pm_debugfs_runlist(struct seq_file *m, void *data)
+-{
+- struct packet_manager *pm = data;
+-
+- if (!pm->allocated) {
+- seq_puts(m, " No active runlist\n");
+- return 0;
+- }
+-
+- seq_hex_dump(m, " ", DUMP_PREFIX_OFFSET, 32, 4,
+- pm->ib_buffer_obj->cpu_ptr, pm->ib_size_bytes, false);
+-
+- return 0;
+-}
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_pasid.c b/drivers/gpu/drm/amd/amdkfd/kfd_pasid.c
+index 1e06de0..6cfe7f1 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_pasid.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_pasid.c
+@@ -32,8 +32,7 @@ int kfd_pasid_init(void)
+ {
+ pasid_limit = KFD_MAX_NUM_OF_PROCESSES;
+
+- pasid_bitmap = kcalloc(BITS_TO_LONGS(pasid_limit), sizeof(long),
+- GFP_KERNEL);
++ pasid_bitmap = kcalloc(BITS_TO_LONGS(pasid_limit), sizeof(long), GFP_KERNEL);
+ if (!pasid_bitmap)
+ return -ENOMEM;
+
+@@ -92,6 +91,6 @@ unsigned int kfd_pasid_alloc(void)
+
+ void kfd_pasid_free(unsigned int pasid)
+ {
+- if (!WARN_ON(pasid == 0 || pasid >= pasid_limit))
+- clear_bit(pasid, pasid_bitmap);
++ BUG_ON(pasid == 0 || pasid >= pasid_limit);
++ clear_bit(pasid, pasid_bitmap);
+ }
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_peerdirect.c b/drivers/gpu/drm/amd/amdkfd/kfd_peerdirect.c
+deleted file mode 100644
+index 543ed83..0000000
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_peerdirect.c
++++ /dev/null
+@@ -1,513 +0,0 @@
+-/*
+- * Copyright 2016 Advanced Micro Devices, Inc.
+- *
+- * Permission is hereby granted, free of charge, to any person obtaining a
+- * copy of this software and associated documentation files (the "Software"),
+- * to deal in the Software without restriction, including without limitation
+- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+- * and/or sell copies of the Software, and to permit persons to whom the
+- * Software is furnished to do so, subject to the following conditions:
+- *
+- * The above copyright notice and this permission notice shall be included in
+- * all copies or substantial portions of the Software.
+- *
+- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+- * OTHER DEALINGS IN THE SOFTWARE.
+- */
+-
+-
+-/* NOTE:
+- *
+- * This file contains logic to dynamically detect and enable PeerDirect
+- * suppor. PeerDirect support is delivered e.g. as part of OFED
+- * from Mellanox. Because we are not able to rely on the fact that the
+- * corresponding OFED will be installed we should:
+- * - copy PeerDirect definitions locally to avoid dependency on
+- * corresponding header file
+- * - try dynamically detect address of PeerDirect function
+- * pointers.
+- *
+- * If dynamic detection failed then PeerDirect support should be
+- * enabled using the standard PeerDirect bridge driver from:
+- * https://github.com/RadeonOpenCompute/ROCnRDMA
+- *
+- *
+- * Logic to support PeerDirect relies only on official public API to be
+- * non-intrusive as much as possible.
+- *
+- **/
+-
+-#include <linux/device.h>
+-#include <linux/export.h>
+-#include <linux/pid.h>
+-#include <linux/err.h>
+-#include <linux/slab.h>
+-#include <linux/scatterlist.h>
+-#include <linux/module.h>
+-
+-#include "kfd_priv.h"
+-
+-
+-
+-/* ----------------------- PeerDirect interface ------------------------------*/
+-
+-/*
+- * Copyright (c) 2013, Mellanox Technologies. All rights reserved.
+- *
+- * This software is available to you under a choice of one of two
+- * licenses. You may choose to be licensed under the terms of the GNU
+- * General Public License (GPL) Version 2, available from the file
+- * COPYING in the main directory of this source tree, or the
+- * OpenIB.org BSD license below:
+- *
+- * Redistribution and use in source and binary forms, with or
+- * without modification, are permitted provided that the following
+- * conditions are met:
+- *
+- * - Redistributions of source code must retain the above
+- * copyright notice, this list of conditions and the following
+- * disclaimer.
+- *
+- * - Redistributions in binary form must reproduce the above
+- * copyright notice, this list of conditions and the following
+- * disclaimer in the documentation and/or other materials
+- * provided with the distribution.
+- *
+- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+- * SOFTWARE.
+- */
+-#define IB_PEER_MEMORY_NAME_MAX 64
+-#define IB_PEER_MEMORY_VER_MAX 16
+-
+-struct peer_memory_client {
+- char name[IB_PEER_MEMORY_NAME_MAX];
+- char version[IB_PEER_MEMORY_VER_MAX];
+- /* acquire return code: 1-mine, 0-not mine */
+- int (*acquire)(unsigned long addr, size_t size,
+- void *peer_mem_private_data,
+- char *peer_mem_name,
+- void **client_context);
+- int (*get_pages)(unsigned long addr,
+- size_t size, int write, int force,
+- struct sg_table *sg_head,
+- void *client_context, void *core_context);
+- int (*dma_map)(struct sg_table *sg_head, void *client_context,
+- struct device *dma_device, int dmasync, int *nmap);
+- int (*dma_unmap)(struct sg_table *sg_head, void *client_context,
+- struct device *dma_device);
+- void (*put_pages)(struct sg_table *sg_head, void *client_context);
+- unsigned long (*get_page_size)(void *client_context);
+- void (*release)(void *client_context);
+- void* (*get_context_private_data)(u64 peer_id);
+- void (*put_context_private_data)(void *context);
+-};
+-
+-typedef int (*invalidate_peer_memory)(void *reg_handle,
+- void *core_context);
+-
+-void *ib_register_peer_memory_client(struct peer_memory_client *peer_client,
+- invalidate_peer_memory *invalidate_callback);
+-void ib_unregister_peer_memory_client(void *reg_handle);
+-
+-
+-/*------------------- PeerDirect bridge driver ------------------------------*/
+-
+-#define AMD_PEER_BRIDGE_DRIVER_VERSION "1.0"
+-#define AMD_PEER_BRIDGE_DRIVER_NAME "amdkfd"
+-
+-
+-static void* (*pfn_ib_register_peer_memory_client)(struct peer_memory_client
+- *peer_client,
+- invalidate_peer_memory
+- *invalidate_callback);
+-
+-static void (*pfn_ib_unregister_peer_memory_client)(void *reg_handle);
+-
+-static const struct amd_rdma_interface *rdma_interface;
+-
+-static invalidate_peer_memory ib_invalidate_callback;
+-static void *ib_reg_handle;
+-
+-struct amd_mem_context {
+- uint64_t va;
+- uint64_t size;
+- struct pid *pid;
+-
+- struct amd_p2p_info *p2p_info;
+-
+- /* Flag that free callback was called */
+- int free_callback_called;
+-
+- /* Context received from PeerDirect call */
+- void *core_context;
+-};
+-
+-
+-static void free_callback(void *client_priv)
+-{
+- struct amd_mem_context *mem_context =
+- (struct amd_mem_context *)client_priv;
+-
+- pr_debug("data 0x%p\n", mem_context);
+-
+- if (!mem_context) {
+- pr_warn("Invalid client context\n");
+- return;
+- }
+-
+- pr_debug("mem_context->core_context 0x%p\n", mem_context->core_context);
+-
+- /* Call back IB stack asking to invalidate memory */
+- (*ib_invalidate_callback) (ib_reg_handle, mem_context->core_context);
+-
+- /* amdkfd will free resources when we return from this callback.
+- * Set flag to inform that there is nothing to do on "put_pages", etc.
+- */
+- ACCESS_ONCE(mem_context->free_callback_called) = 1;
+-}
+-
+-
+-static int amd_acquire(unsigned long addr, size_t size,
+- void *peer_mem_private_data,
+- char *peer_mem_name, void **client_context)
+-{
+- int ret;
+- struct amd_mem_context *mem_context;
+- struct pid *pid;
+-
+- /* Get pointer to structure describing current process */
+- pid = get_task_pid(current, PIDTYPE_PID);
+-
+- pr_debug("addr:0x%lx,size:0x%x, pid 0x%p\n",
+- addr, (unsigned int)size, pid);
+-
+- /* Check if address is handled by AMD GPU driver */
+- ret = rdma_interface->is_gpu_address(addr, pid);
+-
+- if (!ret) {
+- pr_debug("Not GPU Address\n");
+- /* This is not GPU address */
+- return 0;
+- }
+-
+- pr_debug("GPU address\n");
+-
+- /* Initialize context used for operation with given address */
+- mem_context = kzalloc(sizeof(*mem_context), GFP_KERNEL);
+-
+- if (!mem_context)
+- return 0; /* Error case handled as not GPU address */
+-
+- mem_context->free_callback_called = 0;
+- mem_context->va = addr;
+- mem_context->size = size;
+-
+- /* Save PID. It is guaranteed that the function will be
+- * called in the correct process context as opposite to others.
+- */
+- mem_context->pid = pid;
+-
+- pr_debug("Client context %p\n", mem_context);
+-
+- /* Return pointer to allocated context */
+- *client_context = mem_context;
+-
+- /* Return 1 to inform that this address which will be handled
+- * by AMD GPU driver
+- */
+- return 1;
+-}
+-
+-static int amd_get_pages(unsigned long addr, size_t size, int write, int force,
+- struct sg_table *sg_head,
+- void *client_context, void *core_context)
+-{
+- int ret;
+- struct amd_mem_context *mem_context =
+- (struct amd_mem_context *)client_context;
+-
+- pr_debug("addr:0x%lx,size:0x%x, core_context:%p\n",
+- addr, (unsigned int)size, core_context);
+-
+- if (!mem_context) {
+- pr_warn("Invalid client context");
+- return -EINVAL;
+- }
+-
+- pr_debug("pid :0x%p\n", mem_context->pid);
+-
+-
+- if (addr != mem_context->va) {
+- pr_warn("Context address (0x%llx) is not the same\n",
+- mem_context->va);
+- return -EINVAL;
+- }
+-
+- if (size != mem_context->size) {
+- pr_warn("Context size (0x%llx) is not the same\n",
+- mem_context->size);
+- return -EINVAL;
+- }
+-
+- ret = rdma_interface->get_pages(addr,
+- size,
+- mem_context->pid,
+- &mem_context->p2p_info,
+- free_callback,
+- mem_context);
+-
+- if (ret || !mem_context->p2p_info) {
+- pr_err("Could not rdma::get_pages failure: %d\n", ret);
+- return ret;
+- }
+-
+- mem_context->core_context = core_context;
+-
+- /* Note: At this stage it is OK not to fill sg_table */
+- return 0;
+-}
+-
+-
+-static int amd_dma_map(struct sg_table *sg_head, void *client_context,
+- struct device *dma_device, int dmasync, int *nmap)
+-{
+- /*
+- * NOTE/TODO:
+- * We could have potentially three cases for real memory
+- * location:
+- * - all memory in the local
+- * - all memory in the system (RAM)
+- * - memory is spread (s/g) between local and system.
+- *
+- * In the case of all memory in the system we could use
+- * iommu driver to build DMA addresses but not in the case
+- * of local memory because currently iommu driver doesn't
+- * deal with local/device memory addresses (it requires "struct
+- * page").
+- *
+- * Accordingly returning assumes that iommu funcutionality
+- * should be disabled so we can assume that sg_table already
+- * contains DMA addresses.
+- *
+- */
+- struct amd_mem_context *mem_context =
+- (struct amd_mem_context *)client_context;
+-
+- pr_debug("Context 0x%p, sg_head 0x%p\n",
+- client_context, sg_head);
+-
+- pr_debug("pid 0x%p, address 0x%llx, size:0x%llx\n",
+- mem_context->pid,
+- mem_context->va,
+- mem_context->size);
+-
+- if (!mem_context->p2p_info) {
+- pr_err("No sg table were allocated\n");
+- return -EINVAL;
+- }
+-
+- /* Copy information about previosly allocated sg_table */
+- *sg_head = *mem_context->p2p_info->pages;
+-
+- /* Return number of pages */
+- *nmap = mem_context->p2p_info->pages->nents;
+-
+- return 0;
+-}
+-
+-static int amd_dma_unmap(struct sg_table *sg_head, void *client_context,
+- struct device *dma_device)
+-{
+- struct amd_mem_context *mem_context =
+- (struct amd_mem_context *)client_context;
+-
+- pr_debug("Context 0x%p, sg_table 0x%p\n",
+- client_context, sg_head);
+-
+- pr_debug("pid 0x%p, address 0x%llx, size:0x%llx\n",
+- mem_context->pid,
+- mem_context->va,
+- mem_context->size);
+-
+- /* Assume success */
+- return 0;
+-}
+-static void amd_put_pages(struct sg_table *sg_head, void *client_context)
+-{
+- int ret = 0;
+- struct amd_mem_context *mem_context =
+- (struct amd_mem_context *)client_context;
+-
+- pr_debug("sg_head %p client_context: 0x%p\n",
+- sg_head, client_context);
+- pr_debug("pid 0x%p, address 0x%llx, size:0x%llx\n",
+- mem_context->pid,
+- mem_context->va,
+- mem_context->size);
+-
+- pr_debug("mem_context->p2p_info %p\n",
+- mem_context->p2p_info);
+-
+- if (ACCESS_ONCE(mem_context->free_callback_called)) {
+- pr_debug("Free callback was called\n");
+- return;
+- }
+-
+- if (mem_context->p2p_info) {
+- ret = rdma_interface->put_pages(&mem_context->p2p_info);
+- mem_context->p2p_info = NULL;
+-
+- if (ret)
+- pr_err("Failure: %d (callback status %d)\n",
+- ret, mem_context->free_callback_called);
+- } else
+- pr_err("Pointer to p2p info is null\n");
+-}
+-static unsigned long amd_get_page_size(void *client_context)
+-{
+- unsigned long page_size;
+- int result;
+- struct amd_mem_context *mem_context =
+- (struct amd_mem_context *)client_context;
+-
+- pr_debug("context: %p\n", client_context);
+- pr_debug("pid 0x%p, address 0x%llx, size:0x%llx\n",
+- mem_context->pid,
+- mem_context->va,
+- mem_context->size);
+-
+-
+- result = rdma_interface->get_page_size(
+- mem_context->va,
+- mem_context->size,
+- mem_context->pid,
+- &page_size);
+-
+- if (result) {
+- pr_err("Could not get page size. %d\n", result);
+- /* If we failed to get page size then do not know what to do.
+- * Let's return some default value
+- */
+- return PAGE_SIZE;
+- }
+-
+- return page_size;
+-}
+-
+-static void amd_release(void *client_context)
+-{
+- struct amd_mem_context *mem_context =
+- (struct amd_mem_context *)client_context;
+-
+- pr_debug("context: 0x%p\n", client_context);
+- pr_debug("pid 0x%p, address 0x%llx, size:0x%llx\n",
+- mem_context->pid,
+- mem_context->va,
+- mem_context->size);
+-
+- kfree(mem_context);
+-}
+-
+-
+-static struct peer_memory_client amd_mem_client = {
+- .acquire = amd_acquire,
+- .get_pages = amd_get_pages,
+- .dma_map = amd_dma_map,
+- .dma_unmap = amd_dma_unmap,
+- .put_pages = amd_put_pages,
+- .get_page_size = amd_get_page_size,
+- .release = amd_release,
+- .get_context_private_data = NULL,
+- .put_context_private_data = NULL,
+-};
+-
+-/** Initialize PeerDirect interface with RDMA Network stack.
+- *
+- * Because network stack could potentially be loaded later we check
+- * presence of PeerDirect when HSA process is created. If PeerDirect was
+- * already initialized we do nothing otherwise try to detect and register.
+- */
+-void kfd_init_peer_direct(void)
+-{
+- int result;
+-
+- if (pfn_ib_unregister_peer_memory_client) {
+- pr_debug("PeerDirect support was already initialized\n");
+- return;
+- }
+-
+- pr_debug("Try to initialize PeerDirect support\n");
+-
+- pfn_ib_register_peer_memory_client =
+- (void *(*)(struct peer_memory_client *,
+- invalidate_peer_memory *))
+- symbol_request(ib_register_peer_memory_client);
+-
+- pfn_ib_unregister_peer_memory_client = (void (*)(void *))
+- symbol_request(ib_unregister_peer_memory_client);
+-
+- if (!pfn_ib_register_peer_memory_client ||
+- !pfn_ib_unregister_peer_memory_client) {
+- pr_debug("PeerDirect interface was not detected\n");
+- /* Do cleanup */
+- kfd_close_peer_direct();
+- return;
+- }
+-
+- result = amdkfd_query_rdma_interface(&rdma_interface);
+-
+- if (result < 0) {
+- pr_err("Cannot get RDMA Interface (result = %d)\n", result);
+- return;
+- }
+-
+- strcpy(amd_mem_client.name, AMD_PEER_BRIDGE_DRIVER_NAME);
+- strcpy(amd_mem_client.version, AMD_PEER_BRIDGE_DRIVER_VERSION);
+-
+- ib_reg_handle = pfn_ib_register_peer_memory_client(&amd_mem_client,
+- &ib_invalidate_callback);
+-
+- if (!ib_reg_handle) {
+- pr_err("Cannot register peer memory client\n");
+- /* Do cleanup */
+- kfd_close_peer_direct();
+- return;
+- }
+-
+- pr_info("PeerDirect support was initialized successfully\n");
+-}
+-
+-/**
+- * Close connection with PeerDirect interface with RDMA Network stack.
+- *
+- */
+-void kfd_close_peer_direct(void)
+-{
+- if (pfn_ib_unregister_peer_memory_client) {
+- if (ib_reg_handle)
+- pfn_ib_unregister_peer_memory_client(ib_reg_handle);
+-
+- symbol_put(ib_unregister_peer_memory_client);
+- }
+-
+- if (pfn_ib_register_peer_memory_client)
+- symbol_put(ib_register_peer_memory_client);
+-
+-
+- /* Reset pointers to be safe */
+- pfn_ib_unregister_peer_memory_client = NULL;
+- pfn_ib_register_peer_memory_client = NULL;
+- ib_reg_handle = NULL;
+-}
+-
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers.h b/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers.h
+index e50f73d..5b393f3 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers.h
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers.h
+@@ -28,19 +28,112 @@
+ #define PM4_MES_HEADER_DEFINED
+ union PM4_MES_TYPE_3_HEADER {
+ struct {
+- /* reserved */
+- uint32_t reserved1:8;
+- /* IT opcode */
+- uint32_t opcode:8;
+- /* number of DWORDs - 1 in the information body */
+- uint32_t count:14;
+- /* packet identifier. It should be 3 for type 3 packets */
+- uint32_t type:2;
++ uint32_t reserved1:8; /* < reserved */
++ uint32_t opcode:8; /* < IT opcode */
++ uint32_t count:14; /* < number of DWORDs - 1
++ * in the information body.
++ */
++ uint32_t type:2; /* < packet identifier.
++ * It should be 3 for type 3 packets
++ */
+ };
+ uint32_t u32all;
+ };
+ #endif /* PM4_MES_HEADER_DEFINED */
+
++/* --------------------MES_SET_RESOURCES-------------------- */
++
++#ifndef PM4_MES_SET_RESOURCES_DEFINED
++#define PM4_MES_SET_RESOURCES_DEFINED
++enum set_resources_queue_type_enum {
++ queue_type__mes_set_resources__kernel_interface_queue_kiq = 0,
++ queue_type__mes_set_resources__hsa_interface_queue_hiq = 1,
++ queue_type__mes_set_resources__hsa_debug_interface_queue = 4
++};
++
++struct pm4_set_resources {
++ union {
++ union PM4_MES_TYPE_3_HEADER header; /* header */
++ uint32_t ordinal1;
++ };
++
++ union {
++ struct {
++ uint32_t vmid_mask:16;
++ uint32_t unmap_latency:8;
++ uint32_t reserved1:5;
++ enum set_resources_queue_type_enum queue_type:3;
++ } bitfields2;
++ uint32_t ordinal2;
++ };
++
++ uint32_t queue_mask_lo;
++ uint32_t queue_mask_hi;
++ uint32_t gws_mask_lo;
++ uint32_t gws_mask_hi;
++
++ union {
++ struct {
++ uint32_t oac_mask:16;
++ uint32_t reserved2:16;
++ } bitfields7;
++ uint32_t ordinal7;
++ };
++
++ union {
++ struct {
++ uint32_t gds_heap_base:6;
++ uint32_t reserved3:5;
++ uint32_t gds_heap_size:6;
++ uint32_t reserved4:15;
++ } bitfields8;
++ uint32_t ordinal8;
++ };
++
++};
++#endif
++
++/*--------------------MES_RUN_LIST-------------------- */
++
++#ifndef PM4_MES_RUN_LIST_DEFINED
++#define PM4_MES_RUN_LIST_DEFINED
++
++struct pm4_runlist {
++ union {
++ union PM4_MES_TYPE_3_HEADER header; /* header */
++ uint32_t ordinal1;
++ };
++
++ union {
++ struct {
++ uint32_t reserved1:2;
++ uint32_t ib_base_lo:30;
++ } bitfields2;
++ uint32_t ordinal2;
++ };
++
++ union {
++ struct {
++ uint32_t ib_base_hi:16;
++ uint32_t reserved2:16;
++ } bitfields3;
++ uint32_t ordinal3;
++ };
++
++ union {
++ struct {
++ uint32_t ib_size:20;
++ uint32_t chain:1;
++ uint32_t offload_polling:1;
++ uint32_t reserved3:1;
++ uint32_t valid:1;
++ uint32_t reserved4:8;
++ } bitfields4;
++ uint32_t ordinal4;
++ };
++
++};
++#endif
+
+ /*--------------------MES_MAP_PROCESS-------------------- */
+
+@@ -93,58 +186,217 @@ struct pm4_map_process {
+ };
+ #endif
+
+-#ifndef PM4_MES_MAP_PROCESS_DEFINED_KV_SCRATCH
+-#define PM4_MES_MAP_PROCESS_DEFINED_KV_SCRATCH
++/*--------------------MES_MAP_QUEUES--------------------*/
++
++#ifndef PM4_MES_MAP_QUEUES_DEFINED
++#define PM4_MES_MAP_QUEUES_DEFINED
++enum map_queues_queue_sel_enum {
++ queue_sel__mes_map_queues__map_to_specified_queue_slots = 0,
++ queue_sel__mes_map_queues__map_to_hws_determined_queue_slots = 1,
++ queue_sel__mes_map_queues__enable_process_queues = 2
++};
+
+-struct pm4_map_process_scratch_kv {
++enum map_queues_vidmem_enum {
++ vidmem__mes_map_queues__uses_no_video_memory = 0,
++ vidmem__mes_map_queues__uses_video_memory = 1
++};
++
++enum map_queues_alloc_format_enum {
++ alloc_format__mes_map_queues__one_per_pipe = 0,
++ alloc_format__mes_map_queues__all_on_one_pipe = 1
++};
++
++enum map_queues_engine_sel_enum {
++ engine_sel__mes_map_queues__compute = 0,
++ engine_sel__mes_map_queues__sdma0 = 2,
++ engine_sel__mes_map_queues__sdma1 = 3
++};
++
++struct pm4_map_queues {
+ union {
+- union PM4_MES_TYPE_3_HEADER header; /* header */
+- uint32_t ordinal1;
++ union PM4_MES_TYPE_3_HEADER header; /* header */
++ uint32_t ordinal1;
+ };
+
+ union {
+ struct {
+- uint32_t pasid:16;
+- uint32_t reserved1:8;
+- uint32_t diq_enable:1;
+- uint32_t process_quantum:7;
++ uint32_t reserved1:4;
++ enum map_queues_queue_sel_enum queue_sel:2;
++ uint32_t reserved2:2;
++ uint32_t vmid:4;
++ uint32_t reserved3:4;
++ enum map_queues_vidmem_enum vidmem:2;
++ uint32_t reserved4:6;
++ enum map_queues_alloc_format_enum alloc_format:2;
++ enum map_queues_engine_sel_enum engine_sel:3;
++ uint32_t num_queues:3;
+ } bitfields2;
+ uint32_t ordinal2;
+ };
+
++ struct {
++ union {
++ struct {
++ uint32_t is_static:1;
++ uint32_t reserved5:1;
++ uint32_t doorbell_offset:21;
++ uint32_t reserved6:3;
++ uint32_t queue:6;
++ } bitfields3;
++ uint32_t ordinal3;
++ };
++
++ uint32_t mqd_addr_lo;
++ uint32_t mqd_addr_hi;
++ uint32_t wptr_addr_lo;
++ uint32_t wptr_addr_hi;
++
++ } mes_map_queues_ordinals[1]; /* 1..N of these ordinal groups */
++
++};
++#endif
++
++/*--------------------MES_QUERY_STATUS--------------------*/
++
++#ifndef PM4_MES_QUERY_STATUS_DEFINED
++#define PM4_MES_QUERY_STATUS_DEFINED
++enum query_status_interrupt_sel_enum {
++ interrupt_sel__mes_query_status__completion_status = 0,
++ interrupt_sel__mes_query_status__process_status = 1,
++ interrupt_sel__mes_query_status__queue_status = 2
++};
++
++enum query_status_command_enum {
++ command__mes_query_status__interrupt_only = 0,
++ command__mes_query_status__fence_only_immediate = 1,
++ command__mes_query_status__fence_only_after_write_ack = 2,
++ command__mes_query_status__fence_wait_for_write_ack_send_interrupt = 3
++};
++
++enum query_status_engine_sel_enum {
++ engine_sel__mes_query_status__compute = 0,
++ engine_sel__mes_query_status__sdma0_queue = 2,
++ engine_sel__mes_query_status__sdma1_queue = 3
++};
++
++struct pm4_query_status {
++ union {
++ union PM4_MES_TYPE_3_HEADER header; /* header */
++ uint32_t ordinal1;
++ };
++
+ union {
+ struct {
+- uint32_t page_table_base:28;
+- uint32_t reserved2:4;
+- } bitfields3;
++ uint32_t context_id:28;
++ enum query_status_interrupt_sel_enum interrupt_sel:2;
++ enum query_status_command_enum command:2;
++ } bitfields2;
++ uint32_t ordinal2;
++ };
++
++ union {
++ struct {
++ uint32_t pasid:16;
++ uint32_t reserved1:16;
++ } bitfields3a;
++ struct {
++ uint32_t reserved2:2;
++ uint32_t doorbell_offset:21;
++ uint32_t reserved3:3;
++ enum query_status_engine_sel_enum engine_sel:3;
++ uint32_t reserved4:3;
++ } bitfields3b;
+ uint32_t ordinal3;
+ };
+
+- uint32_t reserved3;
+- uint32_t sh_mem_bases;
+- uint32_t sh_mem_config;
+- uint32_t sh_mem_ape1_base;
+- uint32_t sh_mem_ape1_limit;
+- uint32_t sh_hidden_private_base_vmid;
+- uint32_t reserved4;
+- uint32_t reserved5;
+- uint32_t gds_addr_lo;
+- uint32_t gds_addr_hi;
++ uint32_t addr_lo;
++ uint32_t addr_hi;
++ uint32_t data_lo;
++ uint32_t data_hi;
++};
++#endif
++
++/*--------------------MES_UNMAP_QUEUES--------------------*/
++
++#ifndef PM4_MES_UNMAP_QUEUES_DEFINED
++#define PM4_MES_UNMAP_QUEUES_DEFINED
++enum unmap_queues_action_enum {
++ action__mes_unmap_queues__preempt_queues = 0,
++ action__mes_unmap_queues__reset_queues = 1,
++ action__mes_unmap_queues__disable_process_queues = 2
++};
++
++enum unmap_queues_queue_sel_enum {
++ queue_sel__mes_unmap_queues__perform_request_on_specified_queues = 0,
++ queue_sel__mes_unmap_queues__perform_request_on_pasid_queues = 1,
++ queue_sel__mes_unmap_queues__perform_request_on_all_active_queues = 2,
++ queue_sel__mes_unmap_queues__perform_request_on_dynamic_queues_only = 3
++};
++
++enum unmap_queues_engine_sel_enum {
++ engine_sel__mes_unmap_queues__compute = 0,
++ engine_sel__mes_unmap_queues__sdma0 = 2,
++ engine_sel__mes_unmap_queues__sdma1 = 3
++};
++
++struct pm4_unmap_queues {
++ union {
++ union PM4_MES_TYPE_3_HEADER header; /* header */
++ uint32_t ordinal1;
++ };
++
++ union {
++ struct {
++ enum unmap_queues_action_enum action:2;
++ uint32_t reserved1:2;
++ enum unmap_queues_queue_sel_enum queue_sel:2;
++ uint32_t reserved2:20;
++ enum unmap_queues_engine_sel_enum engine_sel:3;
++ uint32_t num_queues:3;
++ } bitfields2;
++ uint32_t ordinal2;
++ };
++
++ union {
++ struct {
++ uint32_t pasid:16;
++ uint32_t reserved3:16;
++ } bitfields3a;
++ struct {
++ uint32_t reserved4:2;
++ uint32_t doorbell_offset0:21;
++ uint32_t reserved5:9;
++ } bitfields3b;
++ uint32_t ordinal3;
++ };
+
+ union {
+ struct {
+- uint32_t num_gws:6;
+ uint32_t reserved6:2;
+- uint32_t num_oac:4;
+- uint32_t reserved7:4;
+- uint32_t gds_size:6;
+- uint32_t num_queues:10;
+- } bitfields14;
+- uint32_t ordinal14;
++ uint32_t doorbell_offset1:21;
++ uint32_t reserved7:9;
++ } bitfields4;
++ uint32_t ordinal4;
++ };
++
++ union {
++ struct {
++ uint32_t reserved8:2;
++ uint32_t doorbell_offset2:21;
++ uint32_t reserved9:9;
++ } bitfields5;
++ uint32_t ordinal5;
++ };
++
++ union {
++ struct {
++ uint32_t reserved10:2;
++ uint32_t doorbell_offset3:21;
++ uint32_t reserved11:9;
++ } bitfields6;
++ uint32_t ordinal6;
+ };
+
+- uint32_t completion_signal_lo32;
+-uint32_t completion_signal_hi32;
+ };
+ #endif
+
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_ai.h b/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_ai.h
+deleted file mode 100644
+index ddad9be..0000000
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_ai.h
++++ /dev/null
+@@ -1,583 +0,0 @@
+-/*
+- * Copyright 2016 Advanced Micro Devices, Inc.
+- *
+- * Permission is hereby granted, free of charge, to any person obtaining a
+- * copy of this software and associated documentation files (the "Software"),
+- * to deal in the Software without restriction, including without limitation
+- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+- * and/or sell copies of the Software, and to permit persons to whom the
+- * Software is furnished to do so, subject to the following conditions:
+- *
+- * The above copyright notice and this permission notice shall be included in
+- * all copies or substantial portions of the Software.
+- *
+- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+- * OTHER DEALINGS IN THE SOFTWARE.
+- *
+- */
+-
+-#ifndef F32_MES_PM4_PACKETS_H
+-#define F32_MES_PM4_PACKETS_H
+-
+-#ifndef PM4_MES_HEADER_DEFINED
+-#define PM4_MES_HEADER_DEFINED
+-union PM4_MES_TYPE_3_HEADER {
+- struct {
+- uint32_t reserved1 : 8; /* < reserved */
+- uint32_t opcode : 8; /* < IT opcode */
+- uint32_t count : 14;/* < number of DWORDs - 1 in the
+- * information body.
+- */
+- uint32_t type : 2; /* < packet identifier.
+- * It should be 3 for type 3 packets
+- */
+- };
+- uint32_t u32All;
+-};
+-#endif /* PM4_MES_HEADER_DEFINED */
+-
+-/*--------------------MES_SET_RESOURCES--------------------*/
+-
+-#ifndef PM4_MES_SET_RESOURCES_DEFINED
+-#define PM4_MES_SET_RESOURCES_DEFINED
+-enum mes_set_resources_queue_type_enum {
+- queue_type__mes_set_resources__kernel_interface_queue_kiq = 0,
+- queue_type__mes_set_resources__hsa_interface_queue_hiq = 1,
+- queue_type__mes_set_resources__hsa_debug_interface_queue = 4
+-};
+-
+-
+-struct pm4_mes_set_resources {
+- union {
+- union PM4_MES_TYPE_3_HEADER header; /* header */
+- uint32_t ordinal1;
+- };
+-
+- union {
+- struct {
+- uint32_t vmid_mask:16;
+- uint32_t unmap_latency:8;
+- uint32_t reserved1:5;
+- enum mes_set_resources_queue_type_enum queue_type:3;
+- } bitfields2;
+- uint32_t ordinal2;
+- };
+-
+- uint32_t queue_mask_lo;
+- uint32_t queue_mask_hi;
+- uint32_t gws_mask_lo;
+- uint32_t gws_mask_hi;
+-
+- union {
+- struct {
+- uint32_t oac_mask:16;
+- uint32_t reserved2:16;
+- } bitfields7;
+- uint32_t ordinal7;
+- };
+-
+- union {
+- struct {
+- uint32_t gds_heap_base:6;
+- uint32_t reserved3:5;
+- uint32_t gds_heap_size:6;
+- uint32_t reserved4:15;
+- } bitfields8;
+- uint32_t ordinal8;
+- };
+-
+-};
+-#endif
+-
+-/*--------------------MES_RUN_LIST--------------------*/
+-
+-#ifndef PM4_MES_RUN_LIST_DEFINED
+-#define PM4_MES_RUN_LIST_DEFINED
+-
+-struct pm4_mes_runlist {
+- union {
+- union PM4_MES_TYPE_3_HEADER header; /* header */
+- uint32_t ordinal1;
+- };
+-
+- union {
+- struct {
+- uint32_t reserved1:2;
+- uint32_t ib_base_lo:30;
+- } bitfields2;
+- uint32_t ordinal2;
+- };
+-
+- uint32_t ib_base_hi;
+-
+- union {
+- struct {
+- uint32_t ib_size:20;
+- uint32_t chain:1;
+- uint32_t offload_polling:1;
+- uint32_t reserved2:1;
+- uint32_t valid:1;
+- uint32_t process_cnt:4;
+- uint32_t reserved3:4;
+- } bitfields4;
+- uint32_t ordinal4;
+- };
+-
+-};
+-#endif
+-
+-/*--------------------MES_MAP_PROCESS--------------------*/
+-
+-#ifndef PM4_MES_MAP_PROCESS_DEFINED
+-#define PM4_MES_MAP_PROCESS_DEFINED
+-
+-struct pm4_mes_map_process {
+- union {
+- union PM4_MES_TYPE_3_HEADER header; /* header */
+- uint32_t ordinal1;
+- };
+-
+- union {
+- struct {
+- uint32_t pasid:16;
+- uint32_t reserved1:8;
+- uint32_t diq_enable:1;
+- uint32_t process_quantum:7;
+- } bitfields2;
+- uint32_t ordinal2;
+- };
+-
+- uint32_t vm_context_page_table_base_addr_lo32;
+-
+- uint32_t vm_context_page_table_base_addr_hi32;
+-
+- uint32_t sh_mem_bases;
+-
+- uint32_t sh_mem_config;
+-
+- uint32_t sq_shader_tba_lo;
+-
+- uint32_t sq_shader_tba_hi;
+-
+- uint32_t sq_shader_tma_lo;
+-
+- uint32_t sq_shader_tma_hi;
+-
+- uint32_t reserved6;
+-
+- uint32_t gds_addr_lo;
+-
+- uint32_t gds_addr_hi;
+-
+- union {
+- struct {
+- uint32_t num_gws:6;
+- uint32_t reserved7:1;
+- uint32_t sdma_enable:1;
+- uint32_t num_oac:4;
+- uint32_t reserved8:4;
+- uint32_t gds_size:6;
+- uint32_t num_queues:10;
+- } bitfields14;
+- uint32_t ordinal14;
+- };
+-
+- uint32_t completion_signal_lo;
+-
+- uint32_t completion_signal_hi;
+-
+-};
+-
+-#endif
+-
+-/*--------------------MES_MAP_PROCESS_VM--------------------*/
+-
+-#ifndef PM4_MES_MAP_PROCESS_VM_DEFINED
+-#define PM4_MES_MAP_PROCESS_VM_DEFINED
+-
+-struct PM4_MES_MAP_PROCESS_VM {
+- union {
+- union PM4_MES_TYPE_3_HEADER header; /* header */
+- uint32_t ordinal1;
+- };
+-
+- uint32_t reserved1;
+-
+- uint32_t vm_context_cntl;
+-
+- uint32_t reserved2;
+-
+- uint32_t vm_context_page_table_end_addr_lo32;
+-
+- uint32_t vm_context_page_table_end_addr_hi32;
+-
+- uint32_t vm_context_page_table_start_addr_lo32;
+-
+- uint32_t vm_context_page_table_start_addr_hi32;
+-
+- uint32_t reserved3;
+-
+- uint32_t reserved4;
+-
+- uint32_t reserved5;
+-
+- uint32_t reserved6;
+-
+- uint32_t reserved7;
+-
+- uint32_t reserved8;
+-
+- uint32_t completion_signal_lo32;
+-
+- uint32_t completion_signal_hi32;
+-
+-};
+-#endif
+-
+-/*--------------------MES_MAP_QUEUES--------------------*/
+-
+-#ifndef PM4_MES_MAP_QUEUES_VI_DEFINED
+-#define PM4_MES_MAP_QUEUES_VI_DEFINED
+-enum mes_map_queues_queue_sel_enum {
+- queue_sel__mes_map_queues__map_to_specified_queue_slots_vi = 0,
+-queue_sel__mes_map_queues__map_to_hws_determined_queue_slots_vi = 1
+-};
+-
+-enum mes_map_queues_queue_type_enum {
+- queue_type__mes_map_queues__normal_compute_vi = 0,
+- queue_type__mes_map_queues__debug_interface_queue_vi = 1,
+- queue_type__mes_map_queues__normal_latency_static_queue_vi = 2,
+-queue_type__mes_map_queues__low_latency_static_queue_vi = 3
+-};
+-
+-enum mes_map_queues_alloc_format_enum {
+- alloc_format__mes_map_queues__one_per_pipe_vi = 0,
+-alloc_format__mes_map_queues__all_on_one_pipe_vi = 1
+-};
+-
+-enum mes_map_queues_engine_sel_enum {
+- engine_sel__mes_map_queues__compute_vi = 0,
+- engine_sel__mes_map_queues__sdma0_vi = 2,
+- engine_sel__mes_map_queues__sdma1_vi = 3
+-};
+-
+-
+-struct pm4_mes_map_queues {
+- union {
+- union PM4_MES_TYPE_3_HEADER header; /* header */
+- uint32_t ordinal1;
+- };
+-
+- union {
+- struct {
+- uint32_t reserved1:4;
+- enum mes_map_queues_queue_sel_enum queue_sel:2;
+- uint32_t reserved2:15;
+- enum mes_map_queues_queue_type_enum queue_type:3;
+- enum mes_map_queues_alloc_format_enum alloc_format:2;
+- enum mes_map_queues_engine_sel_enum engine_sel:3;
+- uint32_t num_queues:3;
+- } bitfields2;
+- uint32_t ordinal2;
+- };
+-
+- union {
+- struct {
+- uint32_t reserved3:1;
+- uint32_t check_disable:1;
+- uint32_t doorbell_offset:26;
+- uint32_t reserved4:4;
+- } bitfields3;
+- uint32_t ordinal3;
+- };
+-
+- uint32_t mqd_addr_lo;
+- uint32_t mqd_addr_hi;
+- uint32_t wptr_addr_lo;
+- uint32_t wptr_addr_hi;
+-};
+-#endif
+-
+-/*--------------------MES_QUERY_STATUS--------------------*/
+-
+-#ifndef PM4_MES_QUERY_STATUS_DEFINED
+-#define PM4_MES_QUERY_STATUS_DEFINED
+-enum mes_query_status_interrupt_sel_enum {
+- interrupt_sel__mes_query_status__completion_status = 0,
+- interrupt_sel__mes_query_status__process_status = 1,
+- interrupt_sel__mes_query_status__queue_status = 2
+-};
+-
+-enum mes_query_status_command_enum {
+- command__mes_query_status__interrupt_only = 0,
+- command__mes_query_status__fence_only_immediate = 1,
+- command__mes_query_status__fence_only_after_write_ack = 2,
+- command__mes_query_status__fence_wait_for_write_ack_send_interrupt = 3
+-};
+-
+-enum mes_query_status_engine_sel_enum {
+- engine_sel__mes_query_status__compute = 0,
+- engine_sel__mes_query_status__sdma0_queue = 2,
+- engine_sel__mes_query_status__sdma1_queue = 3
+-};
+-
+-struct pm4_mes_query_status {
+- union {
+- union PM4_MES_TYPE_3_HEADER header; /* header */
+- uint32_t ordinal1;
+- };
+-
+- union {
+- struct {
+- uint32_t context_id:28;
+- enum mes_query_status_interrupt_sel_enum interrupt_sel:2;
+- enum mes_query_status_command_enum command:2;
+- } bitfields2;
+- uint32_t ordinal2;
+- };
+-
+- union {
+- struct {
+- uint32_t pasid:16;
+- uint32_t reserved1:16;
+- } bitfields3a;
+- struct {
+- uint32_t reserved2:2;
+- uint32_t doorbell_offset:26;
+- enum mes_query_status_engine_sel_enum engine_sel:3;
+- uint32_t reserved3:1;
+- } bitfields3b;
+- uint32_t ordinal3;
+- };
+-
+- uint32_t addr_lo;
+- uint32_t addr_hi;
+- uint32_t data_lo;
+- uint32_t data_hi;
+-};
+-#endif
+-
+-/*--------------------MES_UNMAP_QUEUES--------------------*/
+-
+-#ifndef PM4_MES_UNMAP_QUEUES_DEFINED
+-#define PM4_MES_UNMAP_QUEUES_DEFINED
+-enum mes_unmap_queues_action_enum {
+- action__mes_unmap_queues__preempt_queues = 0,
+- action__mes_unmap_queues__reset_queues = 1,
+- action__mes_unmap_queues__disable_process_queues = 2,
+- action__mes_unmap_queues__reserved = 3
+-};
+-
+-enum mes_unmap_queues_queue_sel_enum {
+- queue_sel__mes_unmap_queues__perform_request_on_specified_queues = 0,
+- queue_sel__mes_unmap_queues__perform_request_on_pasid_queues = 1,
+- queue_sel__mes_unmap_queues__unmap_all_queues = 2,
+- queue_sel__mes_unmap_queues__unmap_all_non_static_queues = 3
+-};
+-
+-enum mes_unmap_queues_engine_sel_enum {
+- engine_sel__mes_unmap_queues__compute = 0,
+- engine_sel__mes_unmap_queues__sdma0 = 2,
+- engine_sel__mes_unmap_queues__sdmal = 3
+-};
+-
+-struct pm4_mes_unmap_queues {
+- union {
+- union PM4_MES_TYPE_3_HEADER header; /* header */
+- uint32_t ordinal1;
+- };
+-
+- union {
+- struct {
+- enum mes_unmap_queues_action_enum action:2;
+- uint32_t reserved1:2;
+- enum mes_unmap_queues_queue_sel_enum queue_sel:2;
+- uint32_t reserved2:20;
+- enum mes_unmap_queues_engine_sel_enum engine_sel:3;
+- uint32_t num_queues:3;
+- } bitfields2;
+- uint32_t ordinal2;
+- };
+-
+- union {
+- struct {
+- uint32_t pasid:16;
+- uint32_t reserved3:16;
+- } bitfields3a;
+- struct {
+- uint32_t reserved4:2;
+- uint32_t doorbell_offset0:26;
+- int32_t reserved5:4;
+- } bitfields3b;
+- uint32_t ordinal3;
+- };
+-
+- union {
+- struct {
+- uint32_t reserved6:2;
+- uint32_t doorbell_offset1:26;
+- uint32_t reserved7:4;
+- } bitfields4;
+- uint32_t ordinal4;
+- };
+-
+- union {
+- struct {
+- uint32_t reserved8:2;
+- uint32_t doorbell_offset2:26;
+- uint32_t reserved9:4;
+- } bitfields5;
+- uint32_t ordinal5;
+- };
+-
+- union {
+- struct {
+- uint32_t reserved10:2;
+- uint32_t doorbell_offset3:26;
+- uint32_t reserved11:4;
+- } bitfields6;
+- uint32_t ordinal6;
+- };
+-};
+-#endif
+-
+-#ifndef PM4_MEC_RELEASE_MEM_DEFINED
+-#define PM4_MEC_RELEASE_MEM_DEFINED
+-
+-enum mec_release_mem_event_index_enum {
+- event_index__mec_release_mem__end_of_pipe = 5,
+- event_index__mec_release_mem__shader_done = 6
+-};
+-
+-enum mec_release_mem_cache_policy_enum {
+- cache_policy__mec_release_mem__lru = 0,
+- cache_policy__mec_release_mem__stream = 1
+-};
+-
+-enum mec_release_mem_pq_exe_status_enum {
+- pq_exe_status__mec_release_mem__default = 0,
+- pq_exe_status__mec_release_mem__phase_update = 1
+-};
+-
+-enum mec_release_mem_dst_sel_enum {
+- dst_sel__mec_release_mem__memory_controller = 0,
+- dst_sel__mec_release_mem__tc_l2 = 1,
+- dst_sel__mec_release_mem__queue_write_pointer_register = 2,
+- dst_sel__mec_release_mem__queue_write_pointer_poll_mask_bit = 3
+-};
+-
+-enum mec_release_mem_int_sel_enum {
+- int_sel__mec_release_mem__none = 0,
+- int_sel__mec_release_mem__send_interrupt_only = 1,
+- int_sel__mec_release_mem__send_interrupt_after_write_confirm = 2,
+- int_sel__mec_release_mem__send_data_after_write_confirm = 3,
+- int_sel__mec_release_mem__unconditionally_send_int_ctxid = 4,
+- int_sel__mec_release_mem__conditionally_send_int_ctxid_based_on_32_bit_compare = 5,
+- int_sel__mec_release_mem__conditionally_send_int_ctxid_based_on_64_bit_compare = 6
+-};
+-
+-enum mec_release_mem_data_sel_enum {
+- data_sel__mec_release_mem__none = 0,
+- data_sel__mec_release_mem__send_32_bit_low = 1,
+- data_sel__mec_release_mem__send_64_bit_data = 2,
+- data_sel__mec_release_mem__send_gpu_clock_counter = 3,
+- data_sel__mec_release_mem__send_cp_perfcounter_hi_lo = 4,
+- data_sel__mec_release_mem__store_gds_data_to_memory = 5
+-};
+-
+-struct pm4_mec_release_mem {
+- union {
+- union PM4_MES_TYPE_3_HEADER header; /*header */
+- unsigned int ordinal1;
+- };
+-
+- union {
+- struct {
+- unsigned int event_type:6;
+- unsigned int reserved1:2;
+- enum mec_release_mem_event_index_enum event_index:4;
+- unsigned int tcl1_vol_action_ena:1;
+- unsigned int tc_vol_action_ena:1;
+- unsigned int reserved2:1;
+- unsigned int tc_wb_action_ena:1;
+- unsigned int tcl1_action_ena:1;
+- unsigned int tc_action_ena:1;
+- uint32_t reserved3:1;
+- uint32_t tc_nc_action_ena:1;
+- uint32_t tc_wc_action_ena:1;
+- uint32_t tc_md_action_ena:1;
+- uint32_t reserved4:3;
+- enum mec_release_mem_cache_policy_enum cache_policy:2;
+- uint32_t reserved5:2;
+- enum mec_release_mem_pq_exe_status_enum pq_exe_status:1;
+- uint32_t reserved6:2;
+- } bitfields2;
+- unsigned int ordinal2;
+- };
+-
+- union {
+- struct {
+- uint32_t reserved7:16;
+- enum mec_release_mem_dst_sel_enum dst_sel:2;
+- uint32_t reserved8:6;
+- enum mec_release_mem_int_sel_enum int_sel:3;
+- uint32_t reserved9:2;
+- enum mec_release_mem_data_sel_enum data_sel:3;
+- } bitfields3;
+- unsigned int ordinal3;
+- };
+-
+- union {
+- struct {
+- uint32_t reserved10:2;
+- unsigned int address_lo_32b:30;
+- } bitfields4;
+- struct {
+- uint32_t reserved11:3;
+- uint32_t address_lo_64b:29;
+- } bitfields4b;
+- uint32_t reserved12;
+- unsigned int ordinal4;
+- };
+-
+- union {
+- uint32_t address_hi;
+- uint32_t reserved13;
+- uint32_t ordinal5;
+- };
+-
+- union {
+- uint32_t data_lo;
+- uint32_t cmp_data_lo;
+- struct {
+- uint32_t dw_offset:16;
+- uint32_t num_dwords:16;
+- } bitfields6c;
+- uint32_t reserved14;
+- uint32_t ordinal6;
+- };
+-
+- union {
+- uint32_t data_hi;
+- uint32_t cmp_data_hi;
+- uint32_t reserved15;
+- uint32_t reserved16;
+- uint32_t ordinal7;
+- };
+-
+- uint32_t int_ctxid;
+-
+-};
+-
+-#endif
+-
+-enum {
+- CACHE_FLUSH_AND_INV_TS_EVENT = 0x00000014
+-};
+-#endif
+-
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_diq.h b/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_diq.h
+index 0b314a8..a0ff348 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_diq.h
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_diq.h
+@@ -77,6 +77,103 @@ struct pm4__indirect_buffer_pasid {
+
+ #endif
+
++/*--------------------_RELEASE_MEM-------------------- */
++
++#ifndef _PM4__RELEASE_MEM_DEFINED
++#define _PM4__RELEASE_MEM_DEFINED
++enum _RELEASE_MEM_event_index_enum {
++ event_index___release_mem__end_of_pipe = 5,
++ event_index___release_mem__shader_done = 6
++};
++
++enum _RELEASE_MEM_cache_policy_enum {
++ cache_policy___release_mem__lru = 0,
++ cache_policy___release_mem__stream = 1,
++ cache_policy___release_mem__bypass = 2
++};
++
++enum _RELEASE_MEM_dst_sel_enum {
++ dst_sel___release_mem__memory_controller = 0,
++ dst_sel___release_mem__tc_l2 = 1,
++ dst_sel___release_mem__queue_write_pointer_register = 2,
++ dst_sel___release_mem__queue_write_pointer_poll_mask_bit = 3
++};
++
++enum _RELEASE_MEM_int_sel_enum {
++ int_sel___release_mem__none = 0,
++ int_sel___release_mem__send_interrupt_only = 1,
++ int_sel___release_mem__send_interrupt_after_write_confirm = 2,
++ int_sel___release_mem__send_data_after_write_confirm = 3
++};
++
++enum _RELEASE_MEM_data_sel_enum {
++ data_sel___release_mem__none = 0,
++ data_sel___release_mem__send_32_bit_low = 1,
++ data_sel___release_mem__send_64_bit_data = 2,
++ data_sel___release_mem__send_gpu_clock_counter = 3,
++ data_sel___release_mem__send_cp_perfcounter_hi_lo = 4,
++ data_sel___release_mem__store_gds_data_to_memory = 5
++};
++
++struct pm4__release_mem {
++ union {
++ union PM4_MES_TYPE_3_HEADER header; /*header */
++ unsigned int ordinal1;
++ };
++
++ union {
++ struct {
++ unsigned int event_type:6;
++ unsigned int reserved1:2;
++ enum _RELEASE_MEM_event_index_enum event_index:4;
++ unsigned int tcl1_vol_action_ena:1;
++ unsigned int tc_vol_action_ena:1;
++ unsigned int reserved2:1;
++ unsigned int tc_wb_action_ena:1;
++ unsigned int tcl1_action_ena:1;
++ unsigned int tc_action_ena:1;
++ unsigned int reserved3:6;
++ unsigned int atc:1;
++ enum _RELEASE_MEM_cache_policy_enum cache_policy:2;
++ unsigned int reserved4:5;
++ } bitfields2;
++ unsigned int ordinal2;
++ };
++
++ union {
++ struct {
++ unsigned int reserved5:16;
++ enum _RELEASE_MEM_dst_sel_enum dst_sel:2;
++ unsigned int reserved6:6;
++ enum _RELEASE_MEM_int_sel_enum int_sel:3;
++ unsigned int reserved7:2;
++ enum _RELEASE_MEM_data_sel_enum data_sel:3;
++ } bitfields3;
++ unsigned int ordinal3;
++ };
++
++ union {
++ struct {
++ unsigned int reserved8:2;
++ unsigned int address_lo_32b:30;
++ } bitfields4;
++ struct {
++ unsigned int reserved9:3;
++ unsigned int address_lo_64b:29;
++ } bitfields5;
++ unsigned int ordinal4;
++ };
++
++ unsigned int address_hi;
++
++ unsigned int data_lo;
++
++ unsigned int data_hi;
++
++};
++#endif
++
++
+ /*--------------------_SET_CONFIG_REG-------------------- */
+
+ #ifndef _PM4__SET_CONFIG_REG_DEFINED
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_vi.h b/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_vi.h
+index 7c8d9b3..08c7219 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_vi.h
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_vi.h
+@@ -30,12 +30,10 @@ union PM4_MES_TYPE_3_HEADER {
+ struct {
+ uint32_t reserved1 : 8; /* < reserved */
+ uint32_t opcode : 8; /* < IT opcode */
+- uint32_t count : 14;/* < Number of DWORDS - 1 in the
+- * information body
+- */
+- uint32_t type : 2; /* < packet identifier
+- * It should be 3 for type 3 packets
+- */
++ uint32_t count : 14;/* < number of DWORDs - 1 in the
++ information body. */
++ uint32_t type : 2; /* < packet identifier.
++ It should be 3 for type 3 packets */
+ };
+ uint32_t u32All;
+ };
+@@ -126,10 +124,9 @@ struct pm4_mes_runlist {
+ uint32_t ib_size:20;
+ uint32_t chain:1;
+ uint32_t offload_polling:1;
+- uint32_t reserved2:1;
++ uint32_t reserved3:1;
+ uint32_t valid:1;
+- uint32_t process_cnt:4;
+- uint32_t reserved3:4;
++ uint32_t reserved4:8;
+ } bitfields4;
+ uint32_t ordinal4;
+ };
+@@ -144,8 +141,8 @@ struct pm4_mes_runlist {
+
+ struct pm4_mes_map_process {
+ union {
+- union PM4_MES_TYPE_3_HEADER header; /* header */
+- uint32_t ordinal1;
++ union PM4_MES_TYPE_3_HEADER header; /* header */
++ uint32_t ordinal1;
+ };
+
+ union {
+@@ -156,48 +153,36 @@ struct pm4_mes_map_process {
+ uint32_t process_quantum:7;
+ } bitfields2;
+ uint32_t ordinal2;
+- };
++};
+
+ union {
+ struct {
+ uint32_t page_table_base:28;
+- uint32_t reserved3:4;
++ uint32_t reserved2:4;
+ } bitfields3;
+ uint32_t ordinal3;
+ };
+
+- uint32_t reserved;
+-
+ uint32_t sh_mem_bases;
+- uint32_t sh_mem_config;
+ uint32_t sh_mem_ape1_base;
+ uint32_t sh_mem_ape1_limit;
+-
+- uint32_t sh_hidden_private_base_vmid;
+-
+- uint32_t reserved2;
+- uint32_t reserved3;
+-
++ uint32_t sh_mem_config;
+ uint32_t gds_addr_lo;
+ uint32_t gds_addr_hi;
+
+ union {
+ struct {
+ uint32_t num_gws:6;
+- uint32_t reserved4:2;
++ uint32_t reserved3:2;
+ uint32_t num_oac:4;
+- uint32_t reserved5:4;
++ uint32_t reserved4:4;
+ uint32_t gds_size:6;
+ uint32_t num_queues:10;
+ } bitfields10;
+ uint32_t ordinal10;
+ };
+
+- uint32_t completion_signal_lo;
+- uint32_t completion_signal_hi;
+-
+ };
+-
+ #endif
+
+ /*--------------------MES_MAP_QUEUES--------------------*/
+@@ -350,7 +335,7 @@ enum mes_unmap_queues_engine_sel_enum {
+ engine_sel__mes_unmap_queues__sdmal = 3
+ };
+
+-struct pm4_mes_unmap_queues {
++struct PM4_MES_UNMAP_QUEUES {
+ union {
+ union PM4_MES_TYPE_3_HEADER header; /* header */
+ uint32_t ordinal1;
+@@ -410,101 +395,4 @@ struct pm4_mes_unmap_queues {
+ };
+ #endif
+
+-#ifndef PM4_MEC_RELEASE_MEM_DEFINED
+-#define PM4_MEC_RELEASE_MEM_DEFINED
+-enum RELEASE_MEM_event_index_enum {
+- event_index___release_mem__end_of_pipe = 5,
+- event_index___release_mem__shader_done = 6
+-};
+-
+-enum RELEASE_MEM_cache_policy_enum {
+- cache_policy___release_mem__lru = 0,
+- cache_policy___release_mem__stream = 1,
+- cache_policy___release_mem__bypass = 2
+-};
+-
+-enum RELEASE_MEM_dst_sel_enum {
+- dst_sel___release_mem__memory_controller = 0,
+- dst_sel___release_mem__tc_l2 = 1,
+- dst_sel___release_mem__queue_write_pointer_register = 2,
+- dst_sel___release_mem__queue_write_pointer_poll_mask_bit = 3
+-};
+-
+-enum RELEASE_MEM_int_sel_enum {
+- int_sel___release_mem__none = 0,
+- int_sel___release_mem__send_interrupt_only = 1,
+- int_sel___release_mem__send_interrupt_after_write_confirm = 2,
+- int_sel___release_mem__send_data_after_write_confirm = 3
+-};
+-
+-enum RELEASE_MEM_data_sel_enum {
+- data_sel___release_mem__none = 0,
+- data_sel___release_mem__send_32_bit_low = 1,
+- data_sel___release_mem__send_64_bit_data = 2,
+- data_sel___release_mem__send_gpu_clock_counter = 3,
+- data_sel___release_mem__send_cp_perfcounter_hi_lo = 4,
+- data_sel___release_mem__store_gds_data_to_memory = 5
+-};
+-
+-struct pm4_mec_release_mem {
+- union {
+- union PM4_MES_TYPE_3_HEADER header; /*header */
+- unsigned int ordinal1;
+- };
+-
+- union {
+- struct {
+- unsigned int event_type:6;
+- unsigned int reserved1:2;
+- enum RELEASE_MEM_event_index_enum event_index:4;
+- unsigned int tcl1_vol_action_ena:1;
+- unsigned int tc_vol_action_ena:1;
+- unsigned int reserved2:1;
+- unsigned int tc_wb_action_ena:1;
+- unsigned int tcl1_action_ena:1;
+- unsigned int tc_action_ena:1;
+- unsigned int reserved3:6;
+- unsigned int atc:1;
+- enum RELEASE_MEM_cache_policy_enum cache_policy:2;
+- unsigned int reserved4:5;
+- } bitfields2;
+- unsigned int ordinal2;
+- };
+-
+- union {
+- struct {
+- unsigned int reserved5:16;
+- enum RELEASE_MEM_dst_sel_enum dst_sel:2;
+- unsigned int reserved6:6;
+- enum RELEASE_MEM_int_sel_enum int_sel:3;
+- unsigned int reserved7:2;
+- enum RELEASE_MEM_data_sel_enum data_sel:3;
+- } bitfields3;
+- unsigned int ordinal3;
+- };
+-
+- union {
+- struct {
+- unsigned int reserved8:2;
+- unsigned int address_lo_32b:30;
+- } bitfields4;
+- struct {
+- unsigned int reserved9:3;
+- unsigned int address_lo_64b:29;
+- } bitfields5;
+- unsigned int ordinal4;
+- };
+-
+- unsigned int address_hi;
+-
+- unsigned int data_lo;
+-
+- unsigned int data_hi;
+-};
+-#endif
+-
+-enum {
+- CACHE_FLUSH_AND_INV_TS_EVENT = 0x00000014
+-};
+-
+ #endif
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+old mode 100755
+new mode 100644
+index 88fdfc9..4750cab
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+@@ -30,49 +30,13 @@
+ #include <linux/atomic.h>
+ #include <linux/workqueue.h>
+ #include <linux/spinlock.h>
+-#include <linux/idr.h>
+ #include <linux/kfd_ioctl.h>
+-#include <linux/pid.h>
+-#include <linux/interval_tree.h>
+-#include <linux/seq_file.h>
+-#include <linux/kref.h>
+-#include <linux/kfifo.h>
+ #include <kgd_kfd_interface.h>
+
+-#include <drm/amd_rdma.h>
+-#include "amd_shared.h"
+-
+ #define KFD_SYSFS_FILE_MODE 0444
+
+-/* GPU ID hash width in bits */
+-#define KFD_GPU_ID_HASH_WIDTH 16
+-
+-/* Use upper bits of mmap offset to store KFD driver specific information.
+- * BITS[63:62] - Encode MMAP type
+- * BITS[61:46] - Encode gpu_id. To identify to which GPU the offset belongs to
+- * BITS[45:40] - Reserved. Not Used.
+- * BITS[39:0] - MMAP offset value. Used by TTM.
+- *
+- * NOTE: struct vm_area_struct.vm_pgoff uses offset in pages. Hence, these
+- * defines are w.r.t to PAGE_SIZE
+- */
+-#define KFD_MMAP_TYPE_SHIFT (62 - PAGE_SHIFT)
+-#define KFD_MMAP_TYPE_MASK (0x3ULL << KFD_MMAP_TYPE_SHIFT)
+-#define KFD_MMAP_TYPE_DOORBELL (0x3ULL << KFD_MMAP_TYPE_SHIFT)
+-#define KFD_MMAP_TYPE_EVENTS (0x2ULL << KFD_MMAP_TYPE_SHIFT)
+-#define KFD_MMAP_TYPE_MAP_BO (0x1ULL << KFD_MMAP_TYPE_SHIFT)
+-#define KFD_MMAP_TYPE_RESERVED_MEM (0x0ULL << KFD_MMAP_TYPE_SHIFT)
+-
+-#define KFD_MMAP_GPU_ID_SHIFT (46 - PAGE_SHIFT)
+-#define KFD_MMAP_GPU_ID_MASK (((1ULL << KFD_GPU_ID_HASH_WIDTH) - 1) \
+- << KFD_MMAP_GPU_ID_SHIFT)
+-#define KFD_MMAP_GPU_ID(gpu_id) ((((uint64_t)gpu_id) << KFD_MMAP_GPU_ID_SHIFT)\
+- & KFD_MMAP_GPU_ID_MASK)
+-#define KFD_MMAP_GPU_ID_GET(offset) ((offset & KFD_MMAP_GPU_ID_MASK) \
+- >> KFD_MMAP_GPU_ID_SHIFT)
+-
+-#define KFD_MMAP_OFFSET_VALUE_MASK (0xFFFFFFFFFFULL >> PAGE_SHIFT)
+-#define KFD_MMAP_OFFSET_VALUE_GET(offset) (offset & KFD_MMAP_OFFSET_VALUE_MASK)
++#define KFD_MMAP_DOORBELL_MASK 0x8000000000000
++#define KFD_MMAP_EVENTS_MASK 0x4000000000000
+
+ /*
+ * When working with cp scheduler we should assign the HIQ manually or via
+@@ -84,6 +48,8 @@
+ #define KFD_CIK_HIQ_PIPE 4
+ #define KFD_CIK_HIQ_QUEUE 0
+
++/* GPU ID hash width in bits */
++#define KFD_GPU_ID_HASH_WIDTH 16
+
+ /* Macro for allocating structures */
+ #define kfd_alloc_struct(ptr_to_struct) \
+@@ -108,42 +74,12 @@ extern int max_num_of_queues_per_device;
+ /* Kernel module parameter to specify the scheduling policy */
+ extern int sched_policy;
+
+-extern int cwsr_enable;
+-
+-/*
+- * Kernel module parameter to specify the maximum process
+- * number per HW scheduler
+- */
+-extern int hws_max_conc_proc;
+-
+ /*
+ * Kernel module parameter to specify whether to send sigterm to HSA process on
+ * unhandled exception
+ */
+ extern int send_sigterm;
+
+-/*
+- * This kernel module is used to simulate large bar machine on non-large bar
+- * enabled machines.
+- */
+-extern int debug_largebar;
+-
+-/*
+- * Ignore CRAT table during KFD initialization, can be used to work around
+- * broken CRAT tables on some AMD systems
+- */
+-extern int ignore_crat;
+-
+-/*
+- * Set sh_mem_config.retry_disable on Vega10
+- */
+-extern int vega10_noretry;
+-
+-/*
+- * Enable privileged mode for all CP queues including user queues
+- */
+-extern int priv_cp_queues;
+-
+ /**
+ * enum kfd_sched_policy
+ *
+@@ -176,28 +112,26 @@ enum cache_policy {
+ cache_policy_noncoherent
+ };
+
+-#define KFD_IS_SOC15(chip) ((chip) >= CHIP_VEGA10)
++enum asic_family_type {
++ CHIP_KAVERI = 0,
++ CHIP_CARRIZO
++};
+
+ struct kfd_event_interrupt_class {
+ bool (*interrupt_isr)(struct kfd_dev *dev,
+- const uint32_t *ih_ring_entry, uint32_t *patched_ihre,
+- bool *patched_flag);
++ const uint32_t *ih_ring_entry);
+ void (*interrupt_wq)(struct kfd_dev *dev,
+- const uint32_t *ih_ring_entry);
++ const uint32_t *ih_ring_entry);
+ };
+
+ struct kfd_device_info {
+- enum amd_asic_type asic_family;
++ unsigned int asic_family;
+ const struct kfd_event_interrupt_class *event_interrupt_class;
+ unsigned int max_pasid_bits;
+ unsigned int max_no_of_hqd;
+- unsigned int doorbell_size;
+ size_t ih_ring_entry_size;
+ uint8_t num_of_watch_points;
+ uint16_t mqd_size_aligned;
+- bool is_need_iommu_device;
+- bool supports_cwsr;
+- bool needs_pci_atomics;
+ };
+
+ struct kfd_mem_obj {
+@@ -205,13 +139,6 @@ struct kfd_mem_obj {
+ uint32_t range_end;
+ uint64_t gpu_addr;
+ uint32_t *cpu_ptr;
+- void *gtt_mem;
+-};
+-
+-struct kfd_vmid_info {
+- uint32_t first_vmid_kfd;
+- uint32_t last_vmid_kfd;
+- uint32_t vmid_num_kfd;
+ };
+
+ struct kfd_dev {
+@@ -238,12 +165,11 @@ struct kfd_dev {
+ */
+
+ struct kgd2kfd_shared_resources shared_resources;
+- struct kfd_vmid_info vm_info;
+
+ const struct kfd2kgd_calls *kfd2kgd;
+ struct mutex doorbell_mutex;
+- unsigned long doorbell_available_index[DIV_ROUND_UP(
+- KFD_MAX_NUM_OF_QUEUES_PER_PROCESS, BITS_PER_LONG)];
++ DECLARE_BITMAP(doorbell_available_index,
++ KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
+
+ void *gtt_mem;
+ uint64_t gtt_start_gpu_addr;
+@@ -253,17 +179,18 @@ struct kfd_dev {
+ unsigned int gtt_sa_chunk_size;
+ unsigned int gtt_sa_num_of_chunks;
+
+- /* QCM Device instance */
+- struct device_queue_manager *dqm;
+-
+- bool init_complete;
+-
+ /* Interrupts */
+- struct kfifo ih_fifo;
+- struct workqueue_struct *ih_wq;
++ void *interrupt_ring;
++ size_t interrupt_ring_size;
++ atomic_t interrupt_ring_rptr;
++ atomic_t interrupt_ring_wptr;
+ struct work_struct interrupt_work;
+ spinlock_t interrupt_lock;
+
++ /* QCM Device instance */
++ struct device_queue_manager *dqm;
++
++ bool init_complete;
+ /*
+ * Interrupts of interest to KFD are copied
+ * from the HW ring into a SW ring.
+@@ -271,32 +198,7 @@ struct kfd_dev {
+ bool interrupts_active;
+
+ /* Debug manager */
+- struct kfd_dbgmgr *dbgmgr;
+-
+- /* MEC firmware version*/
+- uint16_t mec_fw_version;
+-
+- /* Maximum process number mapped to HW scheduler */
+- unsigned int max_proc_per_quantum;
+-
+- /* cwsr */
+- bool cwsr_enabled;
+- struct page *cwsr_pages;
+- uint32_t cwsr_size;
+- uint32_t tma_offset; /*Offset for TMA from the start of cwsr_mem*/
+-
+- /* IB usage */
+- uint32_t ib_size;
+-};
+-
+-struct kfd_ipc_obj;
+-
+-struct kfd_bo {
+- void *mem;
+- struct interval_tree_node it;
+- struct kfd_dev *dev;
+- struct list_head cb_data_head;
+- struct kfd_ipc_obj *kfd_ipc_obj;
++ struct kfd_dbgmgr *dbgmgr;
+ };
+
+ /* KGD2KFD callbacks */
+@@ -319,22 +221,27 @@ void kfd_chardev_exit(void);
+ struct device *kfd_chardev(void);
+
+ /**
+- * enum kfd_unmap_queues_filter
++ * enum kfd_preempt_type_filter
+ *
+- * @KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE: Preempts single queue.
++ * @KFD_PREEMPT_TYPE_FILTER_SINGLE_QUEUE: Preempts single queue.
+ *
+- * @KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES: Preempts all queues in the
++ * @KFD_PRERMPT_TYPE_FILTER_ALL_QUEUES: Preempts all queues in the
+ * running queues list.
+ *
+- * @KFD_UNMAP_QUEUES_FILTER_BY_PASID: Preempts queues that belongs to
++ * @KFD_PRERMPT_TYPE_FILTER_BY_PASID: Preempts queues that belongs to
+ * specific process.
+ *
+ */
+-enum kfd_unmap_queues_filter {
+- KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE,
+- KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES,
+- KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES,
+- KFD_UNMAP_QUEUES_FILTER_BY_PASID
++enum kfd_preempt_type_filter {
++ KFD_PREEMPT_TYPE_FILTER_SINGLE_QUEUE,
++ KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES,
++ KFD_PREEMPT_TYPE_FILTER_DYNAMIC_QUEUES,
++ KFD_PREEMPT_TYPE_FILTER_BY_PASID
++};
++
++enum kfd_preempt_type {
++ KFD_PREEMPT_TYPE_WAVEFRONT,
++ KFD_PREEMPT_TYPE_WAVEFRONT_RESET
+ };
+
+ /**
+@@ -360,11 +267,6 @@ enum kfd_queue_format {
+ KFD_QUEUE_FORMAT_AQL
+ };
+
+-enum KFD_QUEUE_PRIORITY {
+- KFD_QUEUE_PRIORITY_MINIMUM = 0,
+- KFD_QUEUE_PRIORITY_MAXIMUM = 15
+-};
+-
+ /**
+ * struct queue_properties
+ *
+@@ -392,13 +294,13 @@ enum KFD_QUEUE_PRIORITY {
+ * @write_ptr: Defines the number of dwords written to the ring buffer.
+ *
+ * @doorbell_ptr: This field aim is to notify the H/W of new packet written to
+- * the queue ring buffer. This field should be similar to write_ptr and the
+- * user should update this field after he updated the write_ptr.
++ * the queue ring buffer. This field should be similar to write_ptr and the user
++ * should update this field after he updated the write_ptr.
+ *
+ * @doorbell_off: The doorbell offset in the doorbell pci-bar.
+ *
+- * @is_interop: Defines if this is a interop queue. Interop queue means that
+- * the queue can access both graphics and compute resources.
++ * @is_interop: Defines if this is a interop queue. Interop queue means that the
++ * queue can access both graphics and compute resources.
+ *
+ * @is_active: Defines if the queue is active or not.
+ *
+@@ -419,10 +321,9 @@ struct queue_properties {
+ uint32_t queue_percent;
+ uint32_t *read_ptr;
+ uint32_t *write_ptr;
+- void __iomem *doorbell_ptr;
++ uint32_t __iomem *doorbell_ptr;
+ uint32_t doorbell_off;
+ bool is_interop;
+- bool is_evicted; /* true -> queue is evicted */
+ bool is_active;
+ /* Not relevant for user mode queues in cp scheduling */
+ unsigned int vmid;
+@@ -435,12 +336,6 @@ struct queue_properties {
+ uint32_t eop_ring_buffer_size;
+ uint64_t ctx_save_restore_area_address;
+ uint32_t ctx_save_restore_area_size;
+- uint32_t ctl_stack_size;
+- uint64_t tba_addr;
+- uint64_t tma_addr;
+- /* Relevant for CU */
+- uint32_t cu_mask_count; /* Must be a multiple of 32 */
+- uint32_t *cu_mask;
+ };
+
+ /**
+@@ -457,10 +352,9 @@ struct queue_properties {
+ * @properties: The queue properties.
+ *
+ * @mec: Used only in no cp scheduling mode and identifies to micro engine id
+- * that the queue should be execute on.
++ * that the queue should be execute on.
+ *
+- * @pipe: Used only in no cp scheduling mode and identifies the queue's pipe
+- * id.
++ * @pipe: Used only in no cp scheduling mode and identifies the queue's pipe id.
+ *
+ * @queue: Used only in no cp scheduliong mode and identifies the queue's slot.
+ *
+@@ -485,7 +379,6 @@ struct queue {
+ uint32_t queue;
+
+ unsigned int sdma_id;
+- unsigned int doorbell_id;
+
+ struct kfd_process *process;
+ struct kfd_dev *device;
+@@ -502,19 +395,6 @@ enum KFD_MQD_TYPE {
+ KFD_MQD_TYPE_MAX
+ };
+
+-enum KFD_PIPE_PRIORITY {
+- KFD_PIPE_PRIORITY_CS_LOW = 0,
+- KFD_PIPE_PRIORITY_CS_MEDIUM,
+- KFD_PIPE_PRIORITY_CS_HIGH
+-};
+-
+-enum KFD_SPI_PRIORITY {
+- KFD_SPI_PRIORITY_EXTRA_LOW = 0,
+- KFD_SPI_PRIORITY_LOW,
+- KFD_SPI_PRIORITY_MEDIUM,
+- KFD_SPI_PRIORITY_HIGH
+-};
+-
+ struct scheduling_resources {
+ unsigned int vmid_mask;
+ enum kfd_queue_type type;
+@@ -528,6 +408,7 @@ struct scheduling_resources {
+ struct process_queue_manager {
+ /* data */
+ struct kfd_process *process;
++ unsigned int num_concurrent_processes;
+ struct list_head queues;
+ unsigned long *queue_slot_bitmap;
+ };
+@@ -543,13 +424,6 @@ struct qcm_process_device {
+ unsigned int queue_count;
+ unsigned int vmid;
+ bool is_debug;
+- unsigned int evicted; /* eviction counter, 0=active */
+-
+- /* This flag tells if we should reset all wavefronts on
+- * process termination
+- */
+- bool reset_wavefronts;
+-
+ /*
+ * All the memory management data should be here too
+ */
+@@ -562,55 +436,6 @@ struct qcm_process_device {
+ uint32_t gds_size;
+ uint32_t num_gws;
+ uint32_t num_oac;
+- uint32_t sh_hidden_private_base;
+-
+- /*cwsr memory*/
+- uint64_t cwsr_base;
+- uint64_t tba_addr;
+- uint64_t tma_addr;
+- void *cwsr_kaddr;
+- struct page *cwsr_pages;
+-
+- /* IB memory */
+- uint64_t ib_base; /* ib_base+ib_size must be below cwsr_base */
+- void *ib_kaddr;
+-
+- /*doorbell resources per process per device*/
+- unsigned long *doorbell_bitmap;
+-};
+-
+-/* KFD Memory Eviction */
+-struct kfd_eviction_work {
+- struct delayed_work dwork;
+- struct dma_fence *quiesce_fence;
+-};
+-
+-/* Approx. wait time before attempting to restore evicted BOs */
+-#define PROCESS_RESTORE_TIME_MS 100
+-/* Approx. back off time if restore fails due to lack of memory */
+-#define PROCESS_BACK_OFF_TIME_MS 100
+-/* Approx. time before evicting the process again */
+-#define PROCESS_ACTIVE_TIME_MS 10
+-
+-void kfd_evict_bo_worker(struct work_struct *work);
+-void kfd_restore_bo_worker(struct work_struct *work);
+-int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm,
+- struct dma_fence *fence);
+-int quiesce_process_mm(struct kfd_process *p);
+-
+-
+-/* 8 byte handle containing GPU ID in the most significant 4 bytes and
+- * idr_handle in the least significant 4 bytes
+- */
+-#define MAKE_HANDLE(gpu_id, idr_handle) \
+- (((uint64_t)(gpu_id) << 32) + idr_handle)
+-#define GET_GPU_ID(handle) (handle >> 32)
+-#define GET_IDR_HANDLE(handle) (handle & 0xFFFFFFFF)
+-
+-enum kfd_pdd_bound {
+- PDD_UNBOUND = 0,
+- PDD_BOUND,
+- PDD_BOUND_SUSPENDED,
+ };
+
+ /* Data that is per-process-per device. */
+@@ -624,8 +449,6 @@ struct kfd_process_device {
+ /* The device that owns this data. */
+ struct kfd_dev *dev;
+
+- /* The process that owns this kfd_process_device. */
+- struct kfd_process *process;
+
+ /* per-process-per device QCM data structure */
+ struct qcm_process_device qpd;
+@@ -637,27 +460,14 @@ struct kfd_process_device {
+ uint64_t gpuvm_limit;
+ uint64_t scratch_base;
+ uint64_t scratch_limit;
+- uint64_t dgpu_base;
+- uint64_t dgpu_limit;
+-
+- uint64_t sh_hidden_private_base_vmid;
+-
+- /* Is this process/pasid bound to this device? (amd_iommu_bind_pasid)
+- */
+- enum kfd_pdd_bound bound;
+
+- /* VM context for GPUVM allocations */
+- void *vm;
++ /* Is this process/pasid bound to this device? (amd_iommu_bind_pasid) */
++ bool bound;
+
+- /* GPUVM allocations storage */
+- struct idr alloc_idr;
+-
+- /* Flag used to tell the pdd has dequeued from the dqm.
+- * This is used to prevent dev->dqm->ops.process_termination() from
+- * being called twice when it is already called in IOMMU callback
+- * function.
++ /* This flag tells if we should reset all
++ * wavefronts on process termination
+ */
+- bool already_dequeued;
++ bool reset_wavefronts;
+ };
+
+ #define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)
+@@ -670,15 +480,7 @@ struct kfd_process {
+ */
+ struct hlist_node kfd_processes;
+
+- /*
+- * Opaque pointer to mm_struct. We don't hold a reference to
+- * it so it should never be dereferenced from here. This is
+- * only used for looking up processes by their mm.
+- */
+- void *mm;
+-
+- struct kref ref;
+- struct work_struct release_work;
++ struct mm_struct *mm;
+
+ struct mutex mutex;
+
+@@ -686,8 +488,6 @@ struct kfd_process {
+ * In any process, the thread that started main() is the lead
+ * thread and outlives the rest.
+ * It is here because amd_iommu_bind_pasid wants a task_struct.
+- * It can also be used for safely getting a reference to the
+- * mm_struct of the process.
+ */
+ struct task_struct *lead_thread;
+
+@@ -707,8 +507,11 @@ struct kfd_process {
+
+ struct process_queue_manager pqm;
+
+- unsigned long allocated_queue_bitmap[DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,
+- BITS_PER_LONG)];
++ /* The process's queues. */
++ size_t queue_array_size;
++
++ /* Size is queue_array_size, up to MAX_PROCESS_QUEUES. */
++ struct kfd_queue **queues;
+
+ /*Is the user space process 32 bit?*/
+ bool is_32bit_user_mode;
+@@ -717,29 +520,10 @@ struct kfd_process {
+ struct mutex event_mutex;
+ /* All events in process hashed by ID, linked on kfd_event.events. */
+ DECLARE_HASHTABLE(events, 4);
+- /* struct slot_page_header.event_pages */
+- struct list_head signal_event_pages;
++ struct list_head signal_event_pages; /* struct slot_page_header.
++ event_pages */
+ u32 next_nonsignal_event_id;
+ size_t signal_event_count;
+- bool signal_event_limit_reached;
+-
+- struct rb_root_cached bo_interval_tree;
+-
+- /* Information used for memory eviction */
+- void *process_info;
+- /* Eviction fence that is attached to all the BOs of this process. The
+- * fence will be triggered during eviction and new one will be created
+- * during restore
+- */
+- struct dma_fence *ef;
+-
+- /* Work items for evicting and restoring BOs */
+- struct kfd_eviction_work eviction_work;
+- struct delayed_work restore_work;
+- /* Approx. the last timestamp (in jiffies) when the process was
+- * restored after an eviction
+- */
+- unsigned long last_restore_timestamp;
+ };
+
+ /**
+@@ -762,55 +546,21 @@ struct amdkfd_ioctl_desc {
+
+ void kfd_process_create_wq(void);
+ void kfd_process_destroy_wq(void);
+-struct kfd_process *kfd_create_process(struct file *filep);
+-struct kfd_process *kfd_get_process(const struct task_struct *task);
++struct kfd_process *kfd_create_process(const struct task_struct *);
++struct kfd_process *kfd_get_process(const struct task_struct *);
+ struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid);
+-struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm);
+-void kfd_unref_process(struct kfd_process *p);
+-void kfd_suspend_all_processes(void);
+-int kfd_resume_all_processes(void);
+
+ struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
+ struct kfd_process *p);
+-#if defined(CONFIG_AMD_IOMMU_V2_MODULE) || defined(CONFIG_AMD_IOMMU_V2)
+-int kfd_bind_processes_to_device(struct kfd_dev *dev);
+-void kfd_unbind_processes_from_device(struct kfd_dev *dev);
+-#endif
+-void kfd_process_iommu_unbind_callback(struct kfd_dev *dev, unsigned int pasid);
++void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid);
+ struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
+ struct kfd_process *p);
+ struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
+ struct kfd_process *p);
+
+-int kfd_reserved_mem_mmap(struct kfd_process *process,
+- struct vm_area_struct *vma);
+-
+-/* KFD process API for creating and translating handles */
+-int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
+- void *mem, uint64_t start,
+- uint64_t length,
+- struct kfd_ipc_obj *ipc_obj);
+-void *kfd_process_device_translate_handle(struct kfd_process_device *p,
+- int handle);
+-struct kfd_bo *kfd_process_device_find_bo(struct kfd_process_device *pdd,
+- int handle);
+-void *kfd_process_find_bo_from_interval(struct kfd_process *p,
+- uint64_t start_addr,
+- uint64_t last_addr);
+-void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,
+- int handle);
+-
+-void run_rdma_free_callback(struct kfd_bo *buf_obj);
+-struct kfd_process *kfd_lookup_process_by_pid(struct pid *pid);
+-
+-/* kfd dgpu memory */
+-int kfd_unmap_memory_from_gpu(void *mem, struct kfd_process_device *pdd);
+-
+ /* Process device data iterator */
+-struct kfd_process_device *kfd_get_first_process_device_data(
+- struct kfd_process *p);
+-struct kfd_process_device *kfd_get_next_process_device_data(
+- struct kfd_process *p,
++struct kfd_process_device *kfd_get_first_process_device_data(struct kfd_process *p);
++struct kfd_process_device *kfd_get_next_process_device_data(struct kfd_process *p,
+ struct kfd_process_device *pdd);
+ bool kfd_has_process_device_data(struct kfd_process *p);
+
+@@ -823,20 +573,16 @@ unsigned int kfd_pasid_alloc(void);
+ void kfd_pasid_free(unsigned int pasid);
+
+ /* Doorbells */
+-size_t kfd_doorbell_process_slice(struct kfd_dev *kfd);
+-int kfd_doorbell_init(struct kfd_dev *kfd);
+-void kfd_doorbell_fini(struct kfd_dev *kfd);
+-int kfd_doorbell_mmap(struct kfd_dev *kfd, struct kfd_process *process,
+- struct vm_area_struct *vma);
+-void __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
++void kfd_doorbell_init(struct kfd_dev *kfd);
++int kfd_doorbell_mmap(struct kfd_process *process, struct vm_area_struct *vma);
++u32 __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
+ unsigned int *doorbell_off);
+ void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr);
+ u32 read_kernel_doorbell(u32 __iomem *db);
+-void write_kernel_doorbell(void __iomem *db, u32 value);
+-void write_kernel_doorbell64(void __iomem *db, u64 value);
+-unsigned int kfd_doorbell_id_to_offset(struct kfd_dev *kfd,
++void write_kernel_doorbell(u32 __iomem *db, u32 value);
++unsigned int kfd_queue_id_to_doorbell(struct kfd_dev *kfd,
+ struct kfd_process *process,
+- unsigned int doorbell_id);
++ unsigned int queue_id);
+
+ /* GTT Sub-Allocator */
+
+@@ -852,22 +598,16 @@ int kfd_topology_init(void);
+ void kfd_topology_shutdown(void);
+ int kfd_topology_add_device(struct kfd_dev *gpu);
+ int kfd_topology_remove_device(struct kfd_dev *gpu);
+-struct kfd_topology_device *kfd_topology_device_by_proximity_domain(
+- uint32_t proximity_domain);
+ struct kfd_dev *kfd_device_by_id(uint32_t gpu_id);
+ struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev);
+-struct kfd_dev *kfd_device_by_kgd(const struct kgd_dev *kgd);
+-int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_dev **kdev);
+-int kfd_numa_node_to_apic_id(int numa_node_id);
++struct kfd_dev *kfd_topology_enum_kfd_devices(uint8_t idx);
+
+ /* Interrupts */
+ int kfd_interrupt_init(struct kfd_dev *dev);
+ void kfd_interrupt_exit(struct kfd_dev *dev);
+ void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry);
+ bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry);
+-bool interrupt_is_wanted(struct kfd_dev *dev,
+- const uint32_t *ih_ring_entry,
+- uint32_t *patched_ihre, bool *flag);
++bool interrupt_is_wanted(struct kfd_dev *dev, const uint32_t *ih_ring_entry);
+
+ /* Power Management */
+ void kgd2kfd_suspend(struct kfd_dev *kfd);
+@@ -875,10 +615,10 @@ int kgd2kfd_resume(struct kfd_dev *kfd);
+
+ /* amdkfd Apertures */
+ int kfd_init_apertures(struct kfd_process *process);
+-int kfd_set_process_dgpu_aperture(struct kfd_process_device *pdd,
+- uint64_t base, uint64_t limit);
+
+ /* Queue Context Management */
++struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd);
++
+ int init_queue(struct queue **q, const struct queue_properties *properties);
+ void uninit_queue(struct queue *q);
+ void print_queue_properties(struct queue_properties *q);
+@@ -888,20 +628,13 @@ struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type,
+ struct kfd_dev *dev);
+ struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
+ struct kfd_dev *dev);
+-struct mqd_manager *mqd_manager_init_cik_hawaii(enum KFD_MQD_TYPE type,
+- struct kfd_dev *dev);
+ struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
+ struct kfd_dev *dev);
+-struct mqd_manager *mqd_manager_init_vi_tonga(enum KFD_MQD_TYPE type,
+- struct kfd_dev *dev);
+-struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
+- struct kfd_dev *dev);
+ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev);
+ void device_queue_manager_uninit(struct device_queue_manager *dqm);
+ struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
+ enum kfd_queue_type type);
+ void kernel_queue_uninit(struct kernel_queue *kq);
+-int kfd_process_vm_fault(struct device_queue_manager *dqm, unsigned int pasid);
+
+ /* Process Queue Manager */
+ struct process_queue_node {
+@@ -910,36 +643,32 @@ struct process_queue_node {
+ struct list_head process_queue_list;
+ };
+
+-void kfd_process_dequeue_from_device(struct kfd_process_device *pdd);
+-void kfd_process_dequeue_from_all_devices(struct kfd_process *p);
+ int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p);
+ void pqm_uninit(struct process_queue_manager *pqm);
+ int pqm_create_queue(struct process_queue_manager *pqm,
+ struct kfd_dev *dev,
+ struct file *f,
+ struct queue_properties *properties,
++ unsigned int flags,
++ enum kfd_queue_type type,
+ unsigned int *qid);
+ int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid);
+ int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid,
+ struct queue_properties *p);
+-int pqm_set_cu_mask(struct process_queue_manager *pqm, unsigned int qid,
+- struct queue_properties *p);
+ struct kernel_queue *pqm_get_kernel_queue(struct process_queue_manager *pqm,
+ unsigned int qid);
+-int pqm_get_wave_state(struct process_queue_manager *pqm,
+- unsigned int qid,
+- void __user *ctl_stack,
+- u32 *ctl_stack_used_size,
+- u32 *save_area_used_size);
+-int kgd2kfd_quiesce_mm(struct kfd_dev *kfd, struct mm_struct *mm);
+-int kgd2kfd_resume_mm(struct kfd_dev *kfd, struct mm_struct *mm);
++
++int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
++ unsigned int fence_value,
++ unsigned long timeout);
+
+ /* Packet Manager */
+
++#define KFD_HIQ_TIMEOUT (500)
++
+ #define KFD_FENCE_COMPLETED (100)
+ #define KFD_FENCE_INIT (10)
+-
+-struct packet_manager_func;
++#define KFD_UNMAP_LATENCY (150)
+
+ struct packet_manager {
+ struct device_queue_manager *dqm;
+@@ -947,42 +676,9 @@ struct packet_manager {
+ struct mutex lock;
+ bool allocated;
+ struct kfd_mem_obj *ib_buffer_obj;
+- unsigned int ib_size_bytes;
+-
+- struct packet_manager_funcs *pmf;
+-};
+-
+-struct packet_manager_funcs {
+- /* Support different firmware versions for PM4 packets */
+- int (*map_process)(struct packet_manager *pm, uint32_t *buffer,
+- struct qcm_process_device *qpd);
+- int (*runlist)(struct packet_manager *pm, uint32_t *buffer,
+- uint64_t ib, size_t ib_size_in_dwords, bool chain);
+- int (*set_resources)(struct packet_manager *pm, uint32_t *buffer,
+- struct scheduling_resources *res);
+- int (*map_queues)(struct packet_manager *pm, uint32_t *buffer,
+- struct queue *q, bool is_static);
+- int (*unmap_queues)(struct packet_manager *pm, uint32_t *buffer,
+- enum kfd_queue_type type,
+- enum kfd_unmap_queues_filter mode,
+- uint32_t filter_param, bool reset,
+- unsigned int sdma_engine);
+- int (*query_status)(struct packet_manager *pm, uint32_t *buffer,
+- uint64_t fence_address, uint32_t fence_value);
+- uint32_t (*release_mem)(uint64_t gpu_addr, uint32_t *buffer);
+-
+- uint32_t (*get_map_process_packet_size)(void);
+- uint32_t (*get_runlist_packet_size)(void);
+- uint32_t (*get_set_resources_packet_size)(void);
+- uint32_t (*get_map_queues_packet_size)(void);
+- uint32_t (*get_unmap_queues_packet_size)(void);
+- uint32_t (*get_query_status_packet_size)(void);
+- uint32_t (*get_release_mem_packet_size)(void);
+-
+ };
+
+-int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm,
+- uint16_t fw_ver);
++int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm);
+ void pm_uninit(struct packet_manager *pm);
+ int pm_send_set_resources(struct packet_manager *pm,
+ struct scheduling_resources *res);
+@@ -991,55 +687,18 @@ int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address,
+ uint32_t fence_value);
+
+ int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,
+- enum kfd_unmap_queues_filter mode,
++ enum kfd_preempt_type_filter mode,
+ uint32_t filter_param, bool reset,
+ unsigned int sdma_engine);
+
+ void pm_release_ib(struct packet_manager *pm);
+
+-/* Following PM funcs can be shared among CIK and VI */
+-unsigned int pm_build_pm4_header(unsigned int opcode, size_t packet_size);
+-int pm_runlist_vi(struct packet_manager *pm, uint32_t *buffer,
+- uint64_t ib, size_t ib_size_in_dwords, bool chain);
+-int pm_map_queues_vi(struct packet_manager *pm, uint32_t *buffer,
+- struct queue *q, bool is_static);
+-int pm_set_resources_vi(struct packet_manager *pm, uint32_t *buffer,
+- struct scheduling_resources *res);
+-int pm_unmap_queues_vi(struct packet_manager *pm, uint32_t *buffer,
+- enum kfd_queue_type type,
+- enum kfd_unmap_queues_filter filter,
+- uint32_t filter_param, bool reset,
+- unsigned int sdma_engine);
+-int pm_query_status_vi(struct packet_manager *pm, uint32_t *buffer,
+- uint64_t fence_address, uint32_t fence_value);
+-uint32_t pm_release_mem_vi(uint64_t gpu_addr, uint32_t *buffer);
+-
+-uint32_t pm_get_map_process_packet_size_vi(void);
+-uint32_t pm_get_runlist_packet_size_vi(void);
+-uint32_t pm_get_set_resources_packet_size_vi(void);
+-uint32_t pm_get_map_queues_packet_size_vi(void);
+-uint32_t pm_get_unmap_queues_packet_size_vi(void);
+-uint32_t pm_get_query_status_packet_size_vi(void);
+-uint32_t pm_get_release_mem_packet_size_vi(void);
+-
+-
+-void kfd_pm_func_init_vi(struct packet_manager *pm, uint16_t fw_ver);
+-void kfd_pm_func_init_cik(struct packet_manager *pm, uint16_t fw_ver);
+-
+-void kfd_pm_func_init_v9(struct packet_manager *pm, uint16_t fw_ver);
+-
+-
+ uint64_t kfd_get_number_elems(struct kfd_dev *kfd);
+ phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev,
+ struct kfd_process *process);
+-int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
+- unsigned int fence_value,
+- unsigned long timeout_ms);
+
+ /* Events */
+ extern const struct kfd_event_interrupt_class event_interrupt_class_cik;
+-extern const struct kfd_event_interrupt_class event_interrupt_class_v9;
+-
+ extern const struct kfd_device_global_init_class device_global_init_class_cik;
+
+ enum kfd_event_wait_result {
+@@ -1057,55 +716,18 @@ int kfd_wait_on_events(struct kfd_process *p,
+ enum kfd_event_wait_result *wait_result);
+ void kfd_signal_event_interrupt(unsigned int pasid, uint32_t partial_id,
+ uint32_t valid_id_bits);
+-#if defined(CONFIG_AMD_IOMMU_V2_MODULE) || defined(CONFIG_AMD_IOMMU_V2)
+ void kfd_signal_iommu_event(struct kfd_dev *dev,
+ unsigned int pasid, unsigned long address,
+ bool is_write_requested, bool is_execute_requested);
+-#endif
+ void kfd_signal_hw_exception_event(unsigned int pasid);
+ int kfd_set_event(struct kfd_process *p, uint32_t event_id);
+ int kfd_reset_event(struct kfd_process *p, uint32_t event_id);
+ int kfd_event_create(struct file *devkfd, struct kfd_process *p,
+ uint32_t event_type, bool auto_reset, uint32_t node_id,
+ uint32_t *event_id, uint32_t *event_trigger_data,
+- uint64_t *event_page_offset, uint32_t *event_slot_index,
+- void *kern_addr);
++ uint64_t *event_page_offset, uint32_t *event_slot_index);
+ int kfd_event_destroy(struct kfd_process *p, uint32_t event_id);
+-void kfd_free_signal_page_dgpu(struct kfd_process *p, uint64_t handle);
+-
+-void kfd_signal_vm_fault_event(struct kfd_dev *dev, unsigned int pasid,
+- struct kfd_vm_fault_info *info);
+-
+-void kfd_flush_tlb(struct kfd_dev *dev, uint32_t pasid);
+
+ int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p);
+
+-#define KFD_SCRATCH_KV_FW_VER 413
+-
+-/* PeerDirect support */
+-void kfd_init_peer_direct(void);
+-void kfd_close_peer_direct(void);
+-
+-/* IPC Support */
+-int kfd_ipc_init(void);
+-
+-/* Debugfs */
+-#if defined(CONFIG_DEBUG_FS)
+-
+-void kfd_debugfs_init(void);
+-void kfd_debugfs_fini(void);
+-int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data);
+-int pqm_debugfs_mqds(struct seq_file *m, void *data);
+-int kfd_debugfs_hqds_by_device(struct seq_file *m, void *data);
+-int device_queue_manager_debugfs_hqds(struct seq_file *m, void *data);
+-int kfd_debugfs_rls_by_device(struct seq_file *m, void *data);
+-int pm_debugfs_runlist(struct seq_file *m, void *data);
+-
+-#else
+-
+-static inline void kfd_debugfs_init(void) {}
+-static inline void kfd_debugfs_fini(void) {}
+-
+-#endif
+-
+ #endif
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+index c798fa3..035bbc9 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+@@ -24,16 +24,10 @@
+ #include <linux/log2.h>
+ #include <linux/sched.h>
+ #include <linux/sched/mm.h>
+-#include <linux/sched/task.h>
+ #include <linux/slab.h>
+-#if defined(CONFIG_AMD_IOMMU_V2_MODULE) || defined(CONFIG_AMD_IOMMU_V2)
+ #include <linux/amd-iommu.h>
+-#endif
+ #include <linux/notifier.h>
+ #include <linux/compat.h>
+-#include <linux/mman.h>
+-#include <linux/highmem.h>
+-#include "kfd_ipc.h"
+
+ struct mm_struct;
+
+@@ -41,6 +35,13 @@ struct mm_struct;
+ #include "kfd_dbgmgr.h"
+
+ /*
++ * Initial size for the array of queues.
++ * The allocated size is doubled each time
++ * it is exceeded up to MAX_PROCESS_QUEUES.
++ */
++#define INITIAL_QUEUE_ARRAY_SIZE 16
++
++/*
+ * List of struct kfd_process (field kfd_process).
+ * Unique/indexed by mm_struct*
+ */
+@@ -52,16 +53,13 @@ DEFINE_STATIC_SRCU(kfd_processes_srcu);
+
+ static struct workqueue_struct *kfd_process_wq;
+
+-#define MIN_IDR_ID 1
+-#define MAX_IDR_ID 0 /*0 - for unlimited*/
+-
+-static struct kfd_process *find_process(const struct task_struct *thread,
+- bool ref);
+-static void kfd_process_ref_release(struct kref *ref);
+-static struct kfd_process *create_process(const struct task_struct *thread,
+- struct file *filep);
+-static int kfd_process_init_cwsr(struct kfd_process *p, struct file *filep);
++struct kfd_process_release_work {
++ struct work_struct kfd_work;
++ struct kfd_process *p;
++};
+
++static struct kfd_process *find_process(const struct task_struct *thread);
++static struct kfd_process *create_process(const struct task_struct *thread);
+
+ void kfd_process_create_wq(void)
+ {
+@@ -77,144 +75,22 @@ void kfd_process_destroy_wq(void)
+ }
+ }
+
+-static void kfd_process_free_gpuvm(struct kgd_mem *mem,
+- struct kfd_process_device *pdd)
+-{
+- kfd_unmap_memory_from_gpu(mem, pdd);
+- pdd->dev->kfd2kgd->free_memory_of_gpu(pdd->dev->kgd, mem, pdd->vm);
+-}
+-
+-/* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process
+- * This function should be only called right after the process
+- * is created and when kfd_processes_mutex is still being held
+- * to avoid concurrency. Because of that exclusiveness, we do
+- * not need to take p->mutex.
+- */
+-static int kfd_process_alloc_gpuvm(struct kfd_process *p,
+- struct kfd_dev *kdev, uint64_t gpu_va, uint32_t size,
+- void **kptr, struct kfd_process_device *pdd, uint32_t flags)
+-{
+- int err;
+- void *mem = NULL;
+- int handle;
+-
+- err = kdev->kfd2kgd->alloc_memory_of_gpu(kdev->kgd, gpu_va, size,
+- pdd->vm,
+- (struct kgd_mem **)&mem, NULL, flags);
+- if (err)
+- goto err_alloc_mem;
+-
+- err = kdev->kfd2kgd->map_memory_to_gpu(
+- kdev->kgd, (struct kgd_mem *)mem, pdd->vm);
+- if (err)
+- goto err_map_mem;
+-
+- err = kdev->kfd2kgd->sync_memory(kdev->kgd, (struct kgd_mem *) mem,
+- true);
+- if (err) {
+- pr_debug("Sync memory failed, wait interrupted by user signal\n");
+- goto sync_memory_failed;
+- }
+-
+- kfd_flush_tlb(kdev, p->pasid);
+-
+- /* Create an obj handle so kfd_process_device_remove_obj_handle
+- * will take care of the bo removal when the process finishes.
+- * We do not need to take p->mutex, because the process is just
+- * created and the ioctls have not had the chance to run.
+- */
+- handle = kfd_process_device_create_obj_handle(
+- pdd, mem, gpu_va, size, NULL);
+-
+- if (handle < 0) {
+- err = handle;
+- goto free_gpuvm;
+- }
+-
+- if (kptr) {
+- err = kdev->kfd2kgd->map_gtt_bo_to_kernel(kdev->kgd,
+- (struct kgd_mem *)mem, kptr);
+- if (err) {
+- pr_debug("Map GTT BO to kernel failed\n");
+- goto free_obj_handle;
+- }
+- }
+-
+- return err;
+-
+-free_obj_handle:
+- kfd_process_device_remove_obj_handle(pdd, handle);
+-free_gpuvm:
+-sync_memory_failed:
+- kfd_process_free_gpuvm(mem, pdd);
+- return err;
+-
+-err_map_mem:
+- kdev->kfd2kgd->free_memory_of_gpu(kdev->kgd, mem, pdd->vm);
+-err_alloc_mem:
+- *kptr = NULL;
+- return err;
+-}
+-
+-/* kfd_process_reserve_ib_mem - Reserve memory inside the process for IB usage
+- * The memory reserved is for KFD to submit IB to AMDGPU from kernel.
+- * If the memory is reserved successfully, ib_kaddr_assigned will have
+- * the CPU/kernel address. Check ib_kaddr_assigned before accessing the
+- * memory.
+- */
+-static int kfd_process_reserve_ib_mem(struct kfd_process *p)
+-{
+- int ret = 0;
+- struct kfd_process_device *temp, *pdd = NULL;
+- struct kfd_dev *kdev = NULL;
+- struct qcm_process_device *qpd = NULL;
+- void *kaddr;
+- uint32_t flags = ALLOC_MEM_FLAGS_GTT | ALLOC_MEM_FLAGS_NONPAGED |
+- ALLOC_MEM_FLAGS_NO_SUBSTITUTE |
+- ALLOC_MEM_FLAGS_EXECUTE_ACCESS;
+-
+- list_for_each_entry_safe(pdd, temp, &p->per_device_data,
+- per_device_list) {
+- kdev = pdd->dev;
+- qpd = &pdd->qpd;
+- if (!kdev->ib_size || qpd->ib_kaddr)
+- continue;
+-
+- if (qpd->ib_base) { /* is dGPU */
+- ret = kfd_process_alloc_gpuvm(p, kdev,
+- qpd->ib_base, kdev->ib_size,
+- &kaddr, pdd, flags);
+- if (!ret)
+- qpd->ib_kaddr = kaddr;
+- else
+- /* In case of error, the kfd_bos for some pdds
+- * which are already allocated successfully
+- * will be freed in upper level function
+- * i.e. create_process().
+- */
+- return ret;
+- } else {
+- /* FIXME: Support APU */
+- continue;
+- }
+- }
+-
+- return 0;
+-}
+-
+-struct kfd_process *kfd_create_process(struct file *filep)
++struct kfd_process *kfd_create_process(const struct task_struct *thread)
+ {
+ struct kfd_process *process;
+
+- struct task_struct *thread = current;
++ BUG_ON(!kfd_process_wq);
+
+- if (!thread->mm)
++ if (thread->mm == NULL)
+ return ERR_PTR(-EINVAL);
+
+ /* Only the pthreads threading model is supported. */
+ if (thread->group_leader->mm != thread->mm)
+ return ERR_PTR(-EINVAL);
+
++ /* Take mmap_sem because we call __mmu_notifier_register inside */
++ down_write(&thread->mm->mmap_sem);
++
+ /*
+ * take kfd processes mutex before starting of process creation
+ * so there won't be a case where two threads of the same process
+@@ -223,14 +99,17 @@ struct kfd_process *kfd_create_process(struct file *filep)
+ mutex_lock(&kfd_processes_mutex);
+
+ /* A prior open of /dev/kfd could have already created the process. */
+- process = find_process(thread, false);
++ process = find_process(thread);
+ if (process)
+- pr_debug("Process already found\n");
+- else
+- process = create_process(thread, filep);
++ pr_debug("kfd: process already found\n");
++
++ if (!process)
++ process = create_process(thread);
+
+ mutex_unlock(&kfd_processes_mutex);
+
++ up_write(&thread->mm->mmap_sem);
++
+ return process;
+ }
+
+@@ -238,14 +117,14 @@ struct kfd_process *kfd_get_process(const struct task_struct *thread)
+ {
+ struct kfd_process *process;
+
+- if (!thread->mm)
++ if (thread->mm == NULL)
+ return ERR_PTR(-EINVAL);
+
+ /* Only the pthreads threading model is supported. */
+ if (thread->group_leader->mm != thread->mm)
+ return ERR_PTR(-EINVAL);
+
+- process = find_process(thread, false);
++ process = find_process(thread);
+
+ return process;
+ }
+@@ -262,158 +141,81 @@ static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
+ return NULL;
+ }
+
+-static struct kfd_process *find_process(const struct task_struct *thread,
+- bool ref)
++static struct kfd_process *find_process(const struct task_struct *thread)
+ {
+ struct kfd_process *p;
+ int idx;
+
+ idx = srcu_read_lock(&kfd_processes_srcu);
+ p = find_process_by_mm(thread->mm);
+- if (p && ref)
+- kref_get(&p->ref);
+ srcu_read_unlock(&kfd_processes_srcu, idx);
+
+ return p;
+ }
+
+-void kfd_unref_process(struct kfd_process *p)
+-{
+- kref_put(&p->ref, kfd_process_ref_release);
+-}
+-
+-/* This increments the process->ref counter. */
+-struct kfd_process *kfd_lookup_process_by_pid(struct pid *pid)
++static void kfd_process_wq_release(struct work_struct *work)
+ {
+- struct task_struct *task = NULL;
+- struct kfd_process *p = NULL;
+-
+- if (!pid)
+- task = current;
+- else
+- task = get_pid_task(pid, PIDTYPE_PID);
++ struct kfd_process_release_work *my_work;
++ struct kfd_process_device *pdd, *temp;
++ struct kfd_process *p;
+
+- if (task)
+- p = find_process(task, true);
++ my_work = (struct kfd_process_release_work *) work;
+
+- return p;
+-}
++ p = my_work->p;
+
+-static void kfd_process_free_outstanding_kfd_bos(struct kfd_process *p)
+-{
+- struct kfd_process_device *pdd, *peer_pdd;
+- struct kfd_bo *buf_obj;
+- int id;
+-
+- list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
+- /*
+- * Remove all handles from idr and release appropriate
+- * local memory object
+- */
+- idr_for_each_entry(&pdd->alloc_idr, buf_obj, id) {
+- list_for_each_entry(peer_pdd, &p->per_device_data,
+- per_device_list) {
+- peer_pdd->dev->kfd2kgd->unmap_memory_to_gpu(
+- peer_pdd->dev->kgd,
+- buf_obj->mem, peer_pdd->vm);
+- }
+-
+- run_rdma_free_callback(buf_obj);
+- pdd->dev->kfd2kgd->free_memory_of_gpu(
+- pdd->dev->kgd, buf_obj->mem, pdd->vm);
+- kfd_process_device_remove_obj_handle(pdd, id);
+- }
+- }
+-}
++ pr_debug("Releasing process (pasid %d) in workqueue\n",
++ p->pasid);
+
+-static void kfd_process_destroy_pdds(struct kfd_process *p)
+-{
+- struct kfd_process_device *pdd, *temp;
++ mutex_lock(&p->mutex);
+
+ list_for_each_entry_safe(pdd, temp, &p->per_device_data,
+- per_device_list) {
+- kfd_flush_tlb(pdd->dev, p->pasid);
+- /* Destroy the GPUVM VM context */
+- if (pdd->vm) {
+- dma_fence_put(p->ef);
+- pdd->dev->kfd2kgd->destroy_process_vm(
+- pdd->dev->kgd, pdd->vm);
+- }
+- list_del(&pdd->per_device_list);
++ per_device_list) {
++ pr_debug("Releasing pdd (topology id %d) for process (pasid %d) in workqueue\n",
++ pdd->dev->id, p->pasid);
+
+- if (pdd->qpd.cwsr_pages) {
+- kunmap(pdd->qpd.cwsr_pages);
+- __free_pages(pdd->qpd.cwsr_pages,
+- get_order(pdd->dev->cwsr_size));
+- }
++ if (pdd->reset_wavefronts)
++ dbgdev_wave_reset_wavefronts(pdd->dev, p);
+
+- kfree(pdd->qpd.doorbell_bitmap);
+- idr_destroy(&pdd->alloc_idr);
++ amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid);
++ list_del(&pdd->per_device_list);
+
+ kfree(pdd);
+ }
+-}
+-
+-/* No process locking is needed in this function, because the process
+- * is not findable any more. We must assume that no other thread is
+- * using it any more, otherwise we couldn't safely free the process
+- * structure in the end.
+- */
+-static void kfd_process_wq_release(struct work_struct *work)
+-{
+- struct kfd_process *p = container_of(work, struct kfd_process,
+- release_work);
+-#if defined(CONFIG_AMD_IOMMU_V2_MODULE) || defined(CONFIG_AMD_IOMMU_V2)
+- struct kfd_process_device *pdd;
+-
+- pr_debug("Releasing process (pasid %d)\n",
+- p->pasid);
+-
+- list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
+- pr_debug("Releasing pdd (topology id %d) for process (pasid %d)\n",
+- pdd->dev->id, p->pasid);
+-
+- if (pdd->dev->device_info->is_need_iommu_device) {
+- if (pdd->bound == PDD_BOUND) {
+- amd_iommu_unbind_pasid(pdd->dev->pdev,
+- p->pasid);
+- pdd->bound = PDD_UNBOUND;
+- }
+- }
+- }
+-#endif
+-
+- kfd_process_free_outstanding_kfd_bos(p);
+-
+- kfd_process_destroy_pdds(p);
+
+ kfd_event_free_process(p);
+
+ kfd_pasid_free(p->pasid);
+
++ mutex_unlock(&p->mutex);
++
+ mutex_destroy(&p->mutex);
+
+- put_task_struct(p->lead_thread);
++ kfree(p->queues);
+
+ kfree(p);
++
++ kfree(work);
+ }
+
+-static void kfd_process_ref_release(struct kref *ref)
++static void kfd_process_destroy_delayed(struct rcu_head *rcu)
+ {
+- struct kfd_process *p = container_of(ref, struct kfd_process, ref);
++ struct kfd_process_release_work *work;
++ struct kfd_process *p;
+
+- if (WARN_ON(!kfd_process_wq))
+- return;
++ BUG_ON(!kfd_process_wq);
+
+- INIT_WORK(&p->release_work, kfd_process_wq_release);
+- queue_work(kfd_process_wq, &p->release_work);
+-}
++ p = container_of(rcu, struct kfd_process, rcu);
++ BUG_ON(atomic_read(&p->mm->mm_count) <= 0);
+
+-static void kfd_process_destroy_delayed(struct rcu_head *rcu)
+-{
+- struct kfd_process *p = container_of(rcu, struct kfd_process, rcu);
++ mmdrop(p->mm);
++
++ work = kmalloc(sizeof(struct kfd_process_release_work), GFP_ATOMIC);
+
+- kfd_unref_process(p);
++ if (work) {
++ INIT_WORK((struct work_struct *) work, kfd_process_wq_release);
++ work->p = p;
++ queue_work(kfd_process_wq, (struct work_struct *) work);
++ }
+ }
+
+ static void kfd_process_notifier_release(struct mmu_notifier *mn,
+@@ -421,19 +223,13 @@ static void kfd_process_notifier_release(struct mmu_notifier *mn,
+ {
+ struct kfd_process *p;
+ struct kfd_process_device *pdd = NULL;
+- struct kfd_dev *dev = NULL;
+- long status = -EFAULT;
+
+ /*
+ * The kfd_process structure can not be free because the
+ * mmu_notifier srcu is read locked
+ */
+ p = container_of(mn, struct kfd_process, mmu_notifier);
+- if (WARN_ON(p->mm != mm))
+- return;
+-
+- cancel_delayed_work_sync(&p->eviction_work.dwork);
+- cancel_delayed_work_sync(&p->restore_work);
++ BUG_ON(p->mm != mm);
+
+ mutex_lock(&kfd_processes_mutex);
+ hash_del_rcu(&p->kfd_processes);
+@@ -442,46 +238,33 @@ static void kfd_process_notifier_release(struct mmu_notifier *mn,
+
+ mutex_lock(&p->mutex);
+
+- /* Iterate over all process device data structures and if the pdd is in
+- * debug mode,we should first force unregistration, then we will be
+- * able to destroy the queues
+- */
+- list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
+- dev = pdd->dev;
+- mutex_lock(kfd_get_dbgmgr_mutex());
+-
+- if (dev && dev->dbgmgr && (dev->dbgmgr->pasid == p->pasid)) {
+-
+- status = kfd_dbgmgr_unregister(dev->dbgmgr, p);
+- if (status == 0) {
+- kfd_dbgmgr_destroy(dev->dbgmgr);
+- dev->dbgmgr = NULL;
+- }
+- }
+- mutex_unlock(kfd_get_dbgmgr_mutex());
+- }
+-
+- kfd_process_dequeue_from_all_devices(p);
+-
+- /* now we can uninit the pqm: */
++ /* In case our notifier is called before IOMMU notifier */
+ pqm_uninit(&p->pqm);
+
+ /* Iterate over all process device data structure and check
+- * if we should delete debug managers
++ * if we should delete debug managers and reset all wavefronts
+ */
+ list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
+ if ((pdd->dev->dbgmgr) &&
+ (pdd->dev->dbgmgr->pasid == p->pasid))
+ kfd_dbgmgr_destroy(pdd->dev->dbgmgr);
+
++ if (pdd->reset_wavefronts) {
++ pr_warn("amdkfd: Resetting all wave fronts\n");
++ dbgdev_wave_reset_wavefronts(pdd->dev, p);
++ pdd->reset_wavefronts = false;
++ }
+ }
+
+- /* Indicate to other users that MM is no longer valid */
+- p->mm = NULL;
+-
+ mutex_unlock(&p->mutex);
+
+- mmu_notifier_unregister_no_release(&p->mmu_notifier, mm);
++ /*
++ * Because we drop mm_count inside kfd_process_destroy_delayed
++ * and because the mmu_notifier_unregister function also drop
++ * mm_count we need to take an extra count here.
++ */
++ mmgrab(p->mm);
++ mmu_notifier_unregister_no_release(&p->mmu_notifier, p->mm);
+ mmu_notifier_call_srcu(&p->rcu, &kfd_process_destroy_delayed);
+ }
+
+@@ -489,68 +272,7 @@ static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
+ .release = kfd_process_notifier_release,
+ };
+
+-static int kfd_process_init_cwsr(struct kfd_process *p, struct file *filep)
+-{
+- int ret;
+- unsigned long offset;
+- struct kfd_process_device *temp, *pdd = NULL;
+- struct kfd_dev *dev = NULL;
+- struct qcm_process_device *qpd = NULL;
+- void *kaddr;
+- uint32_t flags = ALLOC_MEM_FLAGS_GTT | ALLOC_MEM_FLAGS_NONPAGED |
+- ALLOC_MEM_FLAGS_NO_SUBSTITUTE |
+- ALLOC_MEM_FLAGS_READONLY |
+- ALLOC_MEM_FLAGS_EXECUTE_ACCESS;
+-
+- list_for_each_entry_safe(pdd, temp, &p->per_device_data,
+- per_device_list) {
+- dev = pdd->dev;
+- qpd = &pdd->qpd;
+- if (!dev->cwsr_enabled || qpd->cwsr_kaddr)
+- continue;
+- if (qpd->cwsr_base) {
+- /* cwsr_base is only set for DGPU */
+- ret = kfd_process_alloc_gpuvm(p, dev, qpd->cwsr_base,
+- dev->cwsr_size, &kaddr, pdd, flags);
+- if (!ret) {
+- qpd->cwsr_kaddr = kaddr;
+- qpd->tba_addr = qpd->cwsr_base;
+- } else
+- /* In case of error, the kfd_bos for some pdds
+- * which are already allocated successfully
+- * will be freed in upper level function
+- * i.e. create_process().
+- */
+- return ret;
+- } else {
+- offset = (dev->id |
+- KFD_MMAP_TYPE_RESERVED_MEM) << PAGE_SHIFT;
+- qpd->tba_addr = (uint64_t)vm_mmap(filep, 0,
+- dev->cwsr_size, PROT_READ | PROT_EXEC,
+- MAP_SHARED, offset);
+-
+- if (IS_ERR_VALUE(qpd->tba_addr)) {
+- pr_err("Failure to set tba address. error -%d.\n",
+- (int)qpd->tba_addr);
+- qpd->tba_addr = 0;
+- qpd->cwsr_kaddr = NULL;
+- return -ENOMEM;
+- }
+- }
+-
+- memcpy(qpd->cwsr_kaddr, kmap(dev->cwsr_pages), PAGE_SIZE);
+- kunmap(dev->cwsr_pages);
+-
+- qpd->tma_addr = qpd->tba_addr + dev->tma_offset;
+- pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
+- qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
+- }
+-
+- return 0;
+-}
+-
+-static struct kfd_process *create_process(const struct task_struct *thread,
+- struct file *filep)
++static struct kfd_process *create_process(const struct task_struct *thread)
+ {
+ struct kfd_process *process;
+ int err = -ENOMEM;
+@@ -560,20 +282,22 @@ static struct kfd_process *create_process(const struct task_struct *thread,
+ if (!process)
+ goto err_alloc_process;
+
+- process->bo_interval_tree = RB_ROOT_CACHED;
++ process->queues = kmalloc_array(INITIAL_QUEUE_ARRAY_SIZE,
++ sizeof(process->queues[0]), GFP_KERNEL);
++ if (!process->queues)
++ goto err_alloc_queues;
+
+ process->pasid = kfd_pasid_alloc();
+ if (process->pasid == 0)
+ goto err_alloc_pasid;
+
+- kref_init(&process->ref);
+ mutex_init(&process->mutex);
+
+ process->mm = thread->mm;
+
+ /* register notifier */
+ process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops;
+- err = mmu_notifier_register(&process->mmu_notifier, process->mm);
++ err = __mmu_notifier_register(&process->mmu_notifier, process->mm);
+ if (err)
+ goto err_mmu_notifier;
+
+@@ -581,7 +305,8 @@ static struct kfd_process *create_process(const struct task_struct *thread,
+ (uintptr_t)process->mm);
+
+ process->lead_thread = thread->group_leader;
+- get_task_struct(process->lead_thread);
++
++ process->queue_array_size = INITIAL_QUEUE_ARRAY_SIZE;
+
+ INIT_LIST_HEAD(&process->per_device_data);
+
+@@ -597,28 +322,8 @@ static struct kfd_process *create_process(const struct task_struct *thread,
+ if (err != 0)
+ goto err_init_apertures;
+
+- err = kfd_process_reserve_ib_mem(process);
+- if (err)
+- goto err_reserve_ib_mem;
+- err = kfd_process_init_cwsr(process, filep);
+- if (err)
+- goto err_init_cwsr;
+-
+- INIT_DELAYED_WORK(&process->eviction_work.dwork, kfd_evict_bo_worker);
+- INIT_DELAYED_WORK(&process->restore_work, kfd_restore_bo_worker);
+- process->last_restore_timestamp = get_jiffies_64();
+-
+- /* If PeerDirect interface was not detected try to detect it again
+- * in case if network driver was loaded later.
+- */
+- kfd_init_peer_direct();
+-
+ return process;
+
+-err_init_cwsr:
+-err_reserve_ib_mem:
+- kfd_process_free_outstanding_kfd_bos(process);
+- kfd_process_destroy_pdds(process);
+ err_init_apertures:
+ pqm_uninit(&process->pqm);
+ err_process_pqm_init:
+@@ -629,36 +334,13 @@ static struct kfd_process *create_process(const struct task_struct *thread,
+ mutex_destroy(&process->mutex);
+ kfd_pasid_free(process->pasid);
+ err_alloc_pasid:
++ kfree(process->queues);
++err_alloc_queues:
+ kfree(process);
+ err_alloc_process:
+ return ERR_PTR(err);
+ }
+
+-static int init_doorbell_bitmap(struct qcm_process_device *qpd,
+- struct kfd_dev *dev)
+-{
+- unsigned int i;
+-
+- if (!KFD_IS_SOC15(dev->device_info->asic_family))
+- return 0;
+-
+- qpd->doorbell_bitmap =
+- kzalloc(DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,
+- BITS_PER_BYTE), GFP_KERNEL);
+- if (!qpd->doorbell_bitmap)
+- return -ENOMEM;
+-
+- /* Mask out any reserved doorbells */
+- for (i = 0; i < KFD_MAX_NUM_OF_QUEUES_PER_PROCESS; i++)
+- if ((dev->shared_resources.reserved_doorbell_mask & i) ==
+- dev->shared_resources.reserved_doorbell_val) {
+- set_bit(i, qpd->doorbell_bitmap);
+- pr_debug("reserved doorbell 0x%03x\n", i);
+- }
+-
+- return 0;
+-}
+-
+ struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
+ struct kfd_process *p)
+ {
+@@ -666,9 +348,9 @@ struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
+
+ list_for_each_entry(pdd, &p->per_device_data, per_device_list)
+ if (pdd->dev == dev)
+- return pdd;
++ break;
+
+- return NULL;
++ return pdd;
+ }
+
+ struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
+@@ -677,41 +359,16 @@ struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
+ struct kfd_process_device *pdd = NULL;
+
+ pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
+- if (!pdd)
+- return NULL;
+-
+- pdd->dev = dev;
+- INIT_LIST_HEAD(&pdd->qpd.queues_list);
+- INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
+- pdd->qpd.dqm = dev->dqm;
+- pdd->qpd.pqm = &p->pqm;
+- pdd->qpd.evicted = 0;
+- pdd->process = p;
+- pdd->bound = PDD_UNBOUND;
+- pdd->already_dequeued = false;
+- list_add(&pdd->per_device_list, &p->per_device_data);
+-
+- /* Init idr used for memory handle translation */
+- idr_init(&pdd->alloc_idr);
+- if (init_doorbell_bitmap(&pdd->qpd, dev)) {
+- pr_err("Failed to init doorbell for process\n");
+- goto err_create_pdd;
++ if (pdd != NULL) {
++ pdd->dev = dev;
++ INIT_LIST_HEAD(&pdd->qpd.queues_list);
++ INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
++ pdd->qpd.dqm = dev->dqm;
++ pdd->reset_wavefronts = false;
++ list_add(&pdd->per_device_list, &p->per_device_data);
+ }
+
+- /* Create the GPUVM context for this specific device */
+- if (dev->kfd2kgd->create_process_vm(dev->kgd, &pdd->vm,
+- &p->process_info, &p->ef)) {
+- pr_err("Failed to create process VM object\n");
+- goto err_create_pdd;
+- }
+ return pdd;
+-
+-err_create_pdd:
+- kfree(pdd->qpd.doorbell_bitmap);
+- idr_destroy(&pdd->alloc_idr);
+- list_del(&pdd->per_device_list);
+- kfree(pdd);
+- return NULL;
+ }
+
+ /*
+@@ -725,6 +382,7 @@ struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
+ struct kfd_process *p)
+ {
+ struct kfd_process_device *pdd;
++ int err;
+
+ pdd = kfd_get_process_device_data(dev, p);
+ if (!pdd) {
+@@ -732,89 +390,24 @@ struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
+ return ERR_PTR(-ENOMEM);
+ }
+
+- if (pdd->bound == PDD_BOUND)
++ if (pdd->bound)
+ return pdd;
+
+- if (pdd->bound == PDD_BOUND_SUSPENDED) {
+- pr_err("Binding PDD_BOUND_SUSPENDED pdd is unexpected!\n");
+- return ERR_PTR(-EINVAL);
+- }
++ err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread);
++ if (err < 0)
++ return ERR_PTR(err);
+
+-#if defined(CONFIG_AMD_IOMMU_V2_MODULE) || defined(CONFIG_AMD_IOMMU_V2)
+- if (dev->device_info->is_need_iommu_device) {
+- int err = amd_iommu_bind_pasid(dev->pdev, p->pasid,
+- p->lead_thread);
+- if (err < 0)
+- return ERR_PTR(err);
+- }
+-#endif
+-
+- pdd->bound = PDD_BOUND;
++ pdd->bound = true;
+
+ return pdd;
+ }
+
+-#if defined(CONFIG_AMD_IOMMU_V2_MODULE) || defined(CONFIG_AMD_IOMMU_V2)
+-int kfd_bind_processes_to_device(struct kfd_dev *dev)
++void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid)
+ {
+- struct kfd_process_device *pdd;
+ struct kfd_process *p;
+- unsigned int temp;
+- int err = 0;
+-
+- int idx = srcu_read_lock(&kfd_processes_srcu);
+-
+- hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
+- mutex_lock(&p->mutex);
+- pdd = kfd_get_process_device_data(dev, p);
+- if (pdd->bound != PDD_BOUND_SUSPENDED) {
+- mutex_unlock(&p->mutex);
+- continue;
+- }
+-
+- err = amd_iommu_bind_pasid(dev->pdev, p->pasid,
+- p->lead_thread);
+- if (err < 0) {
+- pr_err("Unexpected pasid %d binding failure\n",
+- p->pasid);
+- mutex_unlock(&p->mutex);
+- break;
+- }
+-
+- pdd->bound = PDD_BOUND;
+- mutex_unlock(&p->mutex);
+- }
+-
+- srcu_read_unlock(&kfd_processes_srcu, idx);
+-
+- return err;
+-}
+-
+-void kfd_unbind_processes_from_device(struct kfd_dev *dev)
+-{
+ struct kfd_process_device *pdd;
+- struct kfd_process *p;
+- unsigned int temp;
+-
+- int idx = srcu_read_lock(&kfd_processes_srcu);
+-
+-
+- hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
+- mutex_lock(&p->mutex);
+- pdd = kfd_get_process_device_data(dev, p);
+
+- if (pdd->bound == PDD_BOUND)
+- pdd->bound = PDD_BOUND_SUSPENDED;
+- mutex_unlock(&p->mutex);
+- }
+-
+- srcu_read_unlock(&kfd_processes_srcu, idx);
+-}
+-
+-void kfd_process_iommu_unbind_callback(struct kfd_dev *dev, unsigned int pasid)
+-{
+- struct kfd_process *p;
+- struct kfd_process_device *pdd;
++ BUG_ON(dev == NULL);
+
+ /*
+ * Look for the process that matches the pasid. If there is no such
+@@ -827,43 +420,43 @@ void kfd_process_iommu_unbind_callback(struct kfd_dev *dev, unsigned int pasid)
+
+ pr_debug("Unbinding process %d from IOMMU\n", pasid);
+
+- mutex_lock(kfd_get_dbgmgr_mutex());
++ if ((dev->dbgmgr) && (dev->dbgmgr->pasid == p->pasid))
++ kfd_dbgmgr_destroy(dev->dbgmgr);
+
+- if (dev->dbgmgr && (dev->dbgmgr->pasid == p->pasid)) {
++ pqm_uninit(&p->pqm);
+
+- if (kfd_dbgmgr_unregister(dev->dbgmgr, p) == 0) {
+- kfd_dbgmgr_destroy(dev->dbgmgr);
+- dev->dbgmgr = NULL;
+- }
+- }
++ pdd = kfd_get_process_device_data(dev, p);
+
+- mutex_unlock(kfd_get_dbgmgr_mutex());
++ if (!pdd) {
++ mutex_unlock(&p->mutex);
++ return;
++ }
+
+- mutex_lock(&p->mutex);
++ if (pdd->reset_wavefronts) {
++ dbgdev_wave_reset_wavefronts(pdd->dev, p);
++ pdd->reset_wavefronts = false;
++ }
+
+- pdd = kfd_get_process_device_data(dev, p);
+- if (pdd)
+- /* For GPU relying on IOMMU, we need to dequeue here
+- * when PASID is still bound.
+- */
+- kfd_process_dequeue_from_device(pdd);
++ /*
++ * Just mark pdd as unbound, because we still need it
++ * to call amd_iommu_unbind_pasid() in when the
++ * process exits.
++ * We don't call amd_iommu_unbind_pasid() here
++ * because the IOMMU called us.
++ */
++ pdd->bound = false;
+
+ mutex_unlock(&p->mutex);
+-
+- kfd_unref_process(p);
+ }
+-#endif /* CONFIG_AMD_IOMMU_V2 */
+
+-struct kfd_process_device *kfd_get_first_process_device_data(
+- struct kfd_process *p)
++struct kfd_process_device *kfd_get_first_process_device_data(struct kfd_process *p)
+ {
+ return list_first_entry(&p->per_device_data,
+ struct kfd_process_device,
+ per_device_list);
+ }
+
+-struct kfd_process_device *kfd_get_next_process_device_data(
+- struct kfd_process *p,
++struct kfd_process_device *kfd_get_next_process_device_data(struct kfd_process *p,
+ struct kfd_process_device *pdd)
+ {
+ if (list_is_last(&pdd->per_device_list, &p->per_device_data))
+@@ -876,272 +469,22 @@ bool kfd_has_process_device_data(struct kfd_process *p)
+ return !(list_empty(&p->per_device_data));
+ }
+
+-/* Create specific handle mapped to mem from process local memory idr
+- * Assumes that the process lock is held.
+- */
+-int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
+- void *mem, uint64_t start,
+- uint64_t length,
+- struct kfd_ipc_obj *ipc_obj)
+-{
+- int handle;
+- struct kfd_bo *buf_obj;
+- struct kfd_process *p;
+-
+- p = pdd->process;
+-
+- buf_obj = kzalloc(sizeof(*buf_obj), GFP_KERNEL);
+-
+- if (!buf_obj)
+- return -ENOMEM;
+-
+- buf_obj->it.start = start;
+- buf_obj->it.last = start + length - 1;
+- interval_tree_insert(&buf_obj->it, &p->bo_interval_tree);
+-
+- buf_obj->mem = mem;
+- buf_obj->dev = pdd->dev;
+- buf_obj->kfd_ipc_obj = ipc_obj;
+-
+- INIT_LIST_HEAD(&buf_obj->cb_data_head);
+-
+- idr_preload(GFP_KERNEL);
+-
+- handle = idr_alloc(&pdd->alloc_idr, buf_obj, MIN_IDR_ID, MAX_IDR_ID,
+- GFP_NOWAIT);
+-
+- idr_preload_end();
+-
+- if (handle < 0)
+- kfree(buf_obj);
+-
+- return handle;
+-}
+-
+-struct kfd_bo *kfd_process_device_find_bo(struct kfd_process_device *pdd,
+- int handle)
+-{
+- if (handle < 0)
+- return NULL;
+-
+- return (struct kfd_bo *)idr_find(&pdd->alloc_idr, handle);
+-}
+-
+-/* Translate specific handle from process local memory idr
+- * Assumes that the process lock is held.
+- */
+-void *kfd_process_device_translate_handle(struct kfd_process_device *pdd,
+- int handle)
+-{
+- struct kfd_bo *buf_obj;
+-
+- buf_obj = kfd_process_device_find_bo(pdd, handle);
+-
+- return buf_obj->mem;
+-}
+-
+-void *kfd_process_find_bo_from_interval(struct kfd_process *p,
+- uint64_t start_addr,
+- uint64_t last_addr)
+-{
+- struct interval_tree_node *it_node;
+- struct kfd_bo *buf_obj;
+-
+- it_node = interval_tree_iter_first(&p->bo_interval_tree,
+- start_addr, last_addr);
+- if (!it_node) {
+- pr_err("0x%llx-0x%llx does not relate to an existing buffer\n",
+- start_addr, last_addr);
+- return NULL;
+- }
+-
+- if (interval_tree_iter_next(it_node, start_addr, last_addr)) {
+- pr_err("0x%llx-0x%llx spans more than a single BO\n",
+- start_addr, last_addr);
+- return NULL;
+- }
+-
+- buf_obj = container_of(it_node, struct kfd_bo, it);
+-
+- return buf_obj;
+-}
+-
+-/* Remove specific handle from process local memory idr
+- * Assumes that the process lock is held.
+- */
+-void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,
+- int handle)
+-{
+- struct kfd_bo *buf_obj;
+- struct kfd_process *p;
+-
+- p = pdd->process;
+-
+- if (handle < 0)
+- return;
+-
+- buf_obj = kfd_process_device_find_bo(pdd, handle);
+-
+- if (buf_obj->kfd_ipc_obj)
+- ipc_obj_put(&buf_obj->kfd_ipc_obj);
+-
+- idr_remove(&pdd->alloc_idr, handle);
+-
+- interval_tree_remove(&buf_obj->it, &p->bo_interval_tree);
+-
+- kfree(buf_obj);
+-}
+-
+-/* This increments the process->ref counter. */
++/* This returns with process->mutex locked. */
+ struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid)
+ {
+- struct kfd_process *p, *ret_p = NULL;
++ struct kfd_process *p;
+ unsigned int temp;
+
+ int idx = srcu_read_lock(&kfd_processes_srcu);
+
+ hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
+ if (p->pasid == pasid) {
+- kref_get(&p->ref);
+- ret_p = p;
++ mutex_lock(&p->mutex);
+ break;
+ }
+ }
+
+ srcu_read_unlock(&kfd_processes_srcu, idx);
+
+- return ret_p;
+-}
+-
+-void kfd_suspend_all_processes(void)
+-{
+- struct kfd_process *p;
+- unsigned int temp;
+- int idx = srcu_read_lock(&kfd_processes_srcu);
+-
+- hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
+- if (cancel_delayed_work_sync(&p->eviction_work.dwork))
+- dma_fence_put(p->eviction_work.quiesce_fence);
+- cancel_delayed_work_sync(&p->restore_work);
+-
+- if (quiesce_process_mm(p))
+- pr_err("Failed to suspend process %d\n", p->pasid);
+- dma_fence_signal(p->ef);
+- dma_fence_put(p->ef);
+- p->ef = NULL;
+- }
+- srcu_read_unlock(&kfd_processes_srcu, idx);
+-}
+-
+-int kfd_resume_all_processes(void)
+-{
+- struct kfd_process *p;
+- unsigned int temp;
+- int ret = 0, idx = srcu_read_lock(&kfd_processes_srcu);
+-
+- hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
+- if (!schedule_delayed_work(&p->restore_work, 0)) {
+- pr_err("Restore process %d failed during resume\n",
+- p->pasid);
+- ret = -EFAULT;
+- }
+- }
+- srcu_read_unlock(&kfd_processes_srcu, idx);
+- return ret;
+-}
+-
+-/* This increments the process->ref counter. */
+-struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm)
+-{
+- struct kfd_process *p;
+-
+- int idx = srcu_read_lock(&kfd_processes_srcu);
+-
+- p = find_process_by_mm(mm);
+- if (p)
+- kref_get(&p->ref);
+-
+- srcu_read_unlock(&kfd_processes_srcu, idx);
+-
+ return p;
+ }
+-
+-int kfd_reserved_mem_mmap(struct kfd_process *process,
+- struct vm_area_struct *vma)
+-{
+- unsigned long pfn, i;
+- int ret = 0;
+- struct kfd_dev *dev = kfd_device_by_id(vma->vm_pgoff);
+- struct kfd_process_device *temp, *pdd = NULL;
+- struct qcm_process_device *qpd = NULL;
+-
+- if (!dev)
+- return -EINVAL;
+- if (((vma->vm_end - vma->vm_start) != dev->cwsr_size) ||
+- (vma->vm_start & (PAGE_SIZE - 1)) ||
+- (vma->vm_end & (PAGE_SIZE - 1))) {
+- pr_err("KFD only support page aligned memory map and correct size.\n");
+- return -EINVAL;
+- }
+-
+- pr_debug("kfd reserved mem mmap been called.\n");
+-
+- list_for_each_entry_safe(pdd, temp, &process->per_device_data,
+- per_device_list) {
+- if (dev == pdd->dev) {
+- qpd = &pdd->qpd;
+- break;
+- }
+- }
+- if (!qpd)
+- return -EINVAL;
+-
+- qpd->cwsr_pages = alloc_pages(GFP_KERNEL | __GFP_HIGHMEM,
+- get_order(dev->cwsr_size));
+- if (!qpd->cwsr_pages) {
+- pr_err("amdkfd: error alloc CWSR isa memory per process.\n");
+- return -ENOMEM;
+- }
+- qpd->cwsr_kaddr = kmap(qpd->cwsr_pages);
+-
+- vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND
+- | VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP;
+- for (i = 0; i < ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT); ++i) {
+- pfn = page_to_pfn(&qpd->cwsr_pages[i]);
+- /* mapping the page to user process */
+- ret = remap_pfn_range(vma, vma->vm_start + (i << PAGE_SHIFT),
+- pfn, PAGE_SIZE, vma->vm_page_prot);
+- if (ret)
+- break;
+- }
+- return ret;
+-}
+-
+-#if defined(CONFIG_DEBUG_FS)
+-
+-int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data)
+-{
+- struct kfd_process *p;
+- unsigned int temp;
+- int r = 0;
+-
+- int idx = srcu_read_lock(&kfd_processes_srcu);
+-
+- hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
+- seq_printf(m, "Process %d PASID %d:\n",
+- p->lead_thread->tgid, p->pasid);
+-
+- mutex_lock(&p->mutex);
+- r = pqm_debugfs_mqds(m, &p->pqm);
+- mutex_unlock(&p->mutex);
+-
+- if (r != 0)
+- break;
+- }
+-
+- srcu_read_unlock(&kfd_processes_srcu, idx);
+-
+- return r;
+-}
+-
+-#endif
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
+index a87fcab..46f497e 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
+@@ -32,9 +32,12 @@ static inline struct process_queue_node *get_queue_by_qid(
+ {
+ struct process_queue_node *pqn;
+
++ BUG_ON(!pqm);
++
+ list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
+- if ((pqn->q && pqn->q->properties.queue_id == qid) ||
+- (pqn->kq && pqn->kq->queue->properties.queue_id == qid))
++ if (pqn->q && pqn->q->properties.queue_id == qid)
++ return pqn;
++ if (pqn->kq && pqn->kq->queue->properties.queue_id == qid)
+ return pqn;
+ }
+
+@@ -46,13 +49,17 @@ static int find_available_queue_slot(struct process_queue_manager *pqm,
+ {
+ unsigned long found;
+
++ BUG_ON(!pqm || !qid);
++
++ pr_debug("kfd: in %s\n", __func__);
++
+ found = find_first_zero_bit(pqm->queue_slot_bitmap,
+ KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
+
+- pr_debug("The new slot id %lu\n", found);
++ pr_debug("kfd: the new slot id %lu\n", found);
+
+ if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) {
+- pr_info("Cannot open more queues for process with pasid %d\n",
++ pr_info("amdkfd: Can not open more queues for process with pasid %d\n",
+ pqm->process->pasid);
+ return -ENOMEM;
+ }
+@@ -63,33 +70,15 @@ static int find_available_queue_slot(struct process_queue_manager *pqm,
+ return 0;
+ }
+
+-void kfd_process_dequeue_from_device(struct kfd_process_device *pdd)
+-{
+- struct kfd_dev *dev = pdd->dev;
+- int retval;
+-
+- if (pdd->already_dequeued)
+- return;
+-
+- retval = dev->dqm->ops.process_termination(dev->dqm, &pdd->qpd);
+- pdd->already_dequeued = true;
+-}
+-
+-void kfd_process_dequeue_from_all_devices(struct kfd_process *p)
+-{
+- struct kfd_process_device *pdd;
+-
+- list_for_each_entry(pdd, &p->per_device_data, per_device_list)
+- kfd_process_dequeue_from_device(pdd);
+-}
+-
+ int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p)
+ {
++ BUG_ON(!pqm);
++
+ INIT_LIST_HEAD(&pqm->queues);
+ pqm->queue_slot_bitmap =
+ kzalloc(DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,
+ BITS_PER_BYTE), GFP_KERNEL);
+- if (!pqm->queue_slot_bitmap)
++ if (pqm->queue_slot_bitmap == NULL)
+ return -ENOMEM;
+ pqm->process = p;
+
+@@ -98,14 +87,25 @@ int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p)
+
+ void pqm_uninit(struct process_queue_manager *pqm)
+ {
++ int retval;
+ struct process_queue_node *pqn, *next;
+
++ BUG_ON(!pqm);
++
++ pr_debug("In func %s\n", __func__);
++
+ list_for_each_entry_safe(pqn, next, &pqm->queues, process_queue_list) {
+- uninit_queue(pqn->q);
+- list_del(&pqn->process_queue_list);
+- kfree(pqn);
++ retval = pqm_destroy_queue(
++ pqm,
++ (pqn->q != NULL) ?
++ pqn->q->properties.queue_id :
++ pqn->kq->queue->properties.queue_id);
++
++ if (retval != 0) {
++ pr_err("kfd: failed to destroy queue\n");
++ return;
++ }
+ }
+-
+ kfree(pqm->queue_slot_bitmap);
+ pqm->queue_slot_bitmap = NULL;
+ }
+@@ -117,39 +117,54 @@ static int create_cp_queue(struct process_queue_manager *pqm,
+ {
+ int retval;
+
++ retval = 0;
++
+ /* Doorbell initialized in user space*/
+ q_properties->doorbell_ptr = NULL;
+
++ q_properties->doorbell_off =
++ kfd_queue_id_to_doorbell(dev, pqm->process, qid);
++
+ /* let DQM handle it*/
+ q_properties->vmid = 0;
+ q_properties->queue_id = qid;
+
+ retval = init_queue(q, q_properties);
+ if (retval != 0)
+- return retval;
++ goto err_init_queue;
+
+ (*q)->device = dev;
+ (*q)->process = pqm->process;
+
+- pr_debug("PQM After init queue");
++ pr_debug("kfd: PQM After init queue");
+
+ return retval;
++
++err_init_queue:
++ return retval;
+ }
+
+ int pqm_create_queue(struct process_queue_manager *pqm,
+ struct kfd_dev *dev,
+ struct file *f,
+ struct queue_properties *properties,
++ unsigned int flags,
++ enum kfd_queue_type type,
+ unsigned int *qid)
+ {
+ int retval;
+ struct kfd_process_device *pdd;
++ struct queue_properties q_properties;
+ struct queue *q;
+ struct process_queue_node *pqn;
+ struct kernel_queue *kq;
+- enum kfd_queue_type type = properties->type;
+- unsigned int max_queues = 127; /* HWS limit */
++ int num_queues = 0;
++ struct queue *cur;
++
++ BUG_ON(!pqm || !dev || !properties || !qid);
+
++ memset(&q_properties, 0, sizeof(struct queue_properties));
++ memcpy(&q_properties, properties, sizeof(struct queue_properties));
+ q = NULL;
+ kq = NULL;
+
+@@ -165,21 +180,24 @@ int pqm_create_queue(struct process_queue_manager *pqm,
+ * If we are just about to create DIQ, the is_debug flag is not set yet
+ * Hence we also check the type as well
+ */
+- if ((pdd->qpd.is_debug) || (type == KFD_QUEUE_TYPE_DIQ))
+- max_queues = dev->device_info->max_no_of_hqd/2;
+-
+- if (pdd->qpd.queue_count >= max_queues)
+- return -ENOSPC;
++ if ((pdd->qpd.is_debug) ||
++ (type == KFD_QUEUE_TYPE_DIQ)) {
++ list_for_each_entry(cur, &pdd->qpd.queues_list, list)
++ num_queues++;
++ if (num_queues >= dev->device_info->max_no_of_hqd/2)
++ return (-ENOSPC);
++ }
+
+ retval = find_available_queue_slot(pqm, qid);
+ if (retval != 0)
+ return retval;
+
+- if (list_empty(&pdd->qpd.queues_list) &&
+- list_empty(&pdd->qpd.priv_queue_list))
++ if (list_empty(&pqm->queues)) {
++ pdd->qpd.pqm = pqm;
+ dev->dqm->ops.register_process(dev->dqm, &pdd->qpd);
++ }
+
+- pqn = kzalloc(sizeof(*pqn), GFP_KERNEL);
++ pqn = kzalloc(sizeof(struct process_queue_node), GFP_KERNEL);
+ if (!pqn) {
+ retval = -ENOMEM;
+ goto err_allocate_pqn;
+@@ -187,35 +205,18 @@ int pqm_create_queue(struct process_queue_manager *pqm,
+
+ switch (type) {
+ case KFD_QUEUE_TYPE_SDMA:
+- if (dev->dqm->sdma_queue_count >= CIK_SDMA_QUEUES) {
+- pr_err("Over-subscription is not allowed for SDMA\n");
+- retval = -EPERM;
+- goto err_create_queue;
+- }
+-
+- retval = create_cp_queue(pqm, dev, &q, properties, f, *qid);
+- if (retval != 0)
+- goto err_create_queue;
+- pqn->q = q;
+- pqn->kq = NULL;
+- retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd,
+- &q->properties.vmid);
+- pr_debug("DQM returned %d for create_queue\n", retval);
+- print_queue(q);
+- break;
+
+ case KFD_QUEUE_TYPE_COMPUTE:
+ /* check if there is over subscription */
+- if ((dev->dqm->sched_policy ==
+- KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION) &&
+- ((dev->dqm->processes_count >= dev->vm_info.vmid_num_kfd) ||
++ if ((sched_policy == KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION) &&
++ ((dev->dqm->processes_count >= VMID_PER_DEVICE) ||
+ (dev->dqm->queue_count >= get_queues_num(dev->dqm)))) {
+- pr_err("Over-subscription is not allowed in radeon_kfd.sched_policy == 1\n");
++ pr_err("kfd: over-subscription is not allowed in radeon_kfd.sched_policy == 1\n");
+ retval = -EPERM;
+ goto err_create_queue;
+ }
+
+- retval = create_cp_queue(pqm, dev, &q, properties, f, *qid);
++ retval = create_cp_queue(pqm, dev, &q, &q_properties, f, *qid);
+ if (retval != 0)
+ goto err_create_queue;
+ pqn->q = q;
+@@ -227,7 +228,7 @@ int pqm_create_queue(struct process_queue_manager *pqm,
+ break;
+ case KFD_QUEUE_TYPE_DIQ:
+ kq = kernel_queue_init(dev, KFD_QUEUE_TYPE_DIQ);
+- if (!kq) {
++ if (kq == NULL) {
+ retval = -ENOMEM;
+ goto err_create_queue;
+ }
+@@ -238,31 +239,23 @@ int pqm_create_queue(struct process_queue_manager *pqm,
+ kq, &pdd->qpd);
+ break;
+ default:
+- WARN(1, "Invalid queue type %d", type);
+- retval = -EINVAL;
++ BUG();
++ break;
+ }
+
+ if (retval != 0) {
+- pr_err("DQM create queue failed\n");
++ pr_debug("Error dqm create queue\n");
+ goto err_create_queue;
+ }
+
+- if (q)
+- /* Return the doorbell offset within the doorbell page
+- * to the caller so it can be passed up to user mode
+- * (in bytes).
+- */
+- properties->doorbell_off =
+- (q->properties.doorbell_off * sizeof(uint32_t)) &
+- (kfd_doorbell_process_slice(dev) - 1);
+-
+- pr_debug("PQM After DQM create queue\n");
++ pr_debug("kfd: PQM After DQM create queue\n");
+
+ list_add(&pqn->process_queue_list, &pqm->queues);
+
+ if (q) {
+- pr_debug("PQM done creating queue\n");
+- print_queue_properties(&q->properties);
++ *properties = q->properties;
++ pr_debug("kfd: PQM done creating queue\n");
++ print_queue_properties(properties);
+ }
+
+ return retval;
+@@ -272,8 +265,7 @@ int pqm_create_queue(struct process_queue_manager *pqm,
+ err_allocate_pqn:
+ /* check if queues list is empty unregister process from device */
+ clear_bit(*qid, pqm->queue_slot_bitmap);
+- if (list_empty(&pdd->qpd.queues_list) &&
+- list_empty(&pdd->qpd.priv_queue_list))
++ if (list_empty(&pqm->queues))
+ dev->dqm->ops.unregister_process(dev->dqm, &pdd->qpd);
+ return retval;
+ }
+@@ -288,11 +280,14 @@ int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid)
+
+ dqm = NULL;
+
++ BUG_ON(!pqm);
+ retval = 0;
+
++ pr_debug("kfd: In Func %s\n", __func__);
++
+ pqn = get_queue_by_qid(pqm, qid);
+- if (!pqn) {
+- pr_err("Queue id does not match any known queue\n");
++ if (pqn == NULL) {
++ pr_err("kfd: queue id does not match any known queue\n");
+ return -EINVAL;
+ }
+
+@@ -301,8 +296,7 @@ int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid)
+ dev = pqn->kq->dev;
+ if (pqn->q)
+ dev = pqn->q->device;
+- if (WARN_ON(!dev))
+- return -ENODEV;
++ BUG_ON(!dev);
+
+ pdd = kfd_get_process_device_data(dev, pqm->process);
+ if (!pdd) {
+@@ -319,9 +313,10 @@ int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid)
+
+ if (pqn->q) {
+ dqm = pqn->q->device->dqm;
+- kfree(pqn->q->properties.cu_mask);
+- pqn->q->properties.cu_mask = NULL;
+ retval = dqm->ops.destroy_queue(dqm, &pdd->qpd, pqn->q);
++ if (retval != 0)
++ return retval;
++
+ uninit_queue(pqn->q);
+ }
+
+@@ -329,8 +324,7 @@ int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid)
+ kfree(pqn);
+ clear_bit(qid, pqm->queue_slot_bitmap);
+
+- if (list_empty(&pdd->qpd.queues_list) &&
+- list_empty(&pdd->qpd.priv_queue_list))
++ if (list_empty(&pqm->queues))
+ dqm->ops.unregister_process(dqm, &pdd->qpd);
+
+ return retval;
+@@ -342,9 +336,12 @@ int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid,
+ int retval;
+ struct process_queue_node *pqn;
+
++ BUG_ON(!pqm);
++
+ pqn = get_queue_by_qid(pqm, qid);
+ if (!pqn) {
+- pr_debug("No queue %d exists for update operation\n", qid);
++ pr_debug("amdkfd: No queue %d exists for update operation\n",
++ qid);
+ return -EFAULT;
+ }
+
+@@ -361,40 +358,14 @@ int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid,
+ return 0;
+ }
+
+-int pqm_set_cu_mask(struct process_queue_manager *pqm, unsigned int qid,
+- struct queue_properties *p)
+-{
+- int retval;
+- struct process_queue_node *pqn;
+-
+- pqn = get_queue_by_qid(pqm, qid);
+- if (!pqn) {
+- pr_debug("No queue %d exists for update operation\n", qid);
+- return -EFAULT;
+- }
+-
+- /* Free the old CU mask memory if it is already allocated, then
+- * allocate memory for the new CU mask.
+- */
+- kfree(pqn->q->properties.cu_mask);
+-
+- pqn->q->properties.cu_mask_count = p->cu_mask_count;
+- pqn->q->properties.cu_mask = p->cu_mask;
+-
+- retval = pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,
+- pqn->q);
+- if (retval != 0)
+- return retval;
+-
+- return 0;
+-}
+-
+ struct kernel_queue *pqm_get_kernel_queue(
+ struct process_queue_manager *pqm,
+ unsigned int qid)
+ {
+ struct process_queue_node *pqn;
+
++ BUG_ON(!pqm);
++
+ pqn = get_queue_by_qid(pqm, qid);
+ if (pqn && pqn->kq)
+ return pqn->kq;
+@@ -402,89 +373,4 @@ struct kernel_queue *pqm_get_kernel_queue(
+ return NULL;
+ }
+
+-int pqm_get_wave_state(struct process_queue_manager *pqm,
+- unsigned int qid,
+- void __user *ctl_stack,
+- u32 *ctl_stack_used_size,
+- u32 *save_area_used_size)
+-{
+- struct process_queue_node *pqn;
+-
+- pqn = get_queue_by_qid(pqm, qid);
+- if (!pqn) {
+- pr_debug("amdkfd: No queue %d exists for operation\n",
+- qid);
+- return -EFAULT;
+- }
+-
+- return pqn->q->device->dqm->ops.get_wave_state(pqn->q->device->dqm,
+- pqn->q,
+- ctl_stack,
+- ctl_stack_used_size,
+- save_area_used_size);
+-}
+-
+-#if defined(CONFIG_DEBUG_FS)
+-
+-int pqm_debugfs_mqds(struct seq_file *m, void *data)
+-{
+- struct process_queue_manager *pqm = data;
+- struct process_queue_node *pqn;
+- struct queue *q;
+- enum KFD_MQD_TYPE mqd_type;
+- struct mqd_manager *mqd_manager;
+- int r = 0;
+-
+- list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
+- if (pqn->q) {
+- q = pqn->q;
+- switch (q->properties.type) {
+- case KFD_QUEUE_TYPE_SDMA:
+- seq_printf(m, " SDMA queue on device %x\n",
+- q->device->id);
+- mqd_type = KFD_MQD_TYPE_SDMA;
+- break;
+- case KFD_QUEUE_TYPE_COMPUTE:
+- seq_printf(m, " Compute queue on device %x\n",
+- q->device->id);
+- mqd_type = KFD_MQD_TYPE_CP;
+- break;
+- default:
+- seq_printf(m,
+- " Bad user queue type %d on device %x\n",
+- q->properties.type, q->device->id);
+- continue;
+- }
+- mqd_manager = q->device->dqm->ops.get_mqd_manager(
+- q->device->dqm, mqd_type);
+- } else if (pqn->kq) {
+- q = pqn->kq->queue;
+- mqd_manager = pqn->kq->mqd;
+- switch (q->properties.type) {
+- case KFD_QUEUE_TYPE_DIQ:
+- seq_printf(m, " DIQ on device %x\n",
+- pqn->kq->dev->id);
+- mqd_type = KFD_MQD_TYPE_HIQ;
+- break;
+- default:
+- seq_printf(m,
+- " Bad kernel queue type %d on device %x\n",
+- q->properties.type,
+- pqn->kq->dev->id);
+- continue;
+- }
+- } else {
+- seq_printf(m,
+- " Weird: Queue node with neither kernel nor user queue\n");
+- continue;
+- }
+-
+- r = mqd_manager->debugfs_show_mqd(m, q->mqd);
+- if (r != 0)
+- break;
+- }
+-
+- return r;
+-}
+
+-#endif
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
+index a5315d4..0ab1970 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
+@@ -65,15 +65,17 @@ void print_queue(struct queue *q)
+
+ int init_queue(struct queue **q, const struct queue_properties *properties)
+ {
+- struct queue *tmp_q;
++ struct queue *tmp;
+
+- tmp_q = kzalloc(sizeof(*tmp_q), GFP_KERNEL);
+- if (!tmp_q)
++ BUG_ON(!q);
++
++ tmp = kzalloc(sizeof(struct queue), GFP_KERNEL);
++ if (!tmp)
+ return -ENOMEM;
+
+- memcpy(&tmp_q->properties, properties, sizeof(*properties));
++ memcpy(&tmp->properties, properties, sizeof(struct queue_properties));
+
+- *q = tmp_q;
++ *q = tmp;
+ return 0;
+ }
+
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_rdma.c b/drivers/gpu/drm/amd/amdkfd/kfd_rdma.c
+deleted file mode 100644
+index 2f5cdb9..0000000
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_rdma.c
++++ /dev/null
+@@ -1,294 +0,0 @@
+-/*
+- * Copyright 2015 Advanced Micro Devices, Inc.
+- *
+- * Permission is hereby granted, free of charge, to any person obtaining a
+- * copy of this software and associated documentation files (the "Software"),
+- * to deal in the Software without restriction, including without limitation
+- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+- * and/or sell copies of the Software, and to permit persons to whom the
+- * Software is furnished to do so, subject to the following conditions:
+- *
+- * The above copyright notice and this permission notice shall be included in
+- * all copies or substantial portions of the Software.
+- *
+- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+- * OTHER DEALINGS IN THE SOFTWARE.
+- */
+-
+-#include <linux/device.h>
+-#include <linux/export.h>
+-#include <linux/pid.h>
+-#include <linux/err.h>
+-#include <linux/slab.h>
+-#include "kfd_priv.h"
+-
+-
+-struct rdma_cb {
+- struct list_head node;
+- struct amd_p2p_info amd_p2p_data;
+- void (*free_callback)(void *client_priv);
+- void *client_priv;
+-};
+-
+-/**
+- * This function makes the pages underlying a range of GPU virtual memory
+- * accessible for DMA operations from another PCIe device
+- *
+- * \param address - The start address in the Unified Virtual Address
+- * space in the specified process
+- * \param length - The length of requested mapping
+- * \param pid - Pointer to structure pid to which address belongs.
+- * Could be NULL for current process address space.
+- * \param p2p_data - On return: Pointer to structure describing
+- * underlying pages/locations
+- * \param free_callback - Pointer to callback which will be called when access
+- * to such memory must be stopped immediately: Memory
+- * was freed, GECC events, etc.
+- * Client should immediately stop any transfer
+- * operations and returned as soon as possible.
+- * After return all resources associated with address
+- * will be release and no access will be allowed.
+- * \param client_priv - Pointer to be passed as parameter on
+- * 'free_callback;
+- *
+- * \return 0 if operation was successful
+- */
+-static int get_pages(uint64_t address, uint64_t length, struct pid *pid,
+- struct amd_p2p_info **amd_p2p_data,
+- void (*free_callback)(void *client_priv),
+- void *client_priv)
+-{
+- struct kfd_bo *buf_obj;
+- struct kgd_mem *mem;
+- struct sg_table *sg_table_tmp;
+- struct kfd_dev *dev;
+- uint64_t last = address + length - 1;
+- uint64_t offset;
+- struct kfd_process *p;
+- struct rdma_cb *rdma_cb_data;
+- int ret = 0;
+-
+- p = kfd_lookup_process_by_pid(pid);
+- if (!p) {
+- pr_err("Could not find the process\n");
+- return -EINVAL;
+- }
+- mutex_lock(&p->mutex);
+-
+- buf_obj = kfd_process_find_bo_from_interval(p, address, last);
+- if (!buf_obj) {
+- pr_err("Cannot find a kfd_bo for the range\n");
+- ret = -EINVAL;
+- goto out;
+- }
+-
+- rdma_cb_data = kmalloc(sizeof(*rdma_cb_data), GFP_KERNEL);
+- if (!rdma_cb_data) {
+- *amd_p2p_data = NULL;
+- ret = -ENOMEM;
+- goto out;
+- }
+-
+- mem = buf_obj->mem;
+- dev = buf_obj->dev;
+- offset = address - buf_obj->it.start;
+-
+- ret = dev->kfd2kgd->pin_get_sg_table_bo(dev->kgd, mem,
+- offset, length, &sg_table_tmp);
+-
+- if (ret) {
+- pr_err("pin_get_sg_table_bo failed.\n");
+- *amd_p2p_data = NULL;
+- goto free_mem;
+- }
+-
+- rdma_cb_data->amd_p2p_data.va = address;
+- rdma_cb_data->amd_p2p_data.size = length;
+- rdma_cb_data->amd_p2p_data.pid = pid;
+- rdma_cb_data->amd_p2p_data.priv = buf_obj;
+- rdma_cb_data->amd_p2p_data.pages = sg_table_tmp;
+-
+- rdma_cb_data->free_callback = free_callback;
+- rdma_cb_data->client_priv = client_priv;
+-
+- list_add(&rdma_cb_data->node, &buf_obj->cb_data_head);
+-
+- *amd_p2p_data = &rdma_cb_data->amd_p2p_data;
+-
+- goto out;
+-
+-free_mem:
+- kfree(rdma_cb_data);
+-out:
+- mutex_unlock(&p->mutex);
+- kfd_unref_process(p);
+-
+- return ret;
+-}
+-
+-static int put_pages_helper(struct amd_p2p_info *p2p_data)
+-{
+- struct kfd_bo *buf_obj;
+- struct kfd_dev *dev;
+- struct sg_table *sg_table_tmp;
+- struct rdma_cb *rdma_cb_data;
+-
+- if (!p2p_data) {
+- pr_err("amd_p2p_info pointer is invalid.\n");
+- return -EINVAL;
+- }
+-
+- rdma_cb_data = container_of(p2p_data, struct rdma_cb, amd_p2p_data);
+-
+- buf_obj = p2p_data->priv;
+- dev = buf_obj->dev;
+- sg_table_tmp = p2p_data->pages;
+-
+- list_del(&rdma_cb_data->node);
+- kfree(rdma_cb_data);
+-
+- dev->kfd2kgd->unpin_put_sg_table_bo(buf_obj->mem, sg_table_tmp);
+-
+-
+- return 0;
+-}
+-
+-void run_rdma_free_callback(struct kfd_bo *buf_obj)
+-{
+- struct rdma_cb *tmp, *rdma_cb_data;
+-
+- list_for_each_entry_safe(rdma_cb_data, tmp,
+- &buf_obj->cb_data_head, node) {
+- if (rdma_cb_data->free_callback)
+- rdma_cb_data->free_callback(
+- rdma_cb_data->client_priv);
+-
+- put_pages_helper(&rdma_cb_data->amd_p2p_data);
+- }
+-}
+-
+-/**
+- *
+- * This function release resources previously allocated by get_pages() call.
+- *
+- * \param p_p2p_data - A pointer to pointer to amd_p2p_info entries
+- * allocated by get_pages() call.
+- *
+- * \return 0 if operation was successful
+- */
+-static int put_pages(struct amd_p2p_info **p_p2p_data)
+-{
+- struct kfd_process *p = NULL;
+- int ret = 0;
+-
+- if (!(*p_p2p_data)) {
+- pr_err("amd_p2p_info pointer is invalid.\n");
+- return -EINVAL;
+- }
+-
+- p = kfd_lookup_process_by_pid((*p_p2p_data)->pid);
+- if (!p) {
+- pr_err("Could not find the process\n");
+- return -EINVAL;
+- }
+-
+- ret = put_pages_helper(*p_p2p_data);
+-
+- if (!ret)
+- *p_p2p_data = NULL;
+-
+- kfd_unref_process(p);
+-
+- return ret;
+-}
+-
+-/**
+- * Check if given address belongs to GPU address space.
+- *
+- * \param address - Address to check
+- * \param pid - Process to which given address belongs.
+- * Could be NULL if current one.
+- *
+- * \return 0 - This is not GPU address managed by AMD driver
+- * 1 - This is GPU address managed by AMD driver
+- */
+-static int is_gpu_address(uint64_t address, struct pid *pid)
+-{
+- struct kfd_bo *buf_obj;
+- struct kfd_process *p;
+-
+- p = kfd_lookup_process_by_pid(pid);
+- if (!p) {
+- pr_debug("Could not find the process\n");
+- return 0;
+- }
+-
+- buf_obj = kfd_process_find_bo_from_interval(p, address, address);
+-
+- kfd_unref_process(p);
+- if (!buf_obj)
+- return 0;
+-
+- return 1;
+-}
+-
+-/**
+- * Return the single page size to be used when building scatter/gather table
+- * for given range.
+- *
+- * \param address - Address
+- * \param length - Range length
+- * \param pid - Process id structure. Could be NULL if current one.
+- * \param page_size - On return: Page size
+- *
+- * \return 0 if operation was successful
+- */
+-static int get_page_size(uint64_t address, uint64_t length, struct pid *pid,
+- unsigned long *page_size)
+-{
+- /*
+- * As local memory is always consecutive, we can assume the local
+- * memory page size to be arbitrary.
+- * Currently we assume the local memory page size to be the same
+- * as system memory, which is 4KB.
+- */
+- *page_size = PAGE_SIZE;
+-
+- return 0;
+-}
+-
+-
+-/**
+- * Singleton object: rdma interface function pointers
+- */
+-static const struct amd_rdma_interface rdma_ops = {
+- .get_pages = get_pages,
+- .put_pages = put_pages,
+- .is_gpu_address = is_gpu_address,
+- .get_page_size = get_page_size,
+-};
+-
+-/**
+- * amdkfd_query_rdma_interface - Return interface (function pointers table) for
+- * rdma interface
+- *
+- *
+- * \param interace - OUT: Pointer to interface
+- *
+- * \return 0 if operation was successful.
+- */
+-int amdkfd_query_rdma_interface(const struct amd_rdma_interface **ops)
+-{
+- *ops = &rdma_ops;
+-
+- return 0;
+-}
+-EXPORT_SYMBOL(amdkfd_query_rdma_interface);
+-
+-
+-
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+index d08e3de..1e50647 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+@@ -28,32 +28,27 @@
+ #include <linux/hash.h>
+ #include <linux/cpufreq.h>
+ #include <linux/log2.h>
+-#include <linux/dmi.h>
+-#include <linux/atomic.h>
+
+ #include "kfd_priv.h"
+ #include "kfd_crat.h"
+ #include "kfd_topology.h"
+-#include "kfd_device_queue_manager.h"
+
+-/* topology_device_list - Master list of all topology devices */
+ static struct list_head topology_device_list;
++static int topology_crat_parsed;
+ static struct kfd_system_properties sys_props;
+
+ static DECLARE_RWSEM(topology_lock);
+-static atomic_t topology_crat_proximity_domain;
+
+-struct kfd_topology_device *kfd_topology_device_by_proximity_domain(
+- uint32_t proximity_domain)
++struct kfd_dev *kfd_device_by_id(uint32_t gpu_id)
+ {
+ struct kfd_topology_device *top_dev;
+- struct kfd_topology_device *device = NULL;
++ struct kfd_dev *device = NULL;
+
+ down_read(&topology_lock);
+
+ list_for_each_entry(top_dev, &topology_device_list, list)
+- if (top_dev->proximity_domain == proximity_domain) {
+- device = top_dev;
++ if (top_dev->gpu_id == gpu_id) {
++ device = top_dev->gpu;
+ break;
+ }
+
+@@ -62,7 +57,7 @@ struct kfd_topology_device *kfd_topology_device_by_proximity_domain(
+ return device;
+ }
+
+-struct kfd_dev *kfd_device_by_id(uint32_t gpu_id)
++struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev)
+ {
+ struct kfd_topology_device *top_dev;
+ struct kfd_dev *device = NULL;
+@@ -70,7 +65,7 @@ struct kfd_dev *kfd_device_by_id(uint32_t gpu_id)
+ down_read(&topology_lock);
+
+ list_for_each_entry(top_dev, &topology_device_list, list)
+- if (top_dev->gpu_id == gpu_id) {
++ if (top_dev->gpu->pdev == pdev) {
+ device = top_dev->gpu;
+ break;
+ }
+@@ -80,49 +75,300 @@ struct kfd_dev *kfd_device_by_id(uint32_t gpu_id)
+ return device;
+ }
+
+-struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev)
++static int kfd_topology_get_crat_acpi(void *crat_image, size_t *size)
+ {
+- struct kfd_topology_device *top_dev;
+- struct kfd_dev *device = NULL;
++ struct acpi_table_header *crat_table;
++ acpi_status status;
+
+- down_read(&topology_lock);
++ if (!size)
++ return -EINVAL;
+
+- list_for_each_entry(top_dev, &topology_device_list, list)
+- if (top_dev->gpu && top_dev->gpu->pdev == pdev) {
+- device = top_dev->gpu;
++ /*
++ * Fetch the CRAT table from ACPI
++ */
++ status = acpi_get_table(CRAT_SIGNATURE, 0, &crat_table);
++ if (status == AE_NOT_FOUND) {
++ pr_warn("CRAT table not found\n");
++ return -ENODATA;
++ } else if (ACPI_FAILURE(status)) {
++ const char *err = acpi_format_exception(status);
++
++ pr_err("CRAT table error: %s\n", err);
++ return -EINVAL;
++ }
++
++ if (*size >= crat_table->length && crat_image != NULL)
++ memcpy(crat_image, crat_table, crat_table->length);
++
++ *size = crat_table->length;
++
++ return 0;
++}
++
++static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev,
++ struct crat_subtype_computeunit *cu)
++{
++ BUG_ON(!dev);
++ BUG_ON(!cu);
++
++ dev->node_props.cpu_cores_count = cu->num_cpu_cores;
++ dev->node_props.cpu_core_id_base = cu->processor_id_low;
++ if (cu->hsa_capability & CRAT_CU_FLAGS_IOMMU_PRESENT)
++ dev->node_props.capability |= HSA_CAP_ATS_PRESENT;
++
++ pr_info("CU CPU: cores=%d id_base=%d\n", cu->num_cpu_cores,
++ cu->processor_id_low);
++}
++
++static void kfd_populated_cu_info_gpu(struct kfd_topology_device *dev,
++ struct crat_subtype_computeunit *cu)
++{
++ BUG_ON(!dev);
++ BUG_ON(!cu);
++
++ dev->node_props.simd_id_base = cu->processor_id_low;
++ dev->node_props.simd_count = cu->num_simd_cores;
++ dev->node_props.lds_size_in_kb = cu->lds_size_in_kb;
++ dev->node_props.max_waves_per_simd = cu->max_waves_simd;
++ dev->node_props.wave_front_size = cu->wave_front_size;
++ dev->node_props.mem_banks_count = cu->num_banks;
++ dev->node_props.array_count = cu->num_arrays;
++ dev->node_props.cu_per_simd_array = cu->num_cu_per_array;
++ dev->node_props.simd_per_cu = cu->num_simd_per_cu;
++ dev->node_props.max_slots_scratch_cu = cu->max_slots_scatch_cu;
++ if (cu->hsa_capability & CRAT_CU_FLAGS_HOT_PLUGGABLE)
++ dev->node_props.capability |= HSA_CAP_HOT_PLUGGABLE;
++ pr_info("CU GPU: simds=%d id_base=%d\n", cu->num_simd_cores,
++ cu->processor_id_low);
++}
++
++/* kfd_parse_subtype_cu is called when the topology mutex is already acquired */
++static int kfd_parse_subtype_cu(struct crat_subtype_computeunit *cu)
++{
++ struct kfd_topology_device *dev;
++ int i = 0;
++
++ BUG_ON(!cu);
++
++ pr_info("Found CU entry in CRAT table with proximity_domain=%d caps=%x\n",
++ cu->proximity_domain, cu->hsa_capability);
++ list_for_each_entry(dev, &topology_device_list, list) {
++ if (cu->proximity_domain == i) {
++ if (cu->flags & CRAT_CU_FLAGS_CPU_PRESENT)
++ kfd_populated_cu_info_cpu(dev, cu);
++
++ if (cu->flags & CRAT_CU_FLAGS_GPU_PRESENT)
++ kfd_populated_cu_info_gpu(dev, cu);
+ break;
+ }
++ i++;
++ }
+
+- up_read(&topology_lock);
++ return 0;
++}
+
+- return device;
++/*
++ * kfd_parse_subtype_mem is called when the topology mutex is
++ * already acquired
++ */
++static int kfd_parse_subtype_mem(struct crat_subtype_memory *mem)
++{
++ struct kfd_mem_properties *props;
++ struct kfd_topology_device *dev;
++ int i = 0;
++
++ BUG_ON(!mem);
++
++ pr_info("Found memory entry in CRAT table with proximity_domain=%d\n",
++ mem->promixity_domain);
++ list_for_each_entry(dev, &topology_device_list, list) {
++ if (mem->promixity_domain == i) {
++ props = kfd_alloc_struct(props);
++ if (props == NULL)
++ return -ENOMEM;
++
++ if (dev->node_props.cpu_cores_count == 0)
++ props->heap_type = HSA_MEM_HEAP_TYPE_FB_PRIVATE;
++ else
++ props->heap_type = HSA_MEM_HEAP_TYPE_SYSTEM;
++
++ if (mem->flags & CRAT_MEM_FLAGS_HOT_PLUGGABLE)
++ props->flags |= HSA_MEM_FLAGS_HOT_PLUGGABLE;
++ if (mem->flags & CRAT_MEM_FLAGS_NON_VOLATILE)
++ props->flags |= HSA_MEM_FLAGS_NON_VOLATILE;
++
++ props->size_in_bytes =
++ ((uint64_t)mem->length_high << 32) +
++ mem->length_low;
++ props->width = mem->width;
++
++ dev->mem_bank_count++;
++ list_add_tail(&props->list, &dev->mem_props);
++
++ break;
++ }
++ i++;
++ }
++
++ return 0;
+ }
+
+-struct kfd_dev *kfd_device_by_kgd(const struct kgd_dev *kgd)
++/*
++ * kfd_parse_subtype_cache is called when the topology mutex
++ * is already acquired
++ */
++static int kfd_parse_subtype_cache(struct crat_subtype_cache *cache)
+ {
+- struct kfd_topology_device *top_dev;
+- struct kfd_dev *device = NULL;
++ struct kfd_cache_properties *props;
++ struct kfd_topology_device *dev;
++ uint32_t id;
+
+- down_read(&topology_lock);
++ BUG_ON(!cache);
++
++ id = cache->processor_id_low;
++
++ pr_info("Found cache entry in CRAT table with processor_id=%d\n", id);
++ list_for_each_entry(dev, &topology_device_list, list)
++ if (id == dev->node_props.cpu_core_id_base ||
++ id == dev->node_props.simd_id_base) {
++ props = kfd_alloc_struct(props);
++ if (props == NULL)
++ return -ENOMEM;
++
++ props->processor_id_low = id;
++ props->cache_level = cache->cache_level;
++ props->cache_size = cache->cache_size;
++ props->cacheline_size = cache->cache_line_size;
++ props->cachelines_per_tag = cache->lines_per_tag;
++ props->cache_assoc = cache->associativity;
++ props->cache_latency = cache->cache_latency;
++
++ if (cache->flags & CRAT_CACHE_FLAGS_DATA_CACHE)
++ props->cache_type |= HSA_CACHE_TYPE_DATA;
++ if (cache->flags & CRAT_CACHE_FLAGS_INST_CACHE)
++ props->cache_type |= HSA_CACHE_TYPE_INSTRUCTION;
++ if (cache->flags & CRAT_CACHE_FLAGS_CPU_CACHE)
++ props->cache_type |= HSA_CACHE_TYPE_CPU;
++ if (cache->flags & CRAT_CACHE_FLAGS_SIMD_CACHE)
++ props->cache_type |= HSA_CACHE_TYPE_HSACU;
++
++ dev->cache_count++;
++ dev->node_props.caches_count++;
++ list_add_tail(&props->list, &dev->cache_props);
+
+- list_for_each_entry(top_dev, &topology_device_list, list)
+- if (top_dev->gpu && top_dev->gpu->kgd == kgd) {
+- device = top_dev->gpu;
+ break;
+ }
+
+- up_read(&topology_lock);
++ return 0;
++}
+
+- return device;
++/*
++ * kfd_parse_subtype_iolink is called when the topology mutex
++ * is already acquired
++ */
++static int kfd_parse_subtype_iolink(struct crat_subtype_iolink *iolink)
++{
++ struct kfd_iolink_properties *props;
++ struct kfd_topology_device *dev;
++ uint32_t i = 0;
++ uint32_t id_from;
++ uint32_t id_to;
++
++ BUG_ON(!iolink);
++
++ id_from = iolink->proximity_domain_from;
++ id_to = iolink->proximity_domain_to;
++
++ pr_info("Found IO link entry in CRAT table with id_from=%d\n", id_from);
++ list_for_each_entry(dev, &topology_device_list, list) {
++ if (id_from == i) {
++ props = kfd_alloc_struct(props);
++ if (props == NULL)
++ return -ENOMEM;
++
++ props->node_from = id_from;
++ props->node_to = id_to;
++ props->ver_maj = iolink->version_major;
++ props->ver_min = iolink->version_minor;
++
++ /*
++ * weight factor (derived from CDIR), currently always 1
++ */
++ props->weight = 1;
++
++ props->min_latency = iolink->minimum_latency;
++ props->max_latency = iolink->maximum_latency;
++ props->min_bandwidth = iolink->minimum_bandwidth_mbs;
++ props->max_bandwidth = iolink->maximum_bandwidth_mbs;
++ props->rec_transfer_size =
++ iolink->recommended_transfer_size;
++
++ dev->io_link_count++;
++ dev->node_props.io_links_count++;
++ list_add_tail(&props->list, &dev->io_link_props);
++
++ break;
++ }
++ i++;
++ }
++
++ return 0;
++}
++
++static int kfd_parse_subtype(struct crat_subtype_generic *sub_type_hdr)
++{
++ struct crat_subtype_computeunit *cu;
++ struct crat_subtype_memory *mem;
++ struct crat_subtype_cache *cache;
++ struct crat_subtype_iolink *iolink;
++ int ret = 0;
++
++ BUG_ON(!sub_type_hdr);
++
++ switch (sub_type_hdr->type) {
++ case CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY:
++ cu = (struct crat_subtype_computeunit *)sub_type_hdr;
++ ret = kfd_parse_subtype_cu(cu);
++ break;
++ case CRAT_SUBTYPE_MEMORY_AFFINITY:
++ mem = (struct crat_subtype_memory *)sub_type_hdr;
++ ret = kfd_parse_subtype_mem(mem);
++ break;
++ case CRAT_SUBTYPE_CACHE_AFFINITY:
++ cache = (struct crat_subtype_cache *)sub_type_hdr;
++ ret = kfd_parse_subtype_cache(cache);
++ break;
++ case CRAT_SUBTYPE_TLB_AFFINITY:
++ /*
++ * For now, nothing to do here
++ */
++ pr_info("Found TLB entry in CRAT table (not processing)\n");
++ break;
++ case CRAT_SUBTYPE_CCOMPUTE_AFFINITY:
++ /*
++ * For now, nothing to do here
++ */
++ pr_info("Found CCOMPUTE entry in CRAT table (not processing)\n");
++ break;
++ case CRAT_SUBTYPE_IOLINK_AFFINITY:
++ iolink = (struct crat_subtype_iolink *)sub_type_hdr;
++ ret = kfd_parse_subtype_iolink(iolink);
++ break;
++ default:
++ pr_warn("Unknown subtype (%d) in CRAT\n",
++ sub_type_hdr->type);
++ }
++
++ return ret;
+ }
+
+-/* Called with write topology_lock acquired */
+ static void kfd_release_topology_device(struct kfd_topology_device *dev)
+ {
+ struct kfd_mem_properties *mem;
+ struct kfd_cache_properties *cache;
+ struct kfd_iolink_properties *iolink;
+- struct kfd_perf_properties *perf;
++
++ BUG_ON(!dev);
+
+ list_del(&dev->list);
+
+@@ -147,40 +393,30 @@ static void kfd_release_topology_device(struct kfd_topology_device *dev)
+ kfree(iolink);
+ }
+
+- while (dev->perf_props.next != &dev->perf_props) {
+- perf = container_of(dev->perf_props.next,
+- struct kfd_perf_properties, list);
+- list_del(&perf->list);
+- kfree(perf);
+- }
+-
+ kfree(dev);
++
++ sys_props.num_devices--;
+ }
+
+-void kfd_release_topology_device_list(struct list_head *device_list)
++static void kfd_release_live_view(void)
+ {
+ struct kfd_topology_device *dev;
+
+- while (!list_empty(device_list)) {
+- dev = list_first_entry(device_list,
+- struct kfd_topology_device, list);
++ while (topology_device_list.next != &topology_device_list) {
++ dev = container_of(topology_device_list.next,
++ struct kfd_topology_device, list);
+ kfd_release_topology_device(dev);
+- }
+ }
+
+-static void kfd_release_live_view(void)
+-{
+- kfd_release_topology_device_list(&topology_device_list);
+ memset(&sys_props, 0, sizeof(sys_props));
+ }
+
+-struct kfd_topology_device *kfd_create_topology_device(
+- struct list_head *device_list)
++static struct kfd_topology_device *kfd_create_topology_device(void)
+ {
+ struct kfd_topology_device *dev;
+
+ dev = kfd_alloc_struct(dev);
+- if (!dev) {
++ if (dev == NULL) {
+ pr_err("No memory to allocate a topology device");
+ return NULL;
+ }
+@@ -188,13 +424,66 @@ struct kfd_topology_device *kfd_create_topology_device(
+ INIT_LIST_HEAD(&dev->mem_props);
+ INIT_LIST_HEAD(&dev->cache_props);
+ INIT_LIST_HEAD(&dev->io_link_props);
+- INIT_LIST_HEAD(&dev->perf_props);
+
+- list_add_tail(&dev->list, device_list);
++ list_add_tail(&dev->list, &topology_device_list);
++ sys_props.num_devices++;
+
+ return dev;
+ }
+
++static int kfd_parse_crat_table(void *crat_image)
++{
++ struct kfd_topology_device *top_dev;
++ struct crat_subtype_generic *sub_type_hdr;
++ uint16_t node_id;
++ int ret;
++ struct crat_header *crat_table = (struct crat_header *)crat_image;
++ uint16_t num_nodes;
++ uint32_t image_len;
++
++ if (!crat_image)
++ return -EINVAL;
++
++ num_nodes = crat_table->num_domains;
++ image_len = crat_table->length;
++
++ pr_info("Parsing CRAT table with %d nodes\n", num_nodes);
++
++ for (node_id = 0; node_id < num_nodes; node_id++) {
++ top_dev = kfd_create_topology_device();
++ if (!top_dev) {
++ kfd_release_live_view();
++ return -ENOMEM;
++ }
++ }
++
++ sys_props.platform_id =
++ (*((uint64_t *)crat_table->oem_id)) & CRAT_OEMID_64BIT_MASK;
++ sys_props.platform_oem = *((uint64_t *)crat_table->oem_table_id);
++ sys_props.platform_rev = crat_table->revision;
++
++ sub_type_hdr = (struct crat_subtype_generic *)(crat_table+1);
++ while ((char *)sub_type_hdr + sizeof(struct crat_subtype_generic) <
++ ((char *)crat_image) + image_len) {
++ if (sub_type_hdr->flags & CRAT_SUBTYPE_FLAGS_ENABLED) {
++ ret = kfd_parse_subtype(sub_type_hdr);
++ if (ret != 0) {
++ kfd_release_live_view();
++ return ret;
++ }
++ }
++
++ sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
++ sub_type_hdr->length);
++ }
++
++ sys_props.generation_count++;
++ topology_crat_parsed = 1;
++
++ return 0;
++}
++
++
+ #define sysfs_show_gen_prop(buffer, fmt, ...) \
+ snprintf(buffer, PAGE_SIZE, "%s"fmt, buffer, __VA_ARGS__)
+ #define sysfs_show_32bit_prop(buffer, name, value) \
+@@ -203,8 +492,6 @@ struct kfd_topology_device *kfd_create_topology_device(
+ sysfs_show_gen_prop(buffer, "%s %llu\n", name, value)
+ #define sysfs_show_32bit_val(buffer, value) \
+ sysfs_show_gen_prop(buffer, "%u\n", value)
+-#define sysfs_show_64bit_val(buffer, value) \
+- sysfs_show_gen_prop(buffer, "%llu\n", value)
+ #define sysfs_show_str_val(buffer, value) \
+ sysfs_show_gen_prop(buffer, "%s\n", value)
+
+@@ -232,17 +519,11 @@ static ssize_t sysprops_show(struct kobject *kobj, struct attribute *attr,
+ return ret;
+ }
+
+-static void kfd_topology_kobj_release(struct kobject *kobj)
+-{
+- kfree(kobj);
+-}
+-
+ static const struct sysfs_ops sysprops_ops = {
+ .show = sysprops_show,
+ };
+
+ static struct kobj_type sysprops_type = {
+- .release = kfd_topology_kobj_release,
+ .sysfs_ops = &sysprops_ops,
+ };
+
+@@ -278,7 +559,6 @@ static const struct sysfs_ops iolink_ops = {
+ };
+
+ static struct kobj_type iolink_type = {
+- .release = kfd_topology_kobj_release,
+ .sysfs_ops = &iolink_ops,
+ };
+
+@@ -287,23 +567,11 @@ static ssize_t mem_show(struct kobject *kobj, struct attribute *attr,
+ {
+ ssize_t ret;
+ struct kfd_mem_properties *mem;
+- uint64_t used_mem;
+
+ /* Making sure that the buffer is an empty string */
+ buffer[0] = 0;
+
+- if (strcmp(attr->name, "used_memory") == 0) {
+- mem = container_of(attr, struct kfd_mem_properties,
+- attr_used);
+- if (mem->gpu) {
+- used_mem = mem->gpu->kfd2kgd->get_vram_usage(mem->gpu->kgd);
+- return sysfs_show_64bit_val(buffer, used_mem);
+- }
+- /* TODO: Report APU/CPU-allocated memory; For now return 0 */
+- return 0;
+- }
+-
+- mem = container_of(attr, struct kfd_mem_properties, attr_props);
++ mem = container_of(attr, struct kfd_mem_properties, attr);
+ sysfs_show_32bit_prop(buffer, "heap_type", mem->heap_type);
+ sysfs_show_64bit_prop(buffer, "size_in_bytes", mem->size_in_bytes);
+ sysfs_show_32bit_prop(buffer, "flags", mem->flags);
+@@ -318,7 +586,6 @@ static const struct sysfs_ops mem_ops = {
+ };
+
+ static struct kobj_type mem_type = {
+- .release = kfd_topology_kobj_release,
+ .sysfs_ops = &mem_ops,
+ };
+
+@@ -326,7 +593,7 @@ static ssize_t kfd_cache_show(struct kobject *kobj, struct attribute *attr,
+ char *buffer)
+ {
+ ssize_t ret;
+- uint32_t i, j;
++ uint32_t i;
+ struct kfd_cache_properties *cache;
+
+ /* Making sure that the buffer is an empty string */
+@@ -344,18 +611,12 @@ static ssize_t kfd_cache_show(struct kobject *kobj, struct attribute *attr,
+ sysfs_show_32bit_prop(buffer, "latency", cache->cache_latency);
+ sysfs_show_32bit_prop(buffer, "type", cache->cache_type);
+ snprintf(buffer, PAGE_SIZE, "%ssibling_map ", buffer);
+- for (i = 0; i < CRAT_SIBLINGMAP_SIZE; i++)
+- for (j = 0; j < sizeof(cache->sibling_map[0])*8; j++) {
+- /* Check each bit */
+- if (cache->sibling_map[i] & (1 << j))
+- ret = snprintf(buffer, PAGE_SIZE,
+- "%s%d%s", buffer, 1, ",");
+- else
+- ret = snprintf(buffer, PAGE_SIZE,
+- "%s%d%s", buffer, 0, ",");
+- }
+- /* Replace the last "," with end of line */
+- *(buffer + strlen(buffer) - 1) = 0xA;
++ for (i = 0; i < KFD_TOPOLOGY_CPU_SIBLINGS; i++)
++ ret = snprintf(buffer, PAGE_SIZE, "%s%d%s",
++ buffer, cache->sibling_map[i],
++ (i == KFD_TOPOLOGY_CPU_SIBLINGS-1) ?
++ "\n" : ",");
++
+ return ret;
+ }
+
+@@ -364,43 +625,9 @@ static const struct sysfs_ops cache_ops = {
+ };
+
+ static struct kobj_type cache_type = {
+- .release = kfd_topology_kobj_release,
+ .sysfs_ops = &cache_ops,
+ };
+
+-/****** Sysfs of Performance Counters ******/
+-
+-struct kfd_perf_attr {
+- struct kobj_attribute attr;
+- uint32_t data;
+-};
+-
+-static ssize_t perf_show(struct kobject *kobj, struct kobj_attribute *attrs,
+- char *buf)
+-{
+- struct kfd_perf_attr *attr;
+-
+- buf[0] = 0;
+- attr = container_of(attrs, struct kfd_perf_attr, attr);
+- if (!attr->data) /* invalid data for PMC */
+- return 0;
+- else
+- return sysfs_show_32bit_val(buf, attr->data);
+-}
+-
+-#define KFD_PERF_DESC(_name, _data) \
+-{ \
+- .attr = __ATTR(_name, 0444, perf_show, NULL), \
+- .data = _data, \
+-}
+-
+-static struct kfd_perf_attr perf_attr_iommu[] = {
+- KFD_PERF_DESC(max_concurrent, 0),
+- KFD_PERF_DESC(num_counters, 0),
+- KFD_PERF_DESC(counter_ids, 0),
+-};
+-/****************************************/
+-
+ static ssize_t node_show(struct kobject *kobj, struct attribute *attr,
+ char *buffer)
+ {
+@@ -408,7 +635,6 @@ static ssize_t node_show(struct kobject *kobj, struct attribute *attr,
+ char public_name[KFD_TOPOLOGY_PUBLIC_NAME_SIZE];
+ uint32_t i;
+ uint32_t log_max_watch_addr;
+- struct kfd_local_mem_info local_mem_info;
+
+ /* Making sure that the buffer is an empty string */
+ buffer[0] = 0;
+@@ -438,8 +664,18 @@ static ssize_t node_show(struct kobject *kobj, struct attribute *attr,
+ dev->node_props.cpu_cores_count);
+ sysfs_show_32bit_prop(buffer, "simd_count",
+ dev->node_props.simd_count);
+- sysfs_show_32bit_prop(buffer, "mem_banks_count",
+- dev->node_props.mem_banks_count);
++
++ if (dev->mem_bank_count < dev->node_props.mem_banks_count) {
++ pr_info_once("kfd: mem_banks_count truncated from %d to %d\n",
++ dev->node_props.mem_banks_count,
++ dev->mem_bank_count);
++ sysfs_show_32bit_prop(buffer, "mem_banks_count",
++ dev->mem_bank_count);
++ } else {
++ sysfs_show_32bit_prop(buffer, "mem_banks_count",
++ dev->node_props.mem_banks_count);
++ }
++
+ sysfs_show_32bit_prop(buffer, "caches_count",
+ dev->node_props.caches_count);
+ sysfs_show_32bit_prop(buffer, "io_links_count",
+@@ -487,28 +723,17 @@ static ssize_t node_show(struct kobject *kobj, struct attribute *attr,
+ HSA_CAP_WATCH_POINTS_TOTALBITS_MASK);
+ }
+
+- if (dev->gpu->device_info->asic_family == CHIP_TONGA)
+- dev->node_props.capability |=
+- HSA_CAP_AQL_QUEUE_DOUBLE_MAP;
+-
+ sysfs_show_32bit_prop(buffer, "max_engine_clk_fcompute",
+- dev->node_props.max_engine_clk_fcompute);
++ dev->gpu->kfd2kgd->get_max_engine_clock_in_mhz(
++ dev->gpu->kgd));
+
+- /*
+- * If the ASIC is CZ, set local memory size to 0 to disable
+- * local memory support
+- */
+- if (dev->gpu->device_info->asic_family != CHIP_CARRIZO) {
+- dev->gpu->kfd2kgd->get_local_mem_info(dev->gpu->kgd,
+- &local_mem_info);
+- sysfs_show_64bit_prop(buffer, "local_mem_size",
+- local_mem_info.local_mem_size_private +
+- local_mem_info.local_mem_size_public);
+- } else
+- sysfs_show_64bit_prop(buffer, "local_mem_size", 0ULL);
++ sysfs_show_64bit_prop(buffer, "local_mem_size",
++ (unsigned long long int) 0);
+
+ sysfs_show_32bit_prop(buffer, "fw_version",
+- dev->gpu->mec_fw_version);
++ dev->gpu->kfd2kgd->get_fw_version(
++ dev->gpu->kgd,
++ KGD_ENGINE_MEC1));
+ sysfs_show_32bit_prop(buffer, "capability",
+ dev->node_props.capability);
+ }
+@@ -522,7 +747,6 @@ static const struct sysfs_ops node_ops = {
+ };
+
+ static struct kobj_type node_type = {
+- .release = kfd_topology_kobj_release,
+ .sysfs_ops = &node_ops,
+ };
+
+@@ -538,7 +762,8 @@ static void kfd_remove_sysfs_node_entry(struct kfd_topology_device *dev)
+ struct kfd_iolink_properties *iolink;
+ struct kfd_cache_properties *cache;
+ struct kfd_mem_properties *mem;
+- struct kfd_perf_properties *perf;
++
++ BUG_ON(!dev);
+
+ if (dev->kobj_iolink) {
+ list_for_each_entry(iolink, &dev->io_link_props, list)
+@@ -567,12 +792,7 @@ static void kfd_remove_sysfs_node_entry(struct kfd_topology_device *dev)
+ if (dev->kobj_mem) {
+ list_for_each_entry(mem, &dev->mem_props, list)
+ if (mem->kobj) {
+- /* TODO: Remove when CPU/APU supported */
+- if (dev->node_props.cpu_cores_count == 0)
+- sysfs_remove_file(mem->kobj,
+- &mem->attr_used);
+- kfd_remove_sysfs_file(mem->kobj,
+- &mem->attr_props);
++ kfd_remove_sysfs_file(mem->kobj, &mem->attr);
+ mem->kobj = NULL;
+ }
+ kobject_del(dev->kobj_mem);
+@@ -580,16 +800,6 @@ static void kfd_remove_sysfs_node_entry(struct kfd_topology_device *dev)
+ dev->kobj_mem = NULL;
+ }
+
+- if (dev->kobj_perf) {
+- list_for_each_entry(perf, &dev->perf_props, list) {
+- kfree(perf->attr_group);
+- perf->attr_group = NULL;
+- }
+- kobject_del(dev->kobj_perf);
+- kobject_put(dev->kobj_perf);
+- dev->kobj_perf = NULL;
+- }
+-
+ if (dev->kobj_node) {
+ sysfs_remove_file(dev->kobj_node, &dev->attr_gpuid);
+ sysfs_remove_file(dev->kobj_node, &dev->attr_name);
+@@ -606,18 +816,15 @@ static int kfd_build_sysfs_node_entry(struct kfd_topology_device *dev,
+ struct kfd_iolink_properties *iolink;
+ struct kfd_cache_properties *cache;
+ struct kfd_mem_properties *mem;
+- struct kfd_perf_properties *perf;
+- uint32_t num_attrs;
+- struct attribute **attrs;
+ int ret;
+ uint32_t i;
+
+- if (WARN_ON(dev->kobj_node))
+- return -EEXIST;
++ BUG_ON(!dev);
+
+ /*
+ * Creating the sysfs folders
+ */
++ BUG_ON(dev->kobj_node);
+ dev->kobj_node = kfd_alloc_struct(dev->kobj_node);
+ if (!dev->kobj_node)
+ return -ENOMEM;
+@@ -639,10 +846,6 @@ static int kfd_build_sysfs_node_entry(struct kfd_topology_device *dev,
+ if (!dev->kobj_iolink)
+ return -ENOMEM;
+
+- dev->kobj_perf = kobject_create_and_add("perf", dev->kobj_node);
+- if (!dev->kobj_perf)
+- return -ENOMEM;
+-
+ /*
+ * Creating sysfs files for node properties
+ */
+@@ -675,23 +878,12 @@ static int kfd_build_sysfs_node_entry(struct kfd_topology_device *dev,
+ if (ret < 0)
+ return ret;
+
+- mem->attr_props.name = "properties";
+- mem->attr_props.mode = KFD_SYSFS_FILE_MODE;
+- sysfs_attr_init(&mem->attr_props);
+- ret = sysfs_create_file(mem->kobj, &mem->attr_props);
++ mem->attr.name = "properties";
++ mem->attr.mode = KFD_SYSFS_FILE_MODE;
++ sysfs_attr_init(&mem->attr);
++ ret = sysfs_create_file(mem->kobj, &mem->attr);
+ if (ret < 0)
+ return ret;
+-
+- /* TODO: Support APU/CPU memory usage */
+- if (dev->node_props.cpu_cores_count == 0) {
+- mem->attr_used.name = "used_memory";
+- mem->attr_used.mode = KFD_SYSFS_FILE_MODE;
+- sysfs_attr_init(&mem->attr_used);
+- ret = sysfs_create_file(mem->kobj, &mem->attr_used);
+- if (ret < 0)
+- return ret;
+- }
+-
+ i++;
+ }
+
+@@ -731,38 +923,11 @@ static int kfd_build_sysfs_node_entry(struct kfd_topology_device *dev,
+ if (ret < 0)
+ return ret;
+ i++;
+- }
+-
+- /* All hardware blocks have the same number of attributes. */
+- num_attrs = sizeof(perf_attr_iommu)/sizeof(struct kfd_perf_attr);
+- list_for_each_entry(perf, &dev->perf_props, list) {
+- perf->attr_group = kzalloc(sizeof(struct kfd_perf_attr)
+- * num_attrs + sizeof(struct attribute_group),
+- GFP_KERNEL);
+- if (!perf->attr_group)
+- return -ENOMEM;
+-
+- attrs = (struct attribute **)(perf->attr_group + 1);
+- if (!strcmp(perf->block_name, "iommu")) {
+- /* Information of IOMMU's num_counters and counter_ids is shown
+- * under /sys/bus/event_source/devices/amd_iommu. We don't
+- * duplicate here.
+- */
+- perf_attr_iommu[0].data = perf->max_concurrent;
+- for (i = 0; i < num_attrs; i++)
+- attrs[i] = &perf_attr_iommu[i].attr.attr;
+- }
+- perf->attr_group->name = perf->block_name;
+- perf->attr_group->attrs = attrs;
+- ret = sysfs_create_group(dev->kobj_perf, perf->attr_group);
+- if (ret < 0)
+- return ret;
+- }
++}
+
+ return 0;
+ }
+
+-/* Called with write topology lock acquired */
+ static int kfd_build_sysfs_node_tree(void)
+ {
+ struct kfd_topology_device *dev;
+@@ -779,7 +944,6 @@ static int kfd_build_sysfs_node_tree(void)
+ return 0;
+ }
+
+-/* Called with write topology lock acquired */
+ static void kfd_remove_sysfs_node_tree(void)
+ {
+ struct kfd_topology_device *dev;
+@@ -793,7 +957,7 @@ static int kfd_topology_update_sysfs(void)
+ int ret;
+
+ pr_info("Creating topology SYSFS entries\n");
+- if (!sys_props.kobj_topology) {
++ if (sys_props.kobj_topology == NULL) {
+ sys_props.kobj_topology =
+ kfd_alloc_struct(sys_props.kobj_topology);
+ if (!sys_props.kobj_topology)
+@@ -851,251 +1015,75 @@ static void kfd_topology_release_sysfs(void)
+ }
+ }
+
+-/* Called with write topology_lock acquired */
+-static void kfd_topology_update_device_list(struct list_head *temp_list,
+- struct list_head *master_list)
+-{
+- while (!list_empty(temp_list)) {
+- list_move_tail(temp_list->next, master_list);
+- sys_props.num_devices++;
+- }
+-}
+-
+-static void kfd_debug_print_topology(void)
+-{
+- struct kfd_topology_device *dev;
+-
+- down_read(&topology_lock);
+-
+- dev = list_last_entry(&topology_device_list,
+- struct kfd_topology_device, list);
+- if (dev) {
+- if (dev->node_props.cpu_cores_count &&
+- dev->node_props.simd_count) {
+- pr_info("Topology: Add APU node [0x%0x:0x%0x]\n",
+- dev->node_props.device_id,
+- dev->node_props.vendor_id);
+- } else if (dev->node_props.cpu_cores_count)
+- pr_info("Topology: Add CPU node\n");
+- else if (dev->node_props.simd_count)
+- pr_info("Topology: Add dGPU node [0x%0x:0x%0x]\n",
+- dev->node_props.device_id,
+- dev->node_props.vendor_id);
+- }
+- up_read(&topology_lock);
+-}
+-
+-/* Helper function for intializing platform_xx members of kfd_system_properties
+- */
+-static void kfd_update_system_properties(void)
+-{
+- struct kfd_topology_device *dev;
+-
+- down_read(&topology_lock);
+- dev = list_last_entry(&topology_device_list,
+- struct kfd_topology_device, list);
+- if (dev) {
+- sys_props.platform_id =
+- (*((uint64_t *)dev->oem_id)) & CRAT_OEMID_64BIT_MASK;
+- sys_props.platform_oem = *((uint64_t *)dev->oem_table_id);
+- sys_props.platform_rev = dev->oem_revision;
+- }
+- up_read(&topology_lock);
+-}
+-
+-static void find_system_memory(const struct dmi_header *dm,
+- void *private)
+-{
+- struct kfd_mem_properties *mem;
+- u16 mem_width, mem_clock;
+- struct kfd_topology_device *kdev =
+- (struct kfd_topology_device *)private;
+- const u8 *dmi_data = (const u8 *)(dm + 1);
+-
+- if (dm->type == DMI_ENTRY_MEM_DEVICE && dm->length >= 0x15) {
+- mem_width = (u16)(*(const u16 *)(dmi_data + 0x6));
+- mem_clock = (u16)(*(const u16 *)(dmi_data + 0x11));
+- list_for_each_entry(mem, &kdev->mem_props, list) {
+- if (mem_width != 0xFFFF && mem_width != 0)
+- mem->width = mem_width;
+- if (mem_clock != 0)
+- mem->mem_clk_max = mem_clock;
+- }
+- }
+-}
+-
+-/*
+- * Performance counters information is not part of CRAT but we would like to
+- * put them in the sysfs under topology directory for Thunk to get the data.
+- * This function is called before updating the sysfs.
+- */
+-static int kfd_add_perf_to_topology(struct kfd_topology_device *kdev)
+-{
+-#if defined(CONFIG_AMD_IOMMU_V2_MODULE) || defined(CONFIG_AMD_IOMMU_V2)
+- struct kfd_perf_properties *props;
+-
+- if (amd_iommu_pc_supported()) {
+- props = kfd_alloc_struct(props);
+- if (!props)
+- return -ENOMEM;
+- strcpy(props->block_name, "iommu");
+- props->max_concurrent = amd_iommu_pc_get_max_banks(0) *
+- amd_iommu_pc_get_max_counters(0); /* assume one iommu */
+- list_add_tail(&props->list, &kdev->perf_props);
+- }
+-#endif
+-
+- return 0;
+-}
+-
+-/* kfd_add_non_crat_information - Add information that is not currently
+- * defined in CRAT but is necessary for KFD topology
+- * @dev - topology device to which addition info is added
+- */
+-static void kfd_add_non_crat_information(struct kfd_topology_device *kdev)
+-{
+- /* Check if CPU only node. */
+- if (!kdev->gpu) {
+- /* Add system memory information */
+- dmi_walk(find_system_memory, kdev);
+- }
+- /* TODO: For GPU node, rearrange code from kfd_topology_add_device */
+-}
+-
+-#ifdef CONFIG_ACPI
+-/* kfd_is_acpi_crat_invalid - CRAT from ACPI is valid only for AMD APU devices.
+- * Ignore CRAT for all other devices. AMD APU is identified if both CPU
+- * and GPU cores are present.
+- * @device_list - topology device list created by parsing ACPI CRAT table.
+- * @return - TRUE if invalid, FALSE is valid.
+- */
+-static bool kfd_is_acpi_crat_invalid(struct list_head *device_list)
+-{
+- struct kfd_topology_device *dev;
+-
+- list_for_each_entry(dev, device_list, list) {
+- if (dev->node_props.cpu_cores_count &&
+- dev->node_props.simd_count)
+- return false;
+- }
+- pr_info("Ignoring ACPI CRAT on non-APU system\n");
+- return true;
+-}
+-#endif
+-
+ int kfd_topology_init(void)
+ {
+ void *crat_image = NULL;
+ size_t image_size = 0;
+ int ret;
+- struct list_head temp_topology_device_list;
+- int cpu_only_node = 0;
+- struct kfd_topology_device *kdev;
+- int proximity_domain;
+-
+- /* topology_device_list - Master list of all topology devices
+- * temp_topology_device_list - temporary list created while parsing CRAT
+- * or VCRAT. Once parsing is complete the contents of list is moved to
+- * topology_device_list
+- */
+
+- /* Initialize the head for the both the lists */
++ /*
++ * Initialize the head for the topology device list
++ */
+ INIT_LIST_HEAD(&topology_device_list);
+- INIT_LIST_HEAD(&temp_topology_device_list);
+ init_rwsem(&topology_lock);
++ topology_crat_parsed = 0;
+
+ memset(&sys_props, 0, sizeof(sys_props));
+
+- /* Proximity domains in ACPI CRAT tables start counting at
+- * 0. The same should be true for virtual CRAT tables created
+- * at this stage. GPUs added later in kfd_topology_add_device
+- * use a counter.
+- */
+- proximity_domain = 0;
+-
+ /*
+- * Get the CRAT image from the ACPI. If ACPI doesn't have one
+- * or if ACPI CRAT is invalid create a virtual CRAT.
+- * NOTE: The current implementation expects all AMD APUs to have
+- * CRAT. If no CRAT is available, it is assumed to be a CPU
++ * Get the CRAT image from the ACPI
+ */
+-#ifdef CONFIG_ACPI
+- ret = kfd_create_crat_image_acpi(&crat_image, &image_size);
+- if (ret == 0) {
+- ret = kfd_parse_crat_table(crat_image,
+- &temp_topology_device_list,
+- proximity_domain);
+- if (ret ||
+- kfd_is_acpi_crat_invalid(&temp_topology_device_list)) {
+-
+- kfd_release_topology_device_list(
+- &temp_topology_device_list);
+- kfd_destroy_crat_image(crat_image);
+- crat_image = NULL;
+- }
+- }
+-#endif
+- if (!crat_image) {
+- ret = kfd_create_crat_image_virtual(&crat_image, &image_size,
+- COMPUTE_UNIT_CPU, NULL,
+- proximity_domain);
+- cpu_only_node = 1;
+- if (ret) {
+- pr_err("Error creating VCRAT table for CPU\n");
+- return ret;
+- }
+-
+- ret = kfd_parse_crat_table(crat_image,
+- &temp_topology_device_list,
+- proximity_domain);
+- if (ret) {
+- pr_err("Error parsing VCRAT table for CPU\n");
++ ret = kfd_topology_get_crat_acpi(crat_image, &image_size);
++ if (ret == 0 && image_size > 0) {
++ pr_info("Found CRAT image with size=%zd\n", image_size);
++ crat_image = kmalloc(image_size, GFP_KERNEL);
++ if (!crat_image) {
++ ret = -ENOMEM;
++ pr_err("No memory for allocating CRAT image\n");
+ goto err;
+ }
+- }
+-
+- kdev = list_first_entry(&temp_topology_device_list,
+- struct kfd_topology_device, list);
+- kfd_add_perf_to_topology(kdev);
+-
+- down_write(&topology_lock);
+- kfd_topology_update_device_list(&temp_topology_device_list,
+- &topology_device_list);
+- atomic_set(&topology_crat_proximity_domain, sys_props.num_devices-1);
+- ret = kfd_topology_update_sysfs();
+- up_write(&topology_lock);
+-
+- if (ret == 0) {
+- sys_props.generation_count++;
+- kfd_update_system_properties();
+- kfd_debug_print_topology();
+- pr_info("Finished initializing topology\n");
+- } else
+- pr_err("Failed to update topology in sysfs ret=%d\n", ret);
+-
+- /* For nodes with GPU, this information gets added
+- * when GPU is detected (kfd_topology_add_device).
+- */
+- if (cpu_only_node) {
+- /* Add additional information to CPU only node created above */
+- down_write(&topology_lock);
+- kdev = list_first_entry(&topology_device_list,
+- struct kfd_topology_device, list);
+- up_write(&topology_lock);
+- kfd_add_non_crat_information(kdev);
++ ret = kfd_topology_get_crat_acpi(crat_image, &image_size);
++
++ if (ret == 0) {
++ down_write(&topology_lock);
++ ret = kfd_parse_crat_table(crat_image);
++ if (ret == 0)
++ ret = kfd_topology_update_sysfs();
++ up_write(&topology_lock);
++ } else {
++ pr_err("Couldn't get CRAT table size from ACPI\n");
++ }
++ kfree(crat_image);
++ } else if (ret == -ENODATA) {
++ ret = 0;
++ } else {
++ pr_err("Couldn't get CRAT table size from ACPI\n");
+ }
+
+ err:
+- kfd_destroy_crat_image(crat_image);
++ pr_info("Finished initializing topology ret=%d\n", ret);
+ return ret;
+ }
+
+ void kfd_topology_shutdown(void)
+ {
+- down_write(&topology_lock);
+ kfd_topology_release_sysfs();
+ kfd_release_live_view();
+- up_write(&topology_lock);
++}
++
++static void kfd_debug_print_topology(void)
++{
++ struct kfd_topology_device *dev;
++ uint32_t i = 0;
++
++ pr_info("DEBUG PRINT OF TOPOLOGY:");
++ list_for_each_entry(dev, &topology_device_list, list) {
++ pr_info("Node: %d\n", i);
++ pr_info("\tGPU assigned: %s\n", (dev->gpu ? "yes" : "no"));
++ pr_info("\tCPU count: %d\n", dev->node_props.cpu_cores_count);
++ pr_info("\tSIMD count: %d", dev->node_props.simd_count);
++ i++;
++ }
+ }
+
+ static uint32_t kfd_generate_gpu_id(struct kfd_dev *gpu)
+@@ -1104,15 +1092,11 @@ static uint32_t kfd_generate_gpu_id(struct kfd_dev *gpu)
+ uint32_t buf[7];
+ uint64_t local_mem_size;
+ int i;
+- struct kfd_local_mem_info local_mem_info;
+
+ if (!gpu)
+ return 0;
+
+- gpu->kfd2kgd->get_local_mem_info(gpu->kgd, &local_mem_info);
+-
+- local_mem_size = local_mem_info.local_mem_size_private +
+- local_mem_info.local_mem_size_public;
++ local_mem_size = gpu->kfd2kgd->get_vmem_size(gpu->kgd);
+
+ buf[0] = gpu->pdev->devfn;
+ buf[1] = gpu->pdev->subsystem_vendor;
+@@ -1127,32 +1111,20 @@ static uint32_t kfd_generate_gpu_id(struct kfd_dev *gpu)
+
+ return hashout;
+ }
+-/* kfd_assign_gpu - Attach @gpu to the correct kfd topology device. If
+- * the GPU device is not already present in the topology device
+- * list then return NULL. This means a new topology device has to
+- * be created for this GPU.
+- * TODO: Rather than assiging @gpu to first topology device withtout
+- * gpu attached, it will better to have more stringent check.
+- */
++
+ static struct kfd_topology_device *kfd_assign_gpu(struct kfd_dev *gpu)
+ {
+ struct kfd_topology_device *dev;
+ struct kfd_topology_device *out_dev = NULL;
+- struct kfd_mem_properties *mem;
+
+- down_write(&topology_lock);
++ BUG_ON(!gpu);
++
+ list_for_each_entry(dev, &topology_device_list, list)
+- if (!dev->gpu && (dev->node_props.simd_count > 0)) {
++ if (dev->gpu == NULL && dev->node_props.simd_count > 0) {
+ dev->gpu = gpu;
+ out_dev = dev;
+-
+- /* Assign mem->gpu */
+- list_for_each_entry(mem, &dev->mem_props, list)
+- mem->gpu = dev->gpu;
+-
+ break;
+ }
+- up_write(&topology_lock);
+
+ return out_dev;
+ }
+@@ -1165,202 +1137,88 @@ static void kfd_notify_gpu_change(uint32_t gpu_id, int arrival)
+ */
+ }
+
+-/* kfd_fill_mem_clk_max_info - Since CRAT doesn't have memory clock info,
+- * patch this after CRAT parsing.
+- */
+-static void kfd_fill_mem_clk_max_info(struct kfd_topology_device *dev)
+-{
+- struct kfd_mem_properties *mem;
+- struct kfd_local_mem_info local_mem_info;
+-
+- if (!dev)
+- return;
+-
+- /* Currently, amdgpu driver (amdgpu_mc) deals only with GPUs with
+- * single bank of VRAM local memory.
+- * for dGPUs - VCRAT reports only one bank of Local Memory
+- * for APUs - If CRAT from ACPI reports more than one bank, then
+- * all the banks will report the same mem_clk_max information
+- */
+- dev->gpu->kfd2kgd->get_local_mem_info(dev->gpu->kgd,
+- &local_mem_info);
+-
+- list_for_each_entry(mem, &dev->mem_props, list)
+- mem->mem_clk_max = local_mem_info.mem_clk_max;
+-}
+-
+-static void kfd_fill_iolink_non_crat_info(struct kfd_topology_device *dev)
+-{
+- struct kfd_iolink_properties *link;
+-
+- if (!dev || !dev->gpu)
+- return;
+-
+- /* GPU only creates direck links so apply flags setting to all */
+- if (dev->gpu->device_info->asic_family == CHIP_HAWAII)
+- list_for_each_entry(link, &dev->io_link_props, list)
+- link->flags = CRAT_IOLINK_FLAGS_ENABLED |
+- CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
+- CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
+-}
+-
+ int kfd_topology_add_device(struct kfd_dev *gpu)
+ {
+ uint32_t gpu_id;
+ struct kfd_topology_device *dev;
+- struct kfd_cu_info cu_info;
+- int res = 0;
+- struct list_head temp_topology_device_list;
+- void *crat_image = NULL;
+- size_t image_size = 0;
+- int proximity_domain;
++ int res;
+
+- INIT_LIST_HEAD(&temp_topology_device_list);
++ BUG_ON(!gpu);
+
+ gpu_id = kfd_generate_gpu_id(gpu);
+
+- pr_debug("Adding new GPU (ID: 0x%x) to topology\n", gpu_id);
+-
+- proximity_domain = atomic_inc_return(&
+- topology_crat_proximity_domain);
++ pr_debug("kfd: Adding new GPU (ID: 0x%x) to topology\n", gpu_id);
+
+- /* Check to see if this gpu device exists in the topology_device_list.
+- * If so, assign the gpu to that device,
+- * else create a Virtual CRAT for this gpu device and then parse that
+- * CRAT to create a new topology device. Once created assign the gpu to
+- * that topology device
++ down_write(&topology_lock);
++ /*
++ * Try to assign the GPU to existing topology device (generated from
++ * CRAT table
+ */
+ dev = kfd_assign_gpu(gpu);
+ if (!dev) {
+- res = kfd_create_crat_image_virtual(&crat_image, &image_size,
+- COMPUTE_UNIT_GPU,
+- gpu, proximity_domain);
+- if (res) {
+- pr_err("Error creating VCRAT for GPU (ID: 0x%x)\n",
+- gpu_id);
+- return res;
+- }
+- res = kfd_parse_crat_table(crat_image,
+- &temp_topology_device_list, proximity_domain);
+- if (res) {
+- pr_err("Error parsing VCRAT for GPU (ID: 0x%x)\n",
+- gpu_id);
++ pr_info("GPU was not found in the current topology. Extending.\n");
++ kfd_debug_print_topology();
++ dev = kfd_create_topology_device();
++ if (!dev) {
++ res = -ENOMEM;
+ goto err;
+ }
++ dev->gpu = gpu;
+
+- down_write(&topology_lock);
+- kfd_topology_update_device_list(&temp_topology_device_list,
+- &topology_device_list);
++ /*
++ * TODO: Make a call to retrieve topology information from the
++ * GPU vBIOS
++ */
+
+- /* Update the SYSFS tree, since we added another topology
+- * device
++ /*
++ * Update the SYSFS tree, since we added another topology device
+ */
+- res = kfd_topology_update_sysfs();
+- up_write(&topology_lock);
+-
+- if (res == 0)
+- sys_props.generation_count++;
+- else
+- pr_err("Failed to update GPU (ID: 0x%x) to sysfs topology. res=%d\n",
+- gpu_id, res);
+- dev = kfd_assign_gpu(gpu);
+- if (!dev) {
+- pr_err("Could not assign GPU\n");
+- res = -ENODEV;
+- goto err;
+- }
++ if (kfd_topology_update_sysfs() < 0)
++ kfd_topology_release_sysfs();
++
+ }
+
+ dev->gpu_id = gpu_id;
+ gpu->id = gpu_id;
+-
+- /* TODO: Move the following lines to function
+- * kfd_add_non_crat_information
+- */
+-
+- /* Fill-in additional information that is not available in CRAT but
+- * needed for the topology
+- */
+-
+- dev->gpu->kfd2kgd->get_cu_info(dev->gpu->kgd, &cu_info);
+- dev->node_props.simd_arrays_per_engine =
+- cu_info.num_shader_arrays_per_engine;
+-
+ dev->node_props.vendor_id = gpu->pdev->vendor;
+ dev->node_props.device_id = gpu->pdev->device;
+- dev->node_props.location_id = PCI_DEVID(gpu->pdev->bus->number,
+- gpu->pdev->devfn);
+- dev->node_props.max_engine_clk_fcompute =
+- dev->gpu->kfd2kgd->get_max_engine_clock_in_mhz(dev->gpu->kgd);
+- dev->node_props.max_engine_clk_ccompute =
+- cpufreq_quick_get_max(0) / 1000;
+-
+- kfd_fill_mem_clk_max_info(dev);
+- kfd_fill_iolink_non_crat_info(dev);
+-
+- switch (dev->gpu->device_info->asic_family) {
+- case CHIP_KAVERI:
+- case CHIP_HAWAII:
+- case CHIP_TONGA:
+- dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_PRE_1_0 <<
+- HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) &
+- HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK);
+- break;
+- case CHIP_CARRIZO:
+- case CHIP_FIJI:
+- case CHIP_POLARIS10:
+- case CHIP_POLARIS11:
+- pr_debug("Adding doorbell packet type capability\n");
+- dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_1_0 <<
+- HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) &
+- HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK);
+- break;
+- case CHIP_VEGA10:
+- case CHIP_RAVEN:
+- dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_2_0 <<
+- HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) &
+- HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK);
+- break;
+- default:
+- BUG();
+- }
+-
+- /* Fix errors in CZ CRAT.
+- * simd_count: Carrizo CRAT reports wrong simd_count, probably because
+- * it doesn't consider masked out CUs
+- * max_waves_per_simd: Carrizo reports wrong max_waves_per_simd.
+- * capability flag: Carrizo CRAT doesn't report IOMMU flags.
++ dev->node_props.location_id = (gpu->pdev->bus->number << 24) +
++ (gpu->pdev->devfn & 0xffffff);
++ /*
++ * TODO: Retrieve max engine clock values from KGD
+ */
++
+ if (dev->gpu->device_info->asic_family == CHIP_CARRIZO) {
+- dev->node_props.simd_count =
+- cu_info.simd_per_cu * cu_info.cu_active_number;
+- dev->node_props.max_waves_per_simd = 10;
+- dev->node_props.capability |= HSA_CAP_ATS_PRESENT;
++ dev->node_props.capability |= HSA_CAP_DOORBELL_PACKET_TYPE;
++ pr_info("amdkfd: adding doorbell packet type capability\n");
+ }
+
+- kfd_debug_print_topology();
++ res = 0;
+
+- if (!res)
+- kfd_notify_gpu_change(gpu_id, 1);
+ err:
+- kfd_destroy_crat_image(crat_image);
++ up_write(&topology_lock);
++
++ if (res == 0)
++ kfd_notify_gpu_change(gpu_id, 1);
++
+ return res;
+ }
+
+ int kfd_topology_remove_device(struct kfd_dev *gpu)
+ {
+- struct kfd_topology_device *dev, *tmp;
++ struct kfd_topology_device *dev;
+ uint32_t gpu_id;
+ int res = -ENODEV;
+
++ BUG_ON(!gpu);
++
+ down_write(&topology_lock);
+
+- list_for_each_entry_safe(dev, tmp, &topology_device_list, list)
++ list_for_each_entry(dev, &topology_device_list, list)
+ if (dev->gpu == gpu) {
+ gpu_id = dev->gpu_id;
+ kfd_remove_sysfs_node_entry(dev);
+ kfd_release_topology_device(dev);
+- sys_props.num_devices--;
+ res = 0;
+ if (kfd_topology_update_sysfs() < 0)
+ kfd_topology_release_sysfs();
+@@ -1375,26 +1233,22 @@ int kfd_topology_remove_device(struct kfd_dev *gpu)
+ return res;
+ }
+
+-/* kfd_topology_enum_kfd_devices - Enumerate through all devices in KFD
+- * topology. If GPU device is found @idx, then valid kfd_dev pointer is
+- * returned through @kdev
+- * Return - 0: On success (@kdev will be NULL for non GPU nodes)
+- * -1: If end of list
++/*
++ * When idx is out of bounds, the function will return NULL
+ */
+-int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_dev **kdev)
++struct kfd_dev *kfd_topology_enum_kfd_devices(uint8_t idx)
+ {
+
+ struct kfd_topology_device *top_dev;
++ struct kfd_dev *device = NULL;
+ uint8_t device_idx = 0;
+
+- *kdev = NULL;
+ down_read(&topology_lock);
+
+ list_for_each_entry(top_dev, &topology_device_list, list) {
+ if (device_idx == idx) {
+- *kdev = top_dev->gpu;
+- up_read(&topology_lock);
+- return 0;
++ device = top_dev->gpu;
++ break;
+ }
+
+ device_idx++;
+@@ -1402,89 +1256,6 @@ int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_dev **kdev)
+
+ up_read(&topology_lock);
+
+- return -1;
+-
+-}
+-
+-static int kfd_cpumask_to_apic_id(const struct cpumask *cpumask)
+-{
+- int first_cpu_of_numa_node;
+-
+- if (!cpumask || (cpumask == cpu_none_mask))
+- return -1;
+- first_cpu_of_numa_node = cpumask_first(cpumask);
+- if (first_cpu_of_numa_node >= nr_cpu_ids)
+- return -1;
+-#ifdef CONFIG_X86_64
+- return cpu_data(first_cpu_of_numa_node).apicid;
+-#else
+- return first_cpu_of_numa_node;
+-#endif
+-}
+-
+-/* kfd_numa_node_to_apic_id - Returns the APIC ID of the first logical processor
+- * of the given NUMA node (numa_node_id)
+- * Return -1 on failure
+- */
+-int kfd_numa_node_to_apic_id(int numa_node_id)
+-{
+- if (numa_node_id == -1) {
+- pr_warn("Invalid NUMA Node. Use online CPU mask\n");
+- return kfd_cpumask_to_apic_id(cpu_online_mask);
+- }
+- return kfd_cpumask_to_apic_id(cpumask_of_node(numa_node_id));
+-}
+-
+-#if defined(CONFIG_DEBUG_FS)
+-
+-int kfd_debugfs_hqds_by_device(struct seq_file *m, void *data)
+-{
+- struct kfd_topology_device *dev;
+- unsigned int i = 0;
+- int r = 0;
+-
+- down_read(&topology_lock);
+-
+- list_for_each_entry(dev, &topology_device_list, list) {
+- if (!dev->gpu) {
+- i++;
+- continue;
+- }
+-
+- seq_printf(m, "Node %u, gpu_id %x:\n", i++, dev->gpu->id);
+- r = device_queue_manager_debugfs_hqds(m, dev->gpu->dqm);
+- if (r != 0)
+- break;
+- }
+-
+- up_read(&topology_lock);
+-
+- return r;
+-}
+-
+-int kfd_debugfs_rls_by_device(struct seq_file *m, void *data)
+-{
+- struct kfd_topology_device *dev;
+- unsigned int i = 0;
+- int r = 0;
+-
+- down_read(&topology_lock);
+-
+- list_for_each_entry(dev, &topology_device_list, list) {
+- if (!dev->gpu) {
+- i++;
+- continue;
+- }
+-
+- seq_printf(m, "Node %u, gpu_id %x:\n", i++, dev->gpu->id);
+- r = pm_debugfs_runlist(m, &dev->gpu->dqm->packets);
+- if (r != 0)
+- break;
+- }
+-
+- up_read(&topology_lock);
++ return device;
+
+- return r;
+ }
+-
+-#endif
+diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
+index f22d420..c3ddb9b 100644
+--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
++++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
+@@ -39,17 +39,8 @@
+ #define HSA_CAP_WATCH_POINTS_SUPPORTED 0x00000080
+ #define HSA_CAP_WATCH_POINTS_TOTALBITS_MASK 0x00000f00
+ #define HSA_CAP_WATCH_POINTS_TOTALBITS_SHIFT 8
+-#define HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK 0x00003000
+-#define HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT 12
+-#define HSA_CAP_RESERVED 0xffffc000
+-
+-#define HSA_CAP_DOORBELL_TYPE_PRE_1_0 0x0
+-#define HSA_CAP_DOORBELL_TYPE_1_0 0x1
+-#define HSA_CAP_DOORBELL_TYPE_2_0 0x2
+-#define HSA_CAP_WATCH_POINTS_TOTALBITS_MASK 0x00000f00
+-#define HSA_CAP_WATCH_POINTS_TOTALBITS_SHIFT 8
++#define HSA_CAP_RESERVED 0xfffff000
+ #define HSA_CAP_DOORBELL_PACKET_TYPE 0x00001000
+-#define HSA_CAP_AQL_QUEUE_DOUBLE_MAP 0x00004000
+
+ struct kfd_node_properties {
+ uint32_t cpu_cores_count;
+@@ -97,11 +88,11 @@ struct kfd_mem_properties {
+ uint32_t width;
+ uint32_t mem_clk_max;
+ struct kobject *kobj;
+- struct kfd_dev *gpu;
+- struct attribute attr_props;
+- struct attribute attr_used;
++ struct attribute attr;
+ };
+
++#define KFD_TOPOLOGY_CPU_SIBLINGS 256
++
+ #define HSA_CACHE_TYPE_DATA 0x00000001
+ #define HSA_CACHE_TYPE_INSTRUCTION 0x00000002
+ #define HSA_CACHE_TYPE_CPU 0x00000004
+@@ -118,7 +109,7 @@ struct kfd_cache_properties {
+ uint32_t cache_assoc;
+ uint32_t cache_latency;
+ uint32_t cache_type;
+- uint8_t sibling_map[CRAT_SIBLINGMAP_SIZE];
++ uint8_t sibling_map[KFD_TOPOLOGY_CPU_SIBLINGS];
+ struct kobject *kobj;
+ struct attribute attr;
+ };
+@@ -141,36 +132,24 @@ struct kfd_iolink_properties {
+ struct attribute attr;
+ };
+
+-struct kfd_perf_properties {
+- struct list_head list;
+- char block_name[16];
+- uint32_t max_concurrent;
+- struct attribute_group *attr_group;
+-};
+-
+ struct kfd_topology_device {
+ struct list_head list;
+ uint32_t gpu_id;
+- uint32_t proximity_domain;
+ struct kfd_node_properties node_props;
++ uint32_t mem_bank_count;
+ struct list_head mem_props;
+ uint32_t cache_count;
+ struct list_head cache_props;
+ uint32_t io_link_count;
+ struct list_head io_link_props;
+- struct list_head perf_props;
+ struct kfd_dev *gpu;
+ struct kobject *kobj_node;
+ struct kobject *kobj_mem;
+ struct kobject *kobj_cache;
+ struct kobject *kobj_iolink;
+- struct kobject *kobj_perf;
+ struct attribute attr_gpuid;
+ struct attribute attr_name;
+ struct attribute attr_props;
+- uint8_t oem_id[CRAT_OEMID_LENGTH];
+- uint8_t oem_table_id[CRAT_OEMTABLEID_LENGTH];
+- uint32_t oem_revision;
+ };
+
+ struct kfd_system_properties {
+@@ -185,14 +164,6 @@ struct kfd_system_properties {
+ struct attribute attr_props;
+ };
+
+-struct kfd_topology_device *kfd_create_topology_device(
+- struct list_head *device_list);
+-void kfd_release_topology_device_list(struct list_head *device_list);
+
+-#if defined(CONFIG_AMD_IOMMU_V2_MODULE) || defined(CONFIG_AMD_IOMMU_V2)
+-extern bool amd_iommu_pc_supported(void);
+-extern u8 amd_iommu_pc_get_max_banks(u16 devid);
+-extern u8 amd_iommu_pc_get_max_counters(u16 devid);
+-#endif
+
+ #endif /* __KFD_TOPOLOGY_H__ */
+diff --git a/drivers/gpu/drm/amd/amdkfd/soc15_int.h b/drivers/gpu/drm/amd/amdkfd/soc15_int.h
+deleted file mode 100644
+index e00d03d..0000000
+--- a/drivers/gpu/drm/amd/amdkfd/soc15_int.h
++++ /dev/null
+@@ -1,84 +0,0 @@
+-/*
+- * Copyright 2016 Advanced Micro Devices, Inc.
+- *
+- * Permission is hereby granted, free of charge, to any person obtaining a
+- * copy of this software and associated documentation files (the "Software"),
+- * to deal in the Software without restriction, including without limitation
+- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+- * and/or sell copies of the Software, and to permit persons to whom the
+- * Software is furnished to do so, subject to the following conditions:
+- *
+- * The above copyright notice and this permission notice shall be included in
+- * all copies or substantial portions of the Software.
+- *
+- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+- * OTHER DEALINGS IN THE SOFTWARE.
+- */
+-
+-#ifndef HSA_SOC15_INT_H_INCLUDED
+-#define HSA_SOC15_INT_H_INCLUDED
+-/*
+- * vega10+ IH clients
+- */
+-enum soc15_ih_client_id {
+- SOC15_IH_CLIENTID_IH = 0x00,
+- SOC15_IH_CLIENTID_ACP = 0x01,
+- SOC15_IH_CLIENTID_ATHUB = 0x02,
+- SOC15_IH_CLIENTID_BIF = 0x03,
+- SOC15_IH_CLIENTID_DCE = 0x04,
+- SOC15_IH_CLIENTID_ISP = 0x05,
+- SOC15_IH_CLIENTID_PCIE0 = 0x06,
+- SOC15_IH_CLIENTID_RLC = 0x07,
+- SOC15_IH_CLIENTID_SDMA0 = 0x08,
+- SOC15_IH_CLIENTID_SDMA1 = 0x09,
+- SOC15_IH_CLIENTID_SE0SH = 0x0a,
+- SOC15_IH_CLIENTID_SE1SH = 0x0b,
+- SOC15_IH_CLIENTID_SE2SH = 0x0c,
+- SOC15_IH_CLIENTID_SE3SH = 0x0d,
+- SOC15_IH_CLIENTID_SYSHUB = 0x0e,
+- SOC15_IH_CLIENTID_THM = 0x0f,
+- SOC15_IH_CLIENTID_UVD = 0x10,
+- SOC15_IH_CLIENTID_VCE0 = 0x11,
+- SOC15_IH_CLIENTID_VMC = 0x12,
+- SOC15_IH_CLIENTID_XDMA = 0x13,
+- SOC15_IH_CLIENTID_GRBM_CP = 0x14,
+- SOC15_IH_CLIENTID_ATS = 0x15,
+- SOC15_IH_CLIENTID_ROM_SMUIO = 0x16,
+- SOC15_IH_CLIENTID_DF = 0x17,
+- SOC15_IH_CLIENTID_VCE1 = 0x18,
+- SOC15_IH_CLIENTID_PWR = 0x19,
+- SOC15_IH_CLIENTID_UTCL2 = 0x1b,
+- SOC15_IH_CLIENTID_EA = 0x1c,
+- SOC15_IH_CLIENTID_UTCL2LOG = 0x1d,
+- SOC15_IH_CLIENTID_MP0 = 0x1e,
+- SOC15_IH_CLIENTID_MP1 = 0x1f,
+-
+- SOC15_IH_CLIENTID_MAX
+-};
+-
+-
+-#define SOC15_INTSRC_CP_END_OF_PIPE 181
+-#define SOC15_INTSRC_CP_BAD_OPCODE 183
+-#define SOC15_INTSRC_SQ_INTERRUPT_MSG 239
+-#define SOC15_INTSRC_VMC_FAULT 0
+-#define SOC15_INTSRC_SDMA_TRAP 224
+-
+-
+-#define SOC15_CLIENT_ID_FROM_IH_ENTRY(entry) (le32_to_cpu(entry[0]) & 0xff)
+-#define SOC15_SOURCE_ID_FROM_IH_ENTRY(entry) (le32_to_cpu(entry[0]) >> 8 & 0xff)
+-#define SOC15_RING_ID_FROM_IH_ENTRY(entry) (le32_to_cpu(entry[0]) >> 16 & 0xff)
+-#define SOC15_VMID_FROM_IH_ENTRY(entry) (le32_to_cpu(entry[0]) >> 24 & 0xf)
+-#define SOC15_VMID_TYPE_FROM_IH_ENTRY(entry) (le32_to_cpu(entry[0]) >> 31 & 0x1)
+-#define SOC15_PASID_FROM_IH_ENTRY(entry) (le32_to_cpu(entry[3]) & 0xffff)
+-#define SOC15_CONTEXT_ID0_FROM_IH_ENTRY(entry) (le32_to_cpu(entry[4]))
+-#define SOC15_CONTEXT_ID1_FROM_IH_ENTRY(entry) (le32_to_cpu(entry[5]))
+-#define SOC15_CONTEXT_ID2_FROM_IH_ENTRY(entry) (le32_to_cpu(entry[6]))
+-#define SOC15_CONTEXT_ID3_FROM_IH_ENTRY(entry) (le32_to_cpu(entry[7]))
+-
+-#endif
+-
+diff --git a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
+old mode 100755
+new mode 100644
+index b6cf2d5..36f3766
+--- a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
++++ b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
+@@ -30,7 +30,6 @@
+
+ #include <linux/types.h>
+ #include <linux/bitmap.h>
+-#include <linux/dma-buf.h>
+
+ struct pci_dev;
+
+@@ -41,46 +40,6 @@ struct kfd_dev;
+ struct kgd_dev;
+
+ struct kgd_mem;
+-struct kfd_process_device;
+-struct amdgpu_bo;
+-
+-enum kfd_preempt_type {
+- KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN = 0,
+- KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
+-};
+-
+-struct kfd_vm_fault_info {
+- uint64_t page_addr;
+- uint32_t vmid;
+- uint32_t mc_id;
+- uint32_t status;
+- bool prot_valid;
+- bool prot_read;
+- bool prot_write;
+- bool prot_exec;
+-};
+-
+-struct kfd_cu_info {
+- uint32_t num_shader_engines;
+- uint32_t num_shader_arrays_per_engine;
+- uint32_t num_cu_per_sh;
+- uint32_t cu_active_number;
+- uint32_t cu_ao_mask;
+- uint32_t simd_per_cu;
+- uint32_t max_waves_per_simd;
+- uint32_t wave_front_size;
+- uint32_t max_scratch_slots_per_cu;
+- uint32_t lds_size;
+- uint32_t cu_bitmap[4][4];
+-};
+-
+-/* For getting GPU local memory information from KGD */
+-struct kfd_local_mem_info {
+- uint64_t local_mem_size_private;
+- uint64_t local_mem_size_public;
+- uint32_t vram_width;
+- uint32_t mem_clk_max;
+-};
+
+ enum kgd_memory_pool {
+ KGD_POOL_SYSTEM_CACHEABLE = 1,
+@@ -113,21 +72,6 @@ struct kgd2kfd_shared_resources {
+ /* Bit n == 1 means Queue n is available for KFD */
+ DECLARE_BITMAP(queue_bitmap, KGD_MAX_QUEUES);
+
+- /* Doorbell assignments (SOC15 and later chips only). Only
+- * specific doorbells are routed to each SDMA engine. Others
+- * are routed to IH and VCN. They are not usable by the CP.
+- *
+- * Any doorbell number D that satisfies the following condition
+- * is reserved: (D & reserved_doorbell_mask) == reserved_doorbell_val
+- *
+- * KFD currently uses 1024 (= 0x3ff) doorbells per process. If
+- * doorbells 0x0f0-0x0f7 and 0x2f-0x2f7 are reserved, that means
+- * mask would be set to 0x1f8 and val set to 0x0f0.
+- */
+- unsigned int sdma_doorbell[2][2];
+- unsigned int reserved_doorbell_mask;
+- unsigned int reserved_doorbell_val;
+-
+ /* Base address of doorbell aperture. */
+ phys_addr_t doorbell_physical_address;
+
+@@ -136,41 +80,8 @@ struct kgd2kfd_shared_resources {
+
+ /* Number of bytes at start of aperture reserved for KGD. */
+ size_t doorbell_start_offset;
+-
+- /* GPUVM address space size in bytes */
+- uint64_t gpuvm_size;
+ };
+
+-struct tile_config {
+- uint32_t *tile_config_ptr;
+- uint32_t *macro_tile_config_ptr;
+- uint32_t num_tile_configs;
+- uint32_t num_macro_tile_configs;
+-
+- uint32_t gb_addr_config;
+- uint32_t num_banks;
+- uint32_t num_ranks;
+-};
+-
+-/*
+- * Allocation flag domains currently only VRAM and GTT domain supported
+- */
+-#define ALLOC_MEM_FLAGS_VRAM (1 << 0)
+-#define ALLOC_MEM_FLAGS_GTT (1 << 1)
+-#define ALLOC_MEM_FLAGS_USERPTR (1 << 2)
+-#define ALLOC_MEM_FLAGS_DOORBELL (1 << 3)
+-
+-/*
+- * Allocation flags attributes/access options.
+- */
+-#define ALLOC_MEM_FLAGS_NONPAGED (1 << 31)
+-#define ALLOC_MEM_FLAGS_READONLY (1 << 30)
+-#define ALLOC_MEM_FLAGS_PUBLIC (1 << 29)
+-#define ALLOC_MEM_FLAGS_NO_SUBSTITUTE (1 << 28)
+-#define ALLOC_MEM_FLAGS_AQL_QUEUE_MEM (1 << 27)
+-#define ALLOC_MEM_FLAGS_EXECUTE_ACCESS (1 << 26)
+-#define ALLOC_MEM_FLAGS_COHERENT (1 << 25)
+-
+ /**
+ * struct kfd2kgd_calls
+ *
+@@ -179,7 +90,7 @@ struct tile_config {
+ *
+ * @free_gtt_mem: Frees a buffer that was allocated on the gart aperture
+ *
+- * @get_local_mem_info: Retrieves information about GPU local memory
++ * @get_vmem_size: Retrieves (physical) size of VRAM
+ *
+ * @get_gpu_clock_counter: Retrieves GPU clock counter
+ *
+@@ -201,12 +112,6 @@ struct tile_config {
+ * @hqd_sdma_load: Loads the SDMA mqd structure to a H/W SDMA hqd slot.
+ * used only for no HWS mode.
+ *
+- * @hqd_dump: Dumps CPC HQD registers to an array of address-value pairs.
+- * Array is allocated with kmalloc, needs to be freed with kfree by caller.
+- *
+- * @hqd_sdma_dump: Dumps SDMA HQD registers to an array of address-value pairs.
+- * Array is allocated with kmalloc, needs to be freed with kfree by caller.
+- *
+ * @hqd_is_occupies: Checks if a hqd slot is occupied.
+ *
+ * @hqd_destroy: Destructs and preempts the queue assigned to that hqd slot.
+@@ -216,34 +121,8 @@ struct tile_config {
+ * @hqd_sdma_destroy: Destructs and preempts the SDMA queue assigned to that
+ * SDMA hqd slot.
+ *
+- * @map_memory_to_gpu: Allocates and pins BO, PD and all related PTs
+- *
+- * @unmap_memory_to_gpu: Releases and unpins BO, PD and all related PTs
+- *
+ * @get_fw_version: Returns FW versions from the header
+ *
+- * @set_num_of_requests: Sets number of Peripheral Page Request (PPR) sent to
+- * IOMMU when address translation failed
+- *
+- * @get_cu_info: Retrieves activated cu info
+- *
+- * @get_dmabuf_info: Returns information about a dmabuf if it was
+- * created by the GPU driver
+- *
+- * @import_dmabuf: Imports a DMA buffer, creating a new kgd_mem object
+- * Supports only DMA buffers created by GPU driver on the same GPU
+- *
+- * @export_dmabuf: Emports a KFD BO for sharing with other process
+- *
+- * @submit_ib: Submits an IB to the engine specified by inserting the IB to
+- * the corresonded ring (ring type).
+- *
+- * @restore_process_bos: Restore all BOs that belongs to the process
+- *
+- * @copy_mem_to_mem: Copies size bytes from source BO to destination BO
+- *
+- * @get_vram_usage: Returns current VRAM usage
+- *
+ * This structure contains function pointers to services that the kgd driver
+ * provides to amdkfd driver.
+ *
+@@ -255,23 +134,11 @@ struct kfd2kgd_calls {
+
+ void (*free_gtt_mem)(struct kgd_dev *kgd, void *mem_obj);
+
+- void(*get_local_mem_info)(struct kgd_dev *kgd,
+- struct kfd_local_mem_info *mem_info);
++ uint64_t (*get_vmem_size)(struct kgd_dev *kgd);
+ uint64_t (*get_gpu_clock_counter)(struct kgd_dev *kgd);
+
+ uint32_t (*get_max_engine_clock_in_mhz)(struct kgd_dev *kgd);
+
+- int (*create_process_vm)(struct kgd_dev *kgd, void **vm,
+- void **process_info, struct dma_fence **ef);
+- void (*destroy_process_vm)(struct kgd_dev *kgd, void *vm);
+-
+- int (*create_process_gpumem)(struct kgd_dev *kgd, uint64_t va, size_t size, void *vm, struct kgd_mem **mem);
+- void (*destroy_process_gpumem)(struct kgd_dev *kgd, struct kgd_mem *mem);
+-
+- uint32_t (*get_process_page_dir)(void *vm);
+-
+- int (*open_graphic_handle)(struct kgd_dev *kgd, uint64_t va, void *vm, int fd, uint32_t handle, struct kgd_mem **mem);
+-
+ /* Register access functions */
+ void (*program_sh_mem_settings)(struct kgd_dev *kgd, uint32_t vmid,
+ uint32_t sh_mem_config, uint32_t sh_mem_ape1_base,
+@@ -284,28 +151,16 @@ struct kfd2kgd_calls {
+ uint32_t hpd_size, uint64_t hpd_gpu_addr);
+
+ int (*init_interrupts)(struct kgd_dev *kgd, uint32_t pipe_id);
+-
+
+ int (*hqd_load)(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
+- uint32_t queue_id, uint32_t __user *wptr,
+- uint32_t wptr_shift, uint32_t wptr_mask,
+- struct mm_struct *mm);
+-
+- int (*hqd_sdma_load)(struct kgd_dev *kgd, void *mqd,
+- uint32_t __user *wptr, struct mm_struct *mm);
+-
+- int (*hqd_dump)(struct kgd_dev *kgd,
+- uint32_t pipe_id, uint32_t queue_id,
+- uint32_t (**dump)[2], uint32_t *n_regs);
++ uint32_t queue_id, uint32_t __user *wptr);
+
+- int (*hqd_sdma_dump)(struct kgd_dev *kgd,
+- uint32_t engine_id, uint32_t queue_id,
+- uint32_t (**dump)[2], uint32_t *n_regs);
++ int (*hqd_sdma_load)(struct kgd_dev *kgd, void *mqd);
+
+ bool (*hqd_is_occupied)(struct kgd_dev *kgd, uint64_t queue_address,
+ uint32_t pipe_id, uint32_t queue_id);
+
+- int (*hqd_destroy)(struct kgd_dev *kgd, void *mqd, uint32_t reset_type,
++ int (*hqd_destroy)(struct kgd_dev *kgd, uint32_t reset_type,
+ unsigned int timeout, uint32_t pipe_id,
+ uint32_t queue_id);
+
+@@ -313,7 +168,7 @@ struct kfd2kgd_calls {
+
+ int (*hqd_sdma_destroy)(struct kgd_dev *kgd, void *mqd,
+ unsigned int timeout);
+-
++
+ int (*address_watch_disable)(struct kgd_dev *kgd);
+ int (*address_watch_execute)(struct kgd_dev *kgd,
+ unsigned int watch_point_id,
+@@ -332,72 +187,11 @@ struct kfd2kgd_calls {
+ uint16_t (*get_atc_vmid_pasid_mapping_pasid)(
+ struct kgd_dev *kgd,
+ uint8_t vmid);
+- uint32_t (*read_vmid_from_vmfault_reg)(struct kgd_dev *kgd);
+ void (*write_vmid_invalidate_request)(struct kgd_dev *kgd,
+ uint8_t vmid);
+
+- int (*invalidate_tlbs)(struct kgd_dev *kgd, uint16_t pasid);
+-
+- int (*sync_memory)(struct kgd_dev *kgd, struct kgd_mem *mem, bool intr);
+-
+- int (*alloc_memory_of_gpu)(struct kgd_dev *kgd, uint64_t va,
+- uint64_t size, void *vm,
+- struct kgd_mem **mem, uint64_t *offset,
+- uint32_t flags);
+- int (*free_memory_of_gpu)(struct kgd_dev *kgd, struct kgd_mem *mem,
+- void *vm);
+- int (*map_memory_to_gpu)(struct kgd_dev *kgd, struct kgd_mem *mem,
+- void *vm);
+- int (*unmap_memory_to_gpu)(struct kgd_dev *kgd, struct kgd_mem *mem,
+- void *vm);
+-
+ uint16_t (*get_fw_version)(struct kgd_dev *kgd,
+ enum kgd_engine_type type);
+-
+- void (*set_num_of_requests)(struct kgd_dev *kgd,
+- uint8_t num_of_requests);
+- int (*alloc_memory_of_scratch)(struct kgd_dev *kgd,
+- uint64_t va, uint32_t vmid);
+- int (*write_config_static_mem)(struct kgd_dev *kgd, bool swizzle_enable,
+- uint8_t element_size, uint8_t index_stride, uint8_t mtype);
+- void (*get_cu_info)(struct kgd_dev *kgd,
+- struct kfd_cu_info *cu_info);
+- int (*mmap_bo)(struct kgd_dev *kgd, struct vm_area_struct *vma);
+- int (*map_gtt_bo_to_kernel)(struct kgd_dev *kgd,
+- struct kgd_mem *mem, void **kptr);
+- void (*set_vm_context_page_table_base)(struct kgd_dev *kgd, uint32_t vmid,
+- uint32_t page_table_base);
+-
+- int (*pin_get_sg_table_bo)(struct kgd_dev *kgd,
+- struct kgd_mem *mem, uint64_t offset,
+- uint64_t size, struct sg_table **ret_sg);
+- void (*unpin_put_sg_table_bo)(struct kgd_mem *mem,
+- struct sg_table *sg);
+-
+- int (*get_dmabuf_info)(struct kgd_dev *kgd, int dma_buf_fd,
+- struct kgd_dev **dma_buf_kgd, uint64_t *bo_size,
+- void *metadata_buffer, size_t buffer_size,
+- uint32_t *metadata_size, uint32_t *flags);
+- int (*import_dmabuf)(struct kgd_dev *kgd, struct dma_buf *dmabuf,
+- uint64_t va, void *vm, struct kgd_mem **mem,
+- uint64_t *size, uint64_t *mmap_offset);
+- int (*export_dmabuf)(struct kgd_dev *kgd, void *vm, struct kgd_mem *mem,
+- struct dma_buf **dmabuf);
+-
+- int (*get_vm_fault_info)(struct kgd_dev *kgd,
+- struct kfd_vm_fault_info *info);
+- int (*submit_ib)(struct kgd_dev *kgd, enum kgd_engine_type engine,
+- uint32_t vmid, uint64_t gpu_addr,
+- uint32_t *ib_cmd, uint32_t ib_len);
+- int (*get_tile_config)(struct kgd_dev *kgd,
+- struct tile_config *config);
+-
+- int (*restore_process_bos)(void *process_info, struct dma_fence **ef);
+- int (*copy_mem_to_mem)(struct kgd_dev *kgd, struct kgd_mem *src_mem,
+- uint64_t src_offset, struct kgd_mem *dst_mem,
+- uint64_t dest_offset, uint64_t size,
+- struct dma_fence **f, uint64_t *actual_size);
+- uint64_t (*get_vram_usage)(struct kgd_dev *kgd);
+ };
+
+ /**
+@@ -416,13 +210,6 @@ struct kfd2kgd_calls {
+ *
+ * @resume: Notifies amdkfd about a resume action done to a kgd device
+ *
+- * @quiesce_mm: Quiesce all user queue access to specified MM address space
+- *
+- * @resume_mm: Resume user queue access to specified MM address space
+- *
+- * @schedule_evict_and_restore_process: Schedules work queue that will prepare
+- * for safe eviction of KFD BOs that belong to the specified process.
+- *
+ * This structure contains function callback pointers so the kgd driver
+ * will notify to the amdkfd about certain status changes.
+ *
+@@ -437,13 +224,9 @@ struct kgd2kfd_calls {
+ void (*interrupt)(struct kfd_dev *kfd, const void *ih_ring_entry);
+ void (*suspend)(struct kfd_dev *kfd);
+ int (*resume)(struct kfd_dev *kfd);
+- int (*quiesce_mm)(struct kfd_dev *kfd, struct mm_struct *mm);
+- int (*resume_mm)(struct kfd_dev *kfd, struct mm_struct *mm);
+- int (*schedule_evict_and_restore_process)(struct mm_struct *mm,
+- struct dma_fence *fence);
+ };
+
+ int kgd2kfd_init(unsigned interface_version,
+ const struct kgd2kfd_calls **g2f);
+
+-#endif /* KGD_KFD_INTERFACE_H_INCLUDED */
++#endif /* KGD_KFD_INTERFACE_H_INCLUDED */
+diff --git a/drivers/gpu/drm/drm_pci.c b/drivers/gpu/drm/drm_pci.c
+index 1235c98..7e5a1fe 100644
+--- a/drivers/gpu/drm/drm_pci.c
++++ b/drivers/gpu/drm/drm_pci.c
+@@ -149,6 +149,7 @@ int drm_pci_set_busid(struct drm_device *dev, struct drm_master *master)
+ master->unique_len = strlen(master->unique);
+ return 0;
+ }
++EXPORT_SYMBOL(drm_pci_set_busid);
+
+ static int drm_pci_irq_by_busid(struct drm_device *dev, struct drm_irq_busid *p)
+ {
+diff --git a/drivers/gpu/drm/radeon/radeon_kfd.c b/drivers/gpu/drm/radeon/radeon_kfd.c
+index c7d2e7a..a2ab6dc 100755
+--- a/drivers/gpu/drm/radeon/radeon_kfd.c
++++ b/drivers/gpu/drm/radeon/radeon_kfd.c
+@@ -75,15 +75,12 @@ static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
+ uint32_t hpd_size, uint64_t hpd_gpu_addr);
+ static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id);
+ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
+- uint32_t queue_id, uint32_t __user *wptr,
+- uint32_t wptr_shift, uint32_t wptr_mask,
+- struct mm_struct *mm);
+-static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
+- uint32_t __user *wptr, struct mm_struct *mm);
++ uint32_t queue_id, uint32_t __user *wptr);
++static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd);
+ static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
+ uint32_t pipe_id, uint32_t queue_id);
+
+-static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd, uint32_t reset_type,
++static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
+ unsigned int timeout, uint32_t pipe_id,
+ uint32_t queue_id);
+ static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd);
+@@ -110,6 +107,7 @@ static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid);
+ static const struct kfd2kgd_calls kfd2kgd = {
+ .init_gtt_mem_allocation = alloc_gtt_mem,
+ .free_gtt_mem = free_gtt_mem,
++ .get_vmem_size = get_vmem_size,
+ .get_gpu_clock_counter = get_gpu_clock_counter,
+ .get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz,
+ .program_sh_mem_settings = kgd_program_sh_mem_settings,
+@@ -484,9 +482,7 @@ static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
+ }
+
+ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
+- uint32_t queue_id, uint32_t __user *wptr,
+- uint32_t wptr_shift, uint32_t wptr_mask,
+- struct mm_struct *mm)
++ uint32_t queue_id, uint32_t __user *wptr)
+ {
+ uint32_t wptr_shadow, is_wptr_shadow_valid;
+ struct cik_mqd *m;
+@@ -562,8 +558,7 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
+ return 0;
+ }
+
+-static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
+- uint32_t __user *wptr, struct mm_struct *mm)
++static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd)
+ {
+ struct cik_sdma_rlc_registers *m;
+ uint32_t sdma_base_addr;
+@@ -641,7 +636,7 @@ static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
+ return false;
+ }
+
+-static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd, uint32_t reset_type,
++static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
+ unsigned int timeout, uint32_t pipe_id,
+ uint32_t queue_id)
+ {
+diff --git a/include/drm/drm_drv.h b/include/drm/drm_drv.h
+index 3053049..f08273c 100644
+--- a/include/drm/drm_drv.h
++++ b/include/drm/drm_drv.h
+@@ -174,6 +174,8 @@ struct drm_driver {
+ * to finalize the device and then freeing the struct themselves.
+ */
+ void (*release) (struct drm_device *);
++
++ int (*set_busid)(struct drm_device *dev, struct drm_master *master);
+
+ /**
+ * @get_vblank_counter:
+diff --git a/include/drm/drm_pci.h b/include/drm/drm_pci.h
+index 6745990..4d5daa8 100644
+--- a/include/drm/drm_pci.h
++++ b/include/drm/drm_pci.h
+@@ -49,6 +49,7 @@ void drm_legacy_pci_exit(struct drm_driver *driver, struct pci_driver *pdriver);
+ int drm_get_pci_dev(struct pci_dev *pdev,
+ const struct pci_device_id *ent,
+ struct drm_driver *driver);
++int drm_pci_set_busid(struct drm_device *dev, struct drm_master *master);
+ #else
+ static inline int drm_get_pci_dev(struct pci_dev *pdev,
+ const struct pci_device_id *ent,
+@@ -56,6 +57,12 @@ static inline int drm_get_pci_dev(struct pci_dev *pdev,
+ {
+ return -ENOSYS;
+ }
++
++static inline int drm_pci_set_busid(struct drm_device *dev,
++ struct drm_master *master)
++{
++ return -ENOSYS;
++}
+ #endif
+
+ #define DRM_PCIE_SPEED_25 1
+--
+2.7.4
+