diff options
Diffstat (limited to 'meta-amd-bsp/recipes-kernel/linux/linux-yocto-4.19.8/3893-Fix-the-messed-up-format.patch')
| -rw-r--r-- | meta-amd-bsp/recipes-kernel/linux/linux-yocto-4.19.8/3893-Fix-the-messed-up-format.patch | 2083 |
1 files changed, 2083 insertions, 0 deletions
diff --git a/meta-amd-bsp/recipes-kernel/linux/linux-yocto-4.19.8/3893-Fix-the-messed-up-format.patch b/meta-amd-bsp/recipes-kernel/linux/linux-yocto-4.19.8/3893-Fix-the-messed-up-format.patch new file mode 100644 index 00000000..41044763 --- /dev/null +++ b/meta-amd-bsp/recipes-kernel/linux/linux-yocto-4.19.8/3893-Fix-the-messed-up-format.patch @@ -0,0 +1,2083 @@ +From a57cc62927d04caa52937b894b89353003eddea6 Mon Sep 17 00:00:00 2001 +From: Yong Zhao <Yong.Zhao@amd.com> +Date: Thu, 26 Sep 2019 16:49:19 -0400 +Subject: [PATCH 3893/4256] Fix the messed up format + +Change-Id: I1fdf5536dca4045c68010b0c78e4a541e916fcaf +Signed-off-by: Yong Zhao <Yong.Zhao@amd.com> +--- + .../drm/amd/amdkfd/kfd_device_queue_manager.c | 1706 ++++++++--------- + 1 file changed, 853 insertions(+), 853 deletions(-) + +diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +index dbcbacb5abd4..787b936a026d 100644 +--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c ++++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +@@ -42,558 +42,558 @@ + #define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2) + + static int set_pasid_vmid_mapping(struct device_queue_manager *dqm, +- unsigned int pasid, unsigned int vmid); ++ unsigned int pasid, unsigned int vmid); + + static int execute_queues_cpsch(struct device_queue_manager *dqm, +- enum kfd_unmap_queues_filter filter, +- uint32_t filter_param, +- uint32_t grace_period); ++ enum kfd_unmap_queues_filter filter, ++ uint32_t filter_param, ++ uint32_t grace_period); + static int unmap_queues_cpsch(struct device_queue_manager *dqm, +- enum kfd_unmap_queues_filter filter, +- uint32_t filter_param, +- uint32_t grace_period); ++ enum kfd_unmap_queues_filter filter, ++ uint32_t filter_param, ++ uint32_t grace_period); + + static int map_queues_cpsch(struct device_queue_manager *dqm); + + static void deallocate_sdma_queue(struct device_queue_manager *dqm, +- struct queue *q); ++ struct queue *q); + + static inline void deallocate_hqd(struct device_queue_manager *dqm, +- struct queue *q); ++ struct queue *q); + static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q); + static int allocate_sdma_queue(struct device_queue_manager *dqm, +- struct queue *q); ++ struct queue *q); + static void kfd_process_hw_exception(struct work_struct *work); + +- static inline ++static inline + enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type) + { +- if (type == KFD_QUEUE_TYPE_SDMA || type == KFD_QUEUE_TYPE_SDMA_XGMI) +- return KFD_MQD_TYPE_SDMA; +- return KFD_MQD_TYPE_CP; ++ if (type == KFD_QUEUE_TYPE_SDMA || type == KFD_QUEUE_TYPE_SDMA_XGMI) ++ return KFD_MQD_TYPE_SDMA; ++ return KFD_MQD_TYPE_CP; + } + + static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe) + { +- int i; +- int pipe_offset = mec * dqm->dev->shared_resources.num_pipe_per_mec +- + pipe * dqm->dev->shared_resources.num_queue_per_pipe; ++ int i; ++ int pipe_offset = mec * dqm->dev->shared_resources.num_pipe_per_mec ++ + pipe * dqm->dev->shared_resources.num_queue_per_pipe; + +- /* queue is available for KFD usage if bit is 1 */ +- for (i = 0; i < dqm->dev->shared_resources.num_queue_per_pipe; ++i) +- if (test_bit(pipe_offset + i, +- dqm->dev->shared_resources.queue_bitmap)) +- return true; +- return false; ++ /* queue is available for KFD usage if bit is 1 */ ++ for (i = 0; i < dqm->dev->shared_resources.num_queue_per_pipe; ++i) ++ if (test_bit(pipe_offset + i, ++ dqm->dev->shared_resources.queue_bitmap)) ++ return true; ++ return false; + } + + unsigned int get_queues_num(struct device_queue_manager *dqm) + { +- return bitmap_weight(dqm->dev->shared_resources.queue_bitmap, +- KGD_MAX_QUEUES); ++ return bitmap_weight(dqm->dev->shared_resources.queue_bitmap, ++ KGD_MAX_QUEUES); + } + + unsigned int get_queues_per_pipe(struct device_queue_manager *dqm) + { +- return dqm->dev->shared_resources.num_queue_per_pipe; ++ return dqm->dev->shared_resources.num_queue_per_pipe; + } + + unsigned int get_pipes_per_mec(struct device_queue_manager *dqm) + { +- return dqm->dev->shared_resources.num_pipe_per_mec; ++ return dqm->dev->shared_resources.num_pipe_per_mec; + } + + static unsigned int get_num_sdma_engines(struct device_queue_manager *dqm) + { +- return dqm->dev->device_info->num_sdma_engines; ++ return dqm->dev->device_info->num_sdma_engines; + } + + static unsigned int get_num_xgmi_sdma_engines(struct device_queue_manager *dqm) + { +- return dqm->dev->device_info->num_xgmi_sdma_engines; ++ return dqm->dev->device_info->num_xgmi_sdma_engines; + } + + unsigned int get_num_sdma_queues(struct device_queue_manager *dqm) + { +- return dqm->dev->device_info->num_sdma_engines +- * dqm->dev->device_info->num_sdma_queues_per_engine; ++ return dqm->dev->device_info->num_sdma_engines ++ * dqm->dev->device_info->num_sdma_queues_per_engine; + } + + unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm) + { +- return dqm->dev->device_info->num_xgmi_sdma_engines +- * dqm->dev->device_info->num_sdma_queues_per_engine; ++ return dqm->dev->device_info->num_xgmi_sdma_engines ++ * dqm->dev->device_info->num_sdma_queues_per_engine; + } + + void program_sh_mem_settings(struct device_queue_manager *dqm, +- struct qcm_process_device *qpd) ++ struct qcm_process_device *qpd) + { +- return dqm->dev->kfd2kgd->program_sh_mem_settings( +- dqm->dev->kgd, qpd->vmid, +- qpd->sh_mem_config, +- qpd->sh_mem_ape1_base, +- qpd->sh_mem_ape1_limit, +- qpd->sh_mem_bases); ++ return dqm->dev->kfd2kgd->program_sh_mem_settings( ++ dqm->dev->kgd, qpd->vmid, ++ qpd->sh_mem_config, ++ qpd->sh_mem_ape1_base, ++ qpd->sh_mem_ape1_limit, ++ qpd->sh_mem_bases); + } + + bool check_if_queues_active(struct device_queue_manager *dqm, +- struct qcm_process_device *qpd) ++ struct qcm_process_device *qpd) + { +- bool busy = false; +- struct queue *q; ++ bool busy = false; ++ struct queue *q; + +- dqm_lock(dqm); +- list_for_each_entry(q, &qpd->queues_list, list) { +- struct mqd_manager *mqd_mgr; +- enum KFD_MQD_TYPE type; ++ dqm_lock(dqm); ++ list_for_each_entry(q, &qpd->queues_list, list) { ++ struct mqd_manager *mqd_mgr; ++ enum KFD_MQD_TYPE type; + +- type = get_mqd_type_from_queue_type(q->properties.type); +- mqd_mgr = dqm->mqd_mgrs[type]; +- if (!mqd_mgr || !mqd_mgr->check_queue_active) +- continue; ++ type = get_mqd_type_from_queue_type(q->properties.type); ++ mqd_mgr = dqm->mqd_mgrs[type]; ++ if (!mqd_mgr || !mqd_mgr->check_queue_active) ++ continue; + +- busy = mqd_mgr->check_queue_active(q); +- if (busy) +- break; +- } +- dqm_unlock(dqm); ++ busy = mqd_mgr->check_queue_active(q); ++ if (busy) ++ break; ++ } ++ dqm_unlock(dqm); + +- return busy; ++ return busy; + } + + static int allocate_doorbell(struct qcm_process_device *qpd, struct queue *q) + { +- struct kfd_dev *dev = qpd->dqm->dev; +- +- if (!KFD_IS_SOC15(dev->device_info->asic_family)) { +- /* On pre-SOC15 chips we need to use the queue ID to +- * preserve the user mode ABI. +- */ +- q->doorbell_id = q->properties.queue_id; +- } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || +- q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { +- /* For SDMA queues on SOC15 with 8-byte doorbell, use static +- * doorbell assignments based on the engine and queue id. +- * The doobell index distance between RLC (2*i) and (2*i+1) +- * for a SDMA engine is 512. +- */ +- uint32_t *idx_offset = +- dev->shared_resources.sdma_doorbell_idx; ++ struct kfd_dev *dev = qpd->dqm->dev; + +- q->doorbell_id = idx_offset[q->properties.sdma_engine_id] +- + (q->properties.sdma_queue_id & 1) +- * KFD_QUEUE_DOORBELL_MIRROR_OFFSET +- + (q->properties.sdma_queue_id >> 1); +- } else { +- /* For CP queues on SOC15 reserve a free doorbell ID */ +- unsigned int found; ++ if (!KFD_IS_SOC15(dev->device_info->asic_family)) { ++ /* On pre-SOC15 chips we need to use the queue ID to ++ * preserve the user mode ABI. ++ */ ++ q->doorbell_id = q->properties.queue_id; ++ } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || ++ q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { ++ /* For SDMA queues on SOC15 with 8-byte doorbell, use static ++ * doorbell assignments based on the engine and queue id. ++ * The doobell index distance between RLC (2*i) and (2*i+1) ++ * for a SDMA engine is 512. ++ */ ++ uint32_t *idx_offset = ++ dev->shared_resources.sdma_doorbell_idx; + +- found = find_first_zero_bit(qpd->doorbell_bitmap, +- KFD_MAX_NUM_OF_QUEUES_PER_PROCESS); +- if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) { +- pr_debug("No doorbells available"); +- return -EBUSY; ++ q->doorbell_id = idx_offset[q->properties.sdma_engine_id] ++ + (q->properties.sdma_queue_id & 1) ++ * KFD_QUEUE_DOORBELL_MIRROR_OFFSET ++ + (q->properties.sdma_queue_id >> 1); ++ } else { ++ /* For CP queues on SOC15 reserve a free doorbell ID */ ++ unsigned int found; ++ ++ found = find_first_zero_bit(qpd->doorbell_bitmap, ++ KFD_MAX_NUM_OF_QUEUES_PER_PROCESS); ++ if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) { ++ pr_debug("No doorbells available"); ++ return -EBUSY; ++ } ++ set_bit(found, qpd->doorbell_bitmap); ++ q->doorbell_id = found; + } +- set_bit(found, qpd->doorbell_bitmap); +- q->doorbell_id = found; +- } + +- q->properties.doorbell_off = +- kfd_doorbell_id_to_offset(dev, q->process, +- q->doorbell_id); ++ q->properties.doorbell_off = ++ kfd_doorbell_id_to_offset(dev, q->process, ++ q->doorbell_id); + +- return 0; ++ return 0; + } + + static void deallocate_doorbell(struct qcm_process_device *qpd, +- struct queue *q) ++ struct queue *q) + { +- unsigned int old; +- struct kfd_dev *dev = qpd->dqm->dev; ++ unsigned int old; ++ struct kfd_dev *dev = qpd->dqm->dev; + +- if (!KFD_IS_SOC15(dev->device_info->asic_family) || +- q->properties.type == KFD_QUEUE_TYPE_SDMA || +- q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) +- return; ++ if (!KFD_IS_SOC15(dev->device_info->asic_family) || ++ q->properties.type == KFD_QUEUE_TYPE_SDMA || ++ q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) ++ return; + +- old = test_and_clear_bit(q->doorbell_id, qpd->doorbell_bitmap); +- WARN_ON(!old); ++ old = test_and_clear_bit(q->doorbell_id, qpd->doorbell_bitmap); ++ WARN_ON(!old); + } + + static int allocate_vmid(struct device_queue_manager *dqm, +- struct qcm_process_device *qpd, +- struct queue *q) ++ struct qcm_process_device *qpd, ++ struct queue *q) + { +- int bit, allocated_vmid; ++ int bit, allocated_vmid; + +- if (dqm->vmid_bitmap == 0) +- return -ENOMEM; ++ if (dqm->vmid_bitmap == 0) ++ return -ENOMEM; + +- bit = ffs(dqm->vmid_bitmap) - 1; +- dqm->vmid_bitmap &= ~(1 << bit); ++ bit = ffs(dqm->vmid_bitmap) - 1; ++ dqm->vmid_bitmap &= ~(1 << bit); + +- allocated_vmid = bit + dqm->dev->vm_info.first_vmid_kfd; +- pr_debug("vmid allocation %d\n", allocated_vmid); +- qpd->vmid = allocated_vmid; +- q->properties.vmid = allocated_vmid; ++ allocated_vmid = bit + dqm->dev->vm_info.first_vmid_kfd; ++ pr_debug("vmid allocation %d\n", allocated_vmid); ++ qpd->vmid = allocated_vmid; ++ q->properties.vmid = allocated_vmid; + +- set_pasid_vmid_mapping(dqm, q->process->pasid, q->properties.vmid); +- program_sh_mem_settings(dqm, qpd); ++ set_pasid_vmid_mapping(dqm, q->process->pasid, q->properties.vmid); ++ program_sh_mem_settings(dqm, qpd); + +- /* qpd->page_table_base is set earlier when register_process() +- * is called, i.e. when the first queue is created. +- */ +- dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->kgd, +- qpd->vmid, +- qpd->page_table_base); +- /* invalidate the VM context after pasid and vmid mapping is set up */ +- kfd_flush_tlb(qpd_to_pdd(qpd)); ++ /* qpd->page_table_base is set earlier when register_process() ++ * is called, i.e. when the first queue is created. ++ */ ++ dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->kgd, ++ qpd->vmid, ++ qpd->page_table_base); ++ /* invalidate the VM context after pasid and vmid mapping is set up */ ++ kfd_flush_tlb(qpd_to_pdd(qpd)); + +- dqm->dev->kfd2kgd->set_scratch_backing_va( +- dqm->dev->kgd, qpd->sh_hidden_private_base, qpd->vmid); ++ dqm->dev->kfd2kgd->set_scratch_backing_va( ++ dqm->dev->kgd, qpd->sh_hidden_private_base, qpd->vmid); + +- return 0; ++ return 0; + } + + static int flush_texture_cache_nocpsch(struct kfd_dev *kdev, +- struct qcm_process_device *qpd) ++ struct qcm_process_device *qpd) + { +- const struct packet_manager_funcs *pmf = qpd->dqm->packets.pmf; +- int ret; ++ const struct packet_manager_funcs *pmf = qpd->dqm->packets.pmf; ++ int ret; + +- if (!qpd->ib_kaddr) +- return -ENOMEM; ++ if (!qpd->ib_kaddr) ++ return -ENOMEM; + +- ret = pmf->release_mem(qpd->ib_base, (uint32_t *)qpd->ib_kaddr); +- if (ret) +- return ret; ++ ret = pmf->release_mem(qpd->ib_base, (uint32_t *)qpd->ib_kaddr); ++ if (ret) ++ return ret; + +- return amdgpu_amdkfd_submit_ib(kdev->kgd, KGD_ENGINE_MEC1, qpd->vmid, +- qpd->ib_base, (uint32_t *)qpd->ib_kaddr, +- pmf->release_mem_size / sizeof(uint32_t)); ++ return amdgpu_amdkfd_submit_ib(kdev->kgd, KGD_ENGINE_MEC1, qpd->vmid, ++ qpd->ib_base, (uint32_t *)qpd->ib_kaddr, ++ pmf->release_mem_size / sizeof(uint32_t)); + } + + static void deallocate_vmid(struct device_queue_manager *dqm, +- struct qcm_process_device *qpd, +- struct queue *q) ++ struct qcm_process_device *qpd, ++ struct queue *q) + { +- int bit = qpd->vmid - dqm->dev->vm_info.first_vmid_kfd; ++ int bit = qpd->vmid - dqm->dev->vm_info.first_vmid_kfd; + +- /* On GFX v7, CP doesn't flush TC at dequeue */ +- if (q->device->device_info->asic_family == CHIP_HAWAII) +- if (flush_texture_cache_nocpsch(q->device, qpd)) +- pr_err("Failed to flush TC\n"); ++ /* On GFX v7, CP doesn't flush TC at dequeue */ ++ if (q->device->device_info->asic_family == CHIP_HAWAII) ++ if (flush_texture_cache_nocpsch(q->device, qpd)) ++ pr_err("Failed to flush TC\n"); + +- kfd_flush_tlb(qpd_to_pdd(qpd)); ++ kfd_flush_tlb(qpd_to_pdd(qpd)); + +- /* Release the vmid mapping */ +- set_pasid_vmid_mapping(dqm, 0, qpd->vmid); ++ /* Release the vmid mapping */ ++ set_pasid_vmid_mapping(dqm, 0, qpd->vmid); + +- dqm->vmid_bitmap |= (1 << bit); +- qpd->vmid = 0; +- q->properties.vmid = 0; ++ dqm->vmid_bitmap |= (1 << bit); ++ qpd->vmid = 0; ++ q->properties.vmid = 0; + } + + static int create_queue_nocpsch(struct device_queue_manager *dqm, +- struct queue *q, +- struct qcm_process_device *qpd) ++ struct queue *q, ++ struct qcm_process_device *qpd) + { +- struct mqd_manager *mqd_mgr; +- int retval; ++ struct mqd_manager *mqd_mgr; ++ int retval; + +- print_queue(q); ++ print_queue(q); + +- dqm_lock(dqm); ++ dqm_lock(dqm); + +- if (dqm->total_queue_count >= max_num_of_queues_per_device) { +- pr_warn("Can't create new usermode queue because %d queues were already created\n", +- dqm->total_queue_count); +- retval = -EPERM; +- goto out_unlock; +- } ++ if (dqm->total_queue_count >= max_num_of_queues_per_device) { ++ pr_warn("Can't create new usermode queue because %d queues were already created\n", ++ dqm->total_queue_count); ++ retval = -EPERM; ++ goto out_unlock; ++ } + +- if (list_empty(&qpd->queues_list)) { +- retval = allocate_vmid(dqm, qpd, q); +- if (retval) +- goto out_unlock; +- } +- q->properties.vmid = qpd->vmid; +- /* +- * Eviction state logic: mark all queues as evicted, even ones +- * not currently active. Restoring inactive queues later only +- * updates the is_evicted flag but is a no-op otherwise. +- */ +- q->properties.is_evicted = !!qpd->evicted; +- +- q->properties.tba_addr = qpd->tba_addr; +- q->properties.tma_addr = qpd->tma_addr; +- +- mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( +- q->properties.type)]; +- if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) { +- retval = allocate_hqd(dqm, q); +- if (retval) +- goto deallocate_vmid; +- pr_debug("Loading mqd to hqd on pipe %d, queue %d\n", +- q->pipe, q->queue); +- } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || +- q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { +- retval = allocate_sdma_queue(dqm, q); +- if (retval) +- goto deallocate_vmid; +- dqm->asic_ops.init_sdma_vm(dqm, q, qpd); +- } +- +- retval = allocate_doorbell(qpd, q); +- if (retval) +- goto out_deallocate_hqd; +- +- /* Temporarily release dqm lock to avoid a circular lock dependency */ +- dqm_unlock(dqm); +- q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties); +- dqm_lock(dqm); +- +- if (!q->mqd_mem_obj) { +- retval = -ENOMEM; +- goto out_deallocate_doorbell; +- } +- mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, +- &q->gart_mqd_addr, &q->properties); +- if (q->properties.is_active) { +- +- if (WARN(q->process->mm != current->mm, +- "should only run in user thread")) +- retval = -EFAULT; +- else +- retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, +- q->queue, &q->properties, current->mm); ++ if (list_empty(&qpd->queues_list)) { ++ retval = allocate_vmid(dqm, qpd, q); ++ if (retval) ++ goto out_unlock; ++ } ++ q->properties.vmid = qpd->vmid; ++ /* ++ * Eviction state logic: mark all queues as evicted, even ones ++ * not currently active. Restoring inactive queues later only ++ * updates the is_evicted flag but is a no-op otherwise. ++ */ ++ q->properties.is_evicted = !!qpd->evicted; ++ ++ q->properties.tba_addr = qpd->tba_addr; ++ q->properties.tma_addr = qpd->tma_addr; ++ ++ mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( ++ q->properties.type)]; ++ if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) { ++ retval = allocate_hqd(dqm, q); ++ if (retval) ++ goto deallocate_vmid; ++ pr_debug("Loading mqd to hqd on pipe %d, queue %d\n", ++ q->pipe, q->queue); ++ } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || ++ q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { ++ retval = allocate_sdma_queue(dqm, q); ++ if (retval) ++ goto deallocate_vmid; ++ dqm->asic_ops.init_sdma_vm(dqm, q, qpd); ++ } ++ ++ retval = allocate_doorbell(qpd, q); + if (retval) +- goto out_free_mqd; +- } ++ goto out_deallocate_hqd; + +- list_add(&q->list, &qpd->queues_list); +- qpd->queue_count++; +- if (q->properties.is_active) +- dqm->queue_count++; ++ /* Temporarily release dqm lock to avoid a circular lock dependency */ ++ dqm_unlock(dqm); ++ q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties); ++ dqm_lock(dqm); ++ ++ if (!q->mqd_mem_obj) { ++ retval = -ENOMEM; ++ goto out_deallocate_doorbell; ++ } ++ mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, ++ &q->gart_mqd_addr, &q->properties); ++ if (q->properties.is_active) { + +- if (q->properties.type == KFD_QUEUE_TYPE_SDMA) +- dqm->sdma_queue_count++; +- else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) +- dqm->xgmi_sdma_queue_count++; ++ if (WARN(q->process->mm != current->mm, ++ "should only run in user thread")) ++ retval = -EFAULT; ++ else ++ retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, ++ q->queue, &q->properties, current->mm); ++ if (retval) ++ goto out_free_mqd; ++ } + +- /* +- * Unconditionally increment this counter, regardless of the queue's +- * type or whether the queue is active. +- */ +- dqm->total_queue_count++; +- pr_debug("Total of %d queues are accountable so far\n", +- dqm->total_queue_count); +- goto out_unlock; ++ list_add(&q->list, &qpd->queues_list); ++ qpd->queue_count++; ++ if (q->properties.is_active) ++ dqm->queue_count++; ++ ++ if (q->properties.type == KFD_QUEUE_TYPE_SDMA) ++ dqm->sdma_queue_count++; ++ else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) ++ dqm->xgmi_sdma_queue_count++; ++ ++ /* ++ * Unconditionally increment this counter, regardless of the queue's ++ * type or whether the queue is active. ++ */ ++ dqm->total_queue_count++; ++ pr_debug("Total of %d queues are accountable so far\n", ++ dqm->total_queue_count); ++ goto out_unlock; + + out_free_mqd: +- mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); ++ mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); + out_deallocate_doorbell: +- deallocate_doorbell(qpd, q); ++ deallocate_doorbell(qpd, q); + out_deallocate_hqd: +- if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) +- deallocate_hqd(dqm, q); +- else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || +- q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) +- deallocate_sdma_queue(dqm, q); ++ if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) ++ deallocate_hqd(dqm, q); ++ else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || ++ q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) ++ deallocate_sdma_queue(dqm, q); + deallocate_vmid: +- if (list_empty(&qpd->queues_list)) +- deallocate_vmid(dqm, qpd, q); ++ if (list_empty(&qpd->queues_list)) ++ deallocate_vmid(dqm, qpd, q); + out_unlock: +- dqm_unlock(dqm); +- return retval; ++ dqm_unlock(dqm); ++ return retval; + } + + static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q) + { +- bool set; +- int pipe, bit, i; ++ bool set; ++ int pipe, bit, i; + +- set = false; ++ set = false; + +- for (pipe = dqm->next_pipe_to_allocate, i = 0; +- i < get_pipes_per_mec(dqm); +- pipe = ((pipe + 1) % get_pipes_per_mec(dqm)), ++i) { ++ for (pipe = dqm->next_pipe_to_allocate, i = 0; ++ i < get_pipes_per_mec(dqm); ++ pipe = ((pipe + 1) % get_pipes_per_mec(dqm)), ++i) { + +- if (!is_pipe_enabled(dqm, 0, pipe)) +- continue; ++ if (!is_pipe_enabled(dqm, 0, pipe)) ++ continue; + +- if (dqm->allocated_queues[pipe] != 0) { +- bit = ffs(dqm->allocated_queues[pipe]) - 1; +- dqm->allocated_queues[pipe] &= ~(1 << bit); +- q->pipe = pipe; +- q->queue = bit; +- set = true; +- break; ++ if (dqm->allocated_queues[pipe] != 0) { ++ bit = ffs(dqm->allocated_queues[pipe]) - 1; ++ dqm->allocated_queues[pipe] &= ~(1 << bit); ++ q->pipe = pipe; ++ q->queue = bit; ++ set = true; ++ break; ++ } + } +- } + +- if (!set) +- return -EBUSY; ++ if (!set) ++ return -EBUSY; + +- pr_debug("hqd slot - pipe %d, queue %d\n", q->pipe, q->queue); +- /* horizontal hqd allocation */ +- dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_per_mec(dqm); ++ pr_debug("hqd slot - pipe %d, queue %d\n", q->pipe, q->queue); ++ /* horizontal hqd allocation */ ++ dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_per_mec(dqm); + +- return 0; ++ return 0; + } + + static inline void deallocate_hqd(struct device_queue_manager *dqm, +- struct queue *q) ++ struct queue *q) + { +- dqm->allocated_queues[q->pipe] |= (1 << q->queue); ++ dqm->allocated_queues[q->pipe] |= (1 << q->queue); + } + + /* Access to DQM has to be locked before calling destroy_queue_nocpsch_locked + * to avoid asynchronized access + */ + static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm, +- struct qcm_process_device *qpd, +- struct queue *q) +-{ +- int retval; +- struct mqd_manager *mqd_mgr; +- +- mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( +- q->properties.type)]; +- +- if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) { +- deallocate_hqd(dqm, q); +- } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { +- dqm->sdma_queue_count--; +- deallocate_sdma_queue(dqm, q); +- } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { +- dqm->xgmi_sdma_queue_count--; +- deallocate_sdma_queue(dqm, q); +- } else { +- pr_debug("q->properties.type %d is invalid\n", +- q->properties.type); +- return -EINVAL; +- } +- dqm->total_queue_count--; +- +- deallocate_doorbell(qpd, q); +- +- retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, +- KFD_PREEMPT_TYPE_WAVEFRONT_RESET, +- KFD_UNMAP_LATENCY_MS, +- q->pipe, q->queue); +- if (retval == -ETIME) +- qpd->reset_wavefronts = true; +- +- mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); +- +- list_del(&q->list); +- if (list_empty(&qpd->queues_list)) { +- if (qpd->reset_wavefronts) { +- pr_warn("Resetting wave fronts (nocpsch) on dev %p\n", +- dqm->dev); +- /* dbgdev_wave_reset_wavefronts has to be called before +- * deallocate_vmid(), i.e. when vmid is still in use. +- */ +- dbgdev_wave_reset_wavefronts(dqm->dev, +- qpd->pqm->process); +- qpd->reset_wavefronts = false; +- } +- +- deallocate_vmid(dqm, qpd, q); +- } +- qpd->queue_count--; +- if (q->properties.is_active) +- dqm->queue_count--; ++ struct qcm_process_device *qpd, ++ struct queue *q) ++{ ++ int retval; ++ struct mqd_manager *mqd_mgr; ++ ++ mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( ++ q->properties.type)]; ++ ++ if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) { ++ deallocate_hqd(dqm, q); ++ } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { ++ dqm->sdma_queue_count--; ++ deallocate_sdma_queue(dqm, q); ++ } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { ++ dqm->xgmi_sdma_queue_count--; ++ deallocate_sdma_queue(dqm, q); ++ } else { ++ pr_debug("q->properties.type %d is invalid\n", ++ q->properties.type); ++ return -EINVAL; ++ } ++ dqm->total_queue_count--; ++ ++ deallocate_doorbell(qpd, q); + +- return retval; ++ retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, ++ KFD_PREEMPT_TYPE_WAVEFRONT_RESET, ++ KFD_UNMAP_LATENCY_MS, ++ q->pipe, q->queue); ++ if (retval == -ETIME) ++ qpd->reset_wavefronts = true; ++ ++ mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); ++ ++ list_del(&q->list); ++ if (list_empty(&qpd->queues_list)) { ++ if (qpd->reset_wavefronts) { ++ pr_warn("Resetting wave fronts (nocpsch) on dev %p\n", ++ dqm->dev); ++ /* dbgdev_wave_reset_wavefronts has to be called before ++ * deallocate_vmid(), i.e. when vmid is still in use. ++ */ ++ dbgdev_wave_reset_wavefronts(dqm->dev, ++ qpd->pqm->process); ++ qpd->reset_wavefronts = false; ++ } ++ ++ deallocate_vmid(dqm, qpd, q); ++ } ++ qpd->queue_count--; ++ if (q->properties.is_active) ++ dqm->queue_count--; ++ ++ return retval; + } + + static int destroy_queue_nocpsch(struct device_queue_manager *dqm, +- struct qcm_process_device *qpd, +- struct queue *q) ++ struct qcm_process_device *qpd, ++ struct queue *q) + { +- int retval; ++ int retval; + +- dqm_lock(dqm); +- retval = destroy_queue_nocpsch_locked(dqm, qpd, q); +- dqm_unlock(dqm); ++ dqm_lock(dqm); ++ retval = destroy_queue_nocpsch_locked(dqm, qpd, q); ++ dqm_unlock(dqm); + +- return retval; ++ return retval; + } + + static int update_queue(struct device_queue_manager *dqm, struct queue *q) + { +- int retval = 0; +- struct mqd_manager *mqd_mgr; +- struct kfd_process_device *pdd; +- bool prev_active = false; ++ int retval = 0; ++ struct mqd_manager *mqd_mgr; ++ struct kfd_process_device *pdd; ++ bool prev_active = false; + +- dqm_lock(dqm); +- pdd = kfd_get_process_device_data(q->device, q->process); +- if (!pdd) { +- retval = -ENODEV; +- goto out_unlock; +- } +- mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( +- q->properties.type)]; +- +- /* Save previous activity state for counters */ +- prev_active = q->properties.is_active; +- +- /* Make sure the queue is unmapped before updating the MQD */ +- if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) { +- retval = unmap_queues_cpsch(dqm, +- KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, +- USE_DEFAULT_GRACE_PERIOD); +- if (retval) { +- pr_err("unmap queue failed\n"); +- goto out_unlock; ++ dqm_lock(dqm); ++ pdd = kfd_get_process_device_data(q->device, q->process); ++ if (!pdd) { ++ retval = -ENODEV; ++ goto out_unlock; + } +- } else if (prev_active && +- (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || +- q->properties.type == KFD_QUEUE_TYPE_SDMA || +- q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { +- retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, +- KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN, +- KFD_UNMAP_LATENCY_MS, q->pipe, q->queue); +- if (retval) { +- pr_err("destroy mqd failed\n"); +- goto out_unlock; ++ mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( ++ q->properties.type)]; ++ ++ /* Save previous activity state for counters */ ++ prev_active = q->properties.is_active; ++ ++ /* Make sure the queue is unmapped before updating the MQD */ ++ if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) { ++ retval = unmap_queues_cpsch(dqm, ++ KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, ++ USE_DEFAULT_GRACE_PERIOD); ++ if (retval) { ++ pr_err("unmap queue failed\n"); ++ goto out_unlock; ++ } ++ } else if (prev_active && ++ (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || ++ q->properties.type == KFD_QUEUE_TYPE_SDMA || ++ q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { ++ retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, ++ KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN, ++ KFD_UNMAP_LATENCY_MS, q->pipe, q->queue); ++ if (retval) { ++ pr_err("destroy mqd failed\n"); ++ goto out_unlock; ++ } + } +- } + +- mqd_mgr->update_mqd(mqd_mgr, q->mqd, &q->properties); ++ mqd_mgr->update_mqd(mqd_mgr, q->mqd, &q->properties); + +- /* +- * check active state vs. the previous state and modify +- * counter accordingly. map_queues_cpsch uses the +- * dqm->queue_count to determine whether a new runlist must be +- * uploaded. +- */ +- if (q->properties.is_active && !prev_active) +- dqm->queue_count++; +- else if (!q->properties.is_active && prev_active) +- dqm->queue_count--; ++ /* ++ * check active state vs. the previous state and modify ++ * counter accordingly. map_queues_cpsch uses the ++ * dqm->queue_count to determine whether a new runlist must be ++ * uploaded. ++ */ ++ if (q->properties.is_active && !prev_active) ++ dqm->queue_count++; ++ else if (!q->properties.is_active && prev_active) ++ dqm->queue_count--; + +- if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) +- retval = map_queues_cpsch(dqm); +- else if (q->properties.is_active && +- (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || +- q->properties.type == KFD_QUEUE_TYPE_SDMA || +- q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { +- if (WARN(q->process->mm != current->mm, ++ if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) ++ retval = map_queues_cpsch(dqm); ++ else if (q->properties.is_active && ++ (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || ++ q->properties.type == KFD_QUEUE_TYPE_SDMA || ++ q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { ++ if (WARN(q->process->mm != current->mm, + "should only run in user thread")) +- retval = -EFAULT; +- else +- retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, +- q->pipe, q->queue, +- &q->properties, current->mm); +- } ++ retval = -EFAULT; ++ else ++ retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, ++ q->pipe, q->queue, ++ &q->properties, current->mm); ++ } + + out_unlock: +- dqm_unlock(dqm); +- return retval; ++ dqm_unlock(dqm); ++ return retval; + } + + /* suspend_single_queue does not lock the dqm like the +@@ -605,22 +605,22 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q) + * multiple times, we will just keep the dqm locked for all of the calls. + */ + static int suspend_single_queue(struct device_queue_manager *dqm, +- struct kfd_process_device *pdd, +- struct queue *q) ++ struct kfd_process_device *pdd, ++ struct queue *q) + { +- int retval = 0; ++ int retval = 0; + +- pr_debug("Suspending PASID %u queue [%i]\n", +- pdd->process->pasid, +- q->properties.queue_id); ++ pr_debug("Suspending PASID %u queue [%i]\n", ++ pdd->process->pasid, ++ q->properties.queue_id); + +- q->properties.is_suspended = true; +- if (q->properties.is_active) { +- dqm->queue_count--; +- q->properties.is_active = false; +- } ++ q->properties.is_suspended = true; ++ if (q->properties.is_active) { ++ dqm->queue_count--; ++ q->properties.is_active = false; ++ } + +- return retval; ++ return retval; + } + + /* resume_single_queue does not lock the dqm like the functions +@@ -632,419 +632,419 @@ static int suspend_single_queue(struct device_queue_manager *dqm, + * multiple times, we will just keep the dqm locked for all of the calls. + */ + static int resume_single_queue(struct device_queue_manager *dqm, +- struct qcm_process_device *qpd, +- struct queue *q) ++ struct qcm_process_device *qpd, ++ struct queue *q) + { +- struct kfd_process_device *pdd; +- uint64_t pd_base; +- int retval = 0; ++ struct kfd_process_device *pdd; ++ uint64_t pd_base; ++ int retval = 0; + +- pdd = qpd_to_pdd(qpd); +- /* Retrieve PD base */ +- pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm); ++ pdd = qpd_to_pdd(qpd); ++ /* Retrieve PD base */ ++ pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm); + +- pr_debug("Restoring from suspend PASID %u queue [%i]\n", +- pdd->process->pasid, +- q->properties.queue_id); ++ pr_debug("Restoring from suspend PASID %u queue [%i]\n", ++ pdd->process->pasid, ++ q->properties.queue_id); + +- q->properties.is_suspended = false; ++ q->properties.is_suspended = false; + +- if (QUEUE_IS_ACTIVE(q->properties)) { +- q->properties.is_active = true; +- dqm->queue_count++; +- } ++ if (QUEUE_IS_ACTIVE(q->properties)) { ++ q->properties.is_active = true; ++ dqm->queue_count++; ++ } + +- return retval; ++ return retval; + } + static int evict_process_queues_nocpsch(struct device_queue_manager *dqm, +- struct qcm_process_device *qpd) ++ struct qcm_process_device *qpd) + { +- struct queue *q; +- struct mqd_manager *mqd_mgr; +- struct kfd_process_device *pdd; +- int retval, ret = 0; ++ struct queue *q; ++ struct mqd_manager *mqd_mgr; ++ struct kfd_process_device *pdd; ++ int retval, ret = 0; + +- dqm_lock(dqm); +- if (qpd->evicted++ > 0) /* already evicted, do nothing */ +- goto out; ++ dqm_lock(dqm); ++ if (qpd->evicted++ > 0) /* already evicted, do nothing */ ++ goto out; + +- pdd = qpd_to_pdd(qpd); +- pr_info_ratelimited("Evicting PASID %u queues\n", +- pdd->process->pasid); ++ pdd = qpd_to_pdd(qpd); ++ pr_info_ratelimited("Evicting PASID %u queues\n", ++ pdd->process->pasid); + +- /* Mark all queues as evicted. Deactivate all active queues on +- * the qpd. +- */ +- list_for_each_entry(q, &qpd->queues_list, list) { +- q->properties.is_evicted = true; +- if (!q->properties.is_active) +- continue; ++ /* Mark all queues as evicted. Deactivate all active queues on ++ * the qpd. ++ */ ++ list_for_each_entry(q, &qpd->queues_list, list) { ++ q->properties.is_evicted = true; ++ if (!q->properties.is_active) ++ continue; + +- mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( +- q->properties.type)]; +- q->properties.is_active = false; +- retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, +- KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN, +- KFD_UNMAP_LATENCY_MS, q->pipe, q->queue); +- if (retval && !ret) +- /* Return the first error, but keep going to +- * maintain a consistent eviction state +- */ +- ret = retval; +- dqm->queue_count--; +- } ++ mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( ++ q->properties.type)]; ++ q->properties.is_active = false; ++ retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, ++ KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN, ++ KFD_UNMAP_LATENCY_MS, q->pipe, q->queue); ++ if (retval && !ret) ++ /* Return the first error, but keep going to ++ * maintain a consistent eviction state ++ */ ++ ret = retval; ++ dqm->queue_count--; ++ } + + out: +- dqm_unlock(dqm); +- return ret; ++ dqm_unlock(dqm); ++ return ret; + } + + static int evict_process_queues_cpsch(struct device_queue_manager *dqm, +- struct qcm_process_device *qpd) ++ struct qcm_process_device *qpd) + { +- struct queue *q; +- struct kfd_process_device *pdd; +- int retval = 0; ++ struct queue *q; ++ struct kfd_process_device *pdd; ++ int retval = 0; + +- dqm_lock(dqm); +- if (qpd->evicted++ > 0) /* already evicted, do nothing */ +- goto out; ++ dqm_lock(dqm); ++ if (qpd->evicted++ > 0) /* already evicted, do nothing */ ++ goto out; + +- pdd = qpd_to_pdd(qpd); +- pr_info_ratelimited("Evicting PASID %u queues\n", +- pdd->process->pasid); ++ pdd = qpd_to_pdd(qpd); ++ pr_info_ratelimited("Evicting PASID %u queues\n", ++ pdd->process->pasid); + +- /* Mark all queues as evicted. Deactivate all active queues on +- * the qpd. +- */ +- list_for_each_entry(q, &qpd->queues_list, list) { +- q->properties.is_evicted = true; +- if (!q->properties.is_active) +- continue; ++ /* Mark all queues as evicted. Deactivate all active queues on ++ * the qpd. ++ */ ++ list_for_each_entry(q, &qpd->queues_list, list) { ++ q->properties.is_evicted = true; ++ if (!q->properties.is_active) ++ continue; + +- q->properties.is_active = false; +- dqm->queue_count--; +- } +- retval = execute_queues_cpsch(dqm, +- qpd->is_debug ? +- KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES : +- KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, +- USE_DEFAULT_GRACE_PERIOD); ++ q->properties.is_active = false; ++ dqm->queue_count--; ++ } ++ retval = execute_queues_cpsch(dqm, ++ qpd->is_debug ? ++ KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES : ++ KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, ++ USE_DEFAULT_GRACE_PERIOD); + + out: +- dqm_unlock(dqm); +- return retval; ++ dqm_unlock(dqm); ++ return retval; + } + + static int restore_process_queues_nocpsch(struct device_queue_manager *dqm, +- struct qcm_process_device *qpd) +-{ +- struct mm_struct *mm = NULL; +- struct queue *q; +- struct mqd_manager *mqd_mgr; +- struct kfd_process_device *pdd; +- uint64_t pd_base; +- int retval, ret = 0; +- +- pdd = qpd_to_pdd(qpd); +- /* Retrieve PD base */ +- pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm); +- +- dqm_lock(dqm); +- if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */ +- goto out; +- if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */ +- qpd->evicted--; +- goto out; +- } +- +- pr_info_ratelimited("Restoring PASID %u queues\n", +- pdd->process->pasid); +- +- /* Update PD Base in QPD */ +- qpd->page_table_base = pd_base; +- pr_debug("Updated PD address to 0x%llx\n", pd_base); +- +- if (!list_empty(&qpd->queues_list)) { +- dqm->dev->kfd2kgd->set_vm_context_page_table_base( +- dqm->dev->kgd, +- qpd->vmid, +- qpd->page_table_base); +- kfd_flush_tlb(pdd); +- } +- +- /* Take a safe reference to the mm_struct, which may otherwise +- * disappear even while the kfd_process is still referenced. +- */ +- mm = get_task_mm(pdd->process->lead_thread); +- if (!mm) { +- ret = -EFAULT; +- goto out; +- } +- +- /* Remove the eviction flags. Activate queues that are not +- * inactive for other reasons. +- */ +- list_for_each_entry(q, &qpd->queues_list, list) { +- q->properties.is_evicted = false; +- if (!QUEUE_IS_ACTIVE(q->properties)) +- continue; ++ struct qcm_process_device *qpd) ++{ ++ struct mm_struct *mm = NULL; ++ struct queue *q; ++ struct mqd_manager *mqd_mgr; ++ struct kfd_process_device *pdd; ++ uint64_t pd_base; ++ int retval, ret = 0; + +- mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( +- q->properties.type)]; +- q->properties.is_active = true; +- retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, +- q->queue, &q->properties, mm); +- if (retval && !ret) +- /* Return the first error, but keep going to +- * maintain a consistent eviction state +- */ +- ret = retval; +- dqm->queue_count++; +- } +- qpd->evicted = 0; ++ pdd = qpd_to_pdd(qpd); ++ /* Retrieve PD base */ ++ pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm); ++ ++ dqm_lock(dqm); ++ if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */ ++ goto out; ++ if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */ ++ qpd->evicted--; ++ goto out; ++ } ++ ++ pr_info_ratelimited("Restoring PASID %u queues\n", ++ pdd->process->pasid); ++ ++ /* Update PD Base in QPD */ ++ qpd->page_table_base = pd_base; ++ pr_debug("Updated PD address to 0x%llx\n", pd_base); ++ ++ if (!list_empty(&qpd->queues_list)) { ++ dqm->dev->kfd2kgd->set_vm_context_page_table_base( ++ dqm->dev->kgd, ++ qpd->vmid, ++ qpd->page_table_base); ++ kfd_flush_tlb(pdd); ++ } ++ ++ /* Take a safe reference to the mm_struct, which may otherwise ++ * disappear even while the kfd_process is still referenced. ++ */ ++ mm = get_task_mm(pdd->process->lead_thread); ++ if (!mm) { ++ ret = -EFAULT; ++ goto out; ++ } ++ ++ /* Remove the eviction flags. Activate queues that are not ++ * inactive for other reasons. ++ */ ++ list_for_each_entry(q, &qpd->queues_list, list) { ++ q->properties.is_evicted = false; ++ if (!QUEUE_IS_ACTIVE(q->properties)) ++ continue; ++ ++ mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( ++ q->properties.type)]; ++ q->properties.is_active = true; ++ retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, ++ q->queue, &q->properties, mm); ++ if (retval && !ret) ++ /* Return the first error, but keep going to ++ * maintain a consistent eviction state ++ */ ++ ret = retval; ++ dqm->queue_count++; ++ } ++ qpd->evicted = 0; + out: +- if (mm) +- mmput(mm); +- dqm_unlock(dqm); +- return ret; ++ if (mm) ++ mmput(mm); ++ dqm_unlock(dqm); ++ return ret; + } + + static int restore_process_queues_cpsch(struct device_queue_manager *dqm, +- struct qcm_process_device *qpd) +-{ +- struct queue *q; +- struct kfd_process_device *pdd; +- uint64_t pd_base; +- int retval = 0; +- +- pdd = qpd_to_pdd(qpd); +- /* Retrieve PD base */ +- pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm); +- +- dqm_lock(dqm); +- if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */ +- goto out; +- if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */ +- qpd->evicted--; +- goto out; +- } +- +- pr_info_ratelimited("Restoring PASID %u queues\n", +- pdd->process->pasid); +- +- /* Update PD Base in QPD */ +- qpd->page_table_base = pd_base; +- pr_debug("Updated PD address to 0x%llx\n", pd_base); +- +- /* activate all active queues on the qpd */ +- list_for_each_entry(q, &qpd->queues_list, list) { +- q->properties.is_evicted = false; +- if (!QUEUE_IS_ACTIVE(q->properties)) +- continue; +- +- q->properties.is_active = true; +- dqm->queue_count++; +- } +- retval = execute_queues_cpsch(dqm, +- KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, +- USE_DEFAULT_GRACE_PERIOD); +- qpd->evicted = 0; ++ struct qcm_process_device *qpd) ++{ ++ struct queue *q; ++ struct kfd_process_device *pdd; ++ uint64_t pd_base; ++ int retval = 0; ++ ++ pdd = qpd_to_pdd(qpd); ++ /* Retrieve PD base */ ++ pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm); ++ ++ dqm_lock(dqm); ++ if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */ ++ goto out; ++ if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */ ++ qpd->evicted--; ++ goto out; ++ } ++ ++ pr_info_ratelimited("Restoring PASID %u queues\n", ++ pdd->process->pasid); ++ ++ /* Update PD Base in QPD */ ++ qpd->page_table_base = pd_base; ++ pr_debug("Updated PD address to 0x%llx\n", pd_base); ++ ++ /* activate all active queues on the qpd */ ++ list_for_each_entry(q, &qpd->queues_list, list) { ++ q->properties.is_evicted = false; ++ if (!QUEUE_IS_ACTIVE(q->properties)) ++ continue; ++ ++ q->properties.is_active = true; ++ dqm->queue_count++; ++ } ++ retval = execute_queues_cpsch(dqm, ++ KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, ++ USE_DEFAULT_GRACE_PERIOD); ++ qpd->evicted = 0; + out: +- dqm_unlock(dqm); +- return retval; ++ dqm_unlock(dqm); ++ return retval; + } + + static int register_process(struct device_queue_manager *dqm, +- struct qcm_process_device *qpd) ++ struct qcm_process_device *qpd) + { +- struct device_process_node *n; +- struct kfd_process_device *pdd; +- uint64_t pd_base; +- int retval; ++ struct device_process_node *n; ++ struct kfd_process_device *pdd; ++ uint64_t pd_base; ++ int retval; + +- n = kzalloc(sizeof(*n), GFP_KERNEL); +- if (!n) +- return -ENOMEM; ++ n = kzalloc(sizeof(*n), GFP_KERNEL); ++ if (!n) ++ return -ENOMEM; + +- n->qpd = qpd; ++ n->qpd = qpd; + +- pdd = qpd_to_pdd(qpd); +- /* Retrieve PD base */ +- pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm); ++ pdd = qpd_to_pdd(qpd); ++ /* Retrieve PD base */ ++ pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm); + +- dqm_lock(dqm); +- list_add(&n->list, &dqm->queues); ++ dqm_lock(dqm); ++ list_add(&n->list, &dqm->queues); + +- /* Update PD Base in QPD */ +- qpd->page_table_base = pd_base; +- pr_debug("Updated PD address to 0x%llx\n", pd_base); ++ /* Update PD Base in QPD */ ++ qpd->page_table_base = pd_base; ++ pr_debug("Updated PD address to 0x%llx\n", pd_base); + +- retval = dqm->asic_ops.update_qpd(dqm, qpd); ++ retval = dqm->asic_ops.update_qpd(dqm, qpd); + +- dqm->processes_count++; +- kfd_inc_compute_active(dqm->dev); ++ dqm->processes_count++; ++ kfd_inc_compute_active(dqm->dev); + +- dqm_unlock(dqm); ++ dqm_unlock(dqm); + +- return retval; ++ return retval; + } + + static int unregister_process(struct device_queue_manager *dqm, +- struct qcm_process_device *qpd) ++ struct qcm_process_device *qpd) + { +- int retval; +- struct device_process_node *cur, *next; ++ int retval; ++ struct device_process_node *cur, *next; + +- pr_debug("qpd->queues_list is %s\n", +- list_empty(&qpd->queues_list) ? "empty" : "not empty"); ++ pr_debug("qpd->queues_list is %s\n", ++ list_empty(&qpd->queues_list) ? "empty" : "not empty"); + +- retval = 0; +- dqm_lock(dqm); ++ retval = 0; ++ dqm_lock(dqm); + +- list_for_each_entry_safe(cur, next, &dqm->queues, list) { +- if (qpd == cur->qpd) { +- list_del(&cur->list); +- kfree(cur); +- dqm->processes_count--; +- kfd_dec_compute_active(dqm->dev); +- goto out; ++ list_for_each_entry_safe(cur, next, &dqm->queues, list) { ++ if (qpd == cur->qpd) { ++ list_del(&cur->list); ++ kfree(cur); ++ dqm->processes_count--; ++ kfd_dec_compute_active(dqm->dev); ++ goto out; ++ } + } +- } +- /* qpd not found in dqm list */ +- retval = 1; ++ /* qpd not found in dqm list */ ++ retval = 1; + out: +- dqm_unlock(dqm); +- return retval; ++ dqm_unlock(dqm); ++ return retval; + } + +- static int ++static int + set_pasid_vmid_mapping(struct device_queue_manager *dqm, unsigned int pasid, +- unsigned int vmid) ++ unsigned int vmid) + { +- return dqm->dev->kfd2kgd->set_pasid_vmid_mapping( +- dqm->dev->kgd, pasid, vmid); ++ return dqm->dev->kfd2kgd->set_pasid_vmid_mapping( ++ dqm->dev->kgd, pasid, vmid); + } + + static void init_interrupts(struct device_queue_manager *dqm) + { +- unsigned int i; ++ unsigned int i; + +- for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++) +- if (is_pipe_enabled(dqm, 0, i)) +- dqm->dev->kfd2kgd->init_interrupts(dqm->dev->kgd, i); ++ for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++) ++ if (is_pipe_enabled(dqm, 0, i)) ++ dqm->dev->kfd2kgd->init_interrupts(dqm->dev->kgd, i); + } + + static int initialize_nocpsch(struct device_queue_manager *dqm) + { +- int pipe, queue; ++ int pipe, queue; + +- pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm)); ++ pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm)); + +- dqm->allocated_queues = kcalloc(get_pipes_per_mec(dqm), +- sizeof(unsigned int), GFP_KERNEL); +- if (!dqm->allocated_queues) +- return -ENOMEM; ++ dqm->allocated_queues = kcalloc(get_pipes_per_mec(dqm), ++ sizeof(unsigned int), GFP_KERNEL); ++ if (!dqm->allocated_queues) ++ return -ENOMEM; + +- mutex_init(&dqm->lock_hidden); +- INIT_LIST_HEAD(&dqm->queues); +- dqm->queue_count = dqm->next_pipe_to_allocate = 0; +- dqm->sdma_queue_count = 0; +- dqm->xgmi_sdma_queue_count = 0; +- dqm->trap_debug_vmid = 0; ++ mutex_init(&dqm->lock_hidden); ++ INIT_LIST_HEAD(&dqm->queues); ++ dqm->queue_count = dqm->next_pipe_to_allocate = 0; ++ dqm->sdma_queue_count = 0; ++ dqm->xgmi_sdma_queue_count = 0; ++ dqm->trap_debug_vmid = 0; + +- for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) { +- int pipe_offset = pipe * get_queues_per_pipe(dqm); ++ for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) { ++ int pipe_offset = pipe * get_queues_per_pipe(dqm); + +- for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) +- if (test_bit(pipe_offset + queue, +- dqm->dev->shared_resources.queue_bitmap)) +- dqm->allocated_queues[pipe] |= 1 << queue; +- } ++ for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) ++ if (test_bit(pipe_offset + queue, ++ dqm->dev->shared_resources.queue_bitmap)) ++ dqm->allocated_queues[pipe] |= 1 << queue; ++ } + +- dqm->vmid_bitmap = (1 << dqm->dev->vm_info.vmid_num_kfd) - 1; +- dqm->sdma_bitmap = ~0ULL >> (64 - get_num_sdma_queues(dqm)); +- dqm->xgmi_sdma_bitmap = ~0ULL >> (64 - get_num_xgmi_sdma_queues(dqm)); ++ dqm->vmid_bitmap = (1 << dqm->dev->vm_info.vmid_num_kfd) - 1; ++ dqm->sdma_bitmap = ~0ULL >> (64 - get_num_sdma_queues(dqm)); ++ dqm->xgmi_sdma_bitmap = ~0ULL >> (64 - get_num_xgmi_sdma_queues(dqm)); + +- return 0; ++ return 0; + } + + static void uninitialize(struct device_queue_manager *dqm) + { +- int i; ++ int i; + +- WARN_ON(dqm->queue_count > 0 || dqm->processes_count > 0); ++ WARN_ON(dqm->queue_count > 0 || dqm->processes_count > 0); + +- kfree(dqm->allocated_queues); +- for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++) +- kfree(dqm->mqd_mgrs[i]); +- mutex_destroy(&dqm->lock_hidden); +- kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem); ++ kfree(dqm->allocated_queues); ++ for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++) ++ kfree(dqm->mqd_mgrs[i]); ++ mutex_destroy(&dqm->lock_hidden); ++ kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem); + } + + static int start_nocpsch(struct device_queue_manager *dqm) + { +- init_interrupts(dqm); +- return pm_init(&dqm->packets, dqm); ++ init_interrupts(dqm); ++ return pm_init(&dqm->packets, dqm); + } + + static int stop_nocpsch(struct device_queue_manager *dqm) + { +- pm_uninit(&dqm->packets); +- return 0; ++ pm_uninit(&dqm->packets); ++ return 0; + } + + static int allocate_sdma_queue(struct device_queue_manager *dqm, +- struct queue *q) +-{ +- int bit; +- +- if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { +- if (dqm->sdma_bitmap == 0) +- return -ENOMEM; +- bit = __ffs64(dqm->sdma_bitmap); +- dqm->sdma_bitmap &= ~(1ULL << bit); +- q->sdma_id = bit; +- q->properties.sdma_engine_id = q->sdma_id % +- get_num_sdma_engines(dqm); +- q->properties.sdma_queue_id = q->sdma_id / +- get_num_sdma_engines(dqm); +- } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { +- if (dqm->xgmi_sdma_bitmap == 0) +- return -ENOMEM; +- bit = __ffs64(dqm->xgmi_sdma_bitmap); +- dqm->xgmi_sdma_bitmap &= ~(1ULL << bit); +- q->sdma_id = bit; +- /* sdma_engine_id is sdma id including +- * both PCIe-optimized SDMAs and XGMI- +- * optimized SDMAs. The calculation below +- * assumes the first N engines are always +- * PCIe-optimized ones +- */ +- q->properties.sdma_engine_id = get_num_sdma_engines(dqm) + +- q->sdma_id % get_num_xgmi_sdma_engines(dqm); +- q->properties.sdma_queue_id = q->sdma_id / +- get_num_xgmi_sdma_engines(dqm); +- } ++ struct queue *q) ++{ ++ int bit; + +- pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id); +- pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id); ++ if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { ++ if (dqm->sdma_bitmap == 0) ++ return -ENOMEM; ++ bit = __ffs64(dqm->sdma_bitmap); ++ dqm->sdma_bitmap &= ~(1ULL << bit); ++ q->sdma_id = bit; ++ q->properties.sdma_engine_id = q->sdma_id % ++ get_num_sdma_engines(dqm); ++ q->properties.sdma_queue_id = q->sdma_id / ++ get_num_sdma_engines(dqm); ++ } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { ++ if (dqm->xgmi_sdma_bitmap == 0) ++ return -ENOMEM; ++ bit = __ffs64(dqm->xgmi_sdma_bitmap); ++ dqm->xgmi_sdma_bitmap &= ~(1ULL << bit); ++ q->sdma_id = bit; ++ /* sdma_engine_id is sdma id including ++ * both PCIe-optimized SDMAs and XGMI- ++ * optimized SDMAs. The calculation below ++ * assumes the first N engines are always ++ * PCIe-optimized ones ++ */ ++ q->properties.sdma_engine_id = get_num_sdma_engines(dqm) + ++ q->sdma_id % get_num_xgmi_sdma_engines(dqm); ++ q->properties.sdma_queue_id = q->sdma_id / ++ get_num_xgmi_sdma_engines(dqm); ++ } ++ ++ pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id); ++ pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id); + +- return 0; ++ return 0; + } + + static void deallocate_sdma_queue(struct device_queue_manager *dqm, +- struct queue *q) ++ struct queue *q) + { +- if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { +- if (q->sdma_id >= get_num_sdma_queues(dqm)) +- return; +- dqm->sdma_bitmap |= (1ULL << q->sdma_id); +- } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { +- if (q->sdma_id >= get_num_xgmi_sdma_queues(dqm)) +- return; +- dqm->xgmi_sdma_bitmap |= (1ULL << q->sdma_id); +- } ++ if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { ++ if (q->sdma_id >= get_num_sdma_queues(dqm)) ++ return; ++ dqm->sdma_bitmap |= (1ULL << q->sdma_id); ++ } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { ++ if (q->sdma_id >= get_num_xgmi_sdma_queues(dqm)) ++ return; ++ dqm->xgmi_sdma_bitmap |= (1ULL << q->sdma_id); ++ } + } + + /* +@@ -1053,60 +1053,60 @@ static void deallocate_sdma_queue(struct device_queue_manager *dqm, + + static int set_sched_resources(struct device_queue_manager *dqm) + { +- int i, mec; +- struct scheduling_resources res; ++ int i, mec; ++ struct scheduling_resources res; + +- res.vmid_mask = dqm->dev->shared_resources.compute_vmid_bitmap; ++ res.vmid_mask = dqm->dev->shared_resources.compute_vmid_bitmap; + +- res.queue_mask = 0; +- for (i = 0; i < KGD_MAX_QUEUES; ++i) { +- mec = (i / dqm->dev->shared_resources.num_queue_per_pipe) +- / dqm->dev->shared_resources.num_pipe_per_mec; ++ res.queue_mask = 0; ++ for (i = 0; i < KGD_MAX_QUEUES; ++i) { ++ mec = (i / dqm->dev->shared_resources.num_queue_per_pipe) ++ / dqm->dev->shared_resources.num_pipe_per_mec; + +- if (!test_bit(i, dqm->dev->shared_resources.queue_bitmap)) +- continue; ++ if (!test_bit(i, dqm->dev->shared_resources.queue_bitmap)) ++ continue; + +- /* only acquire queues from the first MEC */ +- if (mec > 0) +- continue; ++ /* only acquire queues from the first MEC */ ++ if (mec > 0) ++ continue; + +- /* This situation may be hit in the future if a new HW +- * generation exposes more than 64 queues. If so, the +- * definition of res.queue_mask needs updating +- */ +- if (WARN_ON(i >= (sizeof(res.queue_mask)*8))) { +- pr_err("Invalid queue enabled by amdgpu: %d\n", i); +- break; +- } ++ /* This situation may be hit in the future if a new HW ++ * generation exposes more than 64 queues. If so, the ++ * definition of res.queue_mask needs updating ++ */ ++ if (WARN_ON(i >= (sizeof(res.queue_mask)*8))) { ++ pr_err("Invalid queue enabled by amdgpu: %d\n", i); ++ break; ++ } + +- res.queue_mask |= (1ull << i); +- } +- res.gws_mask = ~0ull; +- res.oac_mask = res.gds_heap_base = res.gds_heap_size = 0; ++ res.queue_mask |= (1ull << i); ++ } ++ res.gws_mask = ~0ull; ++ res.oac_mask = res.gds_heap_base = res.gds_heap_size = 0; + +- pr_debug("Scheduling resources:\n" +- "vmid mask: 0x%8X\n" +- "queue mask: 0x%8llX\n", +- res.vmid_mask, res.queue_mask); ++ pr_debug("Scheduling resources:\n" ++ "vmid mask: 0x%8X\n" ++ "queue mask: 0x%8llX\n", ++ res.vmid_mask, res.queue_mask); + +- return pm_send_set_resources(&dqm->packets, &res); ++ return pm_send_set_resources(&dqm->packets, &res); + } + + static int initialize_cpsch(struct device_queue_manager *dqm) + { +- pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm)); ++ pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm)); + +- mutex_init(&dqm->lock_hidden); +- INIT_LIST_HEAD(&dqm->queues); +- dqm->queue_count = dqm->processes_count = 0; +- dqm->sdma_queue_count = 0; +- dqm->xgmi_sdma_queue_count = 0; +- dqm->active_runlist = false; +- dqm->sdma_bitmap = ~0ULL >> (64 - get_num_sdma_queues(dqm)); +- dqm->xgmi_sdma_bitmap = ~0ULL >> (64 - get_num_xgmi_sdma_queues(dqm)); +- dqm->trap_debug_vmid = 0; ++ mutex_init(&dqm->lock_hidden); ++ INIT_LIST_HEAD(&dqm->queues); ++ dqm->queue_count = dqm->processes_count = 0; ++ dqm->sdma_queue_count = 0; ++ dqm->xgmi_sdma_queue_count = 0; ++ dqm->active_runlist = false; ++ dqm->sdma_bitmap = ~0ULL >> (64 - get_num_sdma_queues(dqm)); ++ dqm->xgmi_sdma_bitmap = ~0ULL >> (64 - get_num_xgmi_sdma_queues(dqm)); ++ dqm->trap_debug_vmid = 0; + +- INIT_WORK(&dqm->hw_exception_work, kfd_process_hw_exception); ++ INIT_WORK(&dqm->hw_exception_work, kfd_process_hw_exception); + + if (dqm->dev->kfd2kgd->get_iq_wait_times) + dqm->dev->kfd2kgd->get_iq_wait_times(dqm->dev->kgd, +@@ -1116,141 +1116,140 @@ static int initialize_cpsch(struct device_queue_manager *dqm) + + static int start_cpsch(struct device_queue_manager *dqm) + { +- int retval; ++ int retval; + +- retval = 0; ++ retval = 0; + +- retval = pm_init(&dqm->packets, dqm); +- if (retval) +- goto fail_packet_manager_init; ++ retval = pm_init(&dqm->packets, dqm); ++ if (retval) ++ goto fail_packet_manager_init; + +- retval = set_sched_resources(dqm); +- if (retval) +- goto fail_set_sched_resources; ++ retval = set_sched_resources(dqm); ++ if (retval) ++ goto fail_set_sched_resources; + +- pr_debug("Allocating fence memory\n"); ++ pr_debug("Allocating fence memory\n"); + +- /* allocate fence memory on the gart */ +- retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr), +- &dqm->fence_mem); ++ /* allocate fence memory on the gart */ ++ retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr), ++ &dqm->fence_mem); + +- if (retval) +- goto fail_allocate_vidmem; ++ if (retval) ++ goto fail_allocate_vidmem; + +- dqm->fence_addr = dqm->fence_mem->cpu_ptr; +- dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr; ++ dqm->fence_addr = dqm->fence_mem->cpu_ptr; ++ dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr; + +- init_interrupts(dqm); ++ init_interrupts(dqm); + +- dqm_lock(dqm); +- /* clear hang status when driver try to start the hw scheduler */ +- dqm->is_hws_hang = false; +- execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, +- USE_DEFAULT_GRACE_PERIOD); +- dqm_unlock(dqm); ++ dqm_lock(dqm); ++ /* clear hang status when driver try to start the hw scheduler */ ++ dqm->is_hws_hang = false; ++ execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, ++ USE_DEFAULT_GRACE_PERIOD); ++ dqm_unlock(dqm); + +- return 0; ++ return 0; + fail_allocate_vidmem: + fail_set_sched_resources: +- pm_uninit(&dqm->packets); ++ pm_uninit(&dqm->packets); + fail_packet_manager_init: +- return retval; ++ return retval; + } + + static int stop_cpsch(struct device_queue_manager *dqm) + { +- dqm_lock(dqm); +- unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, ++ dqm_lock(dqm); ++ unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, + USE_DEFAULT_GRACE_PERIOD); +- dqm_unlock(dqm); ++ dqm_unlock(dqm); + +- kfd_gtt_sa_free(dqm->dev, dqm->fence_mem); +- pm_uninit(&dqm->packets); ++ kfd_gtt_sa_free(dqm->dev, dqm->fence_mem); ++ pm_uninit(&dqm->packets); + +- return 0; ++ return 0; + } + + static int create_kernel_queue_cpsch(struct device_queue_manager *dqm, +- struct kernel_queue *kq, +- struct qcm_process_device *qpd) ++ struct kernel_queue *kq, ++ struct qcm_process_device *qpd) + { +- dqm_lock(dqm); +- if (dqm->total_queue_count >= max_num_of_queues_per_device) { +- pr_warn("Can't create new kernel queue because %d queues were already created\n", +- dqm->total_queue_count); +- dqm_unlock(dqm); +- return -EPERM; +- } ++ dqm_lock(dqm); ++ if (dqm->total_queue_count >= max_num_of_queues_per_device) { ++ pr_warn("Can't create new kernel queue because %d queues were already created\n", ++ dqm->total_queue_count); ++ dqm_unlock(dqm); ++ return -EPERM; ++ } + +- /* +- * Unconditionally increment this counter, regardless of the queue's +- * type or whether the queue is active. +- */ +- dqm->total_queue_count++; +- pr_debug("Total of %d queues are accountable so far\n", +- dqm->total_queue_count); ++ /* ++ * Unconditionally increment this counter, regardless of the queue's ++ * type or whether the queue is active. ++ */ ++ dqm->total_queue_count++; ++ pr_debug("Total of %d queues are accountable so far\n", ++ dqm->total_queue_count); + +- list_add(&kq->list, &qpd->priv_queue_list); +- dqm->queue_count++; +- qpd->is_debug = true; +- execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, +- USE_DEFAULT_GRACE_PERIOD); +- dqm_unlock(dqm); ++ list_add(&kq->list, &qpd->priv_queue_list); ++ dqm->queue_count++; ++ qpd->is_debug = true; ++ execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, ++ USE_DEFAULT_GRACE_PERIOD); ++ dqm_unlock(dqm); + +- return 0; ++ return 0; + } + + static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm, +- struct kernel_queue *kq, +- struct qcm_process_device *qpd) +-{ +- dqm_lock(dqm); +- list_del(&kq->list); +- dqm->queue_count--; +- qpd->is_debug = false; +- execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, +- USE_DEFAULT_GRACE_PERIOD); +- /* +- * Unconditionally decrement this counter, regardless of the queue's +- * type. +- */ +- dqm->total_queue_count--; +- pr_debug("Total of %d queues are accountable so far\n", +- dqm->total_queue_count); +- dqm_unlock(dqm); ++ struct kernel_queue *kq, ++ struct qcm_process_device *qpd) ++{ ++ dqm_lock(dqm); ++ list_del(&kq->list); ++ dqm->queue_count--; ++ qpd->is_debug = false; ++ execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, ++ USE_DEFAULT_GRACE_PERIOD); ++ /* ++ * Unconditionally decrement this counter, regardless of the queue's ++ * type. ++ */ ++ dqm->total_queue_count--; ++ pr_debug("Total of %d queues are accountable so far\n", ++ dqm->total_queue_count); ++ dqm_unlock(dqm); + } + + static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, +- struct qcm_process_device *qpd) ++ struct qcm_process_device *qpd) + { +- int retval; +- struct mqd_manager *mqd_mgr; ++ int retval; ++ struct mqd_manager *mqd_mgr; + +- if (dqm->total_queue_count >= max_num_of_queues_per_device) { +- pr_warn("Can't create new usermode queue because %d queues were already created\n", +- dqm->total_queue_count); +- retval = -EPERM; +- goto out; +- } ++ if (dqm->total_queue_count >= max_num_of_queues_per_device) { ++ pr_warn("Can't create new usermode queue because %d queues were already created\n", ++ dqm->total_queue_count); ++ retval = -EPERM; ++ goto out; ++ } + +- if (q->properties.type == KFD_QUEUE_TYPE_SDMA || +- q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { +- dqm_lock(dqm); +- retval = allocate_sdma_queue(dqm, q); +- dqm_unlock(dqm); +- if (retval) +- goto out; +- } ++ if (q->properties.type == KFD_QUEUE_TYPE_SDMA || ++ q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { ++ dqm_lock(dqm); ++ retval = allocate_sdma_queue(dqm, q); ++ dqm_unlock(dqm); ++ if (retval) ++ goto out; ++ } + +- retval = allocate_doorbell(qpd, q); +- if (retval) +- goto out_deallocate_sdma_queue; ++ retval = allocate_doorbell(qpd, q); ++ if (retval) ++ goto out_deallocate_sdma_queue; + +- mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( +- q->properties.type)]; +- q->properties.is_suspended = false; ++ mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( ++ q->properties.type)]; + +- if (q->properties.type == KFD_QUEUE_TYPE_SDMA || ++ if (q->properties.type == KFD_QUEUE_TYPE_SDMA || + q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) + dqm->asic_ops.init_sdma_vm(dqm, q, qpd); + q->properties.tba_addr = qpd->tba_addr; +@@ -1268,6 +1267,7 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, + * updates the is_evicted flag but is a no-op otherwise. + */ + q->properties.is_evicted = !!qpd->evicted; ++ q->properties.is_suspended = false; + mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, + &q->gart_mqd_addr, &q->properties); + +-- +2.17.1 + |