From a57cc62927d04caa52937b894b89353003eddea6 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Thu, 26 Sep 2019 16:49:19 -0400 Subject: [PATCH 3893/4256] Fix the messed up format Change-Id: I1fdf5536dca4045c68010b0c78e4a541e916fcaf Signed-off-by: Yong Zhao --- .../drm/amd/amdkfd/kfd_device_queue_manager.c | 1706 ++++++++--------- 1 file changed, 853 insertions(+), 853 deletions(-) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index dbcbacb5abd4..787b936a026d 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -42,558 +42,558 @@ #define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2) static int set_pasid_vmid_mapping(struct device_queue_manager *dqm, - unsigned int pasid, unsigned int vmid); + unsigned int pasid, unsigned int vmid); static int execute_queues_cpsch(struct device_queue_manager *dqm, - enum kfd_unmap_queues_filter filter, - uint32_t filter_param, - uint32_t grace_period); + enum kfd_unmap_queues_filter filter, + uint32_t filter_param, + uint32_t grace_period); static int unmap_queues_cpsch(struct device_queue_manager *dqm, - enum kfd_unmap_queues_filter filter, - uint32_t filter_param, - uint32_t grace_period); + enum kfd_unmap_queues_filter filter, + uint32_t filter_param, + uint32_t grace_period); static int map_queues_cpsch(struct device_queue_manager *dqm); static void deallocate_sdma_queue(struct device_queue_manager *dqm, - struct queue *q); + struct queue *q); static inline void deallocate_hqd(struct device_queue_manager *dqm, - struct queue *q); + struct queue *q); static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q); static int allocate_sdma_queue(struct device_queue_manager *dqm, - struct queue *q); + struct queue *q); static void kfd_process_hw_exception(struct work_struct *work); - static inline +static inline enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type) { - if (type == KFD_QUEUE_TYPE_SDMA || type == KFD_QUEUE_TYPE_SDMA_XGMI) - return KFD_MQD_TYPE_SDMA; - return KFD_MQD_TYPE_CP; + if (type == KFD_QUEUE_TYPE_SDMA || type == KFD_QUEUE_TYPE_SDMA_XGMI) + return KFD_MQD_TYPE_SDMA; + return KFD_MQD_TYPE_CP; } static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe) { - int i; - int pipe_offset = mec * dqm->dev->shared_resources.num_pipe_per_mec - + pipe * dqm->dev->shared_resources.num_queue_per_pipe; + int i; + int pipe_offset = mec * dqm->dev->shared_resources.num_pipe_per_mec + + pipe * dqm->dev->shared_resources.num_queue_per_pipe; - /* queue is available for KFD usage if bit is 1 */ - for (i = 0; i < dqm->dev->shared_resources.num_queue_per_pipe; ++i) - if (test_bit(pipe_offset + i, - dqm->dev->shared_resources.queue_bitmap)) - return true; - return false; + /* queue is available for KFD usage if bit is 1 */ + for (i = 0; i < dqm->dev->shared_resources.num_queue_per_pipe; ++i) + if (test_bit(pipe_offset + i, + dqm->dev->shared_resources.queue_bitmap)) + return true; + return false; } unsigned int get_queues_num(struct device_queue_manager *dqm) { - return bitmap_weight(dqm->dev->shared_resources.queue_bitmap, - KGD_MAX_QUEUES); + return bitmap_weight(dqm->dev->shared_resources.queue_bitmap, + KGD_MAX_QUEUES); } unsigned int get_queues_per_pipe(struct device_queue_manager *dqm) { - return dqm->dev->shared_resources.num_queue_per_pipe; + return dqm->dev->shared_resources.num_queue_per_pipe; } unsigned int get_pipes_per_mec(struct device_queue_manager *dqm) { - return dqm->dev->shared_resources.num_pipe_per_mec; + return dqm->dev->shared_resources.num_pipe_per_mec; } static unsigned int get_num_sdma_engines(struct device_queue_manager *dqm) { - return dqm->dev->device_info->num_sdma_engines; + return dqm->dev->device_info->num_sdma_engines; } static unsigned int get_num_xgmi_sdma_engines(struct device_queue_manager *dqm) { - return dqm->dev->device_info->num_xgmi_sdma_engines; + return dqm->dev->device_info->num_xgmi_sdma_engines; } unsigned int get_num_sdma_queues(struct device_queue_manager *dqm) { - return dqm->dev->device_info->num_sdma_engines - * dqm->dev->device_info->num_sdma_queues_per_engine; + return dqm->dev->device_info->num_sdma_engines + * dqm->dev->device_info->num_sdma_queues_per_engine; } unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm) { - return dqm->dev->device_info->num_xgmi_sdma_engines - * dqm->dev->device_info->num_sdma_queues_per_engine; + return dqm->dev->device_info->num_xgmi_sdma_engines + * dqm->dev->device_info->num_sdma_queues_per_engine; } void program_sh_mem_settings(struct device_queue_manager *dqm, - struct qcm_process_device *qpd) + struct qcm_process_device *qpd) { - return dqm->dev->kfd2kgd->program_sh_mem_settings( - dqm->dev->kgd, qpd->vmid, - qpd->sh_mem_config, - qpd->sh_mem_ape1_base, - qpd->sh_mem_ape1_limit, - qpd->sh_mem_bases); + return dqm->dev->kfd2kgd->program_sh_mem_settings( + dqm->dev->kgd, qpd->vmid, + qpd->sh_mem_config, + qpd->sh_mem_ape1_base, + qpd->sh_mem_ape1_limit, + qpd->sh_mem_bases); } bool check_if_queues_active(struct device_queue_manager *dqm, - struct qcm_process_device *qpd) + struct qcm_process_device *qpd) { - bool busy = false; - struct queue *q; + bool busy = false; + struct queue *q; - dqm_lock(dqm); - list_for_each_entry(q, &qpd->queues_list, list) { - struct mqd_manager *mqd_mgr; - enum KFD_MQD_TYPE type; + dqm_lock(dqm); + list_for_each_entry(q, &qpd->queues_list, list) { + struct mqd_manager *mqd_mgr; + enum KFD_MQD_TYPE type; - type = get_mqd_type_from_queue_type(q->properties.type); - mqd_mgr = dqm->mqd_mgrs[type]; - if (!mqd_mgr || !mqd_mgr->check_queue_active) - continue; + type = get_mqd_type_from_queue_type(q->properties.type); + mqd_mgr = dqm->mqd_mgrs[type]; + if (!mqd_mgr || !mqd_mgr->check_queue_active) + continue; - busy = mqd_mgr->check_queue_active(q); - if (busy) - break; - } - dqm_unlock(dqm); + busy = mqd_mgr->check_queue_active(q); + if (busy) + break; + } + dqm_unlock(dqm); - return busy; + return busy; } static int allocate_doorbell(struct qcm_process_device *qpd, struct queue *q) { - struct kfd_dev *dev = qpd->dqm->dev; - - if (!KFD_IS_SOC15(dev->device_info->asic_family)) { - /* On pre-SOC15 chips we need to use the queue ID to - * preserve the user mode ABI. - */ - q->doorbell_id = q->properties.queue_id; - } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || - q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { - /* For SDMA queues on SOC15 with 8-byte doorbell, use static - * doorbell assignments based on the engine and queue id. - * The doobell index distance between RLC (2*i) and (2*i+1) - * for a SDMA engine is 512. - */ - uint32_t *idx_offset = - dev->shared_resources.sdma_doorbell_idx; + struct kfd_dev *dev = qpd->dqm->dev; - q->doorbell_id = idx_offset[q->properties.sdma_engine_id] - + (q->properties.sdma_queue_id & 1) - * KFD_QUEUE_DOORBELL_MIRROR_OFFSET - + (q->properties.sdma_queue_id >> 1); - } else { - /* For CP queues on SOC15 reserve a free doorbell ID */ - unsigned int found; + if (!KFD_IS_SOC15(dev->device_info->asic_family)) { + /* On pre-SOC15 chips we need to use the queue ID to + * preserve the user mode ABI. + */ + q->doorbell_id = q->properties.queue_id; + } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || + q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { + /* For SDMA queues on SOC15 with 8-byte doorbell, use static + * doorbell assignments based on the engine and queue id. + * The doobell index distance between RLC (2*i) and (2*i+1) + * for a SDMA engine is 512. + */ + uint32_t *idx_offset = + dev->shared_resources.sdma_doorbell_idx; - found = find_first_zero_bit(qpd->doorbell_bitmap, - KFD_MAX_NUM_OF_QUEUES_PER_PROCESS); - if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) { - pr_debug("No doorbells available"); - return -EBUSY; + q->doorbell_id = idx_offset[q->properties.sdma_engine_id] + + (q->properties.sdma_queue_id & 1) + * KFD_QUEUE_DOORBELL_MIRROR_OFFSET + + (q->properties.sdma_queue_id >> 1); + } else { + /* For CP queues on SOC15 reserve a free doorbell ID */ + unsigned int found; + + found = find_first_zero_bit(qpd->doorbell_bitmap, + KFD_MAX_NUM_OF_QUEUES_PER_PROCESS); + if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) { + pr_debug("No doorbells available"); + return -EBUSY; + } + set_bit(found, qpd->doorbell_bitmap); + q->doorbell_id = found; } - set_bit(found, qpd->doorbell_bitmap); - q->doorbell_id = found; - } - q->properties.doorbell_off = - kfd_doorbell_id_to_offset(dev, q->process, - q->doorbell_id); + q->properties.doorbell_off = + kfd_doorbell_id_to_offset(dev, q->process, + q->doorbell_id); - return 0; + return 0; } static void deallocate_doorbell(struct qcm_process_device *qpd, - struct queue *q) + struct queue *q) { - unsigned int old; - struct kfd_dev *dev = qpd->dqm->dev; + unsigned int old; + struct kfd_dev *dev = qpd->dqm->dev; - if (!KFD_IS_SOC15(dev->device_info->asic_family) || - q->properties.type == KFD_QUEUE_TYPE_SDMA || - q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) - return; + if (!KFD_IS_SOC15(dev->device_info->asic_family) || + q->properties.type == KFD_QUEUE_TYPE_SDMA || + q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) + return; - old = test_and_clear_bit(q->doorbell_id, qpd->doorbell_bitmap); - WARN_ON(!old); + old = test_and_clear_bit(q->doorbell_id, qpd->doorbell_bitmap); + WARN_ON(!old); } static int allocate_vmid(struct device_queue_manager *dqm, - struct qcm_process_device *qpd, - struct queue *q) + struct qcm_process_device *qpd, + struct queue *q) { - int bit, allocated_vmid; + int bit, allocated_vmid; - if (dqm->vmid_bitmap == 0) - return -ENOMEM; + if (dqm->vmid_bitmap == 0) + return -ENOMEM; - bit = ffs(dqm->vmid_bitmap) - 1; - dqm->vmid_bitmap &= ~(1 << bit); + bit = ffs(dqm->vmid_bitmap) - 1; + dqm->vmid_bitmap &= ~(1 << bit); - allocated_vmid = bit + dqm->dev->vm_info.first_vmid_kfd; - pr_debug("vmid allocation %d\n", allocated_vmid); - qpd->vmid = allocated_vmid; - q->properties.vmid = allocated_vmid; + allocated_vmid = bit + dqm->dev->vm_info.first_vmid_kfd; + pr_debug("vmid allocation %d\n", allocated_vmid); + qpd->vmid = allocated_vmid; + q->properties.vmid = allocated_vmid; - set_pasid_vmid_mapping(dqm, q->process->pasid, q->properties.vmid); - program_sh_mem_settings(dqm, qpd); + set_pasid_vmid_mapping(dqm, q->process->pasid, q->properties.vmid); + program_sh_mem_settings(dqm, qpd); - /* qpd->page_table_base is set earlier when register_process() - * is called, i.e. when the first queue is created. - */ - dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->kgd, - qpd->vmid, - qpd->page_table_base); - /* invalidate the VM context after pasid and vmid mapping is set up */ - kfd_flush_tlb(qpd_to_pdd(qpd)); + /* qpd->page_table_base is set earlier when register_process() + * is called, i.e. when the first queue is created. + */ + dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->kgd, + qpd->vmid, + qpd->page_table_base); + /* invalidate the VM context after pasid and vmid mapping is set up */ + kfd_flush_tlb(qpd_to_pdd(qpd)); - dqm->dev->kfd2kgd->set_scratch_backing_va( - dqm->dev->kgd, qpd->sh_hidden_private_base, qpd->vmid); + dqm->dev->kfd2kgd->set_scratch_backing_va( + dqm->dev->kgd, qpd->sh_hidden_private_base, qpd->vmid); - return 0; + return 0; } static int flush_texture_cache_nocpsch(struct kfd_dev *kdev, - struct qcm_process_device *qpd) + struct qcm_process_device *qpd) { - const struct packet_manager_funcs *pmf = qpd->dqm->packets.pmf; - int ret; + const struct packet_manager_funcs *pmf = qpd->dqm->packets.pmf; + int ret; - if (!qpd->ib_kaddr) - return -ENOMEM; + if (!qpd->ib_kaddr) + return -ENOMEM; - ret = pmf->release_mem(qpd->ib_base, (uint32_t *)qpd->ib_kaddr); - if (ret) - return ret; + ret = pmf->release_mem(qpd->ib_base, (uint32_t *)qpd->ib_kaddr); + if (ret) + return ret; - return amdgpu_amdkfd_submit_ib(kdev->kgd, KGD_ENGINE_MEC1, qpd->vmid, - qpd->ib_base, (uint32_t *)qpd->ib_kaddr, - pmf->release_mem_size / sizeof(uint32_t)); + return amdgpu_amdkfd_submit_ib(kdev->kgd, KGD_ENGINE_MEC1, qpd->vmid, + qpd->ib_base, (uint32_t *)qpd->ib_kaddr, + pmf->release_mem_size / sizeof(uint32_t)); } static void deallocate_vmid(struct device_queue_manager *dqm, - struct qcm_process_device *qpd, - struct queue *q) + struct qcm_process_device *qpd, + struct queue *q) { - int bit = qpd->vmid - dqm->dev->vm_info.first_vmid_kfd; + int bit = qpd->vmid - dqm->dev->vm_info.first_vmid_kfd; - /* On GFX v7, CP doesn't flush TC at dequeue */ - if (q->device->device_info->asic_family == CHIP_HAWAII) - if (flush_texture_cache_nocpsch(q->device, qpd)) - pr_err("Failed to flush TC\n"); + /* On GFX v7, CP doesn't flush TC at dequeue */ + if (q->device->device_info->asic_family == CHIP_HAWAII) + if (flush_texture_cache_nocpsch(q->device, qpd)) + pr_err("Failed to flush TC\n"); - kfd_flush_tlb(qpd_to_pdd(qpd)); + kfd_flush_tlb(qpd_to_pdd(qpd)); - /* Release the vmid mapping */ - set_pasid_vmid_mapping(dqm, 0, qpd->vmid); + /* Release the vmid mapping */ + set_pasid_vmid_mapping(dqm, 0, qpd->vmid); - dqm->vmid_bitmap |= (1 << bit); - qpd->vmid = 0; - q->properties.vmid = 0; + dqm->vmid_bitmap |= (1 << bit); + qpd->vmid = 0; + q->properties.vmid = 0; } static int create_queue_nocpsch(struct device_queue_manager *dqm, - struct queue *q, - struct qcm_process_device *qpd) + struct queue *q, + struct qcm_process_device *qpd) { - struct mqd_manager *mqd_mgr; - int retval; + struct mqd_manager *mqd_mgr; + int retval; - print_queue(q); + print_queue(q); - dqm_lock(dqm); + dqm_lock(dqm); - if (dqm->total_queue_count >= max_num_of_queues_per_device) { - pr_warn("Can't create new usermode queue because %d queues were already created\n", - dqm->total_queue_count); - retval = -EPERM; - goto out_unlock; - } + if (dqm->total_queue_count >= max_num_of_queues_per_device) { + pr_warn("Can't create new usermode queue because %d queues were already created\n", + dqm->total_queue_count); + retval = -EPERM; + goto out_unlock; + } - if (list_empty(&qpd->queues_list)) { - retval = allocate_vmid(dqm, qpd, q); - if (retval) - goto out_unlock; - } - q->properties.vmid = qpd->vmid; - /* - * Eviction state logic: mark all queues as evicted, even ones - * not currently active. Restoring inactive queues later only - * updates the is_evicted flag but is a no-op otherwise. - */ - q->properties.is_evicted = !!qpd->evicted; - - q->properties.tba_addr = qpd->tba_addr; - q->properties.tma_addr = qpd->tma_addr; - - mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( - q->properties.type)]; - if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) { - retval = allocate_hqd(dqm, q); - if (retval) - goto deallocate_vmid; - pr_debug("Loading mqd to hqd on pipe %d, queue %d\n", - q->pipe, q->queue); - } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || - q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { - retval = allocate_sdma_queue(dqm, q); - if (retval) - goto deallocate_vmid; - dqm->asic_ops.init_sdma_vm(dqm, q, qpd); - } - - retval = allocate_doorbell(qpd, q); - if (retval) - goto out_deallocate_hqd; - - /* Temporarily release dqm lock to avoid a circular lock dependency */ - dqm_unlock(dqm); - q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties); - dqm_lock(dqm); - - if (!q->mqd_mem_obj) { - retval = -ENOMEM; - goto out_deallocate_doorbell; - } - mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, - &q->gart_mqd_addr, &q->properties); - if (q->properties.is_active) { - - if (WARN(q->process->mm != current->mm, - "should only run in user thread")) - retval = -EFAULT; - else - retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, - q->queue, &q->properties, current->mm); + if (list_empty(&qpd->queues_list)) { + retval = allocate_vmid(dqm, qpd, q); + if (retval) + goto out_unlock; + } + q->properties.vmid = qpd->vmid; + /* + * Eviction state logic: mark all queues as evicted, even ones + * not currently active. Restoring inactive queues later only + * updates the is_evicted flag but is a no-op otherwise. + */ + q->properties.is_evicted = !!qpd->evicted; + + q->properties.tba_addr = qpd->tba_addr; + q->properties.tma_addr = qpd->tma_addr; + + mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( + q->properties.type)]; + if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) { + retval = allocate_hqd(dqm, q); + if (retval) + goto deallocate_vmid; + pr_debug("Loading mqd to hqd on pipe %d, queue %d\n", + q->pipe, q->queue); + } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || + q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { + retval = allocate_sdma_queue(dqm, q); + if (retval) + goto deallocate_vmid; + dqm->asic_ops.init_sdma_vm(dqm, q, qpd); + } + + retval = allocate_doorbell(qpd, q); if (retval) - goto out_free_mqd; - } + goto out_deallocate_hqd; - list_add(&q->list, &qpd->queues_list); - qpd->queue_count++; - if (q->properties.is_active) - dqm->queue_count++; + /* Temporarily release dqm lock to avoid a circular lock dependency */ + dqm_unlock(dqm); + q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties); + dqm_lock(dqm); + + if (!q->mqd_mem_obj) { + retval = -ENOMEM; + goto out_deallocate_doorbell; + } + mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, + &q->gart_mqd_addr, &q->properties); + if (q->properties.is_active) { - if (q->properties.type == KFD_QUEUE_TYPE_SDMA) - dqm->sdma_queue_count++; - else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) - dqm->xgmi_sdma_queue_count++; + if (WARN(q->process->mm != current->mm, + "should only run in user thread")) + retval = -EFAULT; + else + retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, + q->queue, &q->properties, current->mm); + if (retval) + goto out_free_mqd; + } - /* - * Unconditionally increment this counter, regardless of the queue's - * type or whether the queue is active. - */ - dqm->total_queue_count++; - pr_debug("Total of %d queues are accountable so far\n", - dqm->total_queue_count); - goto out_unlock; + list_add(&q->list, &qpd->queues_list); + qpd->queue_count++; + if (q->properties.is_active) + dqm->queue_count++; + + if (q->properties.type == KFD_QUEUE_TYPE_SDMA) + dqm->sdma_queue_count++; + else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) + dqm->xgmi_sdma_queue_count++; + + /* + * Unconditionally increment this counter, regardless of the queue's + * type or whether the queue is active. + */ + dqm->total_queue_count++; + pr_debug("Total of %d queues are accountable so far\n", + dqm->total_queue_count); + goto out_unlock; out_free_mqd: - mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); + mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); out_deallocate_doorbell: - deallocate_doorbell(qpd, q); + deallocate_doorbell(qpd, q); out_deallocate_hqd: - if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) - deallocate_hqd(dqm, q); - else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || - q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) - deallocate_sdma_queue(dqm, q); + if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) + deallocate_hqd(dqm, q); + else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || + q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) + deallocate_sdma_queue(dqm, q); deallocate_vmid: - if (list_empty(&qpd->queues_list)) - deallocate_vmid(dqm, qpd, q); + if (list_empty(&qpd->queues_list)) + deallocate_vmid(dqm, qpd, q); out_unlock: - dqm_unlock(dqm); - return retval; + dqm_unlock(dqm); + return retval; } static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q) { - bool set; - int pipe, bit, i; + bool set; + int pipe, bit, i; - set = false; + set = false; - for (pipe = dqm->next_pipe_to_allocate, i = 0; - i < get_pipes_per_mec(dqm); - pipe = ((pipe + 1) % get_pipes_per_mec(dqm)), ++i) { + for (pipe = dqm->next_pipe_to_allocate, i = 0; + i < get_pipes_per_mec(dqm); + pipe = ((pipe + 1) % get_pipes_per_mec(dqm)), ++i) { - if (!is_pipe_enabled(dqm, 0, pipe)) - continue; + if (!is_pipe_enabled(dqm, 0, pipe)) + continue; - if (dqm->allocated_queues[pipe] != 0) { - bit = ffs(dqm->allocated_queues[pipe]) - 1; - dqm->allocated_queues[pipe] &= ~(1 << bit); - q->pipe = pipe; - q->queue = bit; - set = true; - break; + if (dqm->allocated_queues[pipe] != 0) { + bit = ffs(dqm->allocated_queues[pipe]) - 1; + dqm->allocated_queues[pipe] &= ~(1 << bit); + q->pipe = pipe; + q->queue = bit; + set = true; + break; + } } - } - if (!set) - return -EBUSY; + if (!set) + return -EBUSY; - pr_debug("hqd slot - pipe %d, queue %d\n", q->pipe, q->queue); - /* horizontal hqd allocation */ - dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_per_mec(dqm); + pr_debug("hqd slot - pipe %d, queue %d\n", q->pipe, q->queue); + /* horizontal hqd allocation */ + dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_per_mec(dqm); - return 0; + return 0; } static inline void deallocate_hqd(struct device_queue_manager *dqm, - struct queue *q) + struct queue *q) { - dqm->allocated_queues[q->pipe] |= (1 << q->queue); + dqm->allocated_queues[q->pipe] |= (1 << q->queue); } /* Access to DQM has to be locked before calling destroy_queue_nocpsch_locked * to avoid asynchronized access */ static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm, - struct qcm_process_device *qpd, - struct queue *q) -{ - int retval; - struct mqd_manager *mqd_mgr; - - mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( - q->properties.type)]; - - if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) { - deallocate_hqd(dqm, q); - } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { - dqm->sdma_queue_count--; - deallocate_sdma_queue(dqm, q); - } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { - dqm->xgmi_sdma_queue_count--; - deallocate_sdma_queue(dqm, q); - } else { - pr_debug("q->properties.type %d is invalid\n", - q->properties.type); - return -EINVAL; - } - dqm->total_queue_count--; - - deallocate_doorbell(qpd, q); - - retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, - KFD_PREEMPT_TYPE_WAVEFRONT_RESET, - KFD_UNMAP_LATENCY_MS, - q->pipe, q->queue); - if (retval == -ETIME) - qpd->reset_wavefronts = true; - - mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); - - list_del(&q->list); - if (list_empty(&qpd->queues_list)) { - if (qpd->reset_wavefronts) { - pr_warn("Resetting wave fronts (nocpsch) on dev %p\n", - dqm->dev); - /* dbgdev_wave_reset_wavefronts has to be called before - * deallocate_vmid(), i.e. when vmid is still in use. - */ - dbgdev_wave_reset_wavefronts(dqm->dev, - qpd->pqm->process); - qpd->reset_wavefronts = false; - } - - deallocate_vmid(dqm, qpd, q); - } - qpd->queue_count--; - if (q->properties.is_active) - dqm->queue_count--; + struct qcm_process_device *qpd, + struct queue *q) +{ + int retval; + struct mqd_manager *mqd_mgr; + + mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( + q->properties.type)]; + + if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) { + deallocate_hqd(dqm, q); + } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { + dqm->sdma_queue_count--; + deallocate_sdma_queue(dqm, q); + } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { + dqm->xgmi_sdma_queue_count--; + deallocate_sdma_queue(dqm, q); + } else { + pr_debug("q->properties.type %d is invalid\n", + q->properties.type); + return -EINVAL; + } + dqm->total_queue_count--; + + deallocate_doorbell(qpd, q); - return retval; + retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, + KFD_PREEMPT_TYPE_WAVEFRONT_RESET, + KFD_UNMAP_LATENCY_MS, + q->pipe, q->queue); + if (retval == -ETIME) + qpd->reset_wavefronts = true; + + mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); + + list_del(&q->list); + if (list_empty(&qpd->queues_list)) { + if (qpd->reset_wavefronts) { + pr_warn("Resetting wave fronts (nocpsch) on dev %p\n", + dqm->dev); + /* dbgdev_wave_reset_wavefronts has to be called before + * deallocate_vmid(), i.e. when vmid is still in use. + */ + dbgdev_wave_reset_wavefronts(dqm->dev, + qpd->pqm->process); + qpd->reset_wavefronts = false; + } + + deallocate_vmid(dqm, qpd, q); + } + qpd->queue_count--; + if (q->properties.is_active) + dqm->queue_count--; + + return retval; } static int destroy_queue_nocpsch(struct device_queue_manager *dqm, - struct qcm_process_device *qpd, - struct queue *q) + struct qcm_process_device *qpd, + struct queue *q) { - int retval; + int retval; - dqm_lock(dqm); - retval = destroy_queue_nocpsch_locked(dqm, qpd, q); - dqm_unlock(dqm); + dqm_lock(dqm); + retval = destroy_queue_nocpsch_locked(dqm, qpd, q); + dqm_unlock(dqm); - return retval; + return retval; } static int update_queue(struct device_queue_manager *dqm, struct queue *q) { - int retval = 0; - struct mqd_manager *mqd_mgr; - struct kfd_process_device *pdd; - bool prev_active = false; + int retval = 0; + struct mqd_manager *mqd_mgr; + struct kfd_process_device *pdd; + bool prev_active = false; - dqm_lock(dqm); - pdd = kfd_get_process_device_data(q->device, q->process); - if (!pdd) { - retval = -ENODEV; - goto out_unlock; - } - mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( - q->properties.type)]; - - /* Save previous activity state for counters */ - prev_active = q->properties.is_active; - - /* Make sure the queue is unmapped before updating the MQD */ - if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) { - retval = unmap_queues_cpsch(dqm, - KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, - USE_DEFAULT_GRACE_PERIOD); - if (retval) { - pr_err("unmap queue failed\n"); - goto out_unlock; + dqm_lock(dqm); + pdd = kfd_get_process_device_data(q->device, q->process); + if (!pdd) { + retval = -ENODEV; + goto out_unlock; } - } else if (prev_active && - (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || - q->properties.type == KFD_QUEUE_TYPE_SDMA || - q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { - retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, - KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN, - KFD_UNMAP_LATENCY_MS, q->pipe, q->queue); - if (retval) { - pr_err("destroy mqd failed\n"); - goto out_unlock; + mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( + q->properties.type)]; + + /* Save previous activity state for counters */ + prev_active = q->properties.is_active; + + /* Make sure the queue is unmapped before updating the MQD */ + if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) { + retval = unmap_queues_cpsch(dqm, + KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, + USE_DEFAULT_GRACE_PERIOD); + if (retval) { + pr_err("unmap queue failed\n"); + goto out_unlock; + } + } else if (prev_active && + (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || + q->properties.type == KFD_QUEUE_TYPE_SDMA || + q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { + retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, + KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN, + KFD_UNMAP_LATENCY_MS, q->pipe, q->queue); + if (retval) { + pr_err("destroy mqd failed\n"); + goto out_unlock; + } } - } - mqd_mgr->update_mqd(mqd_mgr, q->mqd, &q->properties); + mqd_mgr->update_mqd(mqd_mgr, q->mqd, &q->properties); - /* - * check active state vs. the previous state and modify - * counter accordingly. map_queues_cpsch uses the - * dqm->queue_count to determine whether a new runlist must be - * uploaded. - */ - if (q->properties.is_active && !prev_active) - dqm->queue_count++; - else if (!q->properties.is_active && prev_active) - dqm->queue_count--; + /* + * check active state vs. the previous state and modify + * counter accordingly. map_queues_cpsch uses the + * dqm->queue_count to determine whether a new runlist must be + * uploaded. + */ + if (q->properties.is_active && !prev_active) + dqm->queue_count++; + else if (!q->properties.is_active && prev_active) + dqm->queue_count--; - if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) - retval = map_queues_cpsch(dqm); - else if (q->properties.is_active && - (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || - q->properties.type == KFD_QUEUE_TYPE_SDMA || - q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { - if (WARN(q->process->mm != current->mm, + if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) + retval = map_queues_cpsch(dqm); + else if (q->properties.is_active && + (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || + q->properties.type == KFD_QUEUE_TYPE_SDMA || + q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { + if (WARN(q->process->mm != current->mm, "should only run in user thread")) - retval = -EFAULT; - else - retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, - q->pipe, q->queue, - &q->properties, current->mm); - } + retval = -EFAULT; + else + retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, + q->pipe, q->queue, + &q->properties, current->mm); + } out_unlock: - dqm_unlock(dqm); - return retval; + dqm_unlock(dqm); + return retval; } /* suspend_single_queue does not lock the dqm like the @@ -605,22 +605,22 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q) * multiple times, we will just keep the dqm locked for all of the calls. */ static int suspend_single_queue(struct device_queue_manager *dqm, - struct kfd_process_device *pdd, - struct queue *q) + struct kfd_process_device *pdd, + struct queue *q) { - int retval = 0; + int retval = 0; - pr_debug("Suspending PASID %u queue [%i]\n", - pdd->process->pasid, - q->properties.queue_id); + pr_debug("Suspending PASID %u queue [%i]\n", + pdd->process->pasid, + q->properties.queue_id); - q->properties.is_suspended = true; - if (q->properties.is_active) { - dqm->queue_count--; - q->properties.is_active = false; - } + q->properties.is_suspended = true; + if (q->properties.is_active) { + dqm->queue_count--; + q->properties.is_active = false; + } - return retval; + return retval; } /* resume_single_queue does not lock the dqm like the functions @@ -632,419 +632,419 @@ static int suspend_single_queue(struct device_queue_manager *dqm, * multiple times, we will just keep the dqm locked for all of the calls. */ static int resume_single_queue(struct device_queue_manager *dqm, - struct qcm_process_device *qpd, - struct queue *q) + struct qcm_process_device *qpd, + struct queue *q) { - struct kfd_process_device *pdd; - uint64_t pd_base; - int retval = 0; + struct kfd_process_device *pdd; + uint64_t pd_base; + int retval = 0; - pdd = qpd_to_pdd(qpd); - /* Retrieve PD base */ - pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm); + pdd = qpd_to_pdd(qpd); + /* Retrieve PD base */ + pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm); - pr_debug("Restoring from suspend PASID %u queue [%i]\n", - pdd->process->pasid, - q->properties.queue_id); + pr_debug("Restoring from suspend PASID %u queue [%i]\n", + pdd->process->pasid, + q->properties.queue_id); - q->properties.is_suspended = false; + q->properties.is_suspended = false; - if (QUEUE_IS_ACTIVE(q->properties)) { - q->properties.is_active = true; - dqm->queue_count++; - } + if (QUEUE_IS_ACTIVE(q->properties)) { + q->properties.is_active = true; + dqm->queue_count++; + } - return retval; + return retval; } static int evict_process_queues_nocpsch(struct device_queue_manager *dqm, - struct qcm_process_device *qpd) + struct qcm_process_device *qpd) { - struct queue *q; - struct mqd_manager *mqd_mgr; - struct kfd_process_device *pdd; - int retval, ret = 0; + struct queue *q; + struct mqd_manager *mqd_mgr; + struct kfd_process_device *pdd; + int retval, ret = 0; - dqm_lock(dqm); - if (qpd->evicted++ > 0) /* already evicted, do nothing */ - goto out; + dqm_lock(dqm); + if (qpd->evicted++ > 0) /* already evicted, do nothing */ + goto out; - pdd = qpd_to_pdd(qpd); - pr_info_ratelimited("Evicting PASID %u queues\n", - pdd->process->pasid); + pdd = qpd_to_pdd(qpd); + pr_info_ratelimited("Evicting PASID %u queues\n", + pdd->process->pasid); - /* Mark all queues as evicted. Deactivate all active queues on - * the qpd. - */ - list_for_each_entry(q, &qpd->queues_list, list) { - q->properties.is_evicted = true; - if (!q->properties.is_active) - continue; + /* Mark all queues as evicted. Deactivate all active queues on + * the qpd. + */ + list_for_each_entry(q, &qpd->queues_list, list) { + q->properties.is_evicted = true; + if (!q->properties.is_active) + continue; - mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( - q->properties.type)]; - q->properties.is_active = false; - retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, - KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN, - KFD_UNMAP_LATENCY_MS, q->pipe, q->queue); - if (retval && !ret) - /* Return the first error, but keep going to - * maintain a consistent eviction state - */ - ret = retval; - dqm->queue_count--; - } + mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( + q->properties.type)]; + q->properties.is_active = false; + retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, + KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN, + KFD_UNMAP_LATENCY_MS, q->pipe, q->queue); + if (retval && !ret) + /* Return the first error, but keep going to + * maintain a consistent eviction state + */ + ret = retval; + dqm->queue_count--; + } out: - dqm_unlock(dqm); - return ret; + dqm_unlock(dqm); + return ret; } static int evict_process_queues_cpsch(struct device_queue_manager *dqm, - struct qcm_process_device *qpd) + struct qcm_process_device *qpd) { - struct queue *q; - struct kfd_process_device *pdd; - int retval = 0; + struct queue *q; + struct kfd_process_device *pdd; + int retval = 0; - dqm_lock(dqm); - if (qpd->evicted++ > 0) /* already evicted, do nothing */ - goto out; + dqm_lock(dqm); + if (qpd->evicted++ > 0) /* already evicted, do nothing */ + goto out; - pdd = qpd_to_pdd(qpd); - pr_info_ratelimited("Evicting PASID %u queues\n", - pdd->process->pasid); + pdd = qpd_to_pdd(qpd); + pr_info_ratelimited("Evicting PASID %u queues\n", + pdd->process->pasid); - /* Mark all queues as evicted. Deactivate all active queues on - * the qpd. - */ - list_for_each_entry(q, &qpd->queues_list, list) { - q->properties.is_evicted = true; - if (!q->properties.is_active) - continue; + /* Mark all queues as evicted. Deactivate all active queues on + * the qpd. + */ + list_for_each_entry(q, &qpd->queues_list, list) { + q->properties.is_evicted = true; + if (!q->properties.is_active) + continue; - q->properties.is_active = false; - dqm->queue_count--; - } - retval = execute_queues_cpsch(dqm, - qpd->is_debug ? - KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES : - KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, - USE_DEFAULT_GRACE_PERIOD); + q->properties.is_active = false; + dqm->queue_count--; + } + retval = execute_queues_cpsch(dqm, + qpd->is_debug ? + KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES : + KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, + USE_DEFAULT_GRACE_PERIOD); out: - dqm_unlock(dqm); - return retval; + dqm_unlock(dqm); + return retval; } static int restore_process_queues_nocpsch(struct device_queue_manager *dqm, - struct qcm_process_device *qpd) -{ - struct mm_struct *mm = NULL; - struct queue *q; - struct mqd_manager *mqd_mgr; - struct kfd_process_device *pdd; - uint64_t pd_base; - int retval, ret = 0; - - pdd = qpd_to_pdd(qpd); - /* Retrieve PD base */ - pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm); - - dqm_lock(dqm); - if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */ - goto out; - if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */ - qpd->evicted--; - goto out; - } - - pr_info_ratelimited("Restoring PASID %u queues\n", - pdd->process->pasid); - - /* Update PD Base in QPD */ - qpd->page_table_base = pd_base; - pr_debug("Updated PD address to 0x%llx\n", pd_base); - - if (!list_empty(&qpd->queues_list)) { - dqm->dev->kfd2kgd->set_vm_context_page_table_base( - dqm->dev->kgd, - qpd->vmid, - qpd->page_table_base); - kfd_flush_tlb(pdd); - } - - /* Take a safe reference to the mm_struct, which may otherwise - * disappear even while the kfd_process is still referenced. - */ - mm = get_task_mm(pdd->process->lead_thread); - if (!mm) { - ret = -EFAULT; - goto out; - } - - /* Remove the eviction flags. Activate queues that are not - * inactive for other reasons. - */ - list_for_each_entry(q, &qpd->queues_list, list) { - q->properties.is_evicted = false; - if (!QUEUE_IS_ACTIVE(q->properties)) - continue; + struct qcm_process_device *qpd) +{ + struct mm_struct *mm = NULL; + struct queue *q; + struct mqd_manager *mqd_mgr; + struct kfd_process_device *pdd; + uint64_t pd_base; + int retval, ret = 0; - mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( - q->properties.type)]; - q->properties.is_active = true; - retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, - q->queue, &q->properties, mm); - if (retval && !ret) - /* Return the first error, but keep going to - * maintain a consistent eviction state - */ - ret = retval; - dqm->queue_count++; - } - qpd->evicted = 0; + pdd = qpd_to_pdd(qpd); + /* Retrieve PD base */ + pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm); + + dqm_lock(dqm); + if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */ + goto out; + if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */ + qpd->evicted--; + goto out; + } + + pr_info_ratelimited("Restoring PASID %u queues\n", + pdd->process->pasid); + + /* Update PD Base in QPD */ + qpd->page_table_base = pd_base; + pr_debug("Updated PD address to 0x%llx\n", pd_base); + + if (!list_empty(&qpd->queues_list)) { + dqm->dev->kfd2kgd->set_vm_context_page_table_base( + dqm->dev->kgd, + qpd->vmid, + qpd->page_table_base); + kfd_flush_tlb(pdd); + } + + /* Take a safe reference to the mm_struct, which may otherwise + * disappear even while the kfd_process is still referenced. + */ + mm = get_task_mm(pdd->process->lead_thread); + if (!mm) { + ret = -EFAULT; + goto out; + } + + /* Remove the eviction flags. Activate queues that are not + * inactive for other reasons. + */ + list_for_each_entry(q, &qpd->queues_list, list) { + q->properties.is_evicted = false; + if (!QUEUE_IS_ACTIVE(q->properties)) + continue; + + mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( + q->properties.type)]; + q->properties.is_active = true; + retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, + q->queue, &q->properties, mm); + if (retval && !ret) + /* Return the first error, but keep going to + * maintain a consistent eviction state + */ + ret = retval; + dqm->queue_count++; + } + qpd->evicted = 0; out: - if (mm) - mmput(mm); - dqm_unlock(dqm); - return ret; + if (mm) + mmput(mm); + dqm_unlock(dqm); + return ret; } static int restore_process_queues_cpsch(struct device_queue_manager *dqm, - struct qcm_process_device *qpd) -{ - struct queue *q; - struct kfd_process_device *pdd; - uint64_t pd_base; - int retval = 0; - - pdd = qpd_to_pdd(qpd); - /* Retrieve PD base */ - pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm); - - dqm_lock(dqm); - if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */ - goto out; - if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */ - qpd->evicted--; - goto out; - } - - pr_info_ratelimited("Restoring PASID %u queues\n", - pdd->process->pasid); - - /* Update PD Base in QPD */ - qpd->page_table_base = pd_base; - pr_debug("Updated PD address to 0x%llx\n", pd_base); - - /* activate all active queues on the qpd */ - list_for_each_entry(q, &qpd->queues_list, list) { - q->properties.is_evicted = false; - if (!QUEUE_IS_ACTIVE(q->properties)) - continue; - - q->properties.is_active = true; - dqm->queue_count++; - } - retval = execute_queues_cpsch(dqm, - KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, - USE_DEFAULT_GRACE_PERIOD); - qpd->evicted = 0; + struct qcm_process_device *qpd) +{ + struct queue *q; + struct kfd_process_device *pdd; + uint64_t pd_base; + int retval = 0; + + pdd = qpd_to_pdd(qpd); + /* Retrieve PD base */ + pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm); + + dqm_lock(dqm); + if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */ + goto out; + if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */ + qpd->evicted--; + goto out; + } + + pr_info_ratelimited("Restoring PASID %u queues\n", + pdd->process->pasid); + + /* Update PD Base in QPD */ + qpd->page_table_base = pd_base; + pr_debug("Updated PD address to 0x%llx\n", pd_base); + + /* activate all active queues on the qpd */ + list_for_each_entry(q, &qpd->queues_list, list) { + q->properties.is_evicted = false; + if (!QUEUE_IS_ACTIVE(q->properties)) + continue; + + q->properties.is_active = true; + dqm->queue_count++; + } + retval = execute_queues_cpsch(dqm, + KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, + USE_DEFAULT_GRACE_PERIOD); + qpd->evicted = 0; out: - dqm_unlock(dqm); - return retval; + dqm_unlock(dqm); + return retval; } static int register_process(struct device_queue_manager *dqm, - struct qcm_process_device *qpd) + struct qcm_process_device *qpd) { - struct device_process_node *n; - struct kfd_process_device *pdd; - uint64_t pd_base; - int retval; + struct device_process_node *n; + struct kfd_process_device *pdd; + uint64_t pd_base; + int retval; - n = kzalloc(sizeof(*n), GFP_KERNEL); - if (!n) - return -ENOMEM; + n = kzalloc(sizeof(*n), GFP_KERNEL); + if (!n) + return -ENOMEM; - n->qpd = qpd; + n->qpd = qpd; - pdd = qpd_to_pdd(qpd); - /* Retrieve PD base */ - pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm); + pdd = qpd_to_pdd(qpd); + /* Retrieve PD base */ + pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm); - dqm_lock(dqm); - list_add(&n->list, &dqm->queues); + dqm_lock(dqm); + list_add(&n->list, &dqm->queues); - /* Update PD Base in QPD */ - qpd->page_table_base = pd_base; - pr_debug("Updated PD address to 0x%llx\n", pd_base); + /* Update PD Base in QPD */ + qpd->page_table_base = pd_base; + pr_debug("Updated PD address to 0x%llx\n", pd_base); - retval = dqm->asic_ops.update_qpd(dqm, qpd); + retval = dqm->asic_ops.update_qpd(dqm, qpd); - dqm->processes_count++; - kfd_inc_compute_active(dqm->dev); + dqm->processes_count++; + kfd_inc_compute_active(dqm->dev); - dqm_unlock(dqm); + dqm_unlock(dqm); - return retval; + return retval; } static int unregister_process(struct device_queue_manager *dqm, - struct qcm_process_device *qpd) + struct qcm_process_device *qpd) { - int retval; - struct device_process_node *cur, *next; + int retval; + struct device_process_node *cur, *next; - pr_debug("qpd->queues_list is %s\n", - list_empty(&qpd->queues_list) ? "empty" : "not empty"); + pr_debug("qpd->queues_list is %s\n", + list_empty(&qpd->queues_list) ? "empty" : "not empty"); - retval = 0; - dqm_lock(dqm); + retval = 0; + dqm_lock(dqm); - list_for_each_entry_safe(cur, next, &dqm->queues, list) { - if (qpd == cur->qpd) { - list_del(&cur->list); - kfree(cur); - dqm->processes_count--; - kfd_dec_compute_active(dqm->dev); - goto out; + list_for_each_entry_safe(cur, next, &dqm->queues, list) { + if (qpd == cur->qpd) { + list_del(&cur->list); + kfree(cur); + dqm->processes_count--; + kfd_dec_compute_active(dqm->dev); + goto out; + } } - } - /* qpd not found in dqm list */ - retval = 1; + /* qpd not found in dqm list */ + retval = 1; out: - dqm_unlock(dqm); - return retval; + dqm_unlock(dqm); + return retval; } - static int +static int set_pasid_vmid_mapping(struct device_queue_manager *dqm, unsigned int pasid, - unsigned int vmid) + unsigned int vmid) { - return dqm->dev->kfd2kgd->set_pasid_vmid_mapping( - dqm->dev->kgd, pasid, vmid); + return dqm->dev->kfd2kgd->set_pasid_vmid_mapping( + dqm->dev->kgd, pasid, vmid); } static void init_interrupts(struct device_queue_manager *dqm) { - unsigned int i; + unsigned int i; - for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++) - if (is_pipe_enabled(dqm, 0, i)) - dqm->dev->kfd2kgd->init_interrupts(dqm->dev->kgd, i); + for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++) + if (is_pipe_enabled(dqm, 0, i)) + dqm->dev->kfd2kgd->init_interrupts(dqm->dev->kgd, i); } static int initialize_nocpsch(struct device_queue_manager *dqm) { - int pipe, queue; + int pipe, queue; - pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm)); + pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm)); - dqm->allocated_queues = kcalloc(get_pipes_per_mec(dqm), - sizeof(unsigned int), GFP_KERNEL); - if (!dqm->allocated_queues) - return -ENOMEM; + dqm->allocated_queues = kcalloc(get_pipes_per_mec(dqm), + sizeof(unsigned int), GFP_KERNEL); + if (!dqm->allocated_queues) + return -ENOMEM; - mutex_init(&dqm->lock_hidden); - INIT_LIST_HEAD(&dqm->queues); - dqm->queue_count = dqm->next_pipe_to_allocate = 0; - dqm->sdma_queue_count = 0; - dqm->xgmi_sdma_queue_count = 0; - dqm->trap_debug_vmid = 0; + mutex_init(&dqm->lock_hidden); + INIT_LIST_HEAD(&dqm->queues); + dqm->queue_count = dqm->next_pipe_to_allocate = 0; + dqm->sdma_queue_count = 0; + dqm->xgmi_sdma_queue_count = 0; + dqm->trap_debug_vmid = 0; - for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) { - int pipe_offset = pipe * get_queues_per_pipe(dqm); + for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) { + int pipe_offset = pipe * get_queues_per_pipe(dqm); - for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) - if (test_bit(pipe_offset + queue, - dqm->dev->shared_resources.queue_bitmap)) - dqm->allocated_queues[pipe] |= 1 << queue; - } + for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) + if (test_bit(pipe_offset + queue, + dqm->dev->shared_resources.queue_bitmap)) + dqm->allocated_queues[pipe] |= 1 << queue; + } - dqm->vmid_bitmap = (1 << dqm->dev->vm_info.vmid_num_kfd) - 1; - dqm->sdma_bitmap = ~0ULL >> (64 - get_num_sdma_queues(dqm)); - dqm->xgmi_sdma_bitmap = ~0ULL >> (64 - get_num_xgmi_sdma_queues(dqm)); + dqm->vmid_bitmap = (1 << dqm->dev->vm_info.vmid_num_kfd) - 1; + dqm->sdma_bitmap = ~0ULL >> (64 - get_num_sdma_queues(dqm)); + dqm->xgmi_sdma_bitmap = ~0ULL >> (64 - get_num_xgmi_sdma_queues(dqm)); - return 0; + return 0; } static void uninitialize(struct device_queue_manager *dqm) { - int i; + int i; - WARN_ON(dqm->queue_count > 0 || dqm->processes_count > 0); + WARN_ON(dqm->queue_count > 0 || dqm->processes_count > 0); - kfree(dqm->allocated_queues); - for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++) - kfree(dqm->mqd_mgrs[i]); - mutex_destroy(&dqm->lock_hidden); - kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem); + kfree(dqm->allocated_queues); + for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++) + kfree(dqm->mqd_mgrs[i]); + mutex_destroy(&dqm->lock_hidden); + kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem); } static int start_nocpsch(struct device_queue_manager *dqm) { - init_interrupts(dqm); - return pm_init(&dqm->packets, dqm); + init_interrupts(dqm); + return pm_init(&dqm->packets, dqm); } static int stop_nocpsch(struct device_queue_manager *dqm) { - pm_uninit(&dqm->packets); - return 0; + pm_uninit(&dqm->packets); + return 0; } static int allocate_sdma_queue(struct device_queue_manager *dqm, - struct queue *q) -{ - int bit; - - if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { - if (dqm->sdma_bitmap == 0) - return -ENOMEM; - bit = __ffs64(dqm->sdma_bitmap); - dqm->sdma_bitmap &= ~(1ULL << bit); - q->sdma_id = bit; - q->properties.sdma_engine_id = q->sdma_id % - get_num_sdma_engines(dqm); - q->properties.sdma_queue_id = q->sdma_id / - get_num_sdma_engines(dqm); - } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { - if (dqm->xgmi_sdma_bitmap == 0) - return -ENOMEM; - bit = __ffs64(dqm->xgmi_sdma_bitmap); - dqm->xgmi_sdma_bitmap &= ~(1ULL << bit); - q->sdma_id = bit; - /* sdma_engine_id is sdma id including - * both PCIe-optimized SDMAs and XGMI- - * optimized SDMAs. The calculation below - * assumes the first N engines are always - * PCIe-optimized ones - */ - q->properties.sdma_engine_id = get_num_sdma_engines(dqm) + - q->sdma_id % get_num_xgmi_sdma_engines(dqm); - q->properties.sdma_queue_id = q->sdma_id / - get_num_xgmi_sdma_engines(dqm); - } + struct queue *q) +{ + int bit; - pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id); - pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id); + if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { + if (dqm->sdma_bitmap == 0) + return -ENOMEM; + bit = __ffs64(dqm->sdma_bitmap); + dqm->sdma_bitmap &= ~(1ULL << bit); + q->sdma_id = bit; + q->properties.sdma_engine_id = q->sdma_id % + get_num_sdma_engines(dqm); + q->properties.sdma_queue_id = q->sdma_id / + get_num_sdma_engines(dqm); + } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { + if (dqm->xgmi_sdma_bitmap == 0) + return -ENOMEM; + bit = __ffs64(dqm->xgmi_sdma_bitmap); + dqm->xgmi_sdma_bitmap &= ~(1ULL << bit); + q->sdma_id = bit; + /* sdma_engine_id is sdma id including + * both PCIe-optimized SDMAs and XGMI- + * optimized SDMAs. The calculation below + * assumes the first N engines are always + * PCIe-optimized ones + */ + q->properties.sdma_engine_id = get_num_sdma_engines(dqm) + + q->sdma_id % get_num_xgmi_sdma_engines(dqm); + q->properties.sdma_queue_id = q->sdma_id / + get_num_xgmi_sdma_engines(dqm); + } + + pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id); + pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id); - return 0; + return 0; } static void deallocate_sdma_queue(struct device_queue_manager *dqm, - struct queue *q) + struct queue *q) { - if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { - if (q->sdma_id >= get_num_sdma_queues(dqm)) - return; - dqm->sdma_bitmap |= (1ULL << q->sdma_id); - } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { - if (q->sdma_id >= get_num_xgmi_sdma_queues(dqm)) - return; - dqm->xgmi_sdma_bitmap |= (1ULL << q->sdma_id); - } + if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { + if (q->sdma_id >= get_num_sdma_queues(dqm)) + return; + dqm->sdma_bitmap |= (1ULL << q->sdma_id); + } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { + if (q->sdma_id >= get_num_xgmi_sdma_queues(dqm)) + return; + dqm->xgmi_sdma_bitmap |= (1ULL << q->sdma_id); + } } /* @@ -1053,60 +1053,60 @@ static void deallocate_sdma_queue(struct device_queue_manager *dqm, static int set_sched_resources(struct device_queue_manager *dqm) { - int i, mec; - struct scheduling_resources res; + int i, mec; + struct scheduling_resources res; - res.vmid_mask = dqm->dev->shared_resources.compute_vmid_bitmap; + res.vmid_mask = dqm->dev->shared_resources.compute_vmid_bitmap; - res.queue_mask = 0; - for (i = 0; i < KGD_MAX_QUEUES; ++i) { - mec = (i / dqm->dev->shared_resources.num_queue_per_pipe) - / dqm->dev->shared_resources.num_pipe_per_mec; + res.queue_mask = 0; + for (i = 0; i < KGD_MAX_QUEUES; ++i) { + mec = (i / dqm->dev->shared_resources.num_queue_per_pipe) + / dqm->dev->shared_resources.num_pipe_per_mec; - if (!test_bit(i, dqm->dev->shared_resources.queue_bitmap)) - continue; + if (!test_bit(i, dqm->dev->shared_resources.queue_bitmap)) + continue; - /* only acquire queues from the first MEC */ - if (mec > 0) - continue; + /* only acquire queues from the first MEC */ + if (mec > 0) + continue; - /* This situation may be hit in the future if a new HW - * generation exposes more than 64 queues. If so, the - * definition of res.queue_mask needs updating - */ - if (WARN_ON(i >= (sizeof(res.queue_mask)*8))) { - pr_err("Invalid queue enabled by amdgpu: %d\n", i); - break; - } + /* This situation may be hit in the future if a new HW + * generation exposes more than 64 queues. If so, the + * definition of res.queue_mask needs updating + */ + if (WARN_ON(i >= (sizeof(res.queue_mask)*8))) { + pr_err("Invalid queue enabled by amdgpu: %d\n", i); + break; + } - res.queue_mask |= (1ull << i); - } - res.gws_mask = ~0ull; - res.oac_mask = res.gds_heap_base = res.gds_heap_size = 0; + res.queue_mask |= (1ull << i); + } + res.gws_mask = ~0ull; + res.oac_mask = res.gds_heap_base = res.gds_heap_size = 0; - pr_debug("Scheduling resources:\n" - "vmid mask: 0x%8X\n" - "queue mask: 0x%8llX\n", - res.vmid_mask, res.queue_mask); + pr_debug("Scheduling resources:\n" + "vmid mask: 0x%8X\n" + "queue mask: 0x%8llX\n", + res.vmid_mask, res.queue_mask); - return pm_send_set_resources(&dqm->packets, &res); + return pm_send_set_resources(&dqm->packets, &res); } static int initialize_cpsch(struct device_queue_manager *dqm) { - pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm)); + pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm)); - mutex_init(&dqm->lock_hidden); - INIT_LIST_HEAD(&dqm->queues); - dqm->queue_count = dqm->processes_count = 0; - dqm->sdma_queue_count = 0; - dqm->xgmi_sdma_queue_count = 0; - dqm->active_runlist = false; - dqm->sdma_bitmap = ~0ULL >> (64 - get_num_sdma_queues(dqm)); - dqm->xgmi_sdma_bitmap = ~0ULL >> (64 - get_num_xgmi_sdma_queues(dqm)); - dqm->trap_debug_vmid = 0; + mutex_init(&dqm->lock_hidden); + INIT_LIST_HEAD(&dqm->queues); + dqm->queue_count = dqm->processes_count = 0; + dqm->sdma_queue_count = 0; + dqm->xgmi_sdma_queue_count = 0; + dqm->active_runlist = false; + dqm->sdma_bitmap = ~0ULL >> (64 - get_num_sdma_queues(dqm)); + dqm->xgmi_sdma_bitmap = ~0ULL >> (64 - get_num_xgmi_sdma_queues(dqm)); + dqm->trap_debug_vmid = 0; - INIT_WORK(&dqm->hw_exception_work, kfd_process_hw_exception); + INIT_WORK(&dqm->hw_exception_work, kfd_process_hw_exception); if (dqm->dev->kfd2kgd->get_iq_wait_times) dqm->dev->kfd2kgd->get_iq_wait_times(dqm->dev->kgd, @@ -1116,141 +1116,140 @@ static int initialize_cpsch(struct device_queue_manager *dqm) static int start_cpsch(struct device_queue_manager *dqm) { - int retval; + int retval; - retval = 0; + retval = 0; - retval = pm_init(&dqm->packets, dqm); - if (retval) - goto fail_packet_manager_init; + retval = pm_init(&dqm->packets, dqm); + if (retval) + goto fail_packet_manager_init; - retval = set_sched_resources(dqm); - if (retval) - goto fail_set_sched_resources; + retval = set_sched_resources(dqm); + if (retval) + goto fail_set_sched_resources; - pr_debug("Allocating fence memory\n"); + pr_debug("Allocating fence memory\n"); - /* allocate fence memory on the gart */ - retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr), - &dqm->fence_mem); + /* allocate fence memory on the gart */ + retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr), + &dqm->fence_mem); - if (retval) - goto fail_allocate_vidmem; + if (retval) + goto fail_allocate_vidmem; - dqm->fence_addr = dqm->fence_mem->cpu_ptr; - dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr; + dqm->fence_addr = dqm->fence_mem->cpu_ptr; + dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr; - init_interrupts(dqm); + init_interrupts(dqm); - dqm_lock(dqm); - /* clear hang status when driver try to start the hw scheduler */ - dqm->is_hws_hang = false; - execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, - USE_DEFAULT_GRACE_PERIOD); - dqm_unlock(dqm); + dqm_lock(dqm); + /* clear hang status when driver try to start the hw scheduler */ + dqm->is_hws_hang = false; + execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, + USE_DEFAULT_GRACE_PERIOD); + dqm_unlock(dqm); - return 0; + return 0; fail_allocate_vidmem: fail_set_sched_resources: - pm_uninit(&dqm->packets); + pm_uninit(&dqm->packets); fail_packet_manager_init: - return retval; + return retval; } static int stop_cpsch(struct device_queue_manager *dqm) { - dqm_lock(dqm); - unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, + dqm_lock(dqm); + unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD); - dqm_unlock(dqm); + dqm_unlock(dqm); - kfd_gtt_sa_free(dqm->dev, dqm->fence_mem); - pm_uninit(&dqm->packets); + kfd_gtt_sa_free(dqm->dev, dqm->fence_mem); + pm_uninit(&dqm->packets); - return 0; + return 0; } static int create_kernel_queue_cpsch(struct device_queue_manager *dqm, - struct kernel_queue *kq, - struct qcm_process_device *qpd) + struct kernel_queue *kq, + struct qcm_process_device *qpd) { - dqm_lock(dqm); - if (dqm->total_queue_count >= max_num_of_queues_per_device) { - pr_warn("Can't create new kernel queue because %d queues were already created\n", - dqm->total_queue_count); - dqm_unlock(dqm); - return -EPERM; - } + dqm_lock(dqm); + if (dqm->total_queue_count >= max_num_of_queues_per_device) { + pr_warn("Can't create new kernel queue because %d queues were already created\n", + dqm->total_queue_count); + dqm_unlock(dqm); + return -EPERM; + } - /* - * Unconditionally increment this counter, regardless of the queue's - * type or whether the queue is active. - */ - dqm->total_queue_count++; - pr_debug("Total of %d queues are accountable so far\n", - dqm->total_queue_count); + /* + * Unconditionally increment this counter, regardless of the queue's + * type or whether the queue is active. + */ + dqm->total_queue_count++; + pr_debug("Total of %d queues are accountable so far\n", + dqm->total_queue_count); - list_add(&kq->list, &qpd->priv_queue_list); - dqm->queue_count++; - qpd->is_debug = true; - execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, - USE_DEFAULT_GRACE_PERIOD); - dqm_unlock(dqm); + list_add(&kq->list, &qpd->priv_queue_list); + dqm->queue_count++; + qpd->is_debug = true; + execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, + USE_DEFAULT_GRACE_PERIOD); + dqm_unlock(dqm); - return 0; + return 0; } static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm, - struct kernel_queue *kq, - struct qcm_process_device *qpd) -{ - dqm_lock(dqm); - list_del(&kq->list); - dqm->queue_count--; - qpd->is_debug = false; - execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, - USE_DEFAULT_GRACE_PERIOD); - /* - * Unconditionally decrement this counter, regardless of the queue's - * type. - */ - dqm->total_queue_count--; - pr_debug("Total of %d queues are accountable so far\n", - dqm->total_queue_count); - dqm_unlock(dqm); + struct kernel_queue *kq, + struct qcm_process_device *qpd) +{ + dqm_lock(dqm); + list_del(&kq->list); + dqm->queue_count--; + qpd->is_debug = false; + execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, + USE_DEFAULT_GRACE_PERIOD); + /* + * Unconditionally decrement this counter, regardless of the queue's + * type. + */ + dqm->total_queue_count--; + pr_debug("Total of %d queues are accountable so far\n", + dqm->total_queue_count); + dqm_unlock(dqm); } static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, - struct qcm_process_device *qpd) + struct qcm_process_device *qpd) { - int retval; - struct mqd_manager *mqd_mgr; + int retval; + struct mqd_manager *mqd_mgr; - if (dqm->total_queue_count >= max_num_of_queues_per_device) { - pr_warn("Can't create new usermode queue because %d queues were already created\n", - dqm->total_queue_count); - retval = -EPERM; - goto out; - } + if (dqm->total_queue_count >= max_num_of_queues_per_device) { + pr_warn("Can't create new usermode queue because %d queues were already created\n", + dqm->total_queue_count); + retval = -EPERM; + goto out; + } - if (q->properties.type == KFD_QUEUE_TYPE_SDMA || - q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { - dqm_lock(dqm); - retval = allocate_sdma_queue(dqm, q); - dqm_unlock(dqm); - if (retval) - goto out; - } + if (q->properties.type == KFD_QUEUE_TYPE_SDMA || + q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { + dqm_lock(dqm); + retval = allocate_sdma_queue(dqm, q); + dqm_unlock(dqm); + if (retval) + goto out; + } - retval = allocate_doorbell(qpd, q); - if (retval) - goto out_deallocate_sdma_queue; + retval = allocate_doorbell(qpd, q); + if (retval) + goto out_deallocate_sdma_queue; - mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( - q->properties.type)]; - q->properties.is_suspended = false; + mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( + q->properties.type)]; - if (q->properties.type == KFD_QUEUE_TYPE_SDMA || + if (q->properties.type == KFD_QUEUE_TYPE_SDMA || q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) dqm->asic_ops.init_sdma_vm(dqm, q, qpd); q->properties.tba_addr = qpd->tba_addr; @@ -1268,6 +1267,7 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, * updates the is_evicted flag but is a no-op otherwise. */ q->properties.is_evicted = !!qpd->evicted; + q->properties.is_suspended = false; mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr, &q->properties); -- 2.17.1