From 0e78bd6127072259996e9cf8919f05e1627f0233 Mon Sep 17 00:00:00 2001 From: Jay Cornwall Date: Thu, 13 Apr 2017 20:01:29 -0500 Subject: [PATCH 1668/4131] drm/amdkfd: Disable CP/SDMA ring/doorbell in MQD The MQD represents an inactive context and should not have ring or doorbell enable bits set. Doing so interferes with HWS which streams the MQD onto the HQD. If enable bits are set this activates the ring or doorbell before the HQD is fully configured. Fixes SDMA faults during queue eviction. Change-Id: I98c5cbe1a91c99d7257cf9f5f4cf610cd7ce06f9 Signed-off-by: Jay Cornwall --- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c | 30 ++++++++++++++++------- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c | 24 ++++++++++++------ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c | 26 ++++++++++++++------ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c | 18 +++----------- drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c | 7 ------ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c | 11 ++------- 6 files changed, 61 insertions(+), 55 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c index 2abf945..5387fca 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c @@ -386,7 +386,7 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, struct amdgpu_device *adev = get_amdgpu_device(kgd); struct cik_mqd *m; uint32_t *mqd_hqd; - uint32_t reg, wptr_val; + uint32_t reg, wptr_val, data; m = get_mqd(mqd); @@ -395,18 +395,23 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, /* HQD registers extend from CP_MQD_BASE_ADDR to CP_MQD_CONTROL. */ mqd_hqd = &m->cp_mqd_base_addr_lo; - for (reg = mmCP_HQD_VMID; reg <= mmCP_MQD_CONTROL; reg++) + for (reg = mmCP_MQD_BASE_ADDR; reg <= mmCP_MQD_CONTROL; reg++) WREG32(reg, mqd_hqd[reg - mmCP_MQD_BASE_ADDR]); /* Copy userspace write pointer value to register. - * Doorbell logic is active and will monitor subsequent changes. + * Activate doorbell logic to monitor subsequent changes. */ + data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control, + CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1); + WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, data); + + if (read_user_wptr(mm, wptr, wptr_val)) WREG32(mmCP_HQD_PQ_WPTR, (wptr_val << wptr_shift) & wptr_mask); - - /* Write CP_HQD_ACTIVE last. */ - for (reg = mmCP_MQD_BASE_ADDR; reg <= mmCP_HQD_ACTIVE; reg++) - WREG32(reg, mqd_hqd[reg - mmCP_MQD_BASE_ADDR]); + + data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1); + WREG32(mmCP_HQD_ACTIVE, data); + release_queue(kgd); return 0; @@ -484,8 +489,11 @@ static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd, RESUME_CTX, 0); WREG32(mmSDMA0_GFX_CONTEXT_CNTL, data); } + + data = REG_SET_FIELD(m->sdma_rlc_doorbell, SDMA0_RLC0_DOORBELL, + ENABLE, 1); + WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, data); - WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, m->sdma_rlc_doorbell); WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR, m->sdma_rlc_rb_rptr); if (read_user_wptr(mm, wptr, data)) @@ -499,7 +507,11 @@ static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd, WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE_HI, m->sdma_rlc_rb_base_hi); WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_LO, m->sdma_rlc_rb_rptr_addr_lo); WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_HI, m->sdma_rlc_rb_rptr_addr_hi); - WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, m->sdma_rlc_rb_cntl); + + data = REG_SET_FIELD(m->sdma_rlc_rb_cntl, SDMA0_RLC0_RB_CNTL, + RB_ENABLE, 1); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, data); + return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c index 452bebd..4cc8052 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c @@ -376,7 +376,7 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, struct amdgpu_device *adev = get_amdgpu_device(kgd); struct vi_mqd *m; uint32_t *mqd_hqd; - uint32_t reg, wptr_val; + uint32_t reg, wptr_val, data; m = get_mqd(mqd); @@ -400,7 +400,7 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, /* HQD registers extend from CP_MQD_BASE_ADDR to CP_HQD_EOP_WPTR_MEM. */ mqd_hqd = &m->cp_mqd_base_addr_lo; - for (reg = mmCP_HQD_VMID; reg <= mmCP_HQD_EOP_CONTROL; reg++) + for (reg = mmCP_MQD_BASE_ADDR; reg <= mmCP_HQD_EOP_CONTROL; reg++) WREG32(reg, mqd_hqd[reg - mmCP_MQD_BASE_ADDR]); /* Tonga errata: EOP RPTR/WPTR should be left unmodified. @@ -418,14 +418,17 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, WREG32(reg, mqd_hqd[reg - mmCP_MQD_BASE_ADDR]); /* Copy userspace write pointer value to register. - * Doorbell logic is active and will monitor subsequent changes. + * Activate doorbell logic to monitor subsequent changes. */ + data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control, + CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1); + WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, data); + if (read_user_wptr(mm, wptr, wptr_val)) WREG32(mmCP_HQD_PQ_WPTR, (wptr_val << wptr_shift) & wptr_mask); - /* Write CP_HQD_ACTIVE last. */ - for (reg = mmCP_MQD_BASE_ADDR; reg <= mmCP_HQD_ACTIVE; reg++) - WREG32(reg, mqd_hqd[reg - mmCP_MQD_BASE_ADDR]); + data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1); + WREG32(mmCP_HQD_ACTIVE, data); release_queue(kgd); @@ -503,7 +506,9 @@ static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd, WREG32(mmSDMA0_GFX_CONTEXT_CNTL, data); } - WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, m->sdmax_rlcx_doorbell); + data = REG_SET_FIELD(m->sdmax_rlcx_doorbell, SDMA0_RLC0_DOORBELL, + ENABLE, 1); + WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, data); WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR, m->sdmax_rlcx_rb_rptr); if (read_user_wptr(mm, wptr, data)) @@ -517,7 +522,10 @@ static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd, WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE_HI, m->sdmax_rlcx_rb_base_hi); WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_LO, m->sdmax_rlcx_rb_rptr_addr_lo); WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_HI, m->sdmax_rlcx_rb_rptr_addr_hi); - WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, m->sdmax_rlcx_rb_cntl); + + data = REG_SET_FIELD(m->sdmax_rlcx_rb_cntl, SDMA0_RLC0_RB_CNTL, + RB_ENABLE, 1); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, data); return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c index a2fe8be..340f0524 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c @@ -483,7 +483,7 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, struct amdgpu_device *adev = get_amdgpu_device(kgd); struct v9_mqd *m; uint32_t *mqd_hqd; - uint32_t reg, hqd_base; + uint32_t reg, hqd_base, data; m = get_mqd(mqd); @@ -508,10 +508,16 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, mqd_hqd = &m->cp_mqd_base_addr_lo; hqd_base = SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR); - for (reg = SOC15_REG_OFFSET(GC, 0, mmCP_HQD_VMID); + for (reg = hqd_base; reg <= SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI); reg++) WREG32(reg, mqd_hqd[reg - hqd_base]); + + /* Activate doorbell logic before triggering WPTR poll. */ + data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control, + CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1); + WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL), data); + if (wptr) { /* Don't read wptr with get_user because the user * context may not be accessible (if this function @@ -556,10 +562,8 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, REG_SET_FIELD(m->cp_hqd_eop_rptr, CP_HQD_EOP_RPTR, INIT_FETCHER, 1)); - /* Write CP_HQD_ACTIVE last. */ - for (reg = SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR); - reg <= SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE); reg++) - WREG32(reg, mqd_hqd[reg - hqd_base]); + data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1); + WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE), data); release_queue(kgd); @@ -635,7 +639,10 @@ static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd, WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL_OFFSET, m->sdmax_rlcx_doorbell_offset); - WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, m->sdmax_rlcx_doorbell); + + data = REG_SET_FIELD(m->sdmax_rlcx_doorbell, SDMA0_RLC0_DOORBELL, + ENABLE, 1); + WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, data); WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR, m->sdmax_rlcx_rb_rptr); WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_HI, m->sdmax_rlcx_rb_rptr_hi); @@ -657,7 +664,10 @@ static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd, WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE_HI, m->sdmax_rlcx_rb_base_hi); WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_LO, m->sdmax_rlcx_rb_rptr_addr_lo); WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_HI, m->sdmax_rlcx_rb_rptr_addr_hi); - WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, m->sdmax_rlcx_rb_cntl); + + data = REG_SET_FIELD(m->sdmax_rlcx_rb_cntl, SDMA0_RLC0_RB_CNTL, + RB_ENABLE, 1); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, data); return 0; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c index a9e3681..41d28b3 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c @@ -264,8 +264,7 @@ static int __update_mqd(struct mqd_manager *mm, void *mqd, m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); - m->cp_hqd_pq_doorbell_control = DOORBELL_EN | - DOORBELL_OFFSET(q->doorbell_off); + m->cp_hqd_pq_doorbell_control = DOORBELL_OFFSET(q->doorbell_off); m->cp_hqd_vmid = q->vmid; @@ -276,13 +275,11 @@ static int __update_mqd(struct mqd_manager *mm, void *mqd, update_cu_mask(mm, mqd, q); set_priority(m, q); - m->cp_hqd_active = 0; q->is_active = false; if (q->queue_size > 0 && q->queue_address != 0 && q->queue_percent > 0 && !q->is_evicted) { - m->cp_hqd_active = 1; q->is_active = true; } @@ -319,9 +316,8 @@ static int update_mqd_sdma(struct mqd_manager *mm, void *mqd, m->sdma_rlc_rb_base_hi = upper_32_bits(q->queue_address >> 8); m->sdma_rlc_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr); m->sdma_rlc_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr); - m->sdma_rlc_doorbell = q->doorbell_off << - SDMA0_RLC0_DOORBELL__OFFSET__SHIFT | - 1 << SDMA0_RLC0_DOORBELL__ENABLE__SHIFT; + m->sdma_rlc_doorbell = + q->doorbell_off << SDMA0_RLC0_DOORBELL__OFFSET__SHIFT; m->sdma_rlc_virtual_addr = q->sdma_vm_addr; @@ -333,9 +329,6 @@ static int update_mqd_sdma(struct mqd_manager *mm, void *mqd, q->queue_address != 0 && q->queue_percent > 0 && !q->is_evicted) { - m->sdma_rlc_rb_cntl |= - 1 << SDMA0_RLC0_RB_CNTL__RB_ENABLE__SHIFT; - q->is_active = true; } @@ -471,18 +464,15 @@ static int update_mqd_hiq(struct mqd_manager *mm, void *mqd, m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); - m->cp_hqd_pq_doorbell_control = DOORBELL_EN | - DOORBELL_OFFSET(q->doorbell_off); + m->cp_hqd_pq_doorbell_control = DOORBELL_OFFSET(q->doorbell_off); m->cp_hqd_vmid = q->vmid; - m->cp_hqd_active = 0; q->is_active = false; if (q->queue_size > 0 && q->queue_address != 0 && q->queue_percent > 0 && !q->is_evicted) { - m->cp_hqd_active = 1; q->is_active = true; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c index 2f12e80..0db4dbf 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c @@ -212,7 +212,6 @@ static int update_mqd(struct mqd_manager *mm, void *mqd, m->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits((uint64_t)q->write_ptr); m->cp_hqd_pq_doorbell_control = - 1 << CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_EN__SHIFT | q->doorbell_off << CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT; pr_debug("kfd: cp_hqd_pq_doorbell_control 0x%x\n", @@ -251,13 +250,11 @@ static int update_mqd(struct mqd_manager *mm, void *mqd, update_cu_mask(mm, mqd, q); - m->cp_hqd_active = 0; q->is_active = false; if (q->queue_size > 0 && q->queue_address != 0 && q->queue_percent > 0 && !q->is_evicted) { - m->cp_hqd_active = 1; q->is_active = true; } @@ -391,7 +388,6 @@ static int update_mqd_sdma(struct mqd_manager *mm, void *mqd, m->sdmax_rlcx_rb_base_hi = upper_32_bits(q->queue_address >> 8); m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr); m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr); - m->sdmax_rlcx_doorbell = 1 << SDMA0_RLC0_DOORBELL__ENABLE__SHIFT; m->sdmax_rlcx_doorbell_offset = q->doorbell_off << SDMA0_RLC0_DOORBELL_OFFSET__OFFSET__SHIFT; @@ -404,9 +400,6 @@ static int update_mqd_sdma(struct mqd_manager *mm, void *mqd, q->queue_address != 0 && q->queue_percent > 0 && !q->is_evicted) { - m->sdmax_rlcx_rb_cntl |= - 1 << SDMA0_RLC0_RB_CNTL__RB_ENABLE__SHIFT; - q->is_active = true; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c index fb30623..0aeebc1 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c @@ -221,7 +221,6 @@ static int __update_mqd(struct mqd_manager *mm, void *mqd, m->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits((uint64_t)q->write_ptr); m->cp_hqd_pq_doorbell_control = - 1 << CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_EN__SHIFT | q->doorbell_off << CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT; pr_debug("kfd: cp_hqd_pq_doorbell_control 0x%x\n", @@ -265,13 +264,11 @@ static int __update_mqd(struct mqd_manager *mm, void *mqd, update_cu_mask(mm, mqd, q); set_priority(m, q); - m->cp_hqd_active = 0; q->is_active = false; if (q->queue_size > 0 && q->queue_address != 0 && q->queue_percent > 0 && !q->is_evicted) { - m->cp_hqd_active = 1; q->is_active = true; } @@ -412,9 +409,8 @@ static int update_mqd_sdma(struct mqd_manager *mm, void *mqd, m->sdmax_rlcx_rb_base_hi = upper_32_bits(q->queue_address >> 8); m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr); m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr); - m->sdmax_rlcx_doorbell = q->doorbell_off << - SDMA0_RLC0_DOORBELL__OFFSET__SHIFT | - 1 << SDMA0_RLC0_DOORBELL__ENABLE__SHIFT; + m->sdmax_rlcx_doorbell = + q->doorbell_off << SDMA0_RLC0_DOORBELL__OFFSET__SHIFT; m->sdmax_rlcx_virtual_addr = q->sdma_vm_addr; @@ -426,9 +422,6 @@ static int update_mqd_sdma(struct mqd_manager *mm, void *mqd, q->queue_address != 0 && q->queue_percent > 0 && !q->is_evicted) { - m->sdmax_rlcx_rb_cntl |= - 1 << SDMA0_RLC0_RB_CNTL__RB_ENABLE__SHIFT; - q->is_active = true; } -- 2.7.4