From 7995d33fb0f9bd70003cd4452fe64ad392750367 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Thu, 23 Nov 2017 18:53:15 -0500 Subject: [PATCH 3050/4131] drm/amdkfd: Add invalidate_tlbs_vmid into kfd2dgd_calls interface Change-Id: I108b989881bd84dce9748645d7d543cfd13bef92 Signed-off-by: Yong Zhao --- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c | 29 +++++++++++++++++------ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c | 29 +++++++++++++++++------ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c | 15 ++++++++++++ drivers/gpu/drm/amd/include/kgd_kfd_interface.h | 1 + 4 files changed, 60 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c index 4faf452..3a4e5ed 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c @@ -147,6 +147,7 @@ static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd, uint8_t vmid); static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd, uint8_t vmid); static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid); +static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid); static void set_num_of_requests(struct kgd_dev *dev, uint8_t num_of_req); static int alloc_memory_of_scratch(struct kgd_dev *kgd, uint64_t va, uint32_t vmid); @@ -216,6 +217,7 @@ static const struct kfd2kgd_calls kfd2kgd = { get_atc_vmid_pasid_mapping_valid, .read_vmid_from_vmfault_reg = read_vmid_from_vmfault_reg, .invalidate_tlbs = invalidate_tlbs, + .invalidate_tlbs_vmid = invalidate_tlbs_vmid, .sync_memory = amdgpu_amdkfd_gpuvm_sync_memory, .alloc_memory_of_gpu = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu, .free_memory_of_gpu = amdgpu_amdkfd_gpuvm_free_memory_of_gpu, @@ -840,23 +842,36 @@ static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid) { struct amdgpu_device *adev = (struct amdgpu_device *) kgd; int vmid; + unsigned int tmp; for (vmid = 0; vmid < 16; vmid++) { if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) continue; - if (RREG32(mmATC_VMID0_PASID_MAPPING + vmid) & - ATC_VMID0_PASID_MAPPING__VALID_MASK) { - if ((RREG32(mmATC_VMID0_PASID_MAPPING + vmid) & - ATC_VMID0_PASID_MAPPING__PASID_MASK) == pasid) { - WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid); - break; - } + + tmp = RREG32(mmATC_VMID0_PASID_MAPPING + vmid); + if ((tmp & ATC_VMID0_PASID_MAPPING__VALID_MASK) && + (tmp & ATC_VMID0_PASID_MAPPING__PASID_MASK) == pasid) { + WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid); + break; } } return 0; } +static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid) +{ + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + + if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) { + pr_err("non kfd vmid\n"); + return 0; + } + + WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid); + return 0; +} + static int write_config_static_mem(struct kgd_dev *kgd, bool swizzle_enable, uint8_t element_size, uint8_t index_stride, uint8_t mtype) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c index 6228dd8..aa23fa3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c @@ -128,6 +128,7 @@ static int write_config_static_mem(struct kgd_dev *kgd, bool swizzle_enable, static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid, uint32_t page_table_base); static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid); +static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid); /* Because of REG_GET_FIELD() being used, we put this function in the * asic specific file. @@ -189,6 +190,7 @@ static const struct kfd2kgd_calls kfd2kgd = { .get_atc_vmid_pasid_mapping_valid = get_atc_vmid_pasid_mapping_valid, .invalidate_tlbs = invalidate_tlbs, + .invalidate_tlbs_vmid = invalidate_tlbs_vmid, .sync_memory = amdgpu_amdkfd_gpuvm_sync_memory, .alloc_memory_of_gpu = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu, .free_memory_of_gpu = amdgpu_amdkfd_gpuvm_free_memory_of_gpu, @@ -811,6 +813,7 @@ static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid) { struct amdgpu_device *adev = (struct amdgpu_device *) kgd; int vmid; + unsigned int tmp; #ifdef V8_SUPPORT_IT_OFFICIAL struct amdgpu_ring *ring = &adev->gfx.kiq.ring; @@ -822,19 +825,31 @@ static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid) for (vmid = 0; vmid < 16; vmid++) { if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) continue; - if (RREG32(mmATC_VMID0_PASID_MAPPING + vmid) & - ATC_VMID0_PASID_MAPPING__VALID_MASK) { - if ((RREG32(mmATC_VMID0_PASID_MAPPING + vmid) & - ATC_VMID0_PASID_MAPPING__PASID_MASK) == pasid) { - WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid); - break; - } + + tmp = RREG32(mmATC_VMID0_PASID_MAPPING + vmid); + if ((tmp & ATC_VMID0_PASID_MAPPING__VALID_MASK) && + (tmp & ATC_VMID0_PASID_MAPPING__PASID_MASK) == pasid) { + WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid); + break; } } return 0; } +static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid) +{ + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + + if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) { + pr_err("non kfd vmid %d\n", vmid); + return 0; + } + + WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid); + return 0; +} + static int kgd_address_watch_disable(struct kgd_dev *kgd) { struct amdgpu_device *adev = get_amdgpu_device(kgd); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c index 0f6b395..8eae0a7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c @@ -165,6 +165,7 @@ static int write_config_static_mem(struct kgd_dev *kgd, bool swizzle_enable, static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid, uint32_t page_table_base); static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid); +static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid); /* Because of REG_GET_FIELD() being used, we put this function in the * asic specific file. @@ -233,6 +234,7 @@ static const struct kfd2kgd_calls kfd2kgd = { .get_atc_vmid_pasid_mapping_valid = get_atc_vmid_pasid_mapping_valid, .invalidate_tlbs = invalidate_tlbs, + .invalidate_tlbs_vmid = invalidate_tlbs_vmid, .sync_memory = amdgpu_amdkfd_gpuvm_sync_memory, .alloc_memory_of_gpu = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu, .free_memory_of_gpu = amdgpu_amdkfd_gpuvm_free_memory_of_gpu, @@ -1016,6 +1018,19 @@ static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid) return 0; } +static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid) +{ + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + + if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) { + pr_err("non kfd vmid %d\n", vmid); + return 0; + } + + write_vmid_invalidate_request(kgd, vmid); + return 0; +} + static int kgd_address_watch_disable(struct kgd_dev *kgd) { struct amdgpu_device *adev = get_amdgpu_device(kgd); diff --git a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h index f94f49b..1da5b4f 100644 --- a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h +++ b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h @@ -345,6 +345,7 @@ struct kfd2kgd_calls { uint32_t (*read_vmid_from_vmfault_reg)(struct kgd_dev *kgd); int (*invalidate_tlbs)(struct kgd_dev *kgd, uint16_t pasid); + int (*invalidate_tlbs_vmid)(struct kgd_dev *kgd, uint16_t vmid); int (*sync_memory)(struct kgd_dev *kgd, struct kgd_mem *mem, bool intr); -- 2.7.4