From fd817d44483158725d4ac55b5a4d1f7fc1eaec7d Mon Sep 17 00:00:00 2001 From: Junwei Zhang Date: Fri, 23 Dec 2016 16:28:35 +0800 Subject: [PATCH 3856/4131] drm/amd/amdkcl: add fence functions Change-Id: I5ecbe63c5fd6765cd28ee570420e347e8283b8d0 Signed-off-by: Junwei Zhang Reviewed-by: Qiang Yu --- drivers/gpu/drm/amd/scheduler/gpu_scheduler.c | 687 ++++++++++++++++++++++++++ drivers/gpu/drm/amd/scheduler/gpu_scheduler.h | 174 +++++++ drivers/gpu/drm/amd/scheduler/sched_fence.c | 181 +++++++ 3 files changed, 1042 insertions(+) create mode 100644 drivers/gpu/drm/amd/scheduler/gpu_scheduler.c create mode 100644 drivers/gpu/drm/amd/scheduler/gpu_scheduler.h create mode 100644 drivers/gpu/drm/amd/scheduler/sched_fence.c diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c new file mode 100644 index 0000000..735a5a4 --- /dev/null +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c @@ -0,0 +1,687 @@ +/* + * Copyright 2015 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * + */ +#include +#include +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) +#include +#endif +#include +#include "gpu_scheduler.h" + +#define CREATE_TRACE_POINTS +#include "gpu_sched_trace.h" + +static bool amd_sched_entity_is_ready(struct amd_sched_entity *entity); +static void amd_sched_wakeup(struct amd_gpu_scheduler *sched); +static void amd_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb); + +/* Initialize a given run queue struct */ +static void amd_sched_rq_init(struct amd_sched_rq *rq) +{ + spin_lock_init(&rq->lock); + INIT_LIST_HEAD(&rq->entities); + rq->current_entity = NULL; +} + +static void amd_sched_rq_add_entity(struct amd_sched_rq *rq, + struct amd_sched_entity *entity) +{ + if (!list_empty(&entity->list)) + return; + spin_lock(&rq->lock); + list_add_tail(&entity->list, &rq->entities); + spin_unlock(&rq->lock); +} + +static void amd_sched_rq_remove_entity(struct amd_sched_rq *rq, + struct amd_sched_entity *entity) +{ + if (list_empty(&entity->list)) + return; + spin_lock(&rq->lock); + list_del_init(&entity->list); + if (rq->current_entity == entity) + rq->current_entity = NULL; + spin_unlock(&rq->lock); +} + +/** + * Select an entity which could provide a job to run + * + * @rq The run queue to check. + * + * Try to find a ready entity, returns NULL if none found. + */ +static struct amd_sched_entity * +amd_sched_rq_select_entity(struct amd_sched_rq *rq) +{ + struct amd_sched_entity *entity; + + spin_lock(&rq->lock); + + entity = rq->current_entity; + if (entity) { + list_for_each_entry_continue(entity, &rq->entities, list) { + if (amd_sched_entity_is_ready(entity)) { + rq->current_entity = entity; + spin_unlock(&rq->lock); + return entity; + } + } + } + + list_for_each_entry(entity, &rq->entities, list) { + + if (amd_sched_entity_is_ready(entity)) { + rq->current_entity = entity; + spin_unlock(&rq->lock); + return entity; + } + + if (entity == rq->current_entity) + break; + } + + spin_unlock(&rq->lock); + + return NULL; +} + +/** + * Init a context entity used by scheduler when submit to HW ring. + * + * @sched The pointer to the scheduler + * @entity The pointer to a valid amd_sched_entity + * @rq The run queue this entity belongs + * @kernel If this is an entity for the kernel + * @jobs The max number of jobs in the job queue + * + * return 0 if succeed. negative error code on failure +*/ +int amd_sched_entity_init(struct amd_gpu_scheduler *sched, + struct amd_sched_entity *entity, + struct amd_sched_rq *rq, + uint32_t jobs) +{ + int r; + + if (!(sched && entity && rq)) + return -EINVAL; + + memset(entity, 0, sizeof(struct amd_sched_entity)); + INIT_LIST_HEAD(&entity->list); + entity->rq = rq; + entity->sched = sched; + + spin_lock_init(&entity->queue_lock); + r = kfifo_alloc(&entity->job_queue, jobs * sizeof(void *), GFP_KERNEL); + if (r) + return r; + + atomic_set(&entity->fence_seq, 0); + entity->fence_context = kcl_fence_context_alloc(2); + + return 0; +} + +/** + * Query if entity is initialized + * + * @sched Pointer to scheduler instance + * @entity The pointer to a valid scheduler entity + * + * return true if entity is initialized, false otherwise +*/ +static bool amd_sched_entity_is_initialized(struct amd_gpu_scheduler *sched, + struct amd_sched_entity *entity) +{ + return entity->sched == sched && + entity->rq != NULL; +} + +/** + * Check if entity is idle + * + * @entity The pointer to a valid scheduler entity + * + * Return true if entity don't has any unscheduled jobs. + */ +static bool amd_sched_entity_is_idle(struct amd_sched_entity *entity) +{ + rmb(); + if (kfifo_is_empty(&entity->job_queue)) + return true; + + return false; +} + +/** + * Check if entity is ready + * + * @entity The pointer to a valid scheduler entity + * + * Return true if entity could provide a job. + */ +static bool amd_sched_entity_is_ready(struct amd_sched_entity *entity) +{ + if (kfifo_is_empty(&entity->job_queue)) + return false; + + if (ACCESS_ONCE(entity->dependency)) + return false; + + return true; +} + +/** + * Destroy a context entity + * + * @sched Pointer to scheduler instance + * @entity The pointer to a valid scheduler entity + * + * Cleanup and free the allocated resources. + */ +void amd_sched_entity_fini(struct amd_gpu_scheduler *sched, + struct amd_sched_entity *entity) +{ + struct amd_sched_rq *rq = entity->rq; + int r; + + if (!amd_sched_entity_is_initialized(sched, entity)) + return; + /** + * The client will not queue more IBs during this fini, consume existing + * queued IBs or discard them on SIGKILL + */ + if ((current->flags & PF_SIGNALED) && current->exit_code == SIGKILL) + r = -ERESTARTSYS; + else + r = wait_event_killable(sched->job_scheduled, + amd_sched_entity_is_idle(entity)); + amd_sched_rq_remove_entity(rq, entity); + if (r) { + struct amd_sched_job *job; + + /* Park the kernel for a moment to make sure it isn't processing + * our enity. + */ + kcl_kthread_park(sched->thread); + kcl_kthread_unpark(sched->thread); + while (kfifo_out(&entity->job_queue, &job, sizeof(job))) + sched->ops->free_job(job); + + } + kfifo_free(&entity->job_queue); +} + +static void amd_sched_entity_wakeup(struct dma_fence *f, struct dma_fence_cb *cb) +{ + struct amd_sched_entity *entity = + container_of(cb, struct amd_sched_entity, cb); + entity->dependency = NULL; + dma_fence_put(f); + amd_sched_wakeup(entity->sched); +} + +static void amd_sched_entity_clear_dep(struct dma_fence *f, struct dma_fence_cb *cb) +{ + struct amd_sched_entity *entity = + container_of(cb, struct amd_sched_entity, cb); + entity->dependency = NULL; + dma_fence_put(f); +} + +bool amd_sched_dependency_optimized(struct dma_fence* fence, + struct amd_sched_entity *entity) +{ + struct amd_gpu_scheduler *sched = entity->sched; + struct amd_sched_fence *s_fence; + + if (!fence || dma_fence_is_signaled(fence)) + return false; + if (fence->context == entity->fence_context) + return true; + s_fence = to_amd_sched_fence(fence); + if (s_fence && s_fence->sched == sched) + return true; + + return false; +} + +static bool amd_sched_entity_add_dependency_cb(struct amd_sched_entity *entity) +{ + struct amd_gpu_scheduler *sched = entity->sched; + struct dma_fence * fence = entity->dependency; + struct amd_sched_fence *s_fence; + + if (fence->context == entity->fence_context) { + /* We can ignore fences from ourself */ + dma_fence_put(entity->dependency); + return false; + } + + s_fence = to_amd_sched_fence(fence); + if (s_fence && s_fence->sched == sched) { + + /* + * Fence is from the same scheduler, only need to wait for + * it to be scheduled + */ + fence = dma_fence_get(&s_fence->scheduled); + dma_fence_put(entity->dependency); + entity->dependency = fence; + if (!dma_fence_add_callback(fence, &entity->cb, + amd_sched_entity_clear_dep)) + return true; + + /* Ignore it when it is already scheduled */ + dma_fence_put(fence); + return false; + } + + if (!dma_fence_add_callback(entity->dependency, &entity->cb, + amd_sched_entity_wakeup)) + return true; + + dma_fence_put(entity->dependency); + return false; +} + +static struct amd_sched_job * +amd_sched_entity_pop_job(struct amd_sched_entity *entity) +{ + struct amd_gpu_scheduler *sched = entity->sched; + struct amd_sched_job *sched_job; + + if (!kfifo_out_peek(&entity->job_queue, &sched_job, sizeof(sched_job))) + return NULL; + + while ((entity->dependency = sched->ops->dependency(sched_job))) + if (amd_sched_entity_add_dependency_cb(entity)) + return NULL; + + return sched_job; +} + +/** + * Helper to submit a job to the job queue + * + * @sched_job The pointer to job required to submit + * + * Returns true if we could submit the job. + */ +static bool amd_sched_entity_in(struct amd_sched_job *sched_job) +{ + struct amd_gpu_scheduler *sched = sched_job->sched; + struct amd_sched_entity *entity = sched_job->s_entity; + bool added, first = false; + + spin_lock(&entity->queue_lock); + added = kfifo_in(&entity->job_queue, &sched_job, + sizeof(sched_job)) == sizeof(sched_job); + + if (added && kfifo_len(&entity->job_queue) == sizeof(sched_job)) + first = true; + + spin_unlock(&entity->queue_lock); + + /* first job wakes up scheduler */ + if (first) { + /* Add the entity to the run queue */ + amd_sched_rq_add_entity(entity->rq, entity); + amd_sched_wakeup(sched); + } + return added; +} + +/* job_finish is called after hw fence signaled, and + * the job had already been deleted from ring_mirror_list + */ +static void amd_sched_job_finish(struct work_struct *work) +{ + struct amd_sched_job *s_job = container_of(work, struct amd_sched_job, + finish_work); + struct amd_gpu_scheduler *sched = s_job->sched; + + /* remove job from ring_mirror_list */ + spin_lock(&sched->job_list_lock); + list_del_init(&s_job->node); + if (sched->timeout != MAX_SCHEDULE_TIMEOUT) { + struct amd_sched_job *next; + + spin_unlock(&sched->job_list_lock); + cancel_delayed_work_sync(&s_job->work_tdr); + spin_lock(&sched->job_list_lock); + + /* queue TDR for next job */ + next = list_first_entry_or_null(&sched->ring_mirror_list, + struct amd_sched_job, node); + + if (next) + schedule_delayed_work(&next->work_tdr, sched->timeout); + } + spin_unlock(&sched->job_list_lock); + sched->ops->free_job(s_job); +} + +static void amd_sched_job_finish_cb(struct dma_fence *f, + struct dma_fence_cb *cb) +{ + struct amd_sched_job *job = container_of(cb, struct amd_sched_job, + finish_cb); + schedule_work(&job->finish_work); +} + +static void amd_sched_job_begin(struct amd_sched_job *s_job) +{ + struct amd_gpu_scheduler *sched = s_job->sched; + + spin_lock(&sched->job_list_lock); + list_add_tail(&s_job->node, &sched->ring_mirror_list); + if (sched->timeout != MAX_SCHEDULE_TIMEOUT && + list_first_entry_or_null(&sched->ring_mirror_list, + struct amd_sched_job, node) == s_job) + schedule_delayed_work(&s_job->work_tdr, sched->timeout); + spin_unlock(&sched->job_list_lock); +} + +static void amd_sched_job_timedout(struct work_struct *work) +{ + struct amd_sched_job *job = container_of(work, struct amd_sched_job, + work_tdr.work); + + job->sched->ops->timedout_job(job); +} + +void amd_sched_hw_job_reset(struct amd_gpu_scheduler *sched) +{ + struct amd_sched_job *s_job; + + spin_lock(&sched->job_list_lock); + list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) { + if (s_job->s_fence->parent && + dma_fence_remove_callback(s_job->s_fence->parent, + &s_job->s_fence->cb)) { + dma_fence_put(s_job->s_fence->parent); + s_job->s_fence->parent = NULL; + atomic_dec(&sched->hw_rq_count); + } + } + spin_unlock(&sched->job_list_lock); +} + +void amd_sched_job_kickout(struct amd_sched_job *s_job) +{ + struct amd_gpu_scheduler *sched = s_job->sched; + + spin_lock(&sched->job_list_lock); + list_del_init(&s_job->node); + spin_unlock(&sched->job_list_lock); +} + +void amd_sched_job_recovery(struct amd_gpu_scheduler *sched) +{ + struct amd_sched_job *s_job, *tmp; + int r; + + spin_lock(&sched->job_list_lock); + s_job = list_first_entry_or_null(&sched->ring_mirror_list, + struct amd_sched_job, node); + if (s_job && sched->timeout != MAX_SCHEDULE_TIMEOUT) + schedule_delayed_work(&s_job->work_tdr, sched->timeout); + + list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) { + struct amd_sched_fence *s_fence = s_job->s_fence; + struct dma_fence *fence; + + spin_unlock(&sched->job_list_lock); + fence = sched->ops->run_job(s_job); + atomic_inc(&sched->hw_rq_count); + if (fence) { + s_fence->parent = dma_fence_get(fence); + r = dma_fence_add_callback(fence, &s_fence->cb, + amd_sched_process_job); + if (r == -ENOENT) + amd_sched_process_job(fence, &s_fence->cb); + else if (r) + DRM_ERROR("fence add callback failed (%d)\n", + r); + dma_fence_put(fence); + } else { + DRM_ERROR("Failed to run job!\n"); + amd_sched_process_job(NULL, &s_fence->cb); + } + spin_lock(&sched->job_list_lock); + } + spin_unlock(&sched->job_list_lock); +} + +/** + * Submit a job to the job queue + * + * @sched_job The pointer to job required to submit + * + * Returns 0 for success, negative error code otherwise. + */ +void amd_sched_entity_push_job(struct amd_sched_job *sched_job) +{ + struct amd_sched_entity *entity = sched_job->s_entity; + + trace_amd_sched_job(sched_job); + dma_fence_add_callback(&sched_job->s_fence->finished, &sched_job->finish_cb, + amd_sched_job_finish_cb); + wait_event(entity->sched->job_scheduled, + amd_sched_entity_in(sched_job)); +} + +/* init a sched_job with basic field */ +int amd_sched_job_init(struct amd_sched_job *job, + struct amd_gpu_scheduler *sched, + struct amd_sched_entity *entity, + void *owner) +{ + job->sched = sched; + job->s_entity = entity; + job->s_fence = amd_sched_fence_create(entity, owner); + if (!job->s_fence) + return -ENOMEM; + job->id = atomic64_inc_return(&sched->job_id_count); + + INIT_WORK(&job->finish_work, amd_sched_job_finish); + INIT_LIST_HEAD(&job->node); + INIT_DELAYED_WORK(&job->work_tdr, amd_sched_job_timedout); + + return 0; +} + +/** + * Return ture if we can push more jobs to the hw. + */ +static bool amd_sched_ready(struct amd_gpu_scheduler *sched) +{ + return atomic_read(&sched->hw_rq_count) < + sched->hw_submission_limit; +} + +/** + * Wake up the scheduler when it is ready + */ +static void amd_sched_wakeup(struct amd_gpu_scheduler *sched) +{ + if (amd_sched_ready(sched)) + wake_up_interruptible(&sched->wake_up_worker); +} + +/** + * Select next entity to process +*/ +static struct amd_sched_entity * +amd_sched_select_entity(struct amd_gpu_scheduler *sched) +{ + struct amd_sched_entity *entity; + int i; + + if (!amd_sched_ready(sched)) + return NULL; + + /* Kernel run queue has higher priority than normal run queue*/ + for (i = AMD_SCHED_PRIORITY_MAX - 1; i >= AMD_SCHED_PRIORITY_MIN; i--) { + entity = amd_sched_rq_select_entity(&sched->sched_rq[i]); + if (entity) + break; + } + + return entity; +} + +static void amd_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb) +{ + struct amd_sched_fence *s_fence = + container_of(cb, struct amd_sched_fence, cb); + struct amd_gpu_scheduler *sched = s_fence->sched; + + atomic_dec(&sched->hw_rq_count); + amd_sched_fence_finished(s_fence); + + trace_amd_sched_process_job(s_fence); + dma_fence_put(&s_fence->finished); + wake_up_interruptible(&sched->wake_up_worker); +} + +static bool amd_sched_blocked(struct amd_gpu_scheduler *sched) +{ + if (kcl_kthread_should_park()) { + kcl_kthread_parkme(); + return true; + } + + return false; +} + +static int amd_sched_main(void *param) +{ + struct sched_param sparam = {.sched_priority = 1}; + struct amd_gpu_scheduler *sched = (struct amd_gpu_scheduler *)param; + int r, count; + + sched_setscheduler(current, SCHED_FIFO, &sparam); + + while (!kthread_should_stop()) { + struct amd_sched_entity *entity = NULL; + struct amd_sched_fence *s_fence; + struct amd_sched_job *sched_job; + struct dma_fence *fence; + + wait_event_interruptible(sched->wake_up_worker, + (!amd_sched_blocked(sched) && + (entity = amd_sched_select_entity(sched))) || + kthread_should_stop()); + + if (!entity) + continue; + + sched_job = amd_sched_entity_pop_job(entity); + if (!sched_job) + continue; + + s_fence = sched_job->s_fence; + + atomic_inc(&sched->hw_rq_count); + amd_sched_job_begin(sched_job); + + fence = sched->ops->run_job(sched_job); + amd_sched_fence_scheduled(s_fence); + if (fence) { + s_fence->parent = dma_fence_get(fence); + r = dma_fence_add_callback(fence, &s_fence->cb, + amd_sched_process_job); + if (r == -ENOENT) + amd_sched_process_job(fence, &s_fence->cb); + else if (r) + DRM_ERROR("fence add callback failed (%d)\n", + r); + dma_fence_put(fence); + } else { + DRM_ERROR("Failed to run job!\n"); + amd_sched_process_job(NULL, &s_fence->cb); + } + + count = kfifo_out(&entity->job_queue, &sched_job, + sizeof(sched_job)); + WARN_ON(count != sizeof(sched_job)); + wake_up(&sched->job_scheduled); + } + return 0; +} + +/** + * Init a gpu scheduler instance + * + * @sched The pointer to the scheduler + * @ops The backend operations for this scheduler. + * @hw_submissions Number of hw submissions to do. + * @name Name used for debugging + * + * Return 0 on success, otherwise error code. +*/ +int amd_sched_init(struct amd_gpu_scheduler *sched, + const struct amd_sched_backend_ops *ops, + unsigned hw_submission, long timeout, const char *name) +{ + int i; + sched->ops = ops; + sched->hw_submission_limit = hw_submission; + sched->name = name; + sched->timeout = timeout; + for (i = AMD_SCHED_PRIORITY_MIN; i < AMD_SCHED_PRIORITY_MAX; i++) + amd_sched_rq_init(&sched->sched_rq[i]); + + init_waitqueue_head(&sched->wake_up_worker); + init_waitqueue_head(&sched->job_scheduled); + INIT_LIST_HEAD(&sched->ring_mirror_list); + spin_lock_init(&sched->job_list_lock); + atomic_set(&sched->hw_rq_count, 0); + atomic64_set(&sched->job_id_count, 0); + + /* Each scheduler will run on a seperate kernel thread */ + sched->thread = kthread_run(amd_sched_main, sched, sched->name); + if (IS_ERR(sched->thread)) { + DRM_ERROR("Failed to create scheduler for %s.\n", name); + return PTR_ERR(sched->thread); + } + + return 0; +} + +/** + * Destroy a gpu scheduler + * + * @sched The pointer to the scheduler + */ +void amd_sched_fini(struct amd_gpu_scheduler *sched) +{ + if (sched->thread) + kthread_stop(sched->thread); +} diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h new file mode 100644 index 0000000..92aef31 --- /dev/null +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h @@ -0,0 +1,174 @@ +/* + * Copyright 2015 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef _GPU_SCHEDULER_H_ +#define _GPU_SCHEDULER_H_ + +#if (defined OS_NAME_RHEL) && (OS_VERSION_MAJOR == 6) +#include +#else +#include +#endif +#include + +struct amd_gpu_scheduler; +struct amd_sched_rq; + +/** + * A scheduler entity is a wrapper around a job queue or a group + * of other entities. Entities take turns emitting jobs from their + * job queues to corresponding hardware ring based on scheduling + * policy. +*/ +struct amd_sched_entity { + struct list_head list; + struct amd_sched_rq *rq; + struct amd_gpu_scheduler *sched; + + spinlock_t queue_lock; + struct kfifo job_queue; + + atomic_t fence_seq; + uint64_t fence_context; + + struct dma_fence *dependency; + struct dma_fence_cb cb; +}; + +/** + * Run queue is a set of entities scheduling command submissions for + * one specific ring. It implements the scheduling policy that selects + * the next entity to emit commands from. +*/ +struct amd_sched_rq { + spinlock_t lock; + struct list_head entities; + struct amd_sched_entity *current_entity; +}; + +struct amd_sched_fence { + struct dma_fence scheduled; + struct dma_fence finished; + struct dma_fence_cb cb; + struct dma_fence *parent; + struct amd_gpu_scheduler *sched; + spinlock_t lock; + void *owner; +}; + +struct amd_sched_job { + struct amd_gpu_scheduler *sched; + struct amd_sched_entity *s_entity; + struct amd_sched_fence *s_fence; + struct dma_fence_cb finish_cb; + struct work_struct finish_work; + struct list_head node; + struct delayed_work work_tdr; + uint64_t id; + atomic_t karma; +}; + +extern const struct dma_fence_ops amd_sched_fence_ops_scheduled; +extern const struct dma_fence_ops amd_sched_fence_ops_finished; +static inline struct amd_sched_fence *to_amd_sched_fence(struct dma_fence *f) +{ + if (f->ops == &amd_sched_fence_ops_scheduled) + return container_of(f, struct amd_sched_fence, scheduled); + + if (f->ops == &amd_sched_fence_ops_finished) + return container_of(f, struct amd_sched_fence, finished); + + return NULL; +} + +static inline bool amd_sched_invalidate_job(struct amd_sched_job *s_job, int threshold) +{ + return (s_job && atomic_inc_return(&s_job->karma) > threshold); +} + +/** + * Define the backend operations called by the scheduler, + * these functions should be implemented in driver side +*/ +struct amd_sched_backend_ops { + struct dma_fence *(*dependency)(struct amd_sched_job *sched_job); + struct dma_fence *(*run_job)(struct amd_sched_job *sched_job); + void (*timedout_job)(struct amd_sched_job *sched_job); + void (*free_job)(struct amd_sched_job *sched_job); +}; + +enum amd_sched_priority { + AMD_SCHED_PRIORITY_MIN, + AMD_SCHED_PRIORITY_NORMAL = AMD_SCHED_PRIORITY_MIN, + AMD_SCHED_PRIORITY_KERNEL, + AMD_SCHED_PRIORITY_MAX +}; + +/** + * One scheduler is implemented for each hardware ring +*/ +struct amd_gpu_scheduler { + const struct amd_sched_backend_ops *ops; + uint32_t hw_submission_limit; + long timeout; + const char *name; + struct amd_sched_rq sched_rq[AMD_SCHED_PRIORITY_MAX]; + wait_queue_head_t wake_up_worker; + wait_queue_head_t job_scheduled; + atomic_t hw_rq_count; + atomic64_t job_id_count; + struct task_struct *thread; + struct list_head ring_mirror_list; + spinlock_t job_list_lock; +}; + +int amd_sched_init(struct amd_gpu_scheduler *sched, + const struct amd_sched_backend_ops *ops, + uint32_t hw_submission, long timeout, const char *name); +void amd_sched_fini(struct amd_gpu_scheduler *sched); + +int amd_sched_entity_init(struct amd_gpu_scheduler *sched, + struct amd_sched_entity *entity, + struct amd_sched_rq *rq, + uint32_t jobs); +void amd_sched_entity_fini(struct amd_gpu_scheduler *sched, + struct amd_sched_entity *entity); +void amd_sched_entity_push_job(struct amd_sched_job *sched_job); + +int amd_sched_fence_slab_init(void); +void amd_sched_fence_slab_fini(void); + +struct amd_sched_fence *amd_sched_fence_create( + struct amd_sched_entity *s_entity, void *owner); +void amd_sched_fence_scheduled(struct amd_sched_fence *fence); +void amd_sched_fence_finished(struct amd_sched_fence *fence); +int amd_sched_job_init(struct amd_sched_job *job, + struct amd_gpu_scheduler *sched, + struct amd_sched_entity *entity, + void *owner); +void amd_sched_hw_job_reset(struct amd_gpu_scheduler *sched); +void amd_sched_job_recovery(struct amd_gpu_scheduler *sched); +bool amd_sched_dependency_optimized(struct dma_fence* fence, + struct amd_sched_entity *entity); +void amd_sched_job_kickout(struct amd_sched_job *s_job); +#endif diff --git a/drivers/gpu/drm/amd/scheduler/sched_fence.c b/drivers/gpu/drm/amd/scheduler/sched_fence.c new file mode 100644 index 0000000..8620615 --- /dev/null +++ b/drivers/gpu/drm/amd/scheduler/sched_fence.c @@ -0,0 +1,181 @@ +/* + * Copyright 2015 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * + */ +#include +#include +#include +#include +#include "gpu_scheduler.h" + +static struct kmem_cache *sched_fence_slab; + +int amd_sched_fence_slab_init(void) +{ + sched_fence_slab = kmem_cache_create( + "amd_sched_fence", sizeof(struct amd_sched_fence), 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!sched_fence_slab) + return -ENOMEM; + + return 0; +} + +void amd_sched_fence_slab_fini(void) +{ + rcu_barrier(); + kmem_cache_destroy(sched_fence_slab); +} + +struct amd_sched_fence *amd_sched_fence_create(struct amd_sched_entity *entity, + void *owner) +{ + struct amd_sched_fence *fence = NULL; + unsigned seq; + + fence = kmem_cache_zalloc(sched_fence_slab, GFP_KERNEL); + if (fence == NULL) + return NULL; + + fence->owner = owner; + fence->sched = entity->sched; + spin_lock_init(&fence->lock); + + seq = atomic_inc_return(&entity->fence_seq); + kcl_fence_init(&fence->scheduled, &amd_sched_fence_ops_scheduled, + &fence->lock, entity->fence_context, seq); + kcl_fence_init(&fence->finished, &amd_sched_fence_ops_finished, + &fence->lock, entity->fence_context + 1, seq); + + return fence; +} + +void amd_sched_fence_scheduled(struct amd_sched_fence *fence) +{ + int ret = dma_fence_signal(&fence->scheduled); + + if (!ret) + DMA_FENCE_TRACE(&fence->scheduled, + "signaled from irq context\n"); + else + DMA_FENCE_TRACE(&fence->scheduled, + "was already signaled\n"); +} + +void amd_sched_fence_finished(struct amd_sched_fence *fence) +{ + int ret = dma_fence_signal(&fence->finished); + + if (!ret) + DMA_FENCE_TRACE(&fence->finished, + "signaled from irq context\n"); + else + DMA_FENCE_TRACE(&fence->finished, + "was already signaled\n"); +} + +static const char *amd_sched_fence_get_driver_name(struct dma_fence *fence) +{ + return "amd_sched"; +} + +static const char *amd_sched_fence_get_timeline_name(struct dma_fence *f) +{ + struct amd_sched_fence *fence = to_amd_sched_fence(f); + return (const char *)fence->sched->name; +} + +static bool amd_sched_fence_enable_signaling(struct dma_fence *f) +{ + return true; +} + +/** + * amd_sched_fence_free - free up the fence memory + * + * @rcu: RCU callback head + * + * Free up the fence memory after the RCU grace period. + */ +static void amd_sched_fence_free(struct rcu_head *rcu) +{ + struct dma_fence *f = container_of(rcu, struct dma_fence, rcu); + struct amd_sched_fence *fence = to_amd_sched_fence(f); + + dma_fence_put(fence->parent); + kmem_cache_free(sched_fence_slab, fence); +} + +/** + * amd_sched_fence_release_scheduled - callback that fence can be freed + * + * @fence: fence + * + * This function is called when the reference count becomes zero. + * It just RCU schedules freeing up the fence. + */ +static void amd_sched_fence_release_scheduled(struct dma_fence *f) +{ + struct amd_sched_fence *fence = to_amd_sched_fence(f); + + call_rcu(&fence->finished.rcu, amd_sched_fence_free); +} + +/** + * amd_sched_fence_release_finished - drop extra reference + * + * @f: fence + * + * Drop the extra reference from the scheduled fence to the base fence. + */ +static void amd_sched_fence_release_finished(struct dma_fence *f) +{ + struct amd_sched_fence *fence = to_amd_sched_fence(f); + + dma_fence_put(&fence->scheduled); +} + +const struct dma_fence_ops amd_sched_fence_ops_scheduled = { + .get_driver_name = amd_sched_fence_get_driver_name, + .get_timeline_name = amd_sched_fence_get_timeline_name, + .enable_signaling = amd_sched_fence_enable_signaling, + .signaled = NULL, +#if defined(BUILD_AS_DKMS) && !defined(OS_NAME_RHEL_7_4) + .wait = kcl_fence_default_wait, +#else + .wait = dma_fence_default_wait, +#endif + .release = amd_sched_fence_release_scheduled, +}; + +const struct dma_fence_ops amd_sched_fence_ops_finished = { + .get_driver_name = amd_sched_fence_get_driver_name, + .get_timeline_name = amd_sched_fence_get_timeline_name, + .enable_signaling = amd_sched_fence_enable_signaling, + .signaled = NULL, +#if defined(BUILD_AS_DKMS) && !defined(OS_NAME_RHEL_7_4) + .wait = kcl_fence_default_wait, +#else + .wait = dma_fence_default_wait, +#endif + .release = amd_sched_fence_release_finished, +}; -- 2.7.4