aboutsummaryrefslogtreecommitdiffstats
path: root/meta-amd-bsp/recipes-kernel/linux/linux-yocto-4.14.71/3856-drm-amd-amdkcl-add-fence-functions.patch
diff options
context:
space:
mode:
Diffstat (limited to 'meta-amd-bsp/recipes-kernel/linux/linux-yocto-4.14.71/3856-drm-amd-amdkcl-add-fence-functions.patch')
-rw-r--r--meta-amd-bsp/recipes-kernel/linux/linux-yocto-4.14.71/3856-drm-amd-amdkcl-add-fence-functions.patch1080
1 files changed, 1080 insertions, 0 deletions
diff --git a/meta-amd-bsp/recipes-kernel/linux/linux-yocto-4.14.71/3856-drm-amd-amdkcl-add-fence-functions.patch b/meta-amd-bsp/recipes-kernel/linux/linux-yocto-4.14.71/3856-drm-amd-amdkcl-add-fence-functions.patch
new file mode 100644
index 00000000..f0af996e
--- /dev/null
+++ b/meta-amd-bsp/recipes-kernel/linux/linux-yocto-4.14.71/3856-drm-amd-amdkcl-add-fence-functions.patch
@@ -0,0 +1,1080 @@
+From fd817d44483158725d4ac55b5a4d1f7fc1eaec7d Mon Sep 17 00:00:00 2001
+From: Junwei Zhang <Jerry.Zhang@amd.com>
+Date: Fri, 23 Dec 2016 16:28:35 +0800
+Subject: [PATCH 3856/4131] drm/amd/amdkcl: add fence functions
+
+Change-Id: I5ecbe63c5fd6765cd28ee570420e347e8283b8d0
+Signed-off-by: Junwei Zhang <Jerry.Zhang@amd.com>
+Reviewed-by: Qiang Yu <Qiang.Yu@amd.com>
+---
+ drivers/gpu/drm/amd/scheduler/gpu_scheduler.c | 687 ++++++++++++++++++++++++++
+ drivers/gpu/drm/amd/scheduler/gpu_scheduler.h | 174 +++++++
+ drivers/gpu/drm/amd/scheduler/sched_fence.c | 181 +++++++
+ 3 files changed, 1042 insertions(+)
+ create mode 100644 drivers/gpu/drm/amd/scheduler/gpu_scheduler.c
+ create mode 100644 drivers/gpu/drm/amd/scheduler/gpu_scheduler.h
+ create mode 100644 drivers/gpu/drm/amd/scheduler/sched_fence.c
+
+diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c
+new file mode 100644
+index 0000000..735a5a4
+--- /dev/null
++++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c
+@@ -0,0 +1,687 @@
++/*
++ * Copyright 2015 Advanced Micro Devices, Inc.
++ *
++ * Permission is hereby granted, free of charge, to any person obtaining a
++ * copy of this software and associated documentation files (the "Software"),
++ * to deal in the Software without restriction, including without limitation
++ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
++ * and/or sell copies of the Software, and to permit persons to whom the
++ * Software is furnished to do so, subject to the following conditions:
++ *
++ * The above copyright notice and this permission notice shall be included in
++ * all copies or substantial portions of the Software.
++ *
++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
++ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
++ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
++ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
++ * OTHER DEALINGS IN THE SOFTWARE.
++ *
++ *
++ */
++#include <linux/kthread.h>
++#include <linux/wait.h>
++#include <linux/sched.h>
++#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
++#include <uapi/linux/sched/types.h>
++#endif
++#include <drm/drmP.h>
++#include "gpu_scheduler.h"
++
++#define CREATE_TRACE_POINTS
++#include "gpu_sched_trace.h"
++
++static bool amd_sched_entity_is_ready(struct amd_sched_entity *entity);
++static void amd_sched_wakeup(struct amd_gpu_scheduler *sched);
++static void amd_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb);
++
++/* Initialize a given run queue struct */
++static void amd_sched_rq_init(struct amd_sched_rq *rq)
++{
++ spin_lock_init(&rq->lock);
++ INIT_LIST_HEAD(&rq->entities);
++ rq->current_entity = NULL;
++}
++
++static void amd_sched_rq_add_entity(struct amd_sched_rq *rq,
++ struct amd_sched_entity *entity)
++{
++ if (!list_empty(&entity->list))
++ return;
++ spin_lock(&rq->lock);
++ list_add_tail(&entity->list, &rq->entities);
++ spin_unlock(&rq->lock);
++}
++
++static void amd_sched_rq_remove_entity(struct amd_sched_rq *rq,
++ struct amd_sched_entity *entity)
++{
++ if (list_empty(&entity->list))
++ return;
++ spin_lock(&rq->lock);
++ list_del_init(&entity->list);
++ if (rq->current_entity == entity)
++ rq->current_entity = NULL;
++ spin_unlock(&rq->lock);
++}
++
++/**
++ * Select an entity which could provide a job to run
++ *
++ * @rq The run queue to check.
++ *
++ * Try to find a ready entity, returns NULL if none found.
++ */
++static struct amd_sched_entity *
++amd_sched_rq_select_entity(struct amd_sched_rq *rq)
++{
++ struct amd_sched_entity *entity;
++
++ spin_lock(&rq->lock);
++
++ entity = rq->current_entity;
++ if (entity) {
++ list_for_each_entry_continue(entity, &rq->entities, list) {
++ if (amd_sched_entity_is_ready(entity)) {
++ rq->current_entity = entity;
++ spin_unlock(&rq->lock);
++ return entity;
++ }
++ }
++ }
++
++ list_for_each_entry(entity, &rq->entities, list) {
++
++ if (amd_sched_entity_is_ready(entity)) {
++ rq->current_entity = entity;
++ spin_unlock(&rq->lock);
++ return entity;
++ }
++
++ if (entity == rq->current_entity)
++ break;
++ }
++
++ spin_unlock(&rq->lock);
++
++ return NULL;
++}
++
++/**
++ * Init a context entity used by scheduler when submit to HW ring.
++ *
++ * @sched The pointer to the scheduler
++ * @entity The pointer to a valid amd_sched_entity
++ * @rq The run queue this entity belongs
++ * @kernel If this is an entity for the kernel
++ * @jobs The max number of jobs in the job queue
++ *
++ * return 0 if succeed. negative error code on failure
++*/
++int amd_sched_entity_init(struct amd_gpu_scheduler *sched,
++ struct amd_sched_entity *entity,
++ struct amd_sched_rq *rq,
++ uint32_t jobs)
++{
++ int r;
++
++ if (!(sched && entity && rq))
++ return -EINVAL;
++
++ memset(entity, 0, sizeof(struct amd_sched_entity));
++ INIT_LIST_HEAD(&entity->list);
++ entity->rq = rq;
++ entity->sched = sched;
++
++ spin_lock_init(&entity->queue_lock);
++ r = kfifo_alloc(&entity->job_queue, jobs * sizeof(void *), GFP_KERNEL);
++ if (r)
++ return r;
++
++ atomic_set(&entity->fence_seq, 0);
++ entity->fence_context = kcl_fence_context_alloc(2);
++
++ return 0;
++}
++
++/**
++ * Query if entity is initialized
++ *
++ * @sched Pointer to scheduler instance
++ * @entity The pointer to a valid scheduler entity
++ *
++ * return true if entity is initialized, false otherwise
++*/
++static bool amd_sched_entity_is_initialized(struct amd_gpu_scheduler *sched,
++ struct amd_sched_entity *entity)
++{
++ return entity->sched == sched &&
++ entity->rq != NULL;
++}
++
++/**
++ * Check if entity is idle
++ *
++ * @entity The pointer to a valid scheduler entity
++ *
++ * Return true if entity don't has any unscheduled jobs.
++ */
++static bool amd_sched_entity_is_idle(struct amd_sched_entity *entity)
++{
++ rmb();
++ if (kfifo_is_empty(&entity->job_queue))
++ return true;
++
++ return false;
++}
++
++/**
++ * Check if entity is ready
++ *
++ * @entity The pointer to a valid scheduler entity
++ *
++ * Return true if entity could provide a job.
++ */
++static bool amd_sched_entity_is_ready(struct amd_sched_entity *entity)
++{
++ if (kfifo_is_empty(&entity->job_queue))
++ return false;
++
++ if (ACCESS_ONCE(entity->dependency))
++ return false;
++
++ return true;
++}
++
++/**
++ * Destroy a context entity
++ *
++ * @sched Pointer to scheduler instance
++ * @entity The pointer to a valid scheduler entity
++ *
++ * Cleanup and free the allocated resources.
++ */
++void amd_sched_entity_fini(struct amd_gpu_scheduler *sched,
++ struct amd_sched_entity *entity)
++{
++ struct amd_sched_rq *rq = entity->rq;
++ int r;
++
++ if (!amd_sched_entity_is_initialized(sched, entity))
++ return;
++ /**
++ * The client will not queue more IBs during this fini, consume existing
++ * queued IBs or discard them on SIGKILL
++ */
++ if ((current->flags & PF_SIGNALED) && current->exit_code == SIGKILL)
++ r = -ERESTARTSYS;
++ else
++ r = wait_event_killable(sched->job_scheduled,
++ amd_sched_entity_is_idle(entity));
++ amd_sched_rq_remove_entity(rq, entity);
++ if (r) {
++ struct amd_sched_job *job;
++
++ /* Park the kernel for a moment to make sure it isn't processing
++ * our enity.
++ */
++ kcl_kthread_park(sched->thread);
++ kcl_kthread_unpark(sched->thread);
++ while (kfifo_out(&entity->job_queue, &job, sizeof(job)))
++ sched->ops->free_job(job);
++
++ }
++ kfifo_free(&entity->job_queue);
++}
++
++static void amd_sched_entity_wakeup(struct dma_fence *f, struct dma_fence_cb *cb)
++{
++ struct amd_sched_entity *entity =
++ container_of(cb, struct amd_sched_entity, cb);
++ entity->dependency = NULL;
++ dma_fence_put(f);
++ amd_sched_wakeup(entity->sched);
++}
++
++static void amd_sched_entity_clear_dep(struct dma_fence *f, struct dma_fence_cb *cb)
++{
++ struct amd_sched_entity *entity =
++ container_of(cb, struct amd_sched_entity, cb);
++ entity->dependency = NULL;
++ dma_fence_put(f);
++}
++
++bool amd_sched_dependency_optimized(struct dma_fence* fence,
++ struct amd_sched_entity *entity)
++{
++ struct amd_gpu_scheduler *sched = entity->sched;
++ struct amd_sched_fence *s_fence;
++
++ if (!fence || dma_fence_is_signaled(fence))
++ return false;
++ if (fence->context == entity->fence_context)
++ return true;
++ s_fence = to_amd_sched_fence(fence);
++ if (s_fence && s_fence->sched == sched)
++ return true;
++
++ return false;
++}
++
++static bool amd_sched_entity_add_dependency_cb(struct amd_sched_entity *entity)
++{
++ struct amd_gpu_scheduler *sched = entity->sched;
++ struct dma_fence * fence = entity->dependency;
++ struct amd_sched_fence *s_fence;
++
++ if (fence->context == entity->fence_context) {
++ /* We can ignore fences from ourself */
++ dma_fence_put(entity->dependency);
++ return false;
++ }
++
++ s_fence = to_amd_sched_fence(fence);
++ if (s_fence && s_fence->sched == sched) {
++
++ /*
++ * Fence is from the same scheduler, only need to wait for
++ * it to be scheduled
++ */
++ fence = dma_fence_get(&s_fence->scheduled);
++ dma_fence_put(entity->dependency);
++ entity->dependency = fence;
++ if (!dma_fence_add_callback(fence, &entity->cb,
++ amd_sched_entity_clear_dep))
++ return true;
++
++ /* Ignore it when it is already scheduled */
++ dma_fence_put(fence);
++ return false;
++ }
++
++ if (!dma_fence_add_callback(entity->dependency, &entity->cb,
++ amd_sched_entity_wakeup))
++ return true;
++
++ dma_fence_put(entity->dependency);
++ return false;
++}
++
++static struct amd_sched_job *
++amd_sched_entity_pop_job(struct amd_sched_entity *entity)
++{
++ struct amd_gpu_scheduler *sched = entity->sched;
++ struct amd_sched_job *sched_job;
++
++ if (!kfifo_out_peek(&entity->job_queue, &sched_job, sizeof(sched_job)))
++ return NULL;
++
++ while ((entity->dependency = sched->ops->dependency(sched_job)))
++ if (amd_sched_entity_add_dependency_cb(entity))
++ return NULL;
++
++ return sched_job;
++}
++
++/**
++ * Helper to submit a job to the job queue
++ *
++ * @sched_job The pointer to job required to submit
++ *
++ * Returns true if we could submit the job.
++ */
++static bool amd_sched_entity_in(struct amd_sched_job *sched_job)
++{
++ struct amd_gpu_scheduler *sched = sched_job->sched;
++ struct amd_sched_entity *entity = sched_job->s_entity;
++ bool added, first = false;
++
++ spin_lock(&entity->queue_lock);
++ added = kfifo_in(&entity->job_queue, &sched_job,
++ sizeof(sched_job)) == sizeof(sched_job);
++
++ if (added && kfifo_len(&entity->job_queue) == sizeof(sched_job))
++ first = true;
++
++ spin_unlock(&entity->queue_lock);
++
++ /* first job wakes up scheduler */
++ if (first) {
++ /* Add the entity to the run queue */
++ amd_sched_rq_add_entity(entity->rq, entity);
++ amd_sched_wakeup(sched);
++ }
++ return added;
++}
++
++/* job_finish is called after hw fence signaled, and
++ * the job had already been deleted from ring_mirror_list
++ */
++static void amd_sched_job_finish(struct work_struct *work)
++{
++ struct amd_sched_job *s_job = container_of(work, struct amd_sched_job,
++ finish_work);
++ struct amd_gpu_scheduler *sched = s_job->sched;
++
++ /* remove job from ring_mirror_list */
++ spin_lock(&sched->job_list_lock);
++ list_del_init(&s_job->node);
++ if (sched->timeout != MAX_SCHEDULE_TIMEOUT) {
++ struct amd_sched_job *next;
++
++ spin_unlock(&sched->job_list_lock);
++ cancel_delayed_work_sync(&s_job->work_tdr);
++ spin_lock(&sched->job_list_lock);
++
++ /* queue TDR for next job */
++ next = list_first_entry_or_null(&sched->ring_mirror_list,
++ struct amd_sched_job, node);
++
++ if (next)
++ schedule_delayed_work(&next->work_tdr, sched->timeout);
++ }
++ spin_unlock(&sched->job_list_lock);
++ sched->ops->free_job(s_job);
++}
++
++static void amd_sched_job_finish_cb(struct dma_fence *f,
++ struct dma_fence_cb *cb)
++{
++ struct amd_sched_job *job = container_of(cb, struct amd_sched_job,
++ finish_cb);
++ schedule_work(&job->finish_work);
++}
++
++static void amd_sched_job_begin(struct amd_sched_job *s_job)
++{
++ struct amd_gpu_scheduler *sched = s_job->sched;
++
++ spin_lock(&sched->job_list_lock);
++ list_add_tail(&s_job->node, &sched->ring_mirror_list);
++ if (sched->timeout != MAX_SCHEDULE_TIMEOUT &&
++ list_first_entry_or_null(&sched->ring_mirror_list,
++ struct amd_sched_job, node) == s_job)
++ schedule_delayed_work(&s_job->work_tdr, sched->timeout);
++ spin_unlock(&sched->job_list_lock);
++}
++
++static void amd_sched_job_timedout(struct work_struct *work)
++{
++ struct amd_sched_job *job = container_of(work, struct amd_sched_job,
++ work_tdr.work);
++
++ job->sched->ops->timedout_job(job);
++}
++
++void amd_sched_hw_job_reset(struct amd_gpu_scheduler *sched)
++{
++ struct amd_sched_job *s_job;
++
++ spin_lock(&sched->job_list_lock);
++ list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) {
++ if (s_job->s_fence->parent &&
++ dma_fence_remove_callback(s_job->s_fence->parent,
++ &s_job->s_fence->cb)) {
++ dma_fence_put(s_job->s_fence->parent);
++ s_job->s_fence->parent = NULL;
++ atomic_dec(&sched->hw_rq_count);
++ }
++ }
++ spin_unlock(&sched->job_list_lock);
++}
++
++void amd_sched_job_kickout(struct amd_sched_job *s_job)
++{
++ struct amd_gpu_scheduler *sched = s_job->sched;
++
++ spin_lock(&sched->job_list_lock);
++ list_del_init(&s_job->node);
++ spin_unlock(&sched->job_list_lock);
++}
++
++void amd_sched_job_recovery(struct amd_gpu_scheduler *sched)
++{
++ struct amd_sched_job *s_job, *tmp;
++ int r;
++
++ spin_lock(&sched->job_list_lock);
++ s_job = list_first_entry_or_null(&sched->ring_mirror_list,
++ struct amd_sched_job, node);
++ if (s_job && sched->timeout != MAX_SCHEDULE_TIMEOUT)
++ schedule_delayed_work(&s_job->work_tdr, sched->timeout);
++
++ list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) {
++ struct amd_sched_fence *s_fence = s_job->s_fence;
++ struct dma_fence *fence;
++
++ spin_unlock(&sched->job_list_lock);
++ fence = sched->ops->run_job(s_job);
++ atomic_inc(&sched->hw_rq_count);
++ if (fence) {
++ s_fence->parent = dma_fence_get(fence);
++ r = dma_fence_add_callback(fence, &s_fence->cb,
++ amd_sched_process_job);
++ if (r == -ENOENT)
++ amd_sched_process_job(fence, &s_fence->cb);
++ else if (r)
++ DRM_ERROR("fence add callback failed (%d)\n",
++ r);
++ dma_fence_put(fence);
++ } else {
++ DRM_ERROR("Failed to run job!\n");
++ amd_sched_process_job(NULL, &s_fence->cb);
++ }
++ spin_lock(&sched->job_list_lock);
++ }
++ spin_unlock(&sched->job_list_lock);
++}
++
++/**
++ * Submit a job to the job queue
++ *
++ * @sched_job The pointer to job required to submit
++ *
++ * Returns 0 for success, negative error code otherwise.
++ */
++void amd_sched_entity_push_job(struct amd_sched_job *sched_job)
++{
++ struct amd_sched_entity *entity = sched_job->s_entity;
++
++ trace_amd_sched_job(sched_job);
++ dma_fence_add_callback(&sched_job->s_fence->finished, &sched_job->finish_cb,
++ amd_sched_job_finish_cb);
++ wait_event(entity->sched->job_scheduled,
++ amd_sched_entity_in(sched_job));
++}
++
++/* init a sched_job with basic field */
++int amd_sched_job_init(struct amd_sched_job *job,
++ struct amd_gpu_scheduler *sched,
++ struct amd_sched_entity *entity,
++ void *owner)
++{
++ job->sched = sched;
++ job->s_entity = entity;
++ job->s_fence = amd_sched_fence_create(entity, owner);
++ if (!job->s_fence)
++ return -ENOMEM;
++ job->id = atomic64_inc_return(&sched->job_id_count);
++
++ INIT_WORK(&job->finish_work, amd_sched_job_finish);
++ INIT_LIST_HEAD(&job->node);
++ INIT_DELAYED_WORK(&job->work_tdr, amd_sched_job_timedout);
++
++ return 0;
++}
++
++/**
++ * Return ture if we can push more jobs to the hw.
++ */
++static bool amd_sched_ready(struct amd_gpu_scheduler *sched)
++{
++ return atomic_read(&sched->hw_rq_count) <
++ sched->hw_submission_limit;
++}
++
++/**
++ * Wake up the scheduler when it is ready
++ */
++static void amd_sched_wakeup(struct amd_gpu_scheduler *sched)
++{
++ if (amd_sched_ready(sched))
++ wake_up_interruptible(&sched->wake_up_worker);
++}
++
++/**
++ * Select next entity to process
++*/
++static struct amd_sched_entity *
++amd_sched_select_entity(struct amd_gpu_scheduler *sched)
++{
++ struct amd_sched_entity *entity;
++ int i;
++
++ if (!amd_sched_ready(sched))
++ return NULL;
++
++ /* Kernel run queue has higher priority than normal run queue*/
++ for (i = AMD_SCHED_PRIORITY_MAX - 1; i >= AMD_SCHED_PRIORITY_MIN; i--) {
++ entity = amd_sched_rq_select_entity(&sched->sched_rq[i]);
++ if (entity)
++ break;
++ }
++
++ return entity;
++}
++
++static void amd_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb)
++{
++ struct amd_sched_fence *s_fence =
++ container_of(cb, struct amd_sched_fence, cb);
++ struct amd_gpu_scheduler *sched = s_fence->sched;
++
++ atomic_dec(&sched->hw_rq_count);
++ amd_sched_fence_finished(s_fence);
++
++ trace_amd_sched_process_job(s_fence);
++ dma_fence_put(&s_fence->finished);
++ wake_up_interruptible(&sched->wake_up_worker);
++}
++
++static bool amd_sched_blocked(struct amd_gpu_scheduler *sched)
++{
++ if (kcl_kthread_should_park()) {
++ kcl_kthread_parkme();
++ return true;
++ }
++
++ return false;
++}
++
++static int amd_sched_main(void *param)
++{
++ struct sched_param sparam = {.sched_priority = 1};
++ struct amd_gpu_scheduler *sched = (struct amd_gpu_scheduler *)param;
++ int r, count;
++
++ sched_setscheduler(current, SCHED_FIFO, &sparam);
++
++ while (!kthread_should_stop()) {
++ struct amd_sched_entity *entity = NULL;
++ struct amd_sched_fence *s_fence;
++ struct amd_sched_job *sched_job;
++ struct dma_fence *fence;
++
++ wait_event_interruptible(sched->wake_up_worker,
++ (!amd_sched_blocked(sched) &&
++ (entity = amd_sched_select_entity(sched))) ||
++ kthread_should_stop());
++
++ if (!entity)
++ continue;
++
++ sched_job = amd_sched_entity_pop_job(entity);
++ if (!sched_job)
++ continue;
++
++ s_fence = sched_job->s_fence;
++
++ atomic_inc(&sched->hw_rq_count);
++ amd_sched_job_begin(sched_job);
++
++ fence = sched->ops->run_job(sched_job);
++ amd_sched_fence_scheduled(s_fence);
++ if (fence) {
++ s_fence->parent = dma_fence_get(fence);
++ r = dma_fence_add_callback(fence, &s_fence->cb,
++ amd_sched_process_job);
++ if (r == -ENOENT)
++ amd_sched_process_job(fence, &s_fence->cb);
++ else if (r)
++ DRM_ERROR("fence add callback failed (%d)\n",
++ r);
++ dma_fence_put(fence);
++ } else {
++ DRM_ERROR("Failed to run job!\n");
++ amd_sched_process_job(NULL, &s_fence->cb);
++ }
++
++ count = kfifo_out(&entity->job_queue, &sched_job,
++ sizeof(sched_job));
++ WARN_ON(count != sizeof(sched_job));
++ wake_up(&sched->job_scheduled);
++ }
++ return 0;
++}
++
++/**
++ * Init a gpu scheduler instance
++ *
++ * @sched The pointer to the scheduler
++ * @ops The backend operations for this scheduler.
++ * @hw_submissions Number of hw submissions to do.
++ * @name Name used for debugging
++ *
++ * Return 0 on success, otherwise error code.
++*/
++int amd_sched_init(struct amd_gpu_scheduler *sched,
++ const struct amd_sched_backend_ops *ops,
++ unsigned hw_submission, long timeout, const char *name)
++{
++ int i;
++ sched->ops = ops;
++ sched->hw_submission_limit = hw_submission;
++ sched->name = name;
++ sched->timeout = timeout;
++ for (i = AMD_SCHED_PRIORITY_MIN; i < AMD_SCHED_PRIORITY_MAX; i++)
++ amd_sched_rq_init(&sched->sched_rq[i]);
++
++ init_waitqueue_head(&sched->wake_up_worker);
++ init_waitqueue_head(&sched->job_scheduled);
++ INIT_LIST_HEAD(&sched->ring_mirror_list);
++ spin_lock_init(&sched->job_list_lock);
++ atomic_set(&sched->hw_rq_count, 0);
++ atomic64_set(&sched->job_id_count, 0);
++
++ /* Each scheduler will run on a seperate kernel thread */
++ sched->thread = kthread_run(amd_sched_main, sched, sched->name);
++ if (IS_ERR(sched->thread)) {
++ DRM_ERROR("Failed to create scheduler for %s.\n", name);
++ return PTR_ERR(sched->thread);
++ }
++
++ return 0;
++}
++
++/**
++ * Destroy a gpu scheduler
++ *
++ * @sched The pointer to the scheduler
++ */
++void amd_sched_fini(struct amd_gpu_scheduler *sched)
++{
++ if (sched->thread)
++ kthread_stop(sched->thread);
++}
+diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h
+new file mode 100644
+index 0000000..92aef31
+--- /dev/null
++++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h
+@@ -0,0 +1,174 @@
++/*
++ * Copyright 2015 Advanced Micro Devices, Inc.
++ *
++ * Permission is hereby granted, free of charge, to any person obtaining a
++ * copy of this software and associated documentation files (the "Software"),
++ * to deal in the Software without restriction, including without limitation
++ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
++ * and/or sell copies of the Software, and to permit persons to whom the
++ * Software is furnished to do so, subject to the following conditions:
++ *
++ * The above copyright notice and this permission notice shall be included in
++ * all copies or substantial portions of the Software.
++ *
++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
++ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
++ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
++ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
++ * OTHER DEALINGS IN THE SOFTWARE.
++ *
++ */
++
++#ifndef _GPU_SCHEDULER_H_
++#define _GPU_SCHEDULER_H_
++
++#if (defined OS_NAME_RHEL) && (OS_VERSION_MAJOR == 6)
++#include <linux/kfifo-new.h>
++#else
++#include <linux/kfifo.h>
++#endif
++#include <kcl/kcl_fence.h>
++
++struct amd_gpu_scheduler;
++struct amd_sched_rq;
++
++/**
++ * A scheduler entity is a wrapper around a job queue or a group
++ * of other entities. Entities take turns emitting jobs from their
++ * job queues to corresponding hardware ring based on scheduling
++ * policy.
++*/
++struct amd_sched_entity {
++ struct list_head list;
++ struct amd_sched_rq *rq;
++ struct amd_gpu_scheduler *sched;
++
++ spinlock_t queue_lock;
++ struct kfifo job_queue;
++
++ atomic_t fence_seq;
++ uint64_t fence_context;
++
++ struct dma_fence *dependency;
++ struct dma_fence_cb cb;
++};
++
++/**
++ * Run queue is a set of entities scheduling command submissions for
++ * one specific ring. It implements the scheduling policy that selects
++ * the next entity to emit commands from.
++*/
++struct amd_sched_rq {
++ spinlock_t lock;
++ struct list_head entities;
++ struct amd_sched_entity *current_entity;
++};
++
++struct amd_sched_fence {
++ struct dma_fence scheduled;
++ struct dma_fence finished;
++ struct dma_fence_cb cb;
++ struct dma_fence *parent;
++ struct amd_gpu_scheduler *sched;
++ spinlock_t lock;
++ void *owner;
++};
++
++struct amd_sched_job {
++ struct amd_gpu_scheduler *sched;
++ struct amd_sched_entity *s_entity;
++ struct amd_sched_fence *s_fence;
++ struct dma_fence_cb finish_cb;
++ struct work_struct finish_work;
++ struct list_head node;
++ struct delayed_work work_tdr;
++ uint64_t id;
++ atomic_t karma;
++};
++
++extern const struct dma_fence_ops amd_sched_fence_ops_scheduled;
++extern const struct dma_fence_ops amd_sched_fence_ops_finished;
++static inline struct amd_sched_fence *to_amd_sched_fence(struct dma_fence *f)
++{
++ if (f->ops == &amd_sched_fence_ops_scheduled)
++ return container_of(f, struct amd_sched_fence, scheduled);
++
++ if (f->ops == &amd_sched_fence_ops_finished)
++ return container_of(f, struct amd_sched_fence, finished);
++
++ return NULL;
++}
++
++static inline bool amd_sched_invalidate_job(struct amd_sched_job *s_job, int threshold)
++{
++ return (s_job && atomic_inc_return(&s_job->karma) > threshold);
++}
++
++/**
++ * Define the backend operations called by the scheduler,
++ * these functions should be implemented in driver side
++*/
++struct amd_sched_backend_ops {
++ struct dma_fence *(*dependency)(struct amd_sched_job *sched_job);
++ struct dma_fence *(*run_job)(struct amd_sched_job *sched_job);
++ void (*timedout_job)(struct amd_sched_job *sched_job);
++ void (*free_job)(struct amd_sched_job *sched_job);
++};
++
++enum amd_sched_priority {
++ AMD_SCHED_PRIORITY_MIN,
++ AMD_SCHED_PRIORITY_NORMAL = AMD_SCHED_PRIORITY_MIN,
++ AMD_SCHED_PRIORITY_KERNEL,
++ AMD_SCHED_PRIORITY_MAX
++};
++
++/**
++ * One scheduler is implemented for each hardware ring
++*/
++struct amd_gpu_scheduler {
++ const struct amd_sched_backend_ops *ops;
++ uint32_t hw_submission_limit;
++ long timeout;
++ const char *name;
++ struct amd_sched_rq sched_rq[AMD_SCHED_PRIORITY_MAX];
++ wait_queue_head_t wake_up_worker;
++ wait_queue_head_t job_scheduled;
++ atomic_t hw_rq_count;
++ atomic64_t job_id_count;
++ struct task_struct *thread;
++ struct list_head ring_mirror_list;
++ spinlock_t job_list_lock;
++};
++
++int amd_sched_init(struct amd_gpu_scheduler *sched,
++ const struct amd_sched_backend_ops *ops,
++ uint32_t hw_submission, long timeout, const char *name);
++void amd_sched_fini(struct amd_gpu_scheduler *sched);
++
++int amd_sched_entity_init(struct amd_gpu_scheduler *sched,
++ struct amd_sched_entity *entity,
++ struct amd_sched_rq *rq,
++ uint32_t jobs);
++void amd_sched_entity_fini(struct amd_gpu_scheduler *sched,
++ struct amd_sched_entity *entity);
++void amd_sched_entity_push_job(struct amd_sched_job *sched_job);
++
++int amd_sched_fence_slab_init(void);
++void amd_sched_fence_slab_fini(void);
++
++struct amd_sched_fence *amd_sched_fence_create(
++ struct amd_sched_entity *s_entity, void *owner);
++void amd_sched_fence_scheduled(struct amd_sched_fence *fence);
++void amd_sched_fence_finished(struct amd_sched_fence *fence);
++int amd_sched_job_init(struct amd_sched_job *job,
++ struct amd_gpu_scheduler *sched,
++ struct amd_sched_entity *entity,
++ void *owner);
++void amd_sched_hw_job_reset(struct amd_gpu_scheduler *sched);
++void amd_sched_job_recovery(struct amd_gpu_scheduler *sched);
++bool amd_sched_dependency_optimized(struct dma_fence* fence,
++ struct amd_sched_entity *entity);
++void amd_sched_job_kickout(struct amd_sched_job *s_job);
++#endif
+diff --git a/drivers/gpu/drm/amd/scheduler/sched_fence.c b/drivers/gpu/drm/amd/scheduler/sched_fence.c
+new file mode 100644
+index 0000000..8620615
+--- /dev/null
++++ b/drivers/gpu/drm/amd/scheduler/sched_fence.c
+@@ -0,0 +1,181 @@
++/*
++ * Copyright 2015 Advanced Micro Devices, Inc.
++ *
++ * Permission is hereby granted, free of charge, to any person obtaining a
++ * copy of this software and associated documentation files (the "Software"),
++ * to deal in the Software without restriction, including without limitation
++ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
++ * and/or sell copies of the Software, and to permit persons to whom the
++ * Software is furnished to do so, subject to the following conditions:
++ *
++ * The above copyright notice and this permission notice shall be included in
++ * all copies or substantial portions of the Software.
++ *
++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
++ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
++ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
++ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
++ * OTHER DEALINGS IN THE SOFTWARE.
++ *
++ *
++ */
++#include <linux/kthread.h>
++#include <linux/wait.h>
++#include <linux/sched.h>
++#include <drm/drmP.h>
++#include "gpu_scheduler.h"
++
++static struct kmem_cache *sched_fence_slab;
++
++int amd_sched_fence_slab_init(void)
++{
++ sched_fence_slab = kmem_cache_create(
++ "amd_sched_fence", sizeof(struct amd_sched_fence), 0,
++ SLAB_HWCACHE_ALIGN, NULL);
++ if (!sched_fence_slab)
++ return -ENOMEM;
++
++ return 0;
++}
++
++void amd_sched_fence_slab_fini(void)
++{
++ rcu_barrier();
++ kmem_cache_destroy(sched_fence_slab);
++}
++
++struct amd_sched_fence *amd_sched_fence_create(struct amd_sched_entity *entity,
++ void *owner)
++{
++ struct amd_sched_fence *fence = NULL;
++ unsigned seq;
++
++ fence = kmem_cache_zalloc(sched_fence_slab, GFP_KERNEL);
++ if (fence == NULL)
++ return NULL;
++
++ fence->owner = owner;
++ fence->sched = entity->sched;
++ spin_lock_init(&fence->lock);
++
++ seq = atomic_inc_return(&entity->fence_seq);
++ kcl_fence_init(&fence->scheduled, &amd_sched_fence_ops_scheduled,
++ &fence->lock, entity->fence_context, seq);
++ kcl_fence_init(&fence->finished, &amd_sched_fence_ops_finished,
++ &fence->lock, entity->fence_context + 1, seq);
++
++ return fence;
++}
++
++void amd_sched_fence_scheduled(struct amd_sched_fence *fence)
++{
++ int ret = dma_fence_signal(&fence->scheduled);
++
++ if (!ret)
++ DMA_FENCE_TRACE(&fence->scheduled,
++ "signaled from irq context\n");
++ else
++ DMA_FENCE_TRACE(&fence->scheduled,
++ "was already signaled\n");
++}
++
++void amd_sched_fence_finished(struct amd_sched_fence *fence)
++{
++ int ret = dma_fence_signal(&fence->finished);
++
++ if (!ret)
++ DMA_FENCE_TRACE(&fence->finished,
++ "signaled from irq context\n");
++ else
++ DMA_FENCE_TRACE(&fence->finished,
++ "was already signaled\n");
++}
++
++static const char *amd_sched_fence_get_driver_name(struct dma_fence *fence)
++{
++ return "amd_sched";
++}
++
++static const char *amd_sched_fence_get_timeline_name(struct dma_fence *f)
++{
++ struct amd_sched_fence *fence = to_amd_sched_fence(f);
++ return (const char *)fence->sched->name;
++}
++
++static bool amd_sched_fence_enable_signaling(struct dma_fence *f)
++{
++ return true;
++}
++
++/**
++ * amd_sched_fence_free - free up the fence memory
++ *
++ * @rcu: RCU callback head
++ *
++ * Free up the fence memory after the RCU grace period.
++ */
++static void amd_sched_fence_free(struct rcu_head *rcu)
++{
++ struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
++ struct amd_sched_fence *fence = to_amd_sched_fence(f);
++
++ dma_fence_put(fence->parent);
++ kmem_cache_free(sched_fence_slab, fence);
++}
++
++/**
++ * amd_sched_fence_release_scheduled - callback that fence can be freed
++ *
++ * @fence: fence
++ *
++ * This function is called when the reference count becomes zero.
++ * It just RCU schedules freeing up the fence.
++ */
++static void amd_sched_fence_release_scheduled(struct dma_fence *f)
++{
++ struct amd_sched_fence *fence = to_amd_sched_fence(f);
++
++ call_rcu(&fence->finished.rcu, amd_sched_fence_free);
++}
++
++/**
++ * amd_sched_fence_release_finished - drop extra reference
++ *
++ * @f: fence
++ *
++ * Drop the extra reference from the scheduled fence to the base fence.
++ */
++static void amd_sched_fence_release_finished(struct dma_fence *f)
++{
++ struct amd_sched_fence *fence = to_amd_sched_fence(f);
++
++ dma_fence_put(&fence->scheduled);
++}
++
++const struct dma_fence_ops amd_sched_fence_ops_scheduled = {
++ .get_driver_name = amd_sched_fence_get_driver_name,
++ .get_timeline_name = amd_sched_fence_get_timeline_name,
++ .enable_signaling = amd_sched_fence_enable_signaling,
++ .signaled = NULL,
++#if defined(BUILD_AS_DKMS) && !defined(OS_NAME_RHEL_7_4)
++ .wait = kcl_fence_default_wait,
++#else
++ .wait = dma_fence_default_wait,
++#endif
++ .release = amd_sched_fence_release_scheduled,
++};
++
++const struct dma_fence_ops amd_sched_fence_ops_finished = {
++ .get_driver_name = amd_sched_fence_get_driver_name,
++ .get_timeline_name = amd_sched_fence_get_timeline_name,
++ .enable_signaling = amd_sched_fence_enable_signaling,
++ .signaled = NULL,
++#if defined(BUILD_AS_DKMS) && !defined(OS_NAME_RHEL_7_4)
++ .wait = kcl_fence_default_wait,
++#else
++ .wait = dma_fence_default_wait,
++#endif
++ .release = amd_sched_fence_release_finished,
++};
+--
+2.7.4
+
generated by cgit 1.2.3-korg (git 2.39.0) at 2024-05-01 09:59:38 +0000