diff options
Diffstat (limited to 'common/recipes-kernel/linux/linux-yocto-4.19.8/0705-drm-scheduler-move-entity-handling-into-separate-fil.patch')
| -rw-r--r-- | common/recipes-kernel/linux/linux-yocto-4.19.8/0705-drm-scheduler-move-entity-handling-into-separate-fil.patch | 1061 |
1 files changed, 1061 insertions, 0 deletions
diff --git a/common/recipes-kernel/linux/linux-yocto-4.19.8/0705-drm-scheduler-move-entity-handling-into-separate-fil.patch b/common/recipes-kernel/linux/linux-yocto-4.19.8/0705-drm-scheduler-move-entity-handling-into-separate-fil.patch new file mode 100644 index 00000000..c4f92298 --- /dev/null +++ b/common/recipes-kernel/linux/linux-yocto-4.19.8/0705-drm-scheduler-move-entity-handling-into-separate-fil.patch @@ -0,0 +1,1061 @@ +From 61e599a3c395e62b2e3d32d1d4f536340138d5a6 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Christian=20K=C3=B6nig?= <christian.koenig@amd.com> +Date: Mon, 6 Aug 2018 14:25:32 +0200 +Subject: [PATCH 0705/2940] drm/scheduler: move entity handling into separate + file +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +This is complex enough on it's own. Move it into a separate C file. + +Signed-off-by: Christian König <christian.koenig@amd.com> +Reviewed-by: Huang Rui <ray.huang@amd.com> +Signed-off-by: Alex Deucher <alexander.deucher@amd.com> +Signed-off-by: Chaudhary Amit Kumar <Chaudharyamit.Kumar@amd.com> +--- + drivers/gpu/drm/scheduler/Makefile | 2 +- + drivers/gpu/drm/scheduler/gpu_scheduler.c | 476 +--------------------- + drivers/gpu/drm/scheduler/sched_entity.c | 459 +++++++++++++++++++++ + include/drm/gpu_scheduler.h | 18 + + 4 files changed, 483 insertions(+), 472 deletions(-) + create mode 100644 drivers/gpu/drm/scheduler/sched_entity.c + +diff --git a/drivers/gpu/drm/scheduler/Makefile b/drivers/gpu/drm/scheduler/Makefile +index 7665883f81d4..f23785d4b3c8 100644 +--- a/drivers/gpu/drm/scheduler/Makefile ++++ b/drivers/gpu/drm/scheduler/Makefile +@@ -20,6 +20,6 @@ + # OTHER DEALINGS IN THE SOFTWARE. + # + # +-gpu-sched-y := gpu_scheduler.o sched_fence.o ++gpu-sched-y := gpu_scheduler.o sched_fence.o sched_entity.o + + obj-$(CONFIG_DRM_SCHED) += gpu-sched.o +diff --git a/drivers/gpu/drm/scheduler/gpu_scheduler.c b/drivers/gpu/drm/scheduler/gpu_scheduler.c +index 1f65ede2b20e..9ca741f3a0bc 100644 +--- a/drivers/gpu/drm/scheduler/gpu_scheduler.c ++++ b/drivers/gpu/drm/scheduler/gpu_scheduler.c +@@ -58,8 +58,6 @@ + #define to_drm_sched_job(sched_job) \ + container_of((sched_job), struct drm_sched_job, queue_node) + +-static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity); +-static void drm_sched_wakeup(struct drm_gpu_scheduler *sched); + static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb); + + /** +@@ -86,8 +84,8 @@ static void drm_sched_rq_init(struct drm_gpu_scheduler *sched, + * + * Adds a scheduler entity to the run queue. + */ +-static void drm_sched_rq_add_entity(struct drm_sched_rq *rq, +- struct drm_sched_entity *entity) ++void drm_sched_rq_add_entity(struct drm_sched_rq *rq, ++ struct drm_sched_entity *entity) + { + if (!list_empty(&entity->list)) + return; +@@ -104,8 +102,8 @@ static void drm_sched_rq_add_entity(struct drm_sched_rq *rq, + * + * Removes a scheduler entity from the run queue. + */ +-static void drm_sched_rq_remove_entity(struct drm_sched_rq *rq, +- struct drm_sched_entity *entity) ++void drm_sched_rq_remove_entity(struct drm_sched_rq *rq, ++ struct drm_sched_entity *entity) + { + if (list_empty(&entity->list)) + return; +@@ -158,336 +156,6 @@ drm_sched_rq_select_entity(struct drm_sched_rq *rq) + return NULL; + } + +-/** +- * drm_sched_entity_init - Init a context entity used by scheduler when +- * submit to HW ring. +- * +- * @entity: scheduler entity to init +- * @rq_list: the list of run queue on which jobs from this +- * entity can be submitted +- * @num_rq_list: number of run queue in rq_list +- * @guilty: atomic_t set to 1 when a job on this queue +- * is found to be guilty causing a timeout +- * +- * Note: the rq_list should have atleast one element to schedule +- * the entity +- * +- * Returns 0 on success or a negative error code on failure. +-*/ +-int drm_sched_entity_init(struct drm_sched_entity *entity, +- struct drm_sched_rq **rq_list, +- unsigned int num_rq_list, +- atomic_t *guilty) +-{ +- int i; +- +- if (!(entity && rq_list && num_rq_list > 0 && rq_list[0])) +- return -EINVAL; +- +- memset(entity, 0, sizeof(struct drm_sched_entity)); +- INIT_LIST_HEAD(&entity->list); +- entity->rq = rq_list[0]; +- entity->guilty = guilty; +- entity->num_rq_list = num_rq_list; +- entity->rq_list = kcalloc(num_rq_list, sizeof(struct drm_sched_rq *), +- GFP_KERNEL); +- if (!entity->rq_list) +- return -ENOMEM; +- +- for (i = 0; i < num_rq_list; ++i) +- entity->rq_list[i] = rq_list[i]; +- entity->last_scheduled = NULL; +- +- spin_lock_init(&entity->rq_lock); +- spsc_queue_init(&entity->job_queue); +- +- atomic_set(&entity->fence_seq, 0); +- entity->fence_context = dma_fence_context_alloc(2); +- +- return 0; +-} +-EXPORT_SYMBOL(drm_sched_entity_init); +- +-/** +- * drm_sched_entity_is_idle - Check if entity is idle +- * +- * @entity: scheduler entity +- * +- * Returns true if the entity does not have any unscheduled jobs. +- */ +-static bool drm_sched_entity_is_idle(struct drm_sched_entity *entity) +-{ +- rmb(); +- +- if (list_empty(&entity->list) || +- spsc_queue_peek(&entity->job_queue) == NULL) +- return true; +- +- return false; +-} +- +-/** +- * drm_sched_entity_is_ready - Check if entity is ready +- * +- * @entity: scheduler entity +- * +- * Return true if entity could provide a job. +- */ +-static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity) +-{ +- if (spsc_queue_peek(&entity->job_queue) == NULL) +- return false; +- +- if (READ_ONCE(entity->dependency)) +- return false; +- +- return true; +-} +- +-/** +- * drm_sched_entity_get_free_sched - Get the rq from rq_list with least load +- * +- * @entity: scheduler entity +- * +- * Return the pointer to the rq with least load. +- */ +-static struct drm_sched_rq * +-drm_sched_entity_get_free_sched(struct drm_sched_entity *entity) +-{ +- struct drm_sched_rq *rq = NULL; +- unsigned int min_jobs = UINT_MAX, num_jobs; +- int i; +- +- for (i = 0; i < entity->num_rq_list; ++i) { +- num_jobs = atomic_read(&entity->rq_list[i]->sched->num_jobs); +- if (num_jobs < min_jobs) { +- min_jobs = num_jobs; +- rq = entity->rq_list[i]; +- } +- } +- +- return rq; +-} +- +-static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f, +- struct dma_fence_cb *cb) +-{ +- struct drm_sched_job *job = container_of(cb, struct drm_sched_job, +- finish_cb); +- drm_sched_fence_finished(job->s_fence); +- WARN_ON(job->s_fence->parent); +- dma_fence_put(&job->s_fence->finished); +- job->sched->ops->free_job(job); +-} +- +- +-/** +- * drm_sched_entity_flush - Flush a context entity +- * +- * @entity: scheduler entity +- * @timeout: time to wait in for Q to become empty in jiffies. +- * +- * Splitting drm_sched_entity_fini() into two functions, The first one does the waiting, +- * removes the entity from the runqueue and returns an error when the process was killed. +- * +- * Returns the remaining time in jiffies left from the input timeout +- */ +-long drm_sched_entity_flush(struct drm_sched_entity *entity, long timeout) +-{ +- struct drm_gpu_scheduler *sched; +- struct task_struct *last_user; +- long ret = timeout; +- +- sched = entity->rq->sched; +- /** +- * The client will not queue more IBs during this fini, consume existing +- * queued IBs or discard them on SIGKILL +- */ +- if (current->flags & PF_EXITING) { +- if (timeout) +- ret = wait_event_timeout( +- sched->job_scheduled, +- drm_sched_entity_is_idle(entity), +- timeout); +- } else +- wait_event_killable(sched->job_scheduled, drm_sched_entity_is_idle(entity)); +- +- +- /* For killed process disable any more IBs enqueue right now */ +- last_user = cmpxchg(&entity->last_user, current->group_leader, NULL); +- if ((!last_user || last_user == current->group_leader) && +- (current->flags & PF_EXITING) && (current->exit_code == SIGKILL)) +- drm_sched_rq_remove_entity(entity->rq, entity); +- +- return ret; +-} +-EXPORT_SYMBOL(drm_sched_entity_flush); +- +-/** +- * drm_sched_entity_cleanup - Destroy a context entity +- * +- * @entity: scheduler entity +- * +- * This should be called after @drm_sched_entity_do_release. It goes over the +- * entity and signals all jobs with an error code if the process was killed. +- * +- */ +-void drm_sched_entity_fini(struct drm_sched_entity *entity) +-{ +- struct drm_gpu_scheduler *sched; +- +- sched = entity->rq->sched; +- drm_sched_rq_remove_entity(entity->rq, entity); +- +- /* Consumption of existing IBs wasn't completed. Forcefully +- * remove them here. +- */ +- if (spsc_queue_peek(&entity->job_queue)) { +- struct drm_sched_job *job; +- int r; +- +- /* Park the kernel for a moment to make sure it isn't processing +- * our enity. +- */ +- kthread_park(sched->thread); +- kthread_unpark(sched->thread); +- if (entity->dependency) { +- dma_fence_remove_callback(entity->dependency, +- &entity->cb); +- dma_fence_put(entity->dependency); +- entity->dependency = NULL; +- } +- +- while ((job = to_drm_sched_job(spsc_queue_pop(&entity->job_queue)))) { +- struct drm_sched_fence *s_fence = job->s_fence; +- drm_sched_fence_scheduled(s_fence); +- dma_fence_set_error(&s_fence->finished, -ESRCH); +- +- /* +- * When pipe is hanged by older entity, new entity might +- * not even have chance to submit it's first job to HW +- * and so entity->last_scheduled will remain NULL +- */ +- if (!entity->last_scheduled) { +- drm_sched_entity_kill_jobs_cb(NULL, &job->finish_cb); +- } else { +- r = dma_fence_add_callback(entity->last_scheduled, &job->finish_cb, +- drm_sched_entity_kill_jobs_cb); +- if (r == -ENOENT) +- drm_sched_entity_kill_jobs_cb(NULL, &job->finish_cb); +- else if (r) +- DRM_ERROR("fence add callback failed (%d)\n", r); +- } +- } +- } +- +- dma_fence_put(entity->last_scheduled); +- entity->last_scheduled = NULL; +- kfree(entity->rq_list); +-} +-EXPORT_SYMBOL(drm_sched_entity_fini); +- +-/** +- * drm_sched_entity_fini - Destroy a context entity +- * +- * @entity: scheduler entity +- * +- * Calls drm_sched_entity_do_release() and drm_sched_entity_cleanup() +- */ +-void drm_sched_entity_destroy(struct drm_sched_entity *entity) +-{ +- drm_sched_entity_flush(entity, MAX_WAIT_SCHED_ENTITY_Q_EMPTY); +- drm_sched_entity_fini(entity); +-} +-EXPORT_SYMBOL(drm_sched_entity_destroy); +- +-static void drm_sched_entity_wakeup(struct dma_fence *f, struct dma_fence_cb *cb) +-{ +- struct drm_sched_entity *entity = +- container_of(cb, struct drm_sched_entity, cb); +- entity->dependency = NULL; +- dma_fence_put(f); +- drm_sched_wakeup(entity->rq->sched); +-} +- +-static void drm_sched_entity_clear_dep(struct dma_fence *f, struct dma_fence_cb *cb) +-{ +- struct drm_sched_entity *entity = +- container_of(cb, struct drm_sched_entity, cb); +- entity->dependency = NULL; +- dma_fence_put(f); +-} +- +-/** +- * drm_sched_entity_set_rq_priority - helper for drm_sched_entity_set_priority +- */ +-static void drm_sched_entity_set_rq_priority(struct drm_sched_rq **rq, +- enum drm_sched_priority priority) +-{ +- *rq = &(*rq)->sched->sched_rq[priority]; +-} +- +-/** +- * drm_sched_entity_set_priority - Sets priority of the entity +- * +- * @entity: scheduler entity +- * @priority: scheduler priority +- * +- * Update the priority of runqueus used for the entity. +- */ +-void drm_sched_entity_set_priority(struct drm_sched_entity *entity, +- enum drm_sched_priority priority) +-{ +- unsigned int i; +- +- spin_lock(&entity->rq_lock); +- +- for (i = 0; i < entity->num_rq_list; ++i) +- drm_sched_entity_set_rq_priority(&entity->rq_list[i], priority); +- +- drm_sched_rq_remove_entity(entity->rq, entity); +- drm_sched_entity_set_rq_priority(&entity->rq, priority); +- drm_sched_rq_add_entity(entity->rq, entity); +- +- spin_unlock(&entity->rq_lock); +-} +-EXPORT_SYMBOL(drm_sched_entity_set_priority); +- +-/** +- * drm_sched_entity_set_rq_priority - helper for drm_sched_entity_set_priority +- */ +-static void drm_sched_entity_set_rq_priority(struct drm_sched_rq **rq, +- enum drm_sched_priority priority) +-{ +- *rq = &(*rq)->sched->sched_rq[priority]; +-} +- +-/** +- * drm_sched_entity_set_priority - Sets priority of the entity +- * +- * @entity: scheduler entity +- * @priority: scheduler priority +- * +- * Update the priority of runqueus used for the entity. +- */ +-void drm_sched_entity_set_priority(struct drm_sched_entity *entity, +- enum drm_sched_priority priority) +-{ +- unsigned int i; +- +- spin_lock(&entity->rq_lock); +- +- for (i = 0; i < entity->num_rq_list; ++i) +- drm_sched_entity_set_rq_priority(&entity->rq_list[i], priority); +- +- drm_sched_rq_remove_entity(entity->rq, entity); +- drm_sched_entity_set_rq_priority(&entity->rq, priority); +- drm_sched_rq_add_entity(entity->rq, entity); +- +- spin_unlock(&entity->rq_lock); +-} +-EXPORT_SYMBOL(drm_sched_entity_set_priority); +- + /** + * drm_sched_dependency_optimized + * +@@ -514,140 +182,6 @@ bool drm_sched_dependency_optimized(struct dma_fence* fence, + } + EXPORT_SYMBOL(drm_sched_dependency_optimized); + +-static bool drm_sched_entity_add_dependency_cb(struct drm_sched_entity *entity) +-{ +- struct drm_gpu_scheduler *sched = entity->rq->sched; +- struct dma_fence * fence = entity->dependency; +- struct drm_sched_fence *s_fence; +- +- if (fence->context == entity->fence_context || +- fence->context == entity->fence_context + 1) { +- /* +- * Fence is a scheduled/finished fence from a job +- * which belongs to the same entity, we can ignore +- * fences from ourself +- */ +- dma_fence_put(entity->dependency); +- return false; +- } +- +- s_fence = to_drm_sched_fence(fence); +- if (s_fence && s_fence->sched == sched) { +- +- /* +- * Fence is from the same scheduler, only need to wait for +- * it to be scheduled +- */ +- fence = dma_fence_get(&s_fence->scheduled); +- dma_fence_put(entity->dependency); +- entity->dependency = fence; +- if (!dma_fence_add_callback(fence, &entity->cb, +- drm_sched_entity_clear_dep)) +- return true; +- +- /* Ignore it when it is already scheduled */ +- dma_fence_put(fence); +- return false; +- } +- +- if (!dma_fence_add_callback(entity->dependency, &entity->cb, +- drm_sched_entity_wakeup)) +- return true; +- +- dma_fence_put(entity->dependency); +- return false; +-} +- +-static struct drm_sched_job * +-drm_sched_entity_pop_job(struct drm_sched_entity *entity) +-{ +- struct drm_gpu_scheduler *sched = entity->rq->sched; +- struct drm_sched_job *sched_job = to_drm_sched_job( +- spsc_queue_peek(&entity->job_queue)); +- +- if (!sched_job) +- return NULL; +- +- while ((entity->dependency = sched->ops->dependency(sched_job, entity))) { +- if (drm_sched_entity_add_dependency_cb(entity)) { +- +- trace_drm_sched_job_wait_dep(sched_job, entity->dependency); +- return NULL; +- } +- } +- +- /* skip jobs from entity that marked guilty */ +- if (entity->guilty && atomic_read(entity->guilty)) +- dma_fence_set_error(&sched_job->s_fence->finished, -ECANCELED); +- +- dma_fence_put(entity->last_scheduled); +- entity->last_scheduled = dma_fence_get(&sched_job->s_fence->finished); +- +- spsc_queue_pop(&entity->job_queue); +- return sched_job; +-} +- +-/** +- * drm_sched_entity_select_rq - select a new rq for the entity +- * +- * @entity: scheduler entity +- * +- * Check all prerequisites and select a new rq for the entity for load +- * balancing. +- */ +-static void drm_sched_entity_select_rq(struct drm_sched_entity *entity) +-{ +- struct dma_fence *fence; +- struct drm_sched_rq *rq; +- +- if (!spsc_queue_count(&entity->job_queue) == 0 || +- entity->num_rq_list <= 1) +- return; +- +- fence = READ_ONCE(entity->last_scheduled); +- if (fence && !dma_fence_is_signaled(fence)) +- return; +- +- rq = drm_sched_entity_get_free_sched(entity); +- spin_lock(&entity->rq_lock); +- drm_sched_rq_remove_entity(entity->rq, entity); +- entity->rq = rq; +- spin_unlock(&entity->rq_lock); +-} +- +-/** +- * drm_sched_entity_push_job - Submit a job to the entity's job queue +- * +- * @sched_job: job to submit +- * @entity: scheduler entity +- * +- * Note: To guarantee that the order of insertion to queue matches +- * the job's fence sequence number this function should be +- * called with drm_sched_job_init under common lock. +- * +- * Returns 0 for success, negative error code otherwise. +- */ +-void drm_sched_entity_push_job(struct drm_sched_job *sched_job, +- struct drm_sched_entity *entity) +-{ +- bool first; +- +- trace_drm_sched_job(sched_job, entity); +- atomic_inc(&entity->rq->sched->num_jobs); +- WRITE_ONCE(entity->last_user, current->group_leader); +- first = spsc_queue_push(&entity->job_queue, &sched_job->queue_node); +- +- /* first job wakes up scheduler */ +- if (first) { +- /* Add the entity to the run queue */ +- spin_lock(&entity->rq_lock); +- drm_sched_rq_add_entity(entity->rq, entity); +- spin_unlock(&entity->rq_lock); +- drm_sched_wakeup(entity->rq->sched); +- } +-} +-EXPORT_SYMBOL(drm_sched_entity_push_job); +- + /* job_finish is called after hw fence signaled + */ + static void drm_sched_job_finish(struct work_struct *work) +@@ -875,7 +409,7 @@ static bool drm_sched_ready(struct drm_gpu_scheduler *sched) + * @sched: scheduler instance + * + */ +-static void drm_sched_wakeup(struct drm_gpu_scheduler *sched) ++void drm_sched_wakeup(struct drm_gpu_scheduler *sched) + { + if (drm_sched_ready(sched)) + wake_up_interruptible(&sched->wake_up_worker); +diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c +new file mode 100644 +index 000000000000..1053f27af9df +--- /dev/null ++++ b/drivers/gpu/drm/scheduler/sched_entity.c +@@ -0,0 +1,459 @@ ++/* ++ * Copyright 2015 Advanced Micro Devices, Inc. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice shall be included in ++ * all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR ++ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ++ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ * ++ */ ++ ++#include <linux/kthread.h> ++#include <drm/gpu_scheduler.h> ++ ++#include "gpu_scheduler_trace.h" ++ ++#define to_drm_sched_job(sched_job) \ ++ container_of((sched_job), struct drm_sched_job, queue_node) ++ ++/** ++ * drm_sched_entity_init - Init a context entity used by scheduler when ++ * submit to HW ring. ++ * ++ * @entity: scheduler entity to init ++ * @rq_list: the list of run queue on which jobs from this ++ * entity can be submitted ++ * @num_rq_list: number of run queue in rq_list ++ * @guilty: atomic_t set to 1 when a job on this queue ++ * is found to be guilty causing a timeout ++ * ++ * Note: the rq_list should have atleast one element to schedule ++ * the entity ++ * ++ * Returns 0 on success or a negative error code on failure. ++*/ ++int drm_sched_entity_init(struct drm_sched_entity *entity, ++ struct drm_sched_rq **rq_list, ++ unsigned int num_rq_list, ++ atomic_t *guilty) ++{ ++ int i; ++ ++ if (!(entity && rq_list && num_rq_list > 0 && rq_list[0])) ++ return -EINVAL; ++ ++ memset(entity, 0, sizeof(struct drm_sched_entity)); ++ INIT_LIST_HEAD(&entity->list); ++ entity->rq = rq_list[0]; ++ entity->guilty = guilty; ++ entity->num_rq_list = num_rq_list; ++ entity->rq_list = kcalloc(num_rq_list, sizeof(struct drm_sched_rq *), ++ GFP_KERNEL); ++ if (!entity->rq_list) ++ return -ENOMEM; ++ ++ for (i = 0; i < num_rq_list; ++i) ++ entity->rq_list[i] = rq_list[i]; ++ entity->last_scheduled = NULL; ++ ++ spin_lock_init(&entity->rq_lock); ++ spsc_queue_init(&entity->job_queue); ++ ++ atomic_set(&entity->fence_seq, 0); ++ entity->fence_context = dma_fence_context_alloc(2); ++ ++ return 0; ++} ++EXPORT_SYMBOL(drm_sched_entity_init); ++ ++/** ++ * drm_sched_entity_is_idle - Check if entity is idle ++ * ++ * @entity: scheduler entity ++ * ++ * Returns true if the entity does not have any unscheduled jobs. ++ */ ++static bool drm_sched_entity_is_idle(struct drm_sched_entity *entity) ++{ ++ rmb(); ++ ++ if (list_empty(&entity->list) || ++ spsc_queue_peek(&entity->job_queue) == NULL) ++ return true; ++ ++ return false; ++} ++ ++/** ++ * drm_sched_entity_is_ready - Check if entity is ready ++ * ++ * @entity: scheduler entity ++ * ++ * Return true if entity could provide a job. ++ */ ++bool drm_sched_entity_is_ready(struct drm_sched_entity *entity) ++{ ++ if (spsc_queue_peek(&entity->job_queue) == NULL) ++ return false; ++ ++ if (READ_ONCE(entity->dependency)) ++ return false; ++ ++ return true; ++} ++ ++/** ++ * drm_sched_entity_get_free_sched - Get the rq from rq_list with least load ++ * ++ * @entity: scheduler entity ++ * ++ * Return the pointer to the rq with least load. ++ */ ++static struct drm_sched_rq * ++drm_sched_entity_get_free_sched(struct drm_sched_entity *entity) ++{ ++ struct drm_sched_rq *rq = NULL; ++ unsigned int min_jobs = UINT_MAX, num_jobs; ++ int i; ++ ++ for (i = 0; i < entity->num_rq_list; ++i) { ++ num_jobs = atomic_read(&entity->rq_list[i]->sched->num_jobs); ++ if (num_jobs < min_jobs) { ++ min_jobs = num_jobs; ++ rq = entity->rq_list[i]; ++ } ++ } ++ ++ return rq; ++} ++ ++static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f, ++ struct dma_fence_cb *cb) ++{ ++ struct drm_sched_job *job = container_of(cb, struct drm_sched_job, ++ finish_cb); ++ drm_sched_fence_finished(job->s_fence); ++ WARN_ON(job->s_fence->parent); ++ dma_fence_put(&job->s_fence->finished); ++ job->sched->ops->free_job(job); ++} ++ ++ ++/** ++ * drm_sched_entity_flush - Flush a context entity ++ * ++ * @entity: scheduler entity ++ * @timeout: time to wait in for Q to become empty in jiffies. ++ * ++ * Splitting drm_sched_entity_fini() into two functions, The first one does the waiting, ++ * removes the entity from the runqueue and returns an error when the process was killed. ++ * ++ * Returns the remaining time in jiffies left from the input timeout ++ */ ++long drm_sched_entity_flush(struct drm_sched_entity *entity, long timeout) ++{ ++ struct drm_gpu_scheduler *sched; ++ struct task_struct *last_user; ++ long ret = timeout; ++ ++ sched = entity->rq->sched; ++ /** ++ * The client will not queue more IBs during this fini, consume existing ++ * queued IBs or discard them on SIGKILL ++ */ ++ if (current->flags & PF_EXITING) { ++ if (timeout) ++ ret = wait_event_timeout( ++ sched->job_scheduled, ++ drm_sched_entity_is_idle(entity), ++ timeout); ++ } else { ++ wait_event_killable(sched->job_scheduled, ++ drm_sched_entity_is_idle(entity)); ++ } ++ ++ /* For killed process disable any more IBs enqueue right now */ ++ last_user = cmpxchg(&entity->last_user, current->group_leader, NULL); ++ if ((!last_user || last_user == current->group_leader) && ++ (current->flags & PF_EXITING) && (current->exit_code == SIGKILL)) ++ drm_sched_rq_remove_entity(entity->rq, entity); ++ ++ return ret; ++} ++EXPORT_SYMBOL(drm_sched_entity_flush); ++ ++/** ++ * drm_sched_entity_cleanup - Destroy a context entity ++ * ++ * @entity: scheduler entity ++ * ++ * This should be called after @drm_sched_entity_do_release. It goes over the ++ * entity and signals all jobs with an error code if the process was killed. ++ * ++ */ ++void drm_sched_entity_fini(struct drm_sched_entity *entity) ++{ ++ struct drm_gpu_scheduler *sched; ++ ++ sched = entity->rq->sched; ++ drm_sched_rq_remove_entity(entity->rq, entity); ++ ++ /* Consumption of existing IBs wasn't completed. Forcefully ++ * remove them here. ++ */ ++ if (spsc_queue_peek(&entity->job_queue)) { ++ struct drm_sched_job *job; ++ int r; ++ ++ /* Park the kernel for a moment to make sure it isn't processing ++ * our enity. ++ */ ++ kthread_park(sched->thread); ++ kthread_unpark(sched->thread); ++ if (entity->dependency) { ++ dma_fence_remove_callback(entity->dependency, ++ &entity->cb); ++ dma_fence_put(entity->dependency); ++ entity->dependency = NULL; ++ } ++ ++ while ((job = to_drm_sched_job(spsc_queue_pop(&entity->job_queue)))) { ++ struct drm_sched_fence *s_fence = job->s_fence; ++ drm_sched_fence_scheduled(s_fence); ++ dma_fence_set_error(&s_fence->finished, -ESRCH); ++ ++ /* ++ * When pipe is hanged by older entity, new entity might ++ * not even have chance to submit it's first job to HW ++ * and so entity->last_scheduled will remain NULL ++ */ ++ if (!entity->last_scheduled) { ++ drm_sched_entity_kill_jobs_cb(NULL, &job->finish_cb); ++ } else { ++ r = dma_fence_add_callback(entity->last_scheduled, &job->finish_cb, ++ drm_sched_entity_kill_jobs_cb); ++ if (r == -ENOENT) ++ drm_sched_entity_kill_jobs_cb(NULL, &job->finish_cb); ++ else if (r) ++ DRM_ERROR("fence add callback failed (%d)\n", r); ++ } ++ } ++ } ++ ++ dma_fence_put(entity->last_scheduled); ++ entity->last_scheduled = NULL; ++ kfree(entity->rq_list); ++} ++EXPORT_SYMBOL(drm_sched_entity_fini); ++ ++/** ++ * drm_sched_entity_fini - Destroy a context entity ++ * ++ * @entity: scheduler entity ++ * ++ * Calls drm_sched_entity_do_release() and drm_sched_entity_cleanup() ++ */ ++void drm_sched_entity_destroy(struct drm_sched_entity *entity) ++{ ++ drm_sched_entity_flush(entity, MAX_WAIT_SCHED_ENTITY_Q_EMPTY); ++ drm_sched_entity_fini(entity); ++} ++EXPORT_SYMBOL(drm_sched_entity_destroy); ++ ++static void drm_sched_entity_wakeup(struct dma_fence *f, struct dma_fence_cb *cb) ++{ ++ struct drm_sched_entity *entity = ++ container_of(cb, struct drm_sched_entity, cb); ++ entity->dependency = NULL; ++ dma_fence_put(f); ++ drm_sched_wakeup(entity->rq->sched); ++} ++ ++static void drm_sched_entity_clear_dep(struct dma_fence *f, struct dma_fence_cb *cb) ++{ ++ struct drm_sched_entity *entity = ++ container_of(cb, struct drm_sched_entity, cb); ++ entity->dependency = NULL; ++ dma_fence_put(f); ++} ++ ++/** ++ * drm_sched_entity_set_rq_priority - helper for drm_sched_entity_set_priority ++ */ ++static void drm_sched_entity_set_rq_priority(struct drm_sched_rq **rq, ++ enum drm_sched_priority priority) ++{ ++ *rq = &(*rq)->sched->sched_rq[priority]; ++} ++ ++/** ++ * drm_sched_entity_set_priority - Sets priority of the entity ++ * ++ * @entity: scheduler entity ++ * @priority: scheduler priority ++ * ++ * Update the priority of runqueus used for the entity. ++ */ ++void drm_sched_entity_set_priority(struct drm_sched_entity *entity, ++ enum drm_sched_priority priority) ++{ ++ unsigned int i; ++ ++ spin_lock(&entity->rq_lock); ++ ++ for (i = 0; i < entity->num_rq_list; ++i) ++ drm_sched_entity_set_rq_priority(&entity->rq_list[i], priority); ++ ++ drm_sched_rq_remove_entity(entity->rq, entity); ++ drm_sched_entity_set_rq_priority(&entity->rq, priority); ++ drm_sched_rq_add_entity(entity->rq, entity); ++ ++ spin_unlock(&entity->rq_lock); ++} ++EXPORT_SYMBOL(drm_sched_entity_set_priority); ++ ++static bool drm_sched_entity_add_dependency_cb(struct drm_sched_entity *entity) ++{ ++ struct drm_gpu_scheduler *sched = entity->rq->sched; ++ struct dma_fence * fence = entity->dependency; ++ struct drm_sched_fence *s_fence; ++ ++ if (fence->context == entity->fence_context || ++ fence->context == entity->fence_context + 1) { ++ /* ++ * Fence is a scheduled/finished fence from a job ++ * which belongs to the same entity, we can ignore ++ * fences from ourself ++ */ ++ dma_fence_put(entity->dependency); ++ return false; ++ } ++ ++ s_fence = to_drm_sched_fence(fence); ++ if (s_fence && s_fence->sched == sched) { ++ ++ /* ++ * Fence is from the same scheduler, only need to wait for ++ * it to be scheduled ++ */ ++ fence = dma_fence_get(&s_fence->scheduled); ++ dma_fence_put(entity->dependency); ++ entity->dependency = fence; ++ if (!dma_fence_add_callback(fence, &entity->cb, ++ drm_sched_entity_clear_dep)) ++ return true; ++ ++ /* Ignore it when it is already scheduled */ ++ dma_fence_put(fence); ++ return false; ++ } ++ ++ if (!dma_fence_add_callback(entity->dependency, &entity->cb, ++ drm_sched_entity_wakeup)) ++ return true; ++ ++ dma_fence_put(entity->dependency); ++ return false; ++} ++ ++struct drm_sched_job *drm_sched_entity_pop_job(struct drm_sched_entity *entity) ++{ ++ struct drm_gpu_scheduler *sched = entity->rq->sched; ++ struct drm_sched_job *sched_job = to_drm_sched_job( ++ spsc_queue_peek(&entity->job_queue)); ++ ++ if (!sched_job) ++ return NULL; ++ ++ while ((entity->dependency = sched->ops->dependency(sched_job, entity))) { ++ if (drm_sched_entity_add_dependency_cb(entity)) { ++ ++ trace_drm_sched_job_wait_dep(sched_job, entity->dependency); ++ return NULL; ++ } ++ } ++ ++ /* skip jobs from entity that marked guilty */ ++ if (entity->guilty && atomic_read(entity->guilty)) ++ dma_fence_set_error(&sched_job->s_fence->finished, -ECANCELED); ++ ++ dma_fence_put(entity->last_scheduled); ++ entity->last_scheduled = dma_fence_get(&sched_job->s_fence->finished); ++ ++ spsc_queue_pop(&entity->job_queue); ++ return sched_job; ++} ++ ++/** ++ * drm_sched_entity_select_rq - select a new rq for the entity ++ * ++ * @entity: scheduler entity ++ * ++ * Check all prerequisites and select a new rq for the entity for load ++ * balancing. ++ */ ++void drm_sched_entity_select_rq(struct drm_sched_entity *entity) ++{ ++ struct dma_fence *fence; ++ struct drm_sched_rq *rq; ++ ++ if (!spsc_queue_count(&entity->job_queue) == 0 || ++ entity->num_rq_list <= 1) ++ return; ++ ++ fence = READ_ONCE(entity->last_scheduled); ++ if (fence && !dma_fence_is_signaled(fence)) ++ return; ++ ++ rq = drm_sched_entity_get_free_sched(entity); ++ spin_lock(&entity->rq_lock); ++ drm_sched_rq_remove_entity(entity->rq, entity); ++ entity->rq = rq; ++ spin_unlock(&entity->rq_lock); ++} ++ ++/** ++ * drm_sched_entity_push_job - Submit a job to the entity's job queue ++ * ++ * @sched_job: job to submit ++ * @entity: scheduler entity ++ * ++ * Note: To guarantee that the order of insertion to queue matches ++ * the job's fence sequence number this function should be ++ * called with drm_sched_job_init under common lock. ++ * ++ * Returns 0 for success, negative error code otherwise. ++ */ ++void drm_sched_entity_push_job(struct drm_sched_job *sched_job, ++ struct drm_sched_entity *entity) ++{ ++ bool first; ++ ++ trace_drm_sched_job(sched_job, entity); ++ atomic_inc(&entity->rq->sched->num_jobs); ++ WRITE_ONCE(entity->last_user, current->group_leader); ++ first = spsc_queue_push(&entity->job_queue, &sched_job->queue_node); ++ ++ /* first job wakes up scheduler */ ++ if (first) { ++ /* Add the entity to the run queue */ ++ spin_lock(&entity->rq_lock); ++ drm_sched_rq_add_entity(entity->rq, entity); ++ spin_unlock(&entity->rq_lock); ++ drm_sched_wakeup(entity->rq->sched); ++ } ++} ++EXPORT_SYMBOL(drm_sched_entity_push_job); +diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h +index 551f92ce2164..fa7ea410b63c 100644 +--- a/include/drm/gpu_scheduler.h ++++ b/include/drm/gpu_scheduler.h +@@ -285,6 +285,21 @@ int drm_sched_init(struct drm_gpu_scheduler *sched, + uint32_t hw_submission, unsigned hang_limit, long timeout, + const char *name); + void drm_sched_fini(struct drm_gpu_scheduler *sched); ++int drm_sched_job_init(struct drm_sched_job *job, ++ struct drm_sched_entity *entity, ++ void *owner); ++void drm_sched_wakeup(struct drm_gpu_scheduler *sched); ++void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, ++ struct drm_sched_job *job); ++void drm_sched_job_recovery(struct drm_gpu_scheduler *sched); ++bool drm_sched_dependency_optimized(struct dma_fence* fence, ++ struct drm_sched_entity *entity); ++void drm_sched_job_kickout(struct drm_sched_job *s_job); ++ ++void drm_sched_rq_add_entity(struct drm_sched_rq *rq, ++ struct drm_sched_entity *entity); ++void drm_sched_rq_remove_entity(struct drm_sched_rq *rq, ++ struct drm_sched_entity *entity); + + int drm_sched_entity_init(struct drm_sched_entity *entity, + struct drm_sched_rq **rq_list, +@@ -293,6 +308,9 @@ int drm_sched_entity_init(struct drm_sched_entity *entity, + long drm_sched_entity_flush(struct drm_sched_entity *entity, long timeout); + void drm_sched_entity_fini(struct drm_sched_entity *entity); + void drm_sched_entity_destroy(struct drm_sched_entity *entity); ++void drm_sched_entity_select_rq(struct drm_sched_entity *entity); ++struct drm_sched_job *drm_sched_entity_pop_job(struct drm_sched_entity *entity); ++bool drm_sched_entity_is_ready(struct drm_sched_entity *entity); + void drm_sched_entity_push_job(struct drm_sched_job *sched_job, + struct drm_sched_entity *entity); + void drm_sched_entity_set_priority(struct drm_sched_entity *entity, +-- +2.17.1 + |