diff options
Diffstat (limited to 'kernel/time/timer.c')
| -rw-r--r-- | kernel/time/timer.c | 61 |
1 files changed, 53 insertions, 8 deletions
diff --git a/kernel/time/timer.c b/kernel/time/timer.c index 786f8c014e7e..09496b7c0001 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -44,6 +44,7 @@ #include <linux/sched/debug.h> #include <linux/slab.h> #include <linux/compat.h> +#include <linux/swait.h> #include <linux/uaccess.h> #include <asm/unistd.h> @@ -197,6 +198,9 @@ EXPORT_SYMBOL(jiffies_64); struct timer_base { raw_spinlock_t lock; struct timer_list *running_timer; +#ifdef CONFIG_PREEMPT_RT_FULL + struct swait_queue_head wait_for_running_timer; +#endif unsigned long clk; unsigned long next_expiry; unsigned int cpu; @@ -213,8 +217,7 @@ static DEFINE_PER_CPU(struct timer_base, timer_bases[NR_BASES]); static DEFINE_STATIC_KEY_FALSE(timers_nohz_active); static DEFINE_MUTEX(timer_keys_mutex); -static void timer_update_keys(struct work_struct *work); -static DECLARE_WORK(timer_update_work, timer_update_keys); +static struct swork_event timer_update_swork; #ifdef CONFIG_SMP unsigned int sysctl_timer_migration = 1; @@ -232,7 +235,7 @@ static void timers_update_migration(void) static inline void timers_update_migration(void) { } #endif /* !CONFIG_SMP */ -static void timer_update_keys(struct work_struct *work) +static void timer_update_keys(struct swork_event *event) { mutex_lock(&timer_keys_mutex); timers_update_migration(); @@ -242,8 +245,16 @@ static void timer_update_keys(struct work_struct *work) void timers_update_nohz(void) { - schedule_work(&timer_update_work); + swork_queue(&timer_update_swork); +} + +static __init int hrtimer_init_thread(void) +{ + WARN_ON(swork_get()); + INIT_SWORK(&timer_update_swork, timer_update_keys); + return 0; } +early_initcall(hrtimer_init_thread); int timer_migration_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, @@ -1178,6 +1189,33 @@ void add_timer_on(struct timer_list *timer, int cpu) } EXPORT_SYMBOL_GPL(add_timer_on); +#ifdef CONFIG_PREEMPT_RT_FULL +/* + * Wait for a running timer + */ +static void wait_for_running_timer(struct timer_list *timer) +{ + struct timer_base *base; + u32 tf = timer->flags; + + if (tf & TIMER_MIGRATING) + return; + + base = get_timer_base(tf); + swait_event(base->wait_for_running_timer, + base->running_timer != timer); +} + +# define wakeup_timer_waiters(b) swake_up_all(&(b)->wait_for_running_timer) +#else +static inline void wait_for_running_timer(struct timer_list *timer) +{ + cpu_relax(); +} + +# define wakeup_timer_waiters(b) do { } while (0) +#endif + /** * del_timer - deactivate a timer. * @timer: the timer to be deactivated @@ -1233,7 +1271,7 @@ int try_to_del_timer_sync(struct timer_list *timer) } EXPORT_SYMBOL(try_to_del_timer_sync); -#ifdef CONFIG_SMP +#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL) /** * del_timer_sync - deactivate a timer and wait for the handler to finish. * @timer: the timer to be deactivated @@ -1293,7 +1331,7 @@ int del_timer_sync(struct timer_list *timer) int ret = try_to_del_timer_sync(timer); if (ret >= 0) return ret; - cpu_relax(); + wait_for_running_timer(timer); } } EXPORT_SYMBOL(del_timer_sync); @@ -1354,13 +1392,16 @@ static void expire_timers(struct timer_base *base, struct hlist_head *head) fn = timer->function; - if (timer->flags & TIMER_IRQSAFE) { + if (!IS_ENABLED(CONFIG_PREEMPT_RT_FULL) && + timer->flags & TIMER_IRQSAFE) { raw_spin_unlock(&base->lock); call_timer_fn(timer, fn); + base->running_timer = NULL; raw_spin_lock(&base->lock); } else { raw_spin_unlock_irq(&base->lock); call_timer_fn(timer, fn); + base->running_timer = NULL; raw_spin_lock_irq(&base->lock); } } @@ -1681,8 +1722,8 @@ static inline void __run_timers(struct timer_base *base) while (levels--) expire_timers(base, heads + levels); } - base->running_timer = NULL; raw_spin_unlock_irq(&base->lock); + wakeup_timer_waiters(base); } /* @@ -1692,6 +1733,7 @@ static __latent_entropy void run_timer_softirq(struct softirq_action *h) { struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]); + irq_work_tick_soft(); __run_timers(base); if (IS_ENABLED(CONFIG_NO_HZ_COMMON)) __run_timers(this_cpu_ptr(&timer_bases[BASE_DEF])); @@ -1927,6 +1969,9 @@ static void __init init_timer_cpu(int cpu) base->cpu = cpu; raw_spin_lock_init(&base->lock); base->clk = jiffies; +#ifdef CONFIG_PREEMPT_RT_FULL + init_swait_queue_head(&base->wait_for_running_timer); +#endif } } |