mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2026-06-21 15:43:21 +02:00
Thomas Gleixner
eddffab828
Evaluating the next expiry time of all clock bases is cache line expensive as the expiry time of the first expiring timer is not cached in the base and requires to access the timer itself, which is definitely in a different cache line. It's way more efficient to keep track of the expiry time on enqueue and dequeue operations as the relevant data is already in the cache at that point. Signed-off-by: Thomas Gleixner <tglx@kernel.org> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Link: https://patch.msgid.link/20260224163431.404839710@kernel.org
114 lines
3.9 KiB
C
114 lines
3.9 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef _LINUX_HRTIMER_DEFS_H
|
|
#define _LINUX_HRTIMER_DEFS_H
|
|
|
|
#include <linux/ktime.h>
|
|
#include <linux/timerqueue.h>
|
|
#include <linux/seqlock.h>
|
|
|
|
#ifdef CONFIG_64BIT
|
|
# define __hrtimer_clock_base_align ____cacheline_aligned
|
|
#else
|
|
# define __hrtimer_clock_base_align
|
|
#endif
|
|
|
|
/**
|
|
* struct hrtimer_clock_base - the timer base for a specific clock
|
|
* @cpu_base: per cpu clock base
|
|
* @index: clock type index for per_cpu support when moving a
|
|
* timer to a base on another cpu.
|
|
* @clockid: clock id for per_cpu support
|
|
* @seq: seqcount around __run_hrtimer
|
|
* @expires_next: Absolute time of the next event in this clock base
|
|
* @running: pointer to the currently running hrtimer
|
|
* @active: red black tree root node for the active timers
|
|
* @offset: offset of this clock to the monotonic base
|
|
*/
|
|
struct hrtimer_clock_base {
|
|
struct hrtimer_cpu_base *cpu_base;
|
|
unsigned int index;
|
|
clockid_t clockid;
|
|
seqcount_raw_spinlock_t seq;
|
|
ktime_t expires_next;
|
|
struct hrtimer *running;
|
|
struct timerqueue_head active;
|
|
ktime_t offset;
|
|
} __hrtimer_clock_base_align;
|
|
|
|
enum hrtimer_base_type {
|
|
HRTIMER_BASE_MONOTONIC,
|
|
HRTIMER_BASE_REALTIME,
|
|
HRTIMER_BASE_BOOTTIME,
|
|
HRTIMER_BASE_TAI,
|
|
HRTIMER_BASE_MONOTONIC_SOFT,
|
|
HRTIMER_BASE_REALTIME_SOFT,
|
|
HRTIMER_BASE_BOOTTIME_SOFT,
|
|
HRTIMER_BASE_TAI_SOFT,
|
|
HRTIMER_MAX_CLOCK_BASES,
|
|
};
|
|
|
|
/**
|
|
* struct hrtimer_cpu_base - the per cpu clock bases
|
|
* @lock: lock protecting the base and associated clock bases and timers
|
|
* @cpu: cpu number
|
|
* @active_bases: Bitfield to mark bases with active timers
|
|
* @clock_was_set_seq: Sequence counter of clock was set events
|
|
* @hres_active: State of high resolution mode
|
|
* @deferred_rearm: A deferred rearm is pending
|
|
* @deferred_needs_update: The deferred rearm must re-evaluate the first timer
|
|
* @hang_detected: The last hrtimer interrupt detected a hang
|
|
* @softirq_activated: displays, if the softirq is raised - update of softirq
|
|
* related settings is not required then.
|
|
* @nr_events: Total number of hrtimer interrupt events
|
|
* @nr_retries: Total number of hrtimer interrupt retries
|
|
* @nr_hangs: Total number of hrtimer interrupt hangs
|
|
* @max_hang_time: Maximum time spent in hrtimer_interrupt
|
|
* @softirq_expiry_lock: Lock which is taken while softirq based hrtimer are expired
|
|
* @online: CPU is online from an hrtimers point of view
|
|
* @timer_waiters: A hrtimer_cancel() waiters for the timer callback to finish.
|
|
* @expires_next: Absolute time of the next event, is required for remote
|
|
* hrtimer enqueue; it is the total first expiry time (hard
|
|
* and soft hrtimer are taken into account)
|
|
* @next_timer: Pointer to the first expiring timer
|
|
* @softirq_expires_next: Time to check, if soft queues needs also to be expired
|
|
* @softirq_next_timer: Pointer to the first expiring softirq based timer
|
|
* @deferred_expires_next: Cached expires next value for deferred rearm
|
|
* @clock_base: Array of clock bases for this cpu
|
|
*
|
|
* Note: next_timer is just an optimization for __remove_hrtimer().
|
|
* Do not dereference the pointer because it is not reliable on
|
|
* cross cpu removals.
|
|
*/
|
|
struct hrtimer_cpu_base {
|
|
raw_spinlock_t lock;
|
|
unsigned int cpu;
|
|
unsigned int active_bases;
|
|
unsigned int clock_was_set_seq;
|
|
bool hres_active;
|
|
bool deferred_rearm;
|
|
bool deferred_needs_update;
|
|
bool hang_detected;
|
|
bool softirq_activated;
|
|
bool online;
|
|
#ifdef CONFIG_HIGH_RES_TIMERS
|
|
unsigned int nr_events;
|
|
unsigned short nr_retries;
|
|
unsigned short nr_hangs;
|
|
unsigned int max_hang_time;
|
|
#endif
|
|
#ifdef CONFIG_PREEMPT_RT
|
|
spinlock_t softirq_expiry_lock;
|
|
atomic_t timer_waiters;
|
|
#endif
|
|
ktime_t expires_next;
|
|
struct hrtimer *next_timer;
|
|
ktime_t softirq_expires_next;
|
|
struct hrtimer *softirq_next_timer;
|
|
ktime_t deferred_expires_next;
|
|
struct hrtimer_clock_base clock_base[HRTIMER_MAX_CLOCK_BASES];
|
|
call_single_data_t csd;
|
|
} ____cacheline_aligned;
|
|
|
|
|
|
#endif
|