Skip to content

L
linux
Commit f726a697 authored by John Stultz's avatar John Stultz Committed by Thomas Gleixner
Browse files
Options

time: Rework timekeeping functions to take timekeeper ptr as argument 

As part of cleaning up the timekeeping code, this patch converts
a number of internal functions to takei a timekeeper ptr as an
argument, so that the internal functions don't access the global
timekeeper structure directly. This allows for further optimizations
to reduce lock hold time later.

This patch has been updated to include more consistent usage of the
timekeeper value, by making sure it is always passed as a argument
to non top-level functions.
Signed-off-by: default avatarJohn Stultz <john.stultz@linaro.org>
Reviewed-by: default avatarIngo Molnar <mingo@kernel.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Richard Cochran <richardcochran@gmail.com>
Cc: Prarit Bhargava <prarit@redhat.com>
Link: http://lkml.kernel.org/r/1342156917-25092-9-git-send-email-john.stultz@linaro.orgSigned-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
parent 2a8c0883
Show whitespace changes
Inline Side-by-side
Showing with 103 additions and 105 deletions
kernel/time/timekeeping.c
View file @ f726a697
......@@ -127,14 +127,14 @@ static void tk_xtime_add(struct timekeeper *tk, const struct timespec *ts)
*
* Unless you're the timekeeping code, you should not be using this!
*/
static void timekeeper_setup_internals(struct clocksource *clock)
static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
{
cycle_t interval;
u64 tmp, ntpinterval;
struct clocksource *old_clock;
old_clock = timekeeper.clock;
timekeeper.clock = clock;
old_clock = tk->clock;
tk->clock = clock;
clock->cycle_last = clock->read(clock);
/* Do the ns -> cycle conversion first, using original mult */
......@@ -147,64 +147,64 @@ static void timekeeper_setup_internals(struct clocksource *clock)
tmp = 1;
interval = (cycle_t) tmp;
timekeeper.cycle_interval = interval;
tk->cycle_interval = interval;
/* Go back from cycles -> shifted ns */
timekeeper.xtime_interval = (u64) interval * clock->mult;
timekeeper.xtime_remainder = ntpinterval - timekeeper.xtime_interval;
timekeeper.raw_interval =
tk->xtime_interval = (u64) interval * clock->mult;
tk->xtime_remainder = ntpinterval - tk->xtime_interval;
tk->raw_interval =
((u64) interval * clock->mult) >> clock->shift;
/* if changing clocks, convert xtime_nsec shift units */
if (old_clock) {
int shift_change = clock->shift - old_clock->shift;
if (shift_change < 0)
timekeeper.xtime_nsec >>= -shift_change;
tk->xtime_nsec >>= -shift_change;
else
timekeeper.xtime_nsec <<= shift_change;
tk->xtime_nsec <<= shift_change;
}
timekeeper.shift = clock->shift;
tk->shift = clock->shift;
timekeeper.ntp_error = 0;
timekeeper.ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
tk->ntp_error = 0;
tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
/*
* The timekeeper keeps its own mult values for the currently
* active clocksource. These value will be adjusted via NTP
* to counteract clock drifting.
*/
timekeeper.mult = clock->mult;
tk->mult = clock->mult;
}
/* Timekeeper helper functions. */
static inline s64 timekeeping_get_ns(void)
static inline s64 timekeeping_get_ns(struct timekeeper *tk)
{
cycle_t cycle_now, cycle_delta;
struct clocksource *clock;
s64 nsec;
/* read clocksource: */
clock = timekeeper.clock;
clock = tk->clock;
cycle_now = clock->read(clock);
/* calculate the delta since the last update_wall_time: */
cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
nsec = cycle_delta * timekeeper.mult + timekeeper.xtime_nsec;
nsec >>= timekeeper.shift;
nsec = cycle_delta * tk->mult + tk->xtime_nsec;
nsec >>= tk->shift;
/* If arch requires, add in gettimeoffset() */
return nsec + arch_gettimeoffset();
}
static inline s64 timekeeping_get_ns_raw(void)
static inline s64 timekeeping_get_ns_raw(struct timekeeper *tk)
{
cycle_t cycle_now, cycle_delta;
struct clocksource *clock;
s64 nsec;
/* read clocksource: */
clock = timekeeper.clock;
clock = tk->clock;
cycle_now = clock->read(clock);
/* calculate the delta since the last update_wall_time: */
......@@ -217,27 +217,26 @@ static inline s64 timekeeping_get_ns_raw(void)
return nsec + arch_gettimeoffset();
}
static void update_rt_offset(void)
static void update_rt_offset(struct timekeeper *tk)
{
struct timespec tmp, *wtm = &timekeeper.wall_to_monotonic;
struct timespec tmp, *wtm = &tk->wall_to_monotonic;
set_normalized_timespec(&tmp, -wtm->tv_sec, -wtm->tv_nsec);
timekeeper.offs_real = timespec_to_ktime(tmp);
tk->offs_real = timespec_to_ktime(tmp);
}
/* must hold write on timekeeper.lock */
static void timekeeping_update(bool clearntp)
static void timekeeping_update(struct timekeeper *tk, bool clearntp)
{
struct timespec xt;
if (clearntp) {
timekeeper.ntp_error = 0;
tk->ntp_error = 0;
ntp_clear();
}
update_rt_offset();
xt = tk_xtime(&timekeeper);
update_vsyscall(&xt, &timekeeper.wall_to_monotonic,
timekeeper.clock, timekeeper.mult);
update_rt_offset(tk);
xt = tk_xtime(tk);
update_vsyscall(&xt, &tk->wall_to_monotonic, tk->clock, tk->mult);
}
......@@ -248,26 +247,26 @@ static void timekeeping_update(bool clearntp)
* update_wall_time(). This is useful before significant clock changes,
* as it avoids having to deal with this time offset explicitly.
*/
static void timekeeping_forward_now(void)
static void timekeeping_forward_now(struct timekeeper *tk)
{
cycle_t cycle_now, cycle_delta;
struct clocksource *clock;
s64 nsec;
clock = timekeeper.clock;
clock = tk->clock;
cycle_now = clock->read(clock);
cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
clock->cycle_last = cycle_now;
timekeeper.xtime_nsec += cycle_delta * timekeeper.mult;
tk->xtime_nsec += cycle_delta * tk->mult;
/* If arch requires, add in gettimeoffset() */
timekeeper.xtime_nsec += arch_gettimeoffset() << timekeeper.shift;
tk->xtime_nsec += arch_gettimeoffset() << tk->shift;
tk_normalize_xtime(&timekeeper);
tk_normalize_xtime(tk);
nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift);
timespec_add_ns(&timekeeper.raw_time, nsec);
timespec_add_ns(&tk->raw_time, nsec);
}
/**
......@@ -287,7 +286,7 @@ void getnstimeofday(struct timespec *ts)
seq = read_seqbegin(&timekeeper.lock);
ts->tv_sec = timekeeper.xtime_sec;
ts->tv_nsec = timekeeping_get_ns();
ts->tv_nsec = timekeeping_get_ns(&timekeeper);
} while (read_seqretry(&timekeeper.lock, seq));
......@@ -306,7 +305,7 @@ ktime_t ktime_get(void)
seq = read_seqbegin(&timekeeper.lock);
secs = timekeeper.xtime_sec +
timekeeper.wall_to_monotonic.tv_sec;
nsecs = timekeeping_get_ns() +
nsecs = timekeeping_get_ns(&timekeeper) +
timekeeper.wall_to_monotonic.tv_nsec;
} while (read_seqretry(&timekeeper.lock, seq));
......@@ -336,7 +335,7 @@ void ktime_get_ts(struct timespec *ts)
do {
seq = read_seqbegin(&timekeeper.lock);
ts->tv_sec = timekeeper.xtime_sec;
ts->tv_nsec = timekeeping_get_ns();
ts->tv_nsec = timekeeping_get_ns(&timekeeper);
tomono = timekeeper.wall_to_monotonic;
} while (read_seqretry(&timekeeper.lock, seq));
......@@ -371,8 +370,8 @@ void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real)
ts_real->tv_sec = timekeeper.xtime_sec;
ts_real->tv_nsec = 0;
nsecs_raw = timekeeping_get_ns_raw();
nsecs_real = timekeeping_get_ns();
nsecs_raw = timekeeping_get_ns_raw(&timekeeper);
nsecs_real = timekeeping_get_ns(&timekeeper);
} while (read_seqretry(&timekeeper.lock, seq));
......@@ -415,7 +414,7 @@ int do_settimeofday(const struct timespec *tv)
write_seqlock_irqsave(&timekeeper.lock, flags);
timekeeping_forward_now();
timekeeping_forward_now(&timekeeper);
xt = tk_xtime(&timekeeper);
ts_delta.tv_sec = tv->tv_sec - xt.tv_sec;
......@@ -426,7 +425,7 @@ int do_settimeofday(const struct timespec *tv)
tk_set_xtime(&timekeeper, tv);
timekeeping_update(true);
timekeeping_update(&timekeeper, true);
write_sequnlock_irqrestore(&timekeeper.lock, flags);
......@@ -453,14 +452,14 @@ int timekeeping_inject_offset(struct timespec *ts)
write_seqlock_irqsave(&timekeeper.lock, flags);
timekeeping_forward_now();
timekeeping_forward_now(&timekeeper);
tk_xtime_add(&timekeeper, ts);
timekeeper.wall_to_monotonic =
timespec_sub(timekeeper.wall_to_monotonic, *ts);
timekeeping_update(true);
timekeeping_update(&timekeeper, true);
write_sequnlock_irqrestore(&timekeeper.lock, flags);
......@@ -485,14 +484,14 @@ static int change_clocksource(void *data)
write_seqlock_irqsave(&timekeeper.lock, flags);
timekeeping_forward_now();
timekeeping_forward_now(&timekeeper);
if (!new->enable || new->enable(new) == 0) {
old = timekeeper.clock;
timekeeper_setup_internals(new);
tk_setup_internals(&timekeeper, new);
if (old->disable)
old->disable(old);
}
timekeeping_update(true);
timekeeping_update(&timekeeper, true);
write_sequnlock_irqrestore(&timekeeper.lock, flags);
......@@ -542,7 +541,7 @@ void getrawmonotonic(struct timespec *ts)
do {
seq = read_seqbegin(&timekeeper.lock);
nsecs = timekeeping_get_ns_raw();
nsecs = timekeeping_get_ns_raw(&timekeeper);
*ts = timekeeper.raw_time;
} while (read_seqretry(&timekeeper.lock, seq));
......@@ -638,7 +637,7 @@ void __init timekeeping_init(void)
clock = clocksource_default_clock();
if (clock->enable)
clock->enable(clock);
timekeeper_setup_internals(clock);
tk_setup_internals(&timekeeper, clock);
tk_set_xtime(&timekeeper, &now);
timekeeper.raw_time.tv_sec = 0;
......@@ -648,7 +647,7 @@ void __init timekeeping_init(void)
set_normalized_timespec(&timekeeper.wall_to_monotonic,
-boot.tv_sec, -boot.tv_nsec);
update_rt_offset();
update_rt_offset(&timekeeper);
timekeeper.total_sleep_time.tv_sec = 0;
timekeeper.total_sleep_time.tv_nsec = 0;
write_sequnlock_irqrestore(&timekeeper.lock, flags);
......@@ -670,7 +669,8 @@ static void update_sleep_time(struct timespec t)
* Takes a timespec offset measuring a suspend interval and properly
* adds the sleep offset to the timekeeping variables.
*/
static void __timekeeping_inject_sleeptime(struct timespec *delta)
static void __timekeeping_inject_sleeptime(struct timekeeper *tk,
struct timespec *delta)
{
if (!timespec_valid(delta)) {
printk(KERN_WARNING "__timekeeping_inject_sleeptime: Invalid "
......@@ -678,10 +678,9 @@ static void __timekeeping_inject_sleeptime(struct timespec *delta)
return;
}
tk_xtime_add(&timekeeper, delta);
timekeeper.wall_to_monotonic =
timespec_sub(timekeeper.wall_to_monotonic, *delta);
update_sleep_time(timespec_add(timekeeper.total_sleep_time, *delta));
tk_xtime_add(tk, delta);
tk->wall_to_monotonic = timespec_sub(tk->wall_to_monotonic, *delta);
update_sleep_time(timespec_add(tk->total_sleep_time, *delta));
}
......@@ -707,11 +706,11 @@ void timekeeping_inject_sleeptime(struct timespec *delta)
write_seqlock_irqsave(&timekeeper.lock, flags);
timekeeping_forward_now();
timekeeping_forward_now(&timekeeper);
__timekeeping_inject_sleeptime(delta);
__timekeeping_inject_sleeptime(&timekeeper, delta);
timekeeping_update(true);
timekeeping_update(&timekeeper, true);
write_sequnlock_irqrestore(&timekeeper.lock, flags);
......@@ -740,7 +739,7 @@ static void timekeeping_resume(void)
if (timespec_compare(&ts, &timekeeping_suspend_time) > 0) {
ts = timespec_sub(ts, timekeeping_suspend_time);
__timekeeping_inject_sleeptime(&ts);
__timekeeping_inject_sleeptime(&timekeeper, &ts);
}
/* re-base the last cycle value */
timekeeper.clock->cycle_last = timekeeper.clock->read(timekeeper.clock);
......@@ -765,7 +764,7 @@ static int timekeeping_suspend(void)
read_persistent_clock(&timekeeping_suspend_time);
write_seqlock_irqsave(&timekeeper.lock, flags);
timekeeping_forward_now();
timekeeping_forward_now(&timekeeper);
timekeeping_suspended = 1;
/*
......@@ -813,7 +812,8 @@ device_initcall(timekeeping_init_ops);
* If the error is already larger, we look ahead even further
* to compensate for late or lost adjustments.
*/
static __always_inline int timekeeping_bigadjust(s64 error, s64 *interval,
static __always_inline int timekeeping_bigadjust(struct timekeeper *tk,
s64 error, s64 *interval,
s64 *offset)
{
s64 tick_error, i;
......@@ -829,7 +829,7 @@ static __always_inline int timekeeping_bigadjust(s64 error, s64 *interval,
* here. This is tuned so that an error of about 1 msec is adjusted
* within about 1 sec (or 2^20 nsec in 2^SHIFT_HZ ticks).
*/
error2 = timekeeper.ntp_error >> (NTP_SCALE_SHIFT + 22 - 2 * SHIFT_HZ);
error2 = tk->ntp_error >> (NTP_SCALE_SHIFT + 22 - 2 * SHIFT_HZ);
error2 = abs(error2);
for (look_ahead = 0; error2 > 0; look_ahead++)
error2 >>= 2;
......@@ -838,8 +838,8 @@ static __always_inline int timekeeping_bigadjust(s64 error, s64 *interval,
* Now calculate the error in (1 << look_ahead) ticks, but first
* remove the single look ahead already included in the error.
*/
tick_error = ntp_tick_length() >> (timekeeper.ntp_error_shift + 1);
tick_error -= timekeeper.xtime_interval >> 1;
tick_error = ntp_tick_length() >> (tk->ntp_error_shift + 1);
tick_error -= tk->xtime_interval >> 1;
error = ((error - tick_error) >> look_ahead) + tick_error;
/* Finally calculate the adjustment shift value. */
......@@ -864,9 +864,9 @@ static __always_inline int timekeeping_bigadjust(s64 error, s64 *interval,
* this is optimized for the most common adjustments of -1,0,1,
* for other values we can do a bit more work.
*/
static void timekeeping_adjust(s64 offset)
static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
{
s64 error, interval = timekeeper.cycle_interval;
s64 error, interval = tk->cycle_interval;
int adj;
/*
......@@ -882,7 +882,7 @@ static void timekeeping_adjust(s64 offset)
*
* Note: It does not "save" on aggravation when reading the code.
*/
error = timekeeper.ntp_error >> (timekeeper.ntp_error_shift - 1);
error = tk->ntp_error >> (tk->ntp_error_shift - 1);
if (error > interval) {
/*
* We now divide error by 4(via shift), which checks if
......@@ -904,7 +904,8 @@ static void timekeeping_adjust(s64 offset)
if (likely(error <= interval))
adj = 1;
else
adj = timekeeping_bigadjust(error, &interval, &offset);
adj = timekeeping_bigadjust(tk, error, &interval,
&offset);
} else if (error < -interval) {
/* See comment above, this is just switched for the negative */
error >>= 2;
......@@ -913,18 +914,17 @@ static void timekeeping_adjust(s64 offset)
interval = -interval;
offset = -offset;
} else
adj = timekeeping_bigadjust(error, &interval, &offset);
} else /* No adjustment needed */
adj = timekeeping_bigadjust(tk, error, &interval,
&offset);
} else
return;
if (unlikely(timekeeper.clock->maxadj &&
(timekeeper.mult + adj >
timekeeper.clock->mult + timekeeper.clock->maxadj))) {
if (unlikely(tk->clock->maxadj &&
(tk->mult + adj > tk->clock->mult + tk->clock->maxadj))) {
printk_once(KERN_WARNING
"Adjusting %s more than 11%% (%ld vs %ld)\n",
timekeeper.clock->name, (long)timekeeper.mult + adj,
(long)timekeeper.clock->mult +
timekeeper.clock->maxadj);
tk->clock->name, (long)tk->mult + adj,
(long)tk->clock->mult + tk->clock->maxadj);
}
/*
* So the following can be confusing.
......@@ -975,11 +975,10 @@ static void timekeeping_adjust(s64 offset)
*
* XXX - TODO: Doc ntp_error calculation.
*/
timekeeper.mult += adj;
timekeeper.xtime_interval += interval;
timekeeper.xtime_nsec -= offset;
timekeeper.ntp_error -= (interval - offset) <<
timekeeper.ntp_error_shift;
tk->mult += adj;
tk->xtime_interval += interval;
tk->xtime_nsec -= offset;
tk->ntp_error -= (interval - offset) << tk->ntp_error_shift;
/*
* It may be possible that when we entered this function, xtime_nsec
......@@ -995,10 +994,10 @@ static void timekeeping_adjust(s64 offset)
* We'll correct this error next time through this function, when
* xtime_nsec is not as small.
*/
if (unlikely((s64)timekeeper.xtime_nsec < 0)) {
s64 neg = -(s64)timekeeper.xtime_nsec;
timekeeper.xtime_nsec = 0;
timekeeper.ntp_error += neg << timekeeper.ntp_error_shift;
if (unlikely((s64)tk->xtime_nsec < 0)) {
s64 neg = -(s64)tk->xtime_nsec;
tk->xtime_nsec = 0;
tk->ntp_error += neg << tk->ntp_error_shift;
}
}
......@@ -1042,37 +1041,36 @@ static inline void accumulate_nsecs_to_secs(struct timekeeper *tk)
*
* Returns the unconsumed cycles.
*/
static cycle_t logarithmic_accumulation(cycle_t offset, u32 shift)
static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset,
u32 shift)
{
u64 raw_nsecs;
/* If the offset is smaller than a shifted interval, do nothing */
if (offset < timekeeper.cycle_interval<<shift)
/* If the offset is smaller then a shifted interval, do nothing */
if (offset < tk->cycle_interval<<shift)
return offset;
/* Accumulate one shifted interval */
offset -= timekeeper.cycle_interval << shift;
timekeeper.clock->cycle_last += timekeeper.cycle_interval << shift;
offset -= tk->cycle_interval << shift;
tk->clock->cycle_last += tk->cycle_interval << shift;
timekeeper.xtime_nsec += timekeeper.xtime_interval << shift;
accumulate_nsecs_to_secs(&timekeeper);
tk->xtime_nsec += tk->xtime_interval << shift;
accumulate_nsecs_to_secs(tk);
/* Accumulate raw time */
raw_nsecs = timekeeper.raw_interval << shift;
raw_nsecs += timekeeper.raw_time.tv_nsec;
raw_nsecs = tk->raw_interval << shift;
raw_nsecs += tk->raw_time.tv_nsec;
if (raw_nsecs >= NSEC_PER_SEC) {
u64 raw_secs = raw_nsecs;
raw_nsecs = do_div(raw_secs, NSEC_PER_SEC);
timekeeper.raw_time.tv_sec += raw_secs;
tk->raw_time.tv_sec += raw_secs;
}
timekeeper.raw_time.tv_nsec = raw_nsecs;
tk->raw_time.tv_nsec = raw_nsecs;
/* Accumulate error between NTP and clock interval */
timekeeper.ntp_error += ntp_tick_length() << shift;
timekeeper.ntp_error -=
(timekeeper.xtime_interval + timekeeper.xtime_remainder) <<
(timekeeper.ntp_error_shift + shift);
tk->ntp_error += ntp_tick_length() << shift;
tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) <<
(tk->ntp_error_shift + shift);
return offset;
}
......@@ -1118,13 +1116,13 @@ static void update_wall_time(void)
maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1;
shift = min(shift, maxshift);
while (offset >= timekeeper.cycle_interval) {
offset = logarithmic_accumulation(offset, shift);
offset = logarithmic_accumulation(&timekeeper, offset, shift);
if(offset < timekeeper.cycle_interval<<shift)
shift--;
}
/* correct the clock when NTP error is too big */
timekeeping_adjust(offset);
timekeeping_adjust(&timekeeper, offset);
/*
......@@ -1147,7 +1145,7 @@ static void update_wall_time(void)
*/
accumulate_nsecs_to_secs(&timekeeper);
timekeeping_update(false);
timekeeping_update(&timekeeper, false);
out:
write_sequnlock_irqrestore(&timekeeper.lock, flags);
......@@ -1198,7 +1196,7 @@ void get_monotonic_boottime(struct timespec *ts)
do {
seq = read_seqbegin(&timekeeper.lock);
ts->tv_sec = timekeeper.xtime_sec;
ts->tv_nsec = timekeeping_get_ns();
ts->tv_nsec = timekeeping_get_ns(&timekeeper);
tomono = timekeeper.wall_to_monotonic;
sleep = timekeeper.total_sleep_time;
......@@ -1330,7 +1328,7 @@ ktime_t ktime_get_update_offsets(ktime_t *offs_real, ktime_t *offs_boot)
seq = read_seqbegin(&timekeeper.lock);
secs = timekeeper.xtime_sec;
nsecs = timekeeping_get_ns();
nsecs = timekeeping_get_ns(&timekeeper);
*offs_real = timekeeper.offs_real;
*offs_boot = timekeeper.offs_boot;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7