Commit 2eebdde6 authored by Richard Cochran's avatar Richard Cochran Committed by David S. Miller

timecounter: keep track of accumulated fractional nanoseconds

The current timecounter implementation will drop a variable amount
of resolution, depending on the magnitude of the time delta. In
other words, reading the clock too often or too close to a time
stamp conversion will introduce errors into the time values. This
patch fixes the issue by introducing a fractional nanosecond field
that accumulates the low order bits.
Reported-by: default avatarJanusz Użycki <j.uzycki@elproma.com.pl>
Signed-off-by: default avatarRichard Cochran <richardcochran@gmail.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent f25a30be
...@@ -240,7 +240,7 @@ void mlx4_en_init_timestamp(struct mlx4_en_dev *mdev) ...@@ -240,7 +240,7 @@ void mlx4_en_init_timestamp(struct mlx4_en_dev *mdev)
{ {
struct mlx4_dev *dev = mdev->dev; struct mlx4_dev *dev = mdev->dev;
unsigned long flags; unsigned long flags;
u64 ns; u64 ns, zero = 0;
rwlock_init(&mdev->clock_lock); rwlock_init(&mdev->clock_lock);
...@@ -265,7 +265,7 @@ void mlx4_en_init_timestamp(struct mlx4_en_dev *mdev) ...@@ -265,7 +265,7 @@ void mlx4_en_init_timestamp(struct mlx4_en_dev *mdev)
/* Calculate period in seconds to call the overflow watchdog - to make /* Calculate period in seconds to call the overflow watchdog - to make
* sure counter is checked at least once every wrap around. * sure counter is checked at least once every wrap around.
*/ */
ns = cyclecounter_cyc2ns(&mdev->cycles, mdev->cycles.mask); ns = cyclecounter_cyc2ns(&mdev->cycles, mdev->cycles.mask, zero, &zero);
do_div(ns, NSEC_PER_SEC / 2 / HZ); do_div(ns, NSEC_PER_SEC / 2 / HZ);
mdev->overflow_period = ns; mdev->overflow_period = ns;
......
...@@ -55,27 +55,32 @@ struct cyclecounter { ...@@ -55,27 +55,32 @@ struct cyclecounter {
* @cycle_last: most recent cycle counter value seen by * @cycle_last: most recent cycle counter value seen by
* timecounter_read() * timecounter_read()
* @nsec: continuously increasing count * @nsec: continuously increasing count
* @mask: bit mask for maintaining the 'frac' field
* @frac: accumulated fractional nanoseconds
*/ */
struct timecounter { struct timecounter {
const struct cyclecounter *cc; const struct cyclecounter *cc;
cycle_t cycle_last; cycle_t cycle_last;
u64 nsec; u64 nsec;
u64 mask;
u64 frac;
}; };
/** /**
* cyclecounter_cyc2ns - converts cycle counter cycles to nanoseconds * cyclecounter_cyc2ns - converts cycle counter cycles to nanoseconds
* @cc: Pointer to cycle counter. * @cc: Pointer to cycle counter.
* @cycles: Cycles * @cycles: Cycles
* * @mask: bit mask for maintaining the 'frac' field
* XXX - This could use some mult_lxl_ll() asm optimization. Same code * @frac: pointer to storage for the fractional nanoseconds.
* as in cyc2ns, but with unsigned result.
*/ */
static inline u64 cyclecounter_cyc2ns(const struct cyclecounter *cc, static inline u64 cyclecounter_cyc2ns(const struct cyclecounter *cc,
cycle_t cycles) cycle_t cycles, u64 mask, u64 *frac)
{ {
u64 ret = (u64)cycles; u64 ns = (u64) cycles;
ret = (ret * cc->mult) >> cc->shift;
return ret; ns = (ns * cc->mult) + *frac;
*frac = ns & mask;
return ns >> cc->shift;
} }
/** /**
......
...@@ -25,6 +25,8 @@ void timecounter_init(struct timecounter *tc, ...@@ -25,6 +25,8 @@ void timecounter_init(struct timecounter *tc,
tc->cc = cc; tc->cc = cc;
tc->cycle_last = cc->read(cc); tc->cycle_last = cc->read(cc);
tc->nsec = start_tstamp; tc->nsec = start_tstamp;
tc->mask = (1ULL << cc->shift) - 1;
tc->frac = 0;
} }
EXPORT_SYMBOL_GPL(timecounter_init); EXPORT_SYMBOL_GPL(timecounter_init);
...@@ -51,7 +53,8 @@ static u64 timecounter_read_delta(struct timecounter *tc) ...@@ -51,7 +53,8 @@ static u64 timecounter_read_delta(struct timecounter *tc)
cycle_delta = (cycle_now - tc->cycle_last) & tc->cc->mask; cycle_delta = (cycle_now - tc->cycle_last) & tc->cc->mask;
/* convert to nanoseconds: */ /* convert to nanoseconds: */
ns_offset = cyclecounter_cyc2ns(tc->cc, cycle_delta); ns_offset = cyclecounter_cyc2ns(tc->cc, cycle_delta,
tc->mask, &tc->frac);
/* update time stamp of timecounter_read_delta() call: */ /* update time stamp of timecounter_read_delta() call: */
tc->cycle_last = cycle_now; tc->cycle_last = cycle_now;
...@@ -72,22 +75,36 @@ u64 timecounter_read(struct timecounter *tc) ...@@ -72,22 +75,36 @@ u64 timecounter_read(struct timecounter *tc)
} }
EXPORT_SYMBOL_GPL(timecounter_read); EXPORT_SYMBOL_GPL(timecounter_read);
/*
* This is like cyclecounter_cyc2ns(), but it is used for computing a
* time previous to the time stored in the cycle counter.
*/
static u64 cc_cyc2ns_backwards(const struct cyclecounter *cc,
cycle_t cycles, u64 mask, u64 frac)
{
u64 ns = (u64) cycles;
ns = ((ns * cc->mult) - frac) >> cc->shift;
return ns;
}
u64 timecounter_cyc2time(struct timecounter *tc, u64 timecounter_cyc2time(struct timecounter *tc,
cycle_t cycle_tstamp) cycle_t cycle_tstamp)
{ {
u64 cycle_delta = (cycle_tstamp - tc->cycle_last) & tc->cc->mask; u64 delta = (cycle_tstamp - tc->cycle_last) & tc->cc->mask;
u64 nsec; u64 nsec = tc->nsec, frac = tc->frac;
/* /*
* Instead of always treating cycle_tstamp as more recent * Instead of always treating cycle_tstamp as more recent
* than tc->cycle_last, detect when it is too far in the * than tc->cycle_last, detect when it is too far in the
* future and treat it as old time stamp instead. * future and treat it as old time stamp instead.
*/ */
if (cycle_delta > tc->cc->mask / 2) { if (delta > tc->cc->mask / 2) {
cycle_delta = (tc->cycle_last - cycle_tstamp) & tc->cc->mask; delta = (tc->cycle_last - cycle_tstamp) & tc->cc->mask;
nsec = tc->nsec - cyclecounter_cyc2ns(tc->cc, cycle_delta); nsec -= cc_cyc2ns_backwards(tc->cc, delta, tc->mask, frac);
} else { } else {
nsec = cyclecounter_cyc2ns(tc->cc, cycle_delta) + tc->nsec; nsec += cyclecounter_cyc2ns(tc->cc, delta, tc->mask, &frac);
} }
return nsec; return nsec;
......
...@@ -152,7 +152,8 @@ void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu) ...@@ -152,7 +152,8 @@ void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu)
return; return;
} }
ns = cyclecounter_cyc2ns(timecounter->cc, cval - now); ns = cyclecounter_cyc2ns(timecounter->cc, cval - now, timecounter->mask,
&timecounter->frac);
timer_arm(timer, ns); timer_arm(timer, ns);
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment