Commit 3aed64f6 authored by Paolo Bonzini's avatar Paolo Bonzini

pvclock: introduce seqcount-like API

The version field in struct pvclock_vcpu_time_info basically implements
a seqcount.  Wrap it with the usual read_begin and read_retry functions,
and use these APIs instead of peppering the code with smp_rmb()s.
While at it, change it to the more pedantically correct virt_rmb().

With this change, __pvclock_read_cycles can be simplified noticeably.
Signed-off-by: default avatarPaolo Bonzini <pbonzini@redhat.com>
parent 96b58526
...@@ -123,9 +123,7 @@ static notrace cycle_t vread_pvclock(int *mode) ...@@ -123,9 +123,7 @@ static notrace cycle_t vread_pvclock(int *mode)
*/ */
do { do {
version = pvti->version; version = pvclock_read_begin(pvti);
smp_rmb();
if (unlikely(!(pvti->flags & PVCLOCK_TSC_STABLE_BIT))) { if (unlikely(!(pvti->flags & PVCLOCK_TSC_STABLE_BIT))) {
*mode = VCLOCK_NONE; *mode = VCLOCK_NONE;
...@@ -137,10 +135,7 @@ static notrace cycle_t vread_pvclock(int *mode) ...@@ -137,10 +135,7 @@ static notrace cycle_t vread_pvclock(int *mode)
pvti_tsc_shift = pvti->tsc_shift; pvti_tsc_shift = pvti->tsc_shift;
pvti_system_time = pvti->system_time; pvti_system_time = pvti->system_time;
pvti_tsc = pvti->tsc_timestamp; pvti_tsc = pvti->tsc_timestamp;
} while (pvclock_read_retry(pvti, version));
/* Make sure that the version double-check is last. */
smp_rmb();
} while (unlikely((version & 1) || version != pvti->version));
delta = tsc - pvti_tsc; delta = tsc - pvti_tsc;
ret = pvti_system_time + ret = pvti_system_time +
......
...@@ -25,6 +25,24 @@ void pvclock_resume(void); ...@@ -25,6 +25,24 @@ void pvclock_resume(void);
void pvclock_touch_watchdogs(void); void pvclock_touch_watchdogs(void);
static __always_inline
unsigned pvclock_read_begin(const struct pvclock_vcpu_time_info *src)
{
unsigned version = src->version & ~1;
/* Make sure that the version is read before the data. */
virt_rmb();
return version;
}
static __always_inline
bool pvclock_read_retry(const struct pvclock_vcpu_time_info *src,
unsigned version)
{
/* Make sure that the version is re-read after the data. */
virt_rmb();
return unlikely(version != src->version);
}
/* /*
* Scale a 64-bit delta by scaling and multiplying by a 32-bit fraction, * Scale a 64-bit delta by scaling and multiplying by a 32-bit fraction,
* yielding a 64-bit result. * yielding a 64-bit result.
...@@ -69,23 +87,12 @@ static inline u64 pvclock_scale_delta(u64 delta, u32 mul_frac, int shift) ...@@ -69,23 +87,12 @@ static inline u64 pvclock_scale_delta(u64 delta, u32 mul_frac, int shift)
} }
static __always_inline static __always_inline
unsigned __pvclock_read_cycles(const struct pvclock_vcpu_time_info *src, cycle_t __pvclock_read_cycles(const struct pvclock_vcpu_time_info *src)
cycle_t *cycles, u8 *flags)
{ {
unsigned version; u64 delta = rdtsc_ordered() - src->tsc_timestamp;
cycle_t offset; cycle_t offset = pvclock_scale_delta(delta, src->tsc_to_system_mul,
u64 delta;
version = src->version;
/* Make the latest version visible */
smp_rmb();
delta = rdtsc_ordered() - src->tsc_timestamp;
offset = pvclock_scale_delta(delta, src->tsc_to_system_mul,
src->tsc_shift); src->tsc_shift);
*cycles = src->system_time + offset; return src->system_time + offset;
*flags = src->flags;
return version;
} }
struct pvclock_vsyscall_time_info { struct pvclock_vsyscall_time_info {
......
...@@ -64,14 +64,9 @@ u8 pvclock_read_flags(struct pvclock_vcpu_time_info *src) ...@@ -64,14 +64,9 @@ u8 pvclock_read_flags(struct pvclock_vcpu_time_info *src)
u8 flags; u8 flags;
do { do {
version = src->version; version = pvclock_read_begin(src);
/* Make the latest version visible */
smp_rmb();
flags = src->flags; flags = src->flags;
/* Make sure that the version double-check is last. */ } while (pvclock_read_retry(src, version));
smp_rmb();
} while ((src->version & 1) || version != src->version);
return flags & valid_flags; return flags & valid_flags;
} }
...@@ -84,10 +79,10 @@ cycle_t pvclock_clocksource_read(struct pvclock_vcpu_time_info *src) ...@@ -84,10 +79,10 @@ cycle_t pvclock_clocksource_read(struct pvclock_vcpu_time_info *src)
u8 flags; u8 flags;
do { do {
version = __pvclock_read_cycles(src, &ret, &flags); version = pvclock_read_begin(src);
/* Make sure that the version double-check is last. */ ret = __pvclock_read_cycles(src);
smp_rmb(); flags = src->flags;
} while ((src->version & 1) || version != src->version); } while (pvclock_read_retry(src, version));
if (unlikely((flags & PVCLOCK_GUEST_STOPPED) != 0)) { if (unlikely((flags & PVCLOCK_GUEST_STOPPED) != 0)) {
src->flags &= ~PVCLOCK_GUEST_STOPPED; src->flags &= ~PVCLOCK_GUEST_STOPPED;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment