Commit 18ad51dd authored by Anton Blanchard's avatar Anton Blanchard Committed by Benjamin Herrenschmidt

powerpc: Add VDSO version of getcpu

We have a request for a fast method of getting CPU and NUMA node IDs
from userspace. This patch implements a getcpu VDSO function,
similar to x86.

Ben suggested we use SPRG3 which is userspace readable. SPRG3 can be
modified by a KVM guest, so we save the SPRG3 value in the paca and
restore it when transitioning from the guest to the host.

I have a glibc patch that implements sched_getcpu on top of this.
Testing on a POWER7:

baseline: 538 cycles
vdso:      30 cycles
Signed-off-by: default avatarAnton Blanchard <anton@samba.org>
Signed-off-by: default avatarBenjamin Herrenschmidt <benh@kernel.crashing.org>
parent e6a74c6e
...@@ -74,6 +74,7 @@ struct kvmppc_host_state { ...@@ -74,6 +74,7 @@ struct kvmppc_host_state {
ulong vmhandler; ulong vmhandler;
ulong scratch0; ulong scratch0;
ulong scratch1; ulong scratch1;
ulong sprg3;
u8 in_guest; u8 in_guest;
u8 restore_hid5; u8 restore_hid5;
u8 napping; u8 napping;
......
...@@ -491,6 +491,7 @@ ...@@ -491,6 +491,7 @@
#define SPRN_SPRG1 0x111 /* Special Purpose Register General 1 */ #define SPRN_SPRG1 0x111 /* Special Purpose Register General 1 */
#define SPRN_SPRG2 0x112 /* Special Purpose Register General 2 */ #define SPRN_SPRG2 0x112 /* Special Purpose Register General 2 */
#define SPRN_SPRG3 0x113 /* Special Purpose Register General 3 */ #define SPRN_SPRG3 0x113 /* Special Purpose Register General 3 */
#define SPRN_USPRG3 0x103 /* SPRG3 userspace read */
#define SPRN_SPRG4 0x114 /* Special Purpose Register General 4 */ #define SPRN_SPRG4 0x114 /* Special Purpose Register General 4 */
#define SPRN_SPRG5 0x115 /* Special Purpose Register General 5 */ #define SPRN_SPRG5 0x115 /* Special Purpose Register General 5 */
#define SPRN_SPRG6 0x116 /* Special Purpose Register General 6 */ #define SPRN_SPRG6 0x116 /* Special Purpose Register General 6 */
...@@ -753,14 +754,14 @@ ...@@ -753,14 +754,14 @@
* 64-bit server: * 64-bit server:
* - SPRG0 unused (reserved for HV on Power4) * - SPRG0 unused (reserved for HV on Power4)
* - SPRG2 scratch for exception vectors * - SPRG2 scratch for exception vectors
* - SPRG3 unused (user visible) * - SPRG3 CPU and NUMA node for VDSO getcpu (user visible)
* - HSPRG0 stores PACA in HV mode * - HSPRG0 stores PACA in HV mode
* - HSPRG1 scratch for "HV" exceptions * - HSPRG1 scratch for "HV" exceptions
* *
* 64-bit embedded * 64-bit embedded
* - SPRG0 generic exception scratch * - SPRG0 generic exception scratch
* - SPRG2 TLB exception stack * - SPRG2 TLB exception stack
* - SPRG3 unused (user visible) * - SPRG3 CPU and NUMA node for VDSO getcpu (user visible)
* - SPRG4 unused (user visible) * - SPRG4 unused (user visible)
* - SPRG6 TLB miss scratch (user visible, sorry !) * - SPRG6 TLB miss scratch (user visible, sorry !)
* - SPRG7 critical exception scratch * - SPRG7 critical exception scratch
......
...@@ -22,6 +22,8 @@ extern unsigned long vdso64_rt_sigtramp; ...@@ -22,6 +22,8 @@ extern unsigned long vdso64_rt_sigtramp;
extern unsigned long vdso32_sigtramp; extern unsigned long vdso32_sigtramp;
extern unsigned long vdso32_rt_sigtramp; extern unsigned long vdso32_rt_sigtramp;
int __cpuinit vdso_getcpu_init(void);
#else /* __ASSEMBLY__ */ #else /* __ASSEMBLY__ */
#ifdef __VDSO64__ #ifdef __VDSO64__
......
...@@ -533,6 +533,7 @@ int main(void) ...@@ -533,6 +533,7 @@ int main(void)
HSTATE_FIELD(HSTATE_VMHANDLER, vmhandler); HSTATE_FIELD(HSTATE_VMHANDLER, vmhandler);
HSTATE_FIELD(HSTATE_SCRATCH0, scratch0); HSTATE_FIELD(HSTATE_SCRATCH0, scratch0);
HSTATE_FIELD(HSTATE_SCRATCH1, scratch1); HSTATE_FIELD(HSTATE_SCRATCH1, scratch1);
HSTATE_FIELD(HSTATE_SPRG3, sprg3);
HSTATE_FIELD(HSTATE_IN_GUEST, in_guest); HSTATE_FIELD(HSTATE_IN_GUEST, in_guest);
HSTATE_FIELD(HSTATE_RESTORE_HID5, restore_hid5); HSTATE_FIELD(HSTATE_RESTORE_HID5, restore_hid5);
HSTATE_FIELD(HSTATE_NAPPING, napping); HSTATE_FIELD(HSTATE_NAPPING, napping);
......
...@@ -48,6 +48,7 @@ ...@@ -48,6 +48,7 @@
#ifdef CONFIG_PPC64 #ifdef CONFIG_PPC64
#include <asm/paca.h> #include <asm/paca.h>
#endif #endif
#include <asm/vdso.h>
#include <asm/debug.h> #include <asm/debug.h>
#ifdef DEBUG #ifdef DEBUG
...@@ -570,6 +571,8 @@ void __devinit start_secondary(void *unused) ...@@ -570,6 +571,8 @@ void __devinit start_secondary(void *unused)
#ifdef CONFIG_PPC64 #ifdef CONFIG_PPC64
if (system_state == SYSTEM_RUNNING) if (system_state == SYSTEM_RUNNING)
vdso_data->processorCount++; vdso_data->processorCount++;
vdso_getcpu_init();
#endif #endif
notify_cpu_starting(cpu); notify_cpu_starting(cpu);
set_cpu_online(cpu, true); set_cpu_online(cpu, true);
......
...@@ -706,6 +706,34 @@ static void __init vdso_setup_syscall_map(void) ...@@ -706,6 +706,34 @@ static void __init vdso_setup_syscall_map(void)
} }
} }
#ifdef CONFIG_PPC64
int __cpuinit vdso_getcpu_init(void)
{
unsigned long cpu, node, val;
/*
* SPRG3 contains the CPU in the bottom 16 bits and the NUMA node in
* the next 16 bits. The VDSO uses this to implement getcpu().
*/
cpu = get_cpu();
WARN_ON_ONCE(cpu > 0xffff);
node = cpu_to_node(cpu);
WARN_ON_ONCE(node > 0xffff);
val = (cpu & 0xfff) | ((node & 0xffff) << 16);
mtspr(SPRN_SPRG3, val);
#ifdef CONFIG_KVM_BOOK3S_HANDLER
get_paca()->kvm_hstate.sprg3 = val;
#endif
put_cpu();
return 0;
}
/* We need to call this before SMP init */
early_initcall(vdso_getcpu_init);
#endif
static int __init vdso_init(void) static int __init vdso_init(void)
{ {
......
# List of files in the vdso, has to be asm only for now # List of files in the vdso, has to be asm only for now
obj-vdso32 = sigtramp.o gettimeofday.o datapage.o cacheflush.o note.o obj-vdso32-$(CONFIG_PPC64) = getcpu.o
obj-vdso32 = sigtramp.o gettimeofday.o datapage.o cacheflush.o note.o \
$(obj-vdso32-y)
# Build rules # Build rules
......
/*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*
* Copyright (C) IBM Corporation, 2012
*
* Author: Anton Blanchard <anton@au.ibm.com>
*/
#include <asm/ppc_asm.h>
#include <asm/vdso.h>
.text
/*
* Exact prototype of getcpu
*
* int __kernel_getcpu(unsigned *cpu, unsigned *node);
*
*/
V_FUNCTION_BEGIN(__kernel_getcpu)
.cfi_startproc
mfspr r5,SPRN_USPRG3
cmpdi cr0,r3,0
cmpdi cr1,r4,0
clrlwi r6,r5,16
rlwinm r7,r5,16,31-15,31-0
beq cr0,1f
stw r6,0(r3)
1: beq cr1,2f
stw r7,0(r4)
2: crclr cr0*4+so
li r3,0 /* always success */
blr
.cfi_endproc
V_FUNCTION_END(__kernel_getcpu)
...@@ -147,6 +147,9 @@ VERSION ...@@ -147,6 +147,9 @@ VERSION
__kernel_sync_dicache_p5; __kernel_sync_dicache_p5;
__kernel_sigtramp32; __kernel_sigtramp32;
__kernel_sigtramp_rt32; __kernel_sigtramp_rt32;
#ifdef CONFIG_PPC64
__kernel_getcpu;
#endif
local: *; local: *;
}; };
......
# List of files in the vdso, has to be asm only for now # List of files in the vdso, has to be asm only for now
obj-vdso64 = sigtramp.o gettimeofday.o datapage.o cacheflush.o note.o obj-vdso64 = sigtramp.o gettimeofday.o datapage.o cacheflush.o note.o getcpu.o
# Build rules # Build rules
......
/*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*
* Copyright (C) IBM Corporation, 2012
*
* Author: Anton Blanchard <anton@au.ibm.com>
*/
#include <asm/ppc_asm.h>
#include <asm/vdso.h>
.text
/*
* Exact prototype of getcpu
*
* int __kernel_getcpu(unsigned *cpu, unsigned *node);
*
*/
V_FUNCTION_BEGIN(__kernel_getcpu)
.cfi_startproc
mfspr r5,SPRN_USPRG3
cmpdi cr0,r3,0
cmpdi cr1,r4,0
clrlwi r6,r5,16
rlwinm r7,r5,16,31-15,31-0
beq cr0,1f
stw r6,0(r3)
1: beq cr1,2f
stw r7,0(r4)
2: crclr cr0*4+so
li r3,0 /* always success */
blr
.cfi_endproc
V_FUNCTION_END(__kernel_getcpu)
...@@ -146,6 +146,7 @@ VERSION ...@@ -146,6 +146,7 @@ VERSION
__kernel_sync_dicache; __kernel_sync_dicache;
__kernel_sync_dicache_p5; __kernel_sync_dicache_p5;
__kernel_sigtramp_rt64; __kernel_sigtramp_rt64;
__kernel_getcpu;
local: *; local: *;
}; };
......
...@@ -1064,6 +1064,10 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206) ...@@ -1064,6 +1064,10 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
mtspr SPRN_DABR,r5 mtspr SPRN_DABR,r5
mtspr SPRN_DABRX,r6 mtspr SPRN_DABRX,r6
/* Restore SPRG3 */
ld r3,HSTATE_SPRG3(r13)
mtspr SPRN_SPRG3,r3
/* /*
* Reload DEC. HDEC interrupts were disabled when * Reload DEC. HDEC interrupts were disabled when
* we reloaded the host's LPCR value. * we reloaded the host's LPCR value.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment