smpboot.c 36.7 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35
/*
 *	x86 SMP booting functions
 *
 *	(c) 1995 Alan Cox, Building #3 <alan@redhat.com>
 *	(c) 1998, 1999, 2000 Ingo Molnar <mingo@redhat.com>
 *
 *	Much of the core SMP work is based on previous work by Thomas Radke, to
 *	whom a great many thanks are extended.
 *
 *	Thanks to Intel for making available several different Pentium,
 *	Pentium Pro and Pentium-II/Xeon MP machines.
 *	Original development of Linux SMP code supported by Caldera.
 *
 *	This code is released under the GNU General Public License version 2 or
 *	later.
 *
 *	Fixes
 *		Felix Koop	:	NR_CPUS used properly
 *		Jose Renau	:	Handle single CPU case.
 *		Alan Cox	:	By repeated request 8) - Total BogoMIPS report.
 *		Greg Wright	:	Fix for kernel stacks panic.
 *		Erich Boleyn	:	MP v1.4 and additional changes.
 *	Matthias Sattler	:	Changes for 2.1 kernel map.
 *	Michel Lespinasse	:	Changes for 2.1 kernel map.
 *	Michael Chastain	:	Change trampoline.S to gnu as.
 *		Alan Cox	:	Dumb bug: 'B' step PPro's are fine
 *		Ingo Molnar	:	Added APIC timers, based on code
 *					from Jose Renau
 *		Ingo Molnar	:	various cleanups and rewrites
 *		Tigran Aivazian	:	fixed "0.00 in /proc/uptime on SMP" bug.
 *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs
 *		Martin J. Bligh	: 	Added support for multi-quad systems
 *		Dave Jones	:	Report invalid combinations of Athlon CPUs.
*		Rusty Russell	:	Hacked into shape for new "hotplug" boot process. */

36 37 38 39 40

/* SMP boot always wants to use real time delay to allow sufficient time for
 * the APs to come online */
#define USE_REAL_TIME_DELAY

Linus Torvalds's avatar
Linus Torvalds committed
41 42 43 44 45 46 47 48 49
#include <linux/module.h>
#include <linux/init.h>
#include <linux/kernel.h>

#include <linux/mm.h>
#include <linux/sched.h>
#include <linux/kernel_stat.h>
#include <linux/smp_lock.h>
#include <linux/bootmem.h>
Zwane Mwaikambo's avatar
Zwane Mwaikambo committed
50 51 52
#include <linux/notifier.h>
#include <linux/cpu.h>
#include <linux/percpu.h>
Linus Torvalds's avatar
Linus Torvalds committed
53 54 55 56 57 58

#include <linux/delay.h>
#include <linux/mc146818rtc.h>
#include <asm/tlbflush.h>
#include <asm/desc.h>
#include <asm/arch_hooks.h>
59
#include <asm/nmi.h>
60
#include <asm/pda.h>
61
#include <asm/genapic.h>
Linus Torvalds's avatar
Linus Torvalds committed
62 63 64 65 66 67

#include <mach_apic.h>
#include <mach_wakecpu.h>
#include <smpboot_hooks.h>

/* Set if we find a B stepping CPU */
Li Shaohua's avatar
Li Shaohua committed
68
static int __devinitdata smp_b_stepping;
Linus Torvalds's avatar
Linus Torvalds committed
69 70 71

/* Number of siblings per CPU package */
int smp_num_siblings = 1;
72
EXPORT_SYMBOL(smp_num_siblings);
73

74 75 76
/* Last level cache ID of each logical CPU */
int cpu_llc_id[NR_CPUS] __cpuinitdata = {[0 ... NR_CPUS-1] = BAD_APICID};

77
/* representing HT siblings of each logical CPU */
78
cpumask_t cpu_sibling_map[NR_CPUS] __read_mostly;
79 80
EXPORT_SYMBOL(cpu_sibling_map);

81
/* representing HT and core siblings of each logical CPU */
82
cpumask_t cpu_core_map[NR_CPUS] __read_mostly;
83 84
EXPORT_SYMBOL(cpu_core_map);

Linus Torvalds's avatar
Linus Torvalds committed
85
/* bitmap of online cpus */
86
cpumask_t cpu_online_map __read_mostly;
87
EXPORT_SYMBOL(cpu_online_map);
Linus Torvalds's avatar
Linus Torvalds committed
88 89 90

cpumask_t cpu_callin_map;
cpumask_t cpu_callout_map;
91
EXPORT_SYMBOL(cpu_callout_map);
92 93
cpumask_t cpu_possible_map;
EXPORT_SYMBOL(cpu_possible_map);
Linus Torvalds's avatar
Linus Torvalds committed
94 95
static cpumask_t smp_commenced_mask;

96 97 98 99 100 101
/* TSC's upper 32 bits can't be written in eariler CPU (before prescott), there
 * is no way to resync one AP against BP. TBD: for prescott and above, we
 * should use IA64's algorithm
 */
static int __devinitdata tsc_sync_disabled;

Linus Torvalds's avatar
Linus Torvalds committed
102 103
/* Per CPU bogomips and other parameters */
struct cpuinfo_x86 cpu_data[NR_CPUS] __cacheline_aligned;
104
EXPORT_SYMBOL(cpu_data);
Linus Torvalds's avatar
Linus Torvalds committed
105

106
u8 x86_cpu_to_apicid[NR_CPUS] __read_mostly =
Linus Torvalds's avatar
Linus Torvalds committed
107 108 109
			{ [0 ... NR_CPUS-1] = 0xff };
EXPORT_SYMBOL(x86_cpu_to_apicid);

110 111
u8 apicid_2_node[MAX_APICID];

Linus Torvalds's avatar
Linus Torvalds committed
112 113 114 115 116 117 118 119 120 121 122
/*
 * Trampoline 80x86 program as an array.
 */

extern unsigned char trampoline_data [];
extern unsigned char trampoline_end  [];
static unsigned char *trampoline_base;
static int trampoline_exec;

static void map_cpu_to_logical_apicid(void);

Zwane Mwaikambo's avatar
Zwane Mwaikambo committed
123 124 125
/* State of each CPU. */
DEFINE_PER_CPU(int, cpu_state) = { 0 };

Linus Torvalds's avatar
Linus Torvalds committed
126 127 128 129 130 131
/*
 * Currently trivial. Write the real->protected mode
 * bootstrap into the page concerned. The caller
 * has made sure it's suitably aligned.
 */

Li Shaohua's avatar
Li Shaohua committed
132
static unsigned long __devinit setup_trampoline(void)
Linus Torvalds's avatar
Linus Torvalds committed
133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161
{
	memcpy(trampoline_base, trampoline_data, trampoline_end - trampoline_data);
	return virt_to_phys(trampoline_base);
}

/*
 * We are called very early to get the low memory for the
 * SMP bootup trampoline page.
 */
void __init smp_alloc_memory(void)
{
	trampoline_base = (void *) alloc_bootmem_low_pages(PAGE_SIZE);
	/*
	 * Has to be in very low memory so we can execute
	 * real-mode AP code.
	 */
	if (__pa(trampoline_base) >= 0x9F000)
		BUG();
	/*
	 * Make the SMP trampoline executable:
	 */
	trampoline_exec = set_kernel_exec((unsigned long)trampoline_base, 1);
}

/*
 * The bootstrap kernel entry code has set these up. Save them for
 * a given CPU
 */

Li Shaohua's avatar
Li Shaohua committed
162
static void __devinit smp_store_cpu_info(int id)
Linus Torvalds's avatar
Linus Torvalds committed
163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186
{
	struct cpuinfo_x86 *c = cpu_data + id;

	*c = boot_cpu_data;
	if (id!=0)
		identify_cpu(c);
	/*
	 * Mask B, Pentium, but not Pentium MMX
	 */
	if (c->x86_vendor == X86_VENDOR_INTEL &&
	    c->x86 == 5 &&
	    c->x86_mask >= 1 && c->x86_mask <= 4 &&
	    c->x86_model <= 3)
		/*
		 * Remember we have B step Pentia with bugs
		 */
		smp_b_stepping = 1;

	/*
	 * Certain Athlons might work (for various values of 'work') in SMP
	 * but they are not certified as MP capable.
	 */
	if ((c->x86_vendor == X86_VENDOR_AMD) && (c->x86 == 6)) {

187 188 189
		if (num_possible_cpus() == 1)
			goto valid_k7;

Linus Torvalds's avatar
Linus Torvalds committed
190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210
		/* Athlon 660/661 is valid. */	
		if ((c->x86_model==6) && ((c->x86_mask==0) || (c->x86_mask==1)))
			goto valid_k7;

		/* Duron 670 is valid */
		if ((c->x86_model==7) && (c->x86_mask==0))
			goto valid_k7;

		/*
		 * Athlon 662, Duron 671, and Athlon >model 7 have capability bit.
		 * It's worth noting that the A5 stepping (662) of some Athlon XP's
		 * have the MP bit set.
		 * See http://www.heise.de/newsticker/data/jow-18.10.01-000 for more.
		 */
		if (((c->x86_model==6) && (c->x86_mask>=2)) ||
		    ((c->x86_model==7) && (c->x86_mask>=1)) ||
		     (c->x86_model> 7))
			if (cpu_has_mp)
				goto valid_k7;

		/* If we get here, it's not a certified SMP capable AMD system. */
211
		add_taint(TAINT_UNSAFE_SMP);
Linus Torvalds's avatar
Linus Torvalds committed
212 213 214 215 216 217 218 219 220 221 222 223 224
	}

valid_k7:
	;
}

/*
 * TSC synchronization.
 *
 * We first check whether all CPUs have their TSC's synchronized,
 * then we print a warning if not, and always resync.
 */

225 226 227 228 229 230 231 232 233 234
static struct {
	atomic_t start_flag;
	atomic_t count_start;
	atomic_t count_stop;
	unsigned long long values[NR_CPUS];
} tsc __initdata = {
	.start_flag = ATOMIC_INIT(0),
	.count_start = ATOMIC_INIT(0),
	.count_stop = ATOMIC_INIT(0),
};
Linus Torvalds's avatar
Linus Torvalds committed
235 236 237

#define NR_LOOPS 5

238
static void __init synchronize_tsc_bp(void)
Linus Torvalds's avatar
Linus Torvalds committed
239 240 241 242 243
{
	int i;
	unsigned long long t0;
	unsigned long long sum, avg;
	long long delta;
244
	unsigned int one_usec;
Linus Torvalds's avatar
Linus Torvalds committed
245 246 247 248 249 250 251
	int buggy = 0;

	printk(KERN_INFO "checking TSC synchronization across %u CPUs: ", num_booting_cpus());

	/* convert from kcyc/sec to cyc/usec */
	one_usec = cpu_khz / 1000;

252
	atomic_set(&tsc.start_flag, 1);
Linus Torvalds's avatar
Linus Torvalds committed
253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268
	wmb();

	/*
	 * We loop a few times to get a primed instruction cache,
	 * then the last pass is more or less synchronized and
	 * the BP and APs set their cycle counters to zero all at
	 * once. This reduces the chance of having random offsets
	 * between the processors, and guarantees that the maximum
	 * delay between the cycle counters is never bigger than
	 * the latency of information-passing (cachelines) between
	 * two CPUs.
	 */
	for (i = 0; i < NR_LOOPS; i++) {
		/*
		 * all APs synchronize but they loop on '== num_cpus'
		 */
269
		while (atomic_read(&tsc.count_start) != num_booting_cpus()-1)
270
			cpu_relax();
271
		atomic_set(&tsc.count_stop, 0);
Linus Torvalds's avatar
Linus Torvalds committed
272 273 274 275
		wmb();
		/*
		 * this lets the APs save their current TSC:
		 */
276
		atomic_inc(&tsc.count_start);
Linus Torvalds's avatar
Linus Torvalds committed
277

278
		rdtscll(tsc.values[smp_processor_id()]);
Linus Torvalds's avatar
Linus Torvalds committed
279 280 281 282 283 284 285 286 287
		/*
		 * We clear the TSC in the last loop:
		 */
		if (i == NR_LOOPS-1)
			write_tsc(0, 0);

		/*
		 * Wait for all APs to leave the synchronization point:
		 */
288
		while (atomic_read(&tsc.count_stop) != num_booting_cpus()-1)
289
			cpu_relax();
290
		atomic_set(&tsc.count_start, 0);
Linus Torvalds's avatar
Linus Torvalds committed
291
		wmb();
292
		atomic_inc(&tsc.count_stop);
Linus Torvalds's avatar
Linus Torvalds committed
293 294 295 296 297
	}

	sum = 0;
	for (i = 0; i < NR_CPUS; i++) {
		if (cpu_isset(i, cpu_callout_map)) {
298
			t0 = tsc.values[i];
Linus Torvalds's avatar
Linus Torvalds committed
299 300 301 302 303 304 305 306 307
			sum += t0;
		}
	}
	avg = sum;
	do_div(avg, num_booting_cpus());

	for (i = 0; i < NR_CPUS; i++) {
		if (!cpu_isset(i, cpu_callout_map))
			continue;
308
		delta = tsc.values[i] - avg;
Linus Torvalds's avatar
Linus Torvalds committed
309 310 311 312 313 314
		if (delta < 0)
			delta = -delta;
		/*
		 * We report bigger than 2 microseconds clock differences.
		 */
		if (delta > 2*one_usec) {
315 316
			long long realdelta;

Linus Torvalds's avatar
Linus Torvalds committed
317 318 319 320 321 322
			if (!buggy) {
				buggy = 1;
				printk("\n");
			}
			realdelta = delta;
			do_div(realdelta, one_usec);
323
			if (tsc.values[i] < avg)
Linus Torvalds's avatar
Linus Torvalds committed
324 325
				realdelta = -realdelta;

326 327
			if (realdelta)
				printk(KERN_INFO "CPU#%d had %Ld usecs TSC "
328
					"skew, fixed it up.\n", i, realdelta);
Linus Torvalds's avatar
Linus Torvalds committed
329 330 331 332 333 334
		}
	}
	if (!buggy)
		printk("passed.\n");
}

335
static void __init synchronize_tsc_ap(void)
Linus Torvalds's avatar
Linus Torvalds committed
336 337 338 339 340 341 342 343
{
	int i;

	/*
	 * Not every cpu is online at the time
	 * this gets called, so we first wait for the BP to
	 * finish SMP initialization:
	 */
344
	while (!atomic_read(&tsc.start_flag))
345
		cpu_relax();
Linus Torvalds's avatar
Linus Torvalds committed
346 347

	for (i = 0; i < NR_LOOPS; i++) {
348 349
		atomic_inc(&tsc.count_start);
		while (atomic_read(&tsc.count_start) != num_booting_cpus())
350
			cpu_relax();
Linus Torvalds's avatar
Linus Torvalds committed
351

352
		rdtscll(tsc.values[smp_processor_id()]);
Linus Torvalds's avatar
Linus Torvalds committed
353 354 355
		if (i == NR_LOOPS-1)
			write_tsc(0, 0);

356 357
		atomic_inc(&tsc.count_stop);
		while (atomic_read(&tsc.count_stop) != num_booting_cpus())
358
			cpu_relax();
Linus Torvalds's avatar
Linus Torvalds committed
359 360 361 362 363 364 365 366
	}
}
#undef NR_LOOPS

extern void calibrate_delay(void);

static atomic_t init_deasserted;

Li Shaohua's avatar
Li Shaohua committed
367
static void __devinit smp_callin(void)
Linus Torvalds's avatar
Linus Torvalds committed
368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451
{
	int cpuid, phys_id;
	unsigned long timeout;

	/*
	 * If waken up by an INIT in an 82489DX configuration
	 * we may get here before an INIT-deassert IPI reaches
	 * our local APIC.  We have to wait for the IPI or we'll
	 * lock up on an APIC access.
	 */
	wait_for_init_deassert(&init_deasserted);

	/*
	 * (This works even if the APIC is not enabled.)
	 */
	phys_id = GET_APIC_ID(apic_read(APIC_ID));
	cpuid = smp_processor_id();
	if (cpu_isset(cpuid, cpu_callin_map)) {
		printk("huh, phys CPU#%d, CPU#%d already present??\n",
					phys_id, cpuid);
		BUG();
	}
	Dprintk("CPU#%d (phys ID: %d) waiting for CALLOUT\n", cpuid, phys_id);

	/*
	 * STARTUP IPIs are fragile beasts as they might sometimes
	 * trigger some glue motherboard logic. Complete APIC bus
	 * silence for 1 second, this overestimates the time the
	 * boot CPU is spending to send the up to 2 STARTUP IPIs
	 * by a factor of two. This should be enough.
	 */

	/*
	 * Waiting 2s total for startup (udelay is not yet working)
	 */
	timeout = jiffies + 2*HZ;
	while (time_before(jiffies, timeout)) {
		/*
		 * Has the boot CPU finished it's STARTUP sequence?
		 */
		if (cpu_isset(cpuid, cpu_callout_map))
			break;
		rep_nop();
	}

	if (!time_before(jiffies, timeout)) {
		printk("BUG: CPU%d started up but did not get a callout!\n",
			cpuid);
		BUG();
	}

	/*
	 * the boot CPU has finished the init stage and is spinning
	 * on callin_map until we finish. We are free to set up this
	 * CPU, first the APIC. (this is probably redundant on most
	 * boards)
	 */

	Dprintk("CALLIN, before setup_local_APIC().\n");
	smp_callin_clear_local_apic();
	setup_local_APIC();
	map_cpu_to_logical_apicid();

	/*
	 * Get our bogomips.
	 */
	calibrate_delay();
	Dprintk("Stack at about %p\n",&cpuid);

	/*
	 * Save our processor parameters
	 */
 	smp_store_cpu_info(cpuid);

	disable_APIC_timer();

	/*
	 * Allow the master to continue.
	 */
	cpu_set(cpuid, cpu_callin_map);

	/*
	 *      Synchronize the TSC with the BP
	 */
452
	if (cpu_has_tsc && cpu_khz && !tsc_sync_disabled)
Linus Torvalds's avatar
Linus Torvalds committed
453 454 455 456 457
		synchronize_tsc_ap();
}

static int cpucount;

458 459 460 461 462 463
/* maps the cpu to the sched domain representing multi-core */
cpumask_t cpu_coregroup_map(int cpu)
{
	struct cpuinfo_x86 *c = cpu_data + cpu;
	/*
	 * For perf, we return last level cache shared map.
464
	 * And for power savings, we return cpu_core_map
465
	 */
466 467 468 469
	if (sched_mc_power_savings || sched_smt_power_savings)
		return cpu_core_map[cpu];
	else
		return c->llc_shared_map;
470 471
}

472 473 474
/* representing cpus for which sibling maps can be computed */
static cpumask_t cpu_sibling_setup_map;

475 476 477 478
static inline void
set_cpu_sibling_map(int cpu)
{
	int i;
479 480 481
	struct cpuinfo_x86 *c = cpu_data;

	cpu_set(cpu, cpu_sibling_setup_map);
482 483

	if (smp_num_siblings > 1) {
484
		for_each_cpu_mask(i, cpu_sibling_setup_map) {
485 486
			if (c[cpu].phys_proc_id == c[i].phys_proc_id &&
			    c[cpu].cpu_core_id == c[i].cpu_core_id) {
487 488
				cpu_set(i, cpu_sibling_map[cpu]);
				cpu_set(cpu, cpu_sibling_map[i]);
489 490
				cpu_set(i, cpu_core_map[cpu]);
				cpu_set(cpu, cpu_core_map[i]);
491 492
				cpu_set(i, c[cpu].llc_shared_map);
				cpu_set(cpu, c[i].llc_shared_map);
493 494 495 496 497 498
			}
		}
	} else {
		cpu_set(cpu, cpu_sibling_map[cpu]);
	}

499 500
	cpu_set(cpu, c[cpu].llc_shared_map);

501
	if (current_cpu_data.x86_max_cores == 1) {
502
		cpu_core_map[cpu] = cpu_sibling_map[cpu];
503 504 505 506 507
		c[cpu].booted_cores = 1;
		return;
	}

	for_each_cpu_mask(i, cpu_sibling_setup_map) {
508 509 510 511 512
		if (cpu_llc_id[cpu] != BAD_APICID &&
		    cpu_llc_id[cpu] == cpu_llc_id[i]) {
			cpu_set(i, c[cpu].llc_shared_map);
			cpu_set(cpu, c[i].llc_shared_map);
		}
513
		if (c[cpu].phys_proc_id == c[i].phys_proc_id) {
514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534
			cpu_set(i, cpu_core_map[cpu]);
			cpu_set(cpu, cpu_core_map[i]);
			/*
			 *  Does this new cpu bringup a new core?
			 */
			if (cpus_weight(cpu_sibling_map[cpu]) == 1) {
				/*
				 * for each core in package, increment
				 * the booted_cores for this new cpu
				 */
				if (first_cpu(cpu_sibling_map[i]) == i)
					c[cpu].booted_cores++;
				/*
				 * increment the core count for all
				 * the other cpus in this package
				 */
				if (i != cpu)
					c[i].booted_cores++;
			} else if (i != cpu && !c[cpu].booted_cores)
				c[cpu].booted_cores = c[i].booted_cores;
		}
535 536 537
	}
}

Linus Torvalds's avatar
Linus Torvalds committed
538 539 540
/*
 * Activate a secondary processor.
 */
Li Shaohua's avatar
Li Shaohua committed
541
static void __devinit start_secondary(void *unused)
Linus Torvalds's avatar
Linus Torvalds committed
542 543
{
	/*
544
	 * Don't put *anything* before secondary_cpu_init(), SMP
Linus Torvalds's avatar
Linus Torvalds committed
545 546 547
	 * booting is too fragile that we want to limit the
	 * things done here to the most necessary things.
	 */
548
	secondary_cpu_init();
549
	preempt_disable();
Linus Torvalds's avatar
Linus Torvalds committed
550 551 552 553 554 555 556 557 558 559 560 561 562 563 564
	smp_callin();
	while (!cpu_isset(smp_processor_id(), smp_commenced_mask))
		rep_nop();
	setup_secondary_APIC_clock();
	if (nmi_watchdog == NMI_IO_APIC) {
		disable_8259A_irq(0);
		enable_NMI_through_LVT0(NULL);
		enable_8259A_irq(0);
	}
	enable_APIC_timer();
	/*
	 * low-memory mappings have been cleared, flush them from
	 * the local TLBs too.
	 */
	local_flush_tlb();
Li Shaohua's avatar
Li Shaohua committed
565

566 567 568 569
	/* This must be done before setting cpu_online_map */
	set_cpu_sibling_map(raw_smp_processor_id());
	wmb();

Li Shaohua's avatar
Li Shaohua committed
570 571 572 573 574 575 576 577 578
	/*
	 * We need to hold call_lock, so there is no inconsistency
	 * between the time smp_call_function() determines number of
	 * IPI receipients, and the time when the determination is made
	 * for which cpus receive the IPI. Holding this
	 * lock helps us to not include this cpu in a currently in progress
	 * smp_call_function().
	 */
	lock_ipi_call_lock();
Linus Torvalds's avatar
Linus Torvalds committed
579
	cpu_set(smp_processor_id(), cpu_online_map);
Li Shaohua's avatar
Li Shaohua committed
580
	unlock_ipi_call_lock();
581
	per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
Linus Torvalds's avatar
Linus Torvalds committed
582 583 584 585 586 587 588 589 590 591 592 593 594 595

	/* We can take interrupts now: we're officially "up". */
	local_irq_enable();

	wmb();
	cpu_idle();
}

/*
 * Everything has been set up for the secondary
 * CPUs - they just need to reload everything
 * from the task structure
 * This function must not return.
 */
Li Shaohua's avatar
Li Shaohua committed
596
void __devinit initialize_secondary(void)
Linus Torvalds's avatar
Linus Torvalds committed
597 598 599 600 601 602 603 604 605 606
{
	/*
	 * We don't actually need to load the full TSS,
	 * basically just the stack pointer and the eip.
	 */

	asm volatile(
		"movl %0,%%esp\n\t"
		"jmp *%1"
		:
607
		:"m" (current->thread.esp),"m" (current->thread.eip));
Linus Torvalds's avatar
Linus Torvalds committed
608 609
}

610
/* Static state in head.S used to set up a CPU */
Linus Torvalds's avatar
Linus Torvalds committed
611 612 613 614
extern struct {
	void * esp;
	unsigned short ss;
} stack_start;
615 616
extern struct i386_pda *start_pda;
extern struct Xgt_desc_struct cpu_gdt_descr;
Linus Torvalds's avatar
Linus Torvalds committed
617 618 619 620

#ifdef CONFIG_NUMA

/* which logical CPUs are on which nodes */
621
cpumask_t node_2_cpu_mask[MAX_NUMNODES] __read_mostly =
Linus Torvalds's avatar
Linus Torvalds committed
622
				{ [0 ... MAX_NUMNODES-1] = CPU_MASK_NONE };
623
EXPORT_SYMBOL(node_2_cpu_mask);
Linus Torvalds's avatar
Linus Torvalds committed
624
/* which node each logical CPU is on */
625
int cpu_2_node[NR_CPUS] __read_mostly = { [0 ... NR_CPUS-1] = 0 };
Linus Torvalds's avatar
Linus Torvalds committed
626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652
EXPORT_SYMBOL(cpu_2_node);

/* set up a mapping between cpu and node. */
static inline void map_cpu_to_node(int cpu, int node)
{
	printk("Mapping cpu %d to node %d\n", cpu, node);
	cpu_set(cpu, node_2_cpu_mask[node]);
	cpu_2_node[cpu] = node;
}

/* undo a mapping between cpu and node. */
static inline void unmap_cpu_to_node(int cpu)
{
	int node;

	printk("Unmapping cpu %d from all nodes\n", cpu);
	for (node = 0; node < MAX_NUMNODES; node ++)
		cpu_clear(cpu, node_2_cpu_mask[node]);
	cpu_2_node[cpu] = 0;
}
#else /* !CONFIG_NUMA */

#define map_cpu_to_node(cpu, node)	({})
#define unmap_cpu_to_node(cpu)	({})

#endif /* CONFIG_NUMA */

653
u8 cpu_2_logical_apicid[NR_CPUS] __read_mostly = { [0 ... NR_CPUS-1] = BAD_APICID };
Linus Torvalds's avatar
Linus Torvalds committed
654 655 656 657 658

static void map_cpu_to_logical_apicid(void)
{
	int cpu = smp_processor_id();
	int apicid = logical_smp_processor_id();
659
	int node = apicid_to_node(apicid);
660 661 662

	if (!node_online(node))
		node = first_online_node;
Linus Torvalds's avatar
Linus Torvalds committed
663 664

	cpu_2_logical_apicid[cpu] = apicid;
665
	map_cpu_to_node(cpu, node);
Linus Torvalds's avatar
Linus Torvalds committed
666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682
}

static void unmap_cpu_to_logical_apicid(int cpu)
{
	cpu_2_logical_apicid[cpu] = BAD_APICID;
	unmap_cpu_to_node(cpu);
}

#if APIC_DEBUG
static inline void __inquire_remote_apic(int apicid)
{
	int i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 };
	char *names[] = { "ID", "VERSION", "SPIV" };
	int timeout, status;

	printk("Inquiring remote APIC #%d...\n", apicid);

683
	for (i = 0; i < ARRAY_SIZE(regs); i++) {
Linus Torvalds's avatar
Linus Torvalds committed
684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717
		printk("... APIC #%d %s: ", apicid, names[i]);

		/*
		 * Wait for idle.
		 */
		apic_wait_icr_idle();

		apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(apicid));
		apic_write_around(APIC_ICR, APIC_DM_REMRD | regs[i]);

		timeout = 0;
		do {
			udelay(100);
			status = apic_read(APIC_ICR) & APIC_ICR_RR_MASK;
		} while (status == APIC_ICR_RR_INPROG && timeout++ < 1000);

		switch (status) {
		case APIC_ICR_RR_VALID:
			status = apic_read(APIC_RRR);
			printk("%08x\n", status);
			break;
		default:
			printk("failed\n");
		}
	}
}
#endif

#ifdef WAKE_SECONDARY_VIA_NMI
/* 
 * Poke the other CPU in the eye via NMI to wake it up. Remember that the normal
 * INIT, INIT, STARTUP sequence will reset the chip hard for us, and this
 * won't ... remember to clear down the APIC, etc later.
 */
Li Shaohua's avatar
Li Shaohua committed
718
static int __devinit
Linus Torvalds's avatar
Linus Torvalds committed
719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763
wakeup_secondary_cpu(int logical_apicid, unsigned long start_eip)
{
	unsigned long send_status = 0, accept_status = 0;
	int timeout, maxlvt;

	/* Target chip */
	apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(logical_apicid));

	/* Boot on the stack */
	/* Kick the second */
	apic_write_around(APIC_ICR, APIC_DM_NMI | APIC_DEST_LOGICAL);

	Dprintk("Waiting for send to finish...\n");
	timeout = 0;
	do {
		Dprintk("+");
		udelay(100);
		send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
	} while (send_status && (timeout++ < 1000));

	/*
	 * Give the other CPU some time to accept the IPI.
	 */
	udelay(200);
	/*
	 * Due to the Pentium erratum 3AP.
	 */
	maxlvt = get_maxlvt();
	if (maxlvt > 3) {
		apic_read_around(APIC_SPIV);
		apic_write(APIC_ESR, 0);
	}
	accept_status = (apic_read(APIC_ESR) & 0xEF);
	Dprintk("NMI sent.\n");

	if (send_status)
		printk("APIC never delivered???\n");
	if (accept_status)
		printk("APIC delivery error (%lx).\n", accept_status);

	return (send_status | accept_status);
}
#endif	/* WAKE_SECONDARY_VIA_NMI */

#ifdef WAKE_SECONDARY_VIA_INIT
Li Shaohua's avatar
Li Shaohua committed
764
static int __devinit
Linus Torvalds's avatar
Linus Torvalds committed
765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898
wakeup_secondary_cpu(int phys_apicid, unsigned long start_eip)
{
	unsigned long send_status = 0, accept_status = 0;
	int maxlvt, timeout, num_starts, j;

	/*
	 * Be paranoid about clearing APIC errors.
	 */
	if (APIC_INTEGRATED(apic_version[phys_apicid])) {
		apic_read_around(APIC_SPIV);
		apic_write(APIC_ESR, 0);
		apic_read(APIC_ESR);
	}

	Dprintk("Asserting INIT.\n");

	/*
	 * Turn INIT on target chip
	 */
	apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));

	/*
	 * Send IPI
	 */
	apic_write_around(APIC_ICR, APIC_INT_LEVELTRIG | APIC_INT_ASSERT
				| APIC_DM_INIT);

	Dprintk("Waiting for send to finish...\n");
	timeout = 0;
	do {
		Dprintk("+");
		udelay(100);
		send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
	} while (send_status && (timeout++ < 1000));

	mdelay(10);

	Dprintk("Deasserting INIT.\n");

	/* Target chip */
	apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));

	/* Send IPI */
	apic_write_around(APIC_ICR, APIC_INT_LEVELTRIG | APIC_DM_INIT);

	Dprintk("Waiting for send to finish...\n");
	timeout = 0;
	do {
		Dprintk("+");
		udelay(100);
		send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
	} while (send_status && (timeout++ < 1000));

	atomic_set(&init_deasserted, 1);

	/*
	 * Should we send STARTUP IPIs ?
	 *
	 * Determine this based on the APIC version.
	 * If we don't have an integrated APIC, don't send the STARTUP IPIs.
	 */
	if (APIC_INTEGRATED(apic_version[phys_apicid]))
		num_starts = 2;
	else
		num_starts = 0;

	/*
	 * Run STARTUP IPI loop.
	 */
	Dprintk("#startup loops: %d.\n", num_starts);

	maxlvt = get_maxlvt();

	for (j = 1; j <= num_starts; j++) {
		Dprintk("Sending STARTUP #%d.\n",j);
		apic_read_around(APIC_SPIV);
		apic_write(APIC_ESR, 0);
		apic_read(APIC_ESR);
		Dprintk("After apic_write.\n");

		/*
		 * STARTUP IPI
		 */

		/* Target chip */
		apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));

		/* Boot on the stack */
		/* Kick the second */
		apic_write_around(APIC_ICR, APIC_DM_STARTUP
					| (start_eip >> 12));

		/*
		 * Give the other CPU some time to accept the IPI.
		 */
		udelay(300);

		Dprintk("Startup point 1.\n");

		Dprintk("Waiting for send to finish...\n");
		timeout = 0;
		do {
			Dprintk("+");
			udelay(100);
			send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
		} while (send_status && (timeout++ < 1000));

		/*
		 * Give the other CPU some time to accept the IPI.
		 */
		udelay(200);
		/*
		 * Due to the Pentium erratum 3AP.
		 */
		if (maxlvt > 3) {
			apic_read_around(APIC_SPIV);
			apic_write(APIC_ESR, 0);
		}
		accept_status = (apic_read(APIC_ESR) & 0xEF);
		if (send_status || accept_status)
			break;
	}
	Dprintk("After Startup.\n");

	if (send_status)
		printk("APIC never delivered???\n");
	if (accept_status)
		printk("APIC delivery error (%lx).\n", accept_status);

	return (send_status | accept_status);
}
#endif	/* WAKE_SECONDARY_VIA_INIT */

extern cpumask_t cpu_initialized;
899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919
static inline int alloc_cpu_id(void)
{
	cpumask_t	tmp_map;
	int cpu;
	cpus_complement(tmp_map, cpu_present_map);
	cpu = first_cpu(tmp_map);
	if (cpu >= NR_CPUS)
		return -ENODEV;
	return cpu;
}

#ifdef CONFIG_HOTPLUG_CPU
static struct task_struct * __devinitdata cpu_idle_tasks[NR_CPUS];
static inline struct task_struct * alloc_idle_task(int cpu)
{
	struct task_struct *idle;

	if ((idle = cpu_idle_tasks[cpu]) != NULL) {
		/* initialize thread_struct.  we really want to avoid destroy
		 * idle tread
		 */
920
		idle->thread.esp = (unsigned long)task_pt_regs(idle);
921 922 923 924 925 926 927 928 929 930 931 932
		init_idle(idle, cpu);
		return idle;
	}
	idle = fork_idle(cpu);

	if (!IS_ERR(idle))
		cpu_idle_tasks[cpu] = idle;
	return idle;
}
#else
#define alloc_idle_task(cpu) fork_idle(cpu)
#endif
Linus Torvalds's avatar
Linus Torvalds committed
933

934
static int __devinit do_boot_cpu(int apicid, int cpu)
Linus Torvalds's avatar
Linus Torvalds committed
935 936 937 938 939 940 941 942
/*
 * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad
 * (ie clustered apic addressing mode), this is a LOGICAL apic ID.
 * Returns zero if CPU booted OK, else error code from wakeup_secondary_cpu.
 */
{
	struct task_struct *idle;
	unsigned long boot_error;
943
	int timeout;
Linus Torvalds's avatar
Linus Torvalds committed
944 945 946 947 948 949 950
	unsigned long start_eip;
	unsigned short nmi_high = 0, nmi_low = 0;

	/*
	 * We can't use kernel_thread since we must avoid to
	 * reschedule the child.
	 */
951
	idle = alloc_idle_task(cpu);
Linus Torvalds's avatar
Linus Torvalds committed
952 953
	if (IS_ERR(idle))
		panic("failed fork for CPU %d", cpu);
954 955 956 957 958 959 960 961 962

	/* Pre-allocate and initialize the CPU's GDT and PDA so it
	   doesn't have to do any memory allocation during the
	   delicate CPU-bringup phase. */
	if (!init_gdt(cpu, idle)) {
		printk(KERN_INFO "Couldn't allocate GDT/PDA for CPU %d\n", cpu);
		return -1;	/* ? */
	}

Linus Torvalds's avatar
Linus Torvalds committed
963 964 965 966
	idle->thread.eip = (unsigned long) start_secondary;
	/* start_eip had better be page-aligned! */
	start_eip = setup_trampoline();

967 968 969
	++cpucount;
	alternatives_smp_switch(1);

Linus Torvalds's avatar
Linus Torvalds committed
970 971 972 973 974
	/* So we see what's up   */
	printk("Booting processor %d/%d eip %lx\n", cpu, apicid, start_eip);
	/* Stack for startup_32 can be just as for start_secondary onwards */
	stack_start.esp = (void *) idle->thread.esp;

975 976 977
	start_pda = cpu_pda(cpu);
	cpu_gdt_descr = per_cpu(cpu_gdt_descr, cpu);

Linus Torvalds's avatar
Linus Torvalds committed
978 979
	irq_ctx_init(cpu);

980
	x86_cpu_to_apicid[cpu] = apicid;
Linus Torvalds's avatar
Linus Torvalds committed
981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033
	/*
	 * This grunge runs the startup process for
	 * the targeted processor.
	 */

	atomic_set(&init_deasserted, 0);

	Dprintk("Setting warm reset code and vector.\n");

	store_NMI_vector(&nmi_high, &nmi_low);

	smpboot_setup_warm_reset_vector(start_eip);

	/*
	 * Starting actual IPI sequence...
	 */
	boot_error = wakeup_secondary_cpu(apicid, start_eip);

	if (!boot_error) {
		/*
		 * allow APs to start initializing.
		 */
		Dprintk("Before Callout %d.\n", cpu);
		cpu_set(cpu, cpu_callout_map);
		Dprintk("After Callout %d.\n", cpu);

		/*
		 * Wait 5s total for a response
		 */
		for (timeout = 0; timeout < 50000; timeout++) {
			if (cpu_isset(cpu, cpu_callin_map))
				break;	/* It has booted */
			udelay(100);
		}

		if (cpu_isset(cpu, cpu_callin_map)) {
			/* number CPUs logically, starting from 1 (BSP is 0) */
			Dprintk("OK.\n");
			printk("CPU%d: ", cpu);
			print_cpu_info(&cpu_data[cpu]);
			Dprintk("CPU has booted.\n");
		} else {
			boot_error= 1;
			if (*((volatile unsigned char *)trampoline_base)
					== 0xA5)
				/* trampoline started but...? */
				printk("Stuck ??\n");
			else
				/* trampoline code not run */
				printk("Not responding.\n");
			inquire_remote_apic(apicid);
		}
	}
1034

Linus Torvalds's avatar
Linus Torvalds committed
1035 1036 1037 1038 1039 1040
	if (boot_error) {
		/* Try to put things back the way they were before ... */
		unmap_cpu_to_logical_apicid(cpu);
		cpu_clear(cpu, cpu_callout_map); /* was set here (do_boot_cpu()) */
		cpu_clear(cpu, cpu_initialized); /* was set by cpu_init() */
		cpucount--;
1041 1042 1043
	} else {
		x86_cpu_to_apicid[cpu] = apicid;
		cpu_set(cpu, cpu_present_map);
Linus Torvalds's avatar
Linus Torvalds committed
1044 1045 1046 1047 1048 1049 1050 1051
	}

	/* mark "stuck" area as not stuck */
	*((volatile unsigned long *)trampoline_base) = 0;

	return boot_error;
}

1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071
#ifdef CONFIG_HOTPLUG_CPU
void cpu_exit_clear(void)
{
	int cpu = raw_smp_processor_id();

	idle_task_exit();

	cpucount --;
	cpu_uninit();
	irq_ctx_exit(cpu);

	cpu_clear(cpu, cpu_callout_map);
	cpu_clear(cpu, cpu_callin_map);

	cpu_clear(cpu, smp_commenced_mask);
	unmap_cpu_to_logical_apicid(cpu);
}

struct warm_boot_cpu_info {
	struct completion *complete;
1072
	struct work_struct task;
1073 1074 1075 1076
	int apicid;
	int cpu;
};

1077
static void __cpuinit do_warm_boot_cpu(struct work_struct *work)
1078
{
1079 1080
	struct warm_boot_cpu_info *info =
		container_of(work, struct warm_boot_cpu_info, task);
1081 1082 1083 1084
	do_boot_cpu(info->apicid, info->cpu);
	complete(info->complete);
}

1085
static int __cpuinit __smp_prepare_cpu(int cpu)
1086
{
1087
	DECLARE_COMPLETION_ONSTACK(done);
1088 1089
	struct warm_boot_cpu_info info;
	int	apicid, ret;
1090
	struct Xgt_desc_struct *cpu_gdt_descr = &per_cpu(cpu_gdt_descr, cpu);
1091 1092 1093 1094 1095 1096 1097

	apicid = x86_cpu_to_apicid[cpu];
	if (apicid == BAD_APICID) {
		ret = -ENODEV;
		goto exit;
	}

1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109
	/*
	 * the CPU isn't initialized at boot time, allocate gdt table here.
	 * cpu_init will initialize it
	 */
	if (!cpu_gdt_descr->address) {
		cpu_gdt_descr->address = get_zeroed_page(GFP_KERNEL);
		if (!cpu_gdt_descr->address)
			printk(KERN_CRIT "CPU%d failed to allocate GDT\n", cpu);
			ret = -ENOMEM;
			goto exit;
	}

1110 1111 1112
	info.complete = &done;
	info.apicid = apicid;
	info.cpu = cpu;
1113
	INIT_WORK(&info.task, do_warm_boot_cpu);
1114 1115 1116 1117

	tsc_sync_disabled = 1;

	/* init low mem mapping */
1118
	clone_pgd_range(swapper_pg_dir, swapper_pg_dir + USER_PGD_PTRS,
1119
			min_t(unsigned long, KERNEL_PGD_PTRS, USER_PGD_PTRS));
1120
	flush_tlb_all();
1121
	schedule_work(&info.task);
1122 1123 1124 1125 1126 1127 1128 1129 1130 1131
	wait_for_completion(&done);

	tsc_sync_disabled = 0;
	zap_low_mappings();
	ret = 0;
exit:
	return ret;
}
#endif

1132
static void smp_tune_scheduling(void)
Linus Torvalds's avatar
Linus Torvalds committed
1133 1134 1135
{
	unsigned long cachesize;       /* kB   */

1136
	if (cpu_khz) {
Linus Torvalds's avatar
Linus Torvalds committed
1137
		cachesize = boot_cpu_data.x86_cache_size;
1138 1139 1140

		if (cachesize > 0)
			max_cache_size = cachesize * 1024;
Linus Torvalds's avatar
Linus Torvalds committed
1141 1142 1143 1144 1145 1146 1147 1148 1149 1150
	}
}

/*
 * Cycle through the processors sending APIC IPIs to boot each.
 */

static int boot_cpu_logical_apicid;
/* Where the IO area was mapped on multiquad, always 0 otherwise */
void *xquad_portio;
1151 1152 1153
#ifdef CONFIG_X86_NUMAQ
EXPORT_SYMBOL(xquad_portio);
#endif
Linus Torvalds's avatar
Linus Torvalds committed
1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166

static void __init smp_boot_cpus(unsigned int max_cpus)
{
	int apicid, cpu, bit, kicked;
	unsigned long bogosum = 0;

	/*
	 * Setup boot CPU information
	 */
	smp_store_cpu_info(0); /* Final full version of the data */
	printk("CPU%d: ", 0);
	print_cpu_info(&cpu_data[0]);

1167
	boot_cpu_physical_apicid = GET_APIC_ID(apic_read(APIC_ID));
Linus Torvalds's avatar
Linus Torvalds committed
1168 1169 1170 1171 1172 1173
	boot_cpu_logical_apicid = logical_smp_processor_id();
	x86_cpu_to_apicid[0] = boot_cpu_physical_apicid;

	current_thread_info()->cpu = 0;
	smp_tune_scheduling();

1174
	set_cpu_sibling_map(0);
1175

Linus Torvalds's avatar
Linus Torvalds committed
1176 1177 1178 1179 1180 1181
	/*
	 * If we couldn't find an SMP configuration at boot time,
	 * get out of here now!
	 */
	if (!smp_found_config && !acpi_lapic) {
		printk(KERN_NOTICE "SMP motherboard not detected.\n");
1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214
		smpboot_clear_io_apic_irqs();
		phys_cpu_present_map = physid_mask_of_physid(0);
		if (APIC_init_uniprocessor())
			printk(KERN_NOTICE "Local APIC not detected."
					   " Using dummy APIC emulation.\n");
		map_cpu_to_logical_apicid();
		cpu_set(0, cpu_sibling_map[0]);
		cpu_set(0, cpu_core_map[0]);
		return;
	}

	/*
	 * Should not be necessary because the MP table should list the boot
	 * CPU too, but we do it for the sake of robustness anyway.
	 * Makes no sense to do this check in clustered apic mode, so skip it
	 */
	if (!check_phys_apicid_present(boot_cpu_physical_apicid)) {
		printk("weird, boot CPU (#%d) not listed by the BIOS.\n",
				boot_cpu_physical_apicid);
		physid_set(hard_smp_processor_id(), phys_cpu_present_map);
	}

	/*
	 * If we couldn't find a local APIC, then get out of here now!
	 */
	if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid]) && !cpu_has_apic) {
		printk(KERN_ERR "BIOS bug, local APIC #%d not detected!...\n",
			boot_cpu_physical_apicid);
		printk(KERN_ERR "... forcing use of dummy APIC emulation. (tell your hw vendor)\n");
		smpboot_clear_io_apic_irqs();
		phys_cpu_present_map = physid_mask_of_physid(0);
		cpu_set(0, cpu_sibling_map[0]);
		cpu_set(0, cpu_core_map[0]);
Linus Torvalds's avatar
Linus Torvalds committed
1215 1216 1217
		return;
	}

1218 1219
	verify_local_APIC();

Linus Torvalds's avatar
Linus Torvalds committed
1220 1221 1222
	/*
	 * If SMP should be disabled, then really disable it!
	 */
1223 1224 1225 1226 1227 1228 1229
	if (!max_cpus) {
		smp_found_config = 0;
		printk(KERN_INFO "SMP mode deactivated, forcing use of dummy APIC emulation.\n");
		smpboot_clear_io_apic_irqs();
		phys_cpu_present_map = physid_mask_of_physid(0);
		cpu_set(0, cpu_sibling_map[0]);
		cpu_set(0, cpu_core_map[0]);
Linus Torvalds's avatar
Linus Torvalds committed
1230 1231 1232
		return;
	}

1233 1234 1235 1236 1237
	connect_bsp_APIC();
	setup_local_APIC();
	map_cpu_to_logical_apicid();


Linus Torvalds's avatar
Linus Torvalds committed
1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262
	setup_portio_remap();

	/*
	 * Scan the CPU present map and fire up the other CPUs via do_boot_cpu
	 *
	 * In clustered apic mode, phys_cpu_present_map is a constructed thus:
	 * bits 0-3 are quad0, 4-7 are quad1, etc. A perverse twist on the 
	 * clustered apic ID.
	 */
	Dprintk("CPU present map: %lx\n", physids_coerce(phys_cpu_present_map));

	kicked = 1;
	for (bit = 0; kicked < NR_CPUS && bit < MAX_APICS; bit++) {
		apicid = cpu_present_to_apicid(bit);
		/*
		 * Don't even attempt to start the boot CPU!
		 */
		if ((apicid == boot_cpu_apicid) || (apicid == BAD_APICID))
			continue;

		if (!check_apicid_present(bit))
			continue;
		if (max_cpus <= cpucount+1)
			continue;

1263
		if (((cpu = alloc_cpu_id()) <= 0) || do_boot_cpu(apicid, cpu))
Linus Torvalds's avatar
Linus Torvalds committed
1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309
			printk("CPU #%d not responding - cannot use it.\n",
								apicid);
		else
			++kicked;
	}

	/*
	 * Cleanup possible dangling ends...
	 */
	smpboot_restore_warm_reset_vector();

	/*
	 * Allow the user to impress friends.
	 */
	Dprintk("Before bogomips.\n");
	for (cpu = 0; cpu < NR_CPUS; cpu++)
		if (cpu_isset(cpu, cpu_callout_map))
			bogosum += cpu_data[cpu].loops_per_jiffy;
	printk(KERN_INFO
		"Total of %d processors activated (%lu.%02lu BogoMIPS).\n",
		cpucount+1,
		bogosum/(500000/HZ),
		(bogosum/(5000/HZ))%100);
	
	Dprintk("Before bogocount - setting activated=1.\n");

	if (smp_b_stepping)
		printk(KERN_WARNING "WARNING: SMP operation may be unreliable with B stepping processors.\n");

	/*
	 * Don't taint if we are running SMP kernel on a single non-MP
	 * approved Athlon
	 */
	if (tainted & TAINT_UNSAFE_SMP) {
		if (cpucount)
			printk (KERN_INFO "WARNING: This combination of AMD processors is not suitable for SMP.\n");
		else
			tainted &= ~TAINT_UNSAFE_SMP;
	}

	Dprintk("Boot done.\n");

	/*
	 * construct cpu_sibling_map[], so that we can tell sibling CPUs
	 * efficiently.
	 */
1310
	for (cpu = 0; cpu < NR_CPUS; cpu++) {
Linus Torvalds's avatar
Linus Torvalds committed
1311
		cpus_clear(cpu_sibling_map[cpu]);
1312 1313
		cpus_clear(cpu_core_map[cpu]);
	}
Linus Torvalds's avatar
Linus Torvalds committed
1314

1315 1316
	cpu_set(0, cpu_sibling_map[0]);
	cpu_set(0, cpu_core_map[0]);
Linus Torvalds's avatar
Linus Torvalds committed
1317

1318 1319 1320 1321
	smpboot_setup_io_apic();

	setup_boot_APIC_clock();

Linus Torvalds's avatar
Linus Torvalds committed
1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332
	/*
	 * Synchronize the TSC with the AP
	 */
	if (cpu_has_tsc && cpucount && cpu_khz)
		synchronize_tsc_bp();
}

/* These are wrappers to interface to the new boot process.  Someone
   who understands all this stuff should rewrite it properly. --RR 15/Jul/02 */
void __init smp_prepare_cpus(unsigned int max_cpus)
{
Zwane Mwaikambo's avatar
Zwane Mwaikambo committed
1333 1334 1335
	smp_commenced_mask = cpumask_of_cpu(0);
	cpu_callin_map = cpumask_of_cpu(0);
	mb();
Linus Torvalds's avatar
Linus Torvalds committed
1336 1337 1338 1339 1340 1341 1342
	smp_boot_cpus(max_cpus);
}

void __devinit smp_prepare_boot_cpu(void)
{
	cpu_set(smp_processor_id(), cpu_online_map);
	cpu_set(smp_processor_id(), cpu_callout_map);
1343
	cpu_set(smp_processor_id(), cpu_present_map);
1344
	cpu_set(smp_processor_id(), cpu_possible_map);
1345
	per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
Linus Torvalds's avatar
Linus Torvalds committed
1346 1347
}

Zwane Mwaikambo's avatar
Zwane Mwaikambo committed
1348
#ifdef CONFIG_HOTPLUG_CPU
1349 1350
static void
remove_siblinginfo(int cpu)
Linus Torvalds's avatar
Linus Torvalds committed
1351
{
1352
	int sibling;
1353
	struct cpuinfo_x86 *c = cpu_data;
1354

1355 1356 1357 1358 1359 1360 1361 1362 1363
	for_each_cpu_mask(sibling, cpu_core_map[cpu]) {
		cpu_clear(cpu, cpu_core_map[sibling]);
		/*
		 * last thread sibling in this cpu core going down
		 */
		if (cpus_weight(cpu_sibling_map[cpu]) == 1)
			c[sibling].booted_cores--;
	}
			
1364 1365 1366 1367
	for_each_cpu_mask(sibling, cpu_sibling_map[cpu])
		cpu_clear(cpu, cpu_sibling_map[sibling]);
	cpus_clear(cpu_sibling_map[cpu]);
	cpus_clear(cpu_core_map[cpu]);
1368 1369
	c[cpu].phys_proc_id = 0;
	c[cpu].cpu_core_id = 0;
1370
	cpu_clear(cpu, cpu_sibling_setup_map);
Zwane Mwaikambo's avatar
Zwane Mwaikambo committed
1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387
}

int __cpu_disable(void)
{
	cpumask_t map = cpu_online_map;
	int cpu = smp_processor_id();

	/*
	 * Perhaps use cpufreq to drop frequency, but that could go
	 * into generic code.
 	 *
	 * We won't take down the boot processor on i386 due to some
	 * interrupts only being able to be serviced by the BSP.
	 * Especially so if we're not using an IOAPIC	-zwane
	 */
	if (cpu == 0)
		return -EBUSY;
1388 1389
	if (nmi_watchdog == NMI_LOCAL_APIC)
		stop_apic_nmi_watchdog(NULL);
1390
	clear_local_APIC();
Zwane Mwaikambo's avatar
Zwane Mwaikambo committed
1391 1392 1393 1394 1395
	/* Allow any queued timer interrupts to get serviced */
	local_irq_enable();
	mdelay(1);
	local_irq_disable();

1396 1397
	remove_siblinginfo(cpu);

Zwane Mwaikambo's avatar
Zwane Mwaikambo committed
1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411
	cpu_clear(cpu, map);
	fixup_irqs(map);
	/* It's now safe to remove this processor from the online map */
	cpu_clear(cpu, cpu_online_map);
	return 0;
}

void __cpu_die(unsigned int cpu)
{
	/* We don't do anything here: idle task is faking death itself. */
	unsigned int i;

	for (i = 0; i < 10; i++) {
		/* They ack this in play_dead by setting CPU_DEAD */
1412 1413
		if (per_cpu(cpu_state, cpu) == CPU_DEAD) {
			printk ("CPU %d is now offline\n", cpu);
1414 1415
			if (1 == num_online_cpus())
				alternatives_smp_switch(0);
Zwane Mwaikambo's avatar
Zwane Mwaikambo committed
1416
			return;
1417
		}
1418
		msleep(100);
Linus Torvalds's avatar
Linus Torvalds committed
1419
	}
Zwane Mwaikambo's avatar
Zwane Mwaikambo committed
1420 1421 1422 1423 1424 1425 1426
 	printk(KERN_ERR "CPU %u didn't die...\n", cpu);
}
#else /* ... !CONFIG_HOTPLUG_CPU */
int __cpu_disable(void)
{
	return -ENOSYS;
}
Linus Torvalds's avatar
Linus Torvalds committed
1427

Zwane Mwaikambo's avatar
Zwane Mwaikambo committed
1428 1429 1430 1431 1432 1433 1434 1435 1436
void __cpu_die(unsigned int cpu)
{
	/* We said "no" in __cpu_disable */
	BUG();
}
#endif /* CONFIG_HOTPLUG_CPU */

int __devinit __cpu_up(unsigned int cpu)
{
1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452
#ifdef CONFIG_HOTPLUG_CPU
	int ret=0;

	/*
	 * We do warm boot only on cpus that had booted earlier
	 * Otherwise cold boot is all handled from smp_boot_cpus().
	 * cpu_callin_map is set during AP kickstart process. Its reset
	 * when a cpu is taken offline from cpu_exit_clear().
	 */
	if (!cpu_isset(cpu, cpu_callin_map))
		ret = __smp_prepare_cpu(cpu);

	if (ret)
		return -EIO;
#endif

Linus Torvalds's avatar
Linus Torvalds committed
1453 1454
	/* In case one didn't come up */
	if (!cpu_isset(cpu, cpu_callin_map)) {
Zwane Mwaikambo's avatar
Zwane Mwaikambo committed
1455
		printk(KERN_DEBUG "skipping cpu%d, didn't come online\n", cpu);
Linus Torvalds's avatar
Linus Torvalds committed
1456 1457 1458 1459 1460
		local_irq_enable();
		return -EIO;
	}

	local_irq_enable();
1461
	per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
Linus Torvalds's avatar
Linus Torvalds committed
1462 1463 1464
	/* Unleash the CPU! */
	cpu_set(cpu, smp_commenced_mask);
	while (!cpu_isset(cpu, cpu_online_map))
1465
		cpu_relax();
1466 1467 1468 1469 1470 1471

#ifdef CONFIG_X86_GENERICARCH
	if (num_online_cpus() > 8 && genapic == &apic_default)
		panic("Default flat APIC routing can't be used with > 8 cpus\n");
#endif

Linus Torvalds's avatar
Linus Torvalds committed
1472 1473 1474 1475 1476 1477 1478 1479 1480
	return 0;
}

void __init smp_cpus_done(unsigned int max_cpus)
{
#ifdef CONFIG_X86_IO_APIC
	setup_ioapic_dest();
#endif
	zap_low_mappings();
1481
#ifndef CONFIG_HOTPLUG_CPU
Linus Torvalds's avatar
Linus Torvalds committed
1482 1483 1484 1485
	/*
	 * Disable executability of the SMP trampoline:
	 */
	set_kernel_exec((unsigned long)trampoline_base, trampoline_exec);
1486
#endif
Linus Torvalds's avatar
Linus Torvalds committed
1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508
}

void __init smp_intr_init(void)
{
	/*
	 * IRQ0 must be given a fixed assignment and initialized,
	 * because it's used before the IO-APIC is set up.
	 */
	set_intr_gate(FIRST_DEVICE_VECTOR, interrupt[0]);

	/*
	 * The reschedule interrupt is a CPU-to-CPU reschedule-helper
	 * IPI, driven by wakeup.
	 */
	set_intr_gate(RESCHEDULE_VECTOR, reschedule_interrupt);

	/* IPI for invalidation */
	set_intr_gate(INVALIDATE_TLB_VECTOR, invalidate_interrupt);

	/* IPI for generic function call */
	set_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt);
}
1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521

/*
 * If the BIOS enumerates physical processors before logical,
 * maxcpus=N at enumeration-time can be used to disable HT.
 */
static int __init parse_maxcpus(char *arg)
{
	extern unsigned int maxcpus;

	maxcpus = simple_strtoul(arg, NULL, 0);
	return 0;
}
early_param("maxcpus", parse_maxcpus);