From: Wei, G. <gan...@in...> - 2012-01-05 14:54:04
Attachments:
mwait-smp-up-for-tboot.patch
|
tboot may be trying to put APs waiting in MWAIT loops before launching Xen. Xen could check the new flag field in v6 tboot shared page for the hint. If TB_FLAG_AP_WAKE_SUPPORT bit in flag field is set, Xen BSP have to write the monitored memory(g_tboot_shared->ap_wake_trigger) to bring APs out of MWAIT loops. The sipi vector should be written in g_tboot_shared->ap_wake_addr before waking up APs. Signed-off-by: Joseph Cihula <jos...@in...> Signed-off-by: Shane Wang <sha...@in...> Signed-off-by: Gang Wei <gan...@in...> diff -r cbf1ce3afd74 xen/arch/x86/smpboot.c --- a/xen/arch/x86/smpboot.c Sat Dec 31 16:18:55 2011 +0800 +++ b/xen/arch/x86/smpboot.c Sat Dec 31 18:50:14 2011 +0800 @@ -586,7 +586,9 @@ smpboot_setup_warm_reset_vector(start_eip); /* Starting actual IPI sequence... */ - boot_error = wakeup_secondary_cpu(apicid, start_eip); + boot_error = tboot_wake_ap(apicid, start_eip); + if ( boot_error ) + boot_error = wakeup_secondary_cpu(apicid, start_eip); if ( !boot_error ) { diff -r cbf1ce3afd74 xen/arch/x86/tboot.c --- a/xen/arch/x86/tboot.c Sat Dec 31 16:18:55 2011 +0800 +++ b/xen/arch/x86/tboot.c Sat Dec 31 18:50:14 2011 +0800 @@ -123,6 +123,10 @@ printk(" shutdown_entry: 0x%08x\n", tboot_shared->shutdown_entry); printk(" tboot_base: 0x%08x\n", tboot_shared->tboot_base); printk(" tboot_size: 0x%x\n", tboot_shared->tboot_size); + if ( tboot_shared->version >= 5 ) + printk(" num_in_wfs: %u\n", tboot_shared->num_in_wfs); + if ( tboot_shared->version >= 6 ) + printk(" flags: 0x%08x\n", tboot_shared->flags); /* these will be needed by tboot_protect_mem_regions() and/or tboot_parse_dmar_table(), so get them now */ @@ -529,6 +533,19 @@ panic("Memory integrity was lost on resume (%d)\n", error); } +int tboot_wake_ap(int apicid, unsigned long sipi_vec) +{ + if ( tboot_in_measured_env() && g_tboot_shared->version >= 6 && + (g_tboot_shared->flags & TB_FLAG_AP_WAKE_SUPPORT) ) + { + printk("waking AP %d w/ monitor write\n", apicid); + g_tboot_shared->ap_wake_addr = sipi_vec; + g_tboot_shared->ap_wake_trigger = apicid; + return 0; + } + return -1; +} + /* * Local variables: * mode: C diff -r cbf1ce3afd74 xen/include/asm-x86/tboot.h --- a/xen/include/asm-x86/tboot.h Sat Dec 31 16:18:55 2011 +0800 +++ b/xen/include/asm-x86/tboot.h Sat Dec 31 18:50:14 2011 +0800 @@ -85,7 +85,7 @@ typedef struct __packed { /* version 3+ fields: */ uuid_t uuid; /* {663C8DFF-E8B3-4b82-AABF-19EA4D057A08} */ - uint32_t version; /* Version number; currently supports 0.4 */ + uint32_t version; /* Version number; currently supports 0.6 */ uint32_t log_addr; /* physical addr of tb_log_t log */ uint32_t shutdown_entry; /* entry point for tboot shutdown */ uint32_t shutdown_type; /* type of shutdown (TB_SHUTDOWN_*) */ @@ -99,6 +99,13 @@ /* version 4+ fields: */ /* populated by tboot; will be encrypted */ uint8_t s3_key[TB_KEY_SIZE]; + /* version 5+ fields: */ + uint8_t reserved_align[3]; /* used to 4byte-align num_in_wfs */ + uint32_t num_in_wfs; /* number of processors in wait-for-SIPI */ + /* version 6+ fields: */ + uint32_t flags; + uint64_t ap_wake_addr; /* phys addr of kernel/VMM SIPI vector */ + uint32_t ap_wake_trigger; /* kernel/VMM writes APIC ID to wake AP */ } tboot_shared_t; #define TB_SHUTDOWN_REBOOT 0 @@ -107,6 +114,9 @@ #define TB_SHUTDOWN_S3 3 #define TB_SHUTDOWN_HALT 4 +#define TB_FLAG_AP_WAKE_SUPPORT 0x00000001 /* kernel/VMM use INIT-SIPI-SIPI + if clear, ap_wake_* if set */ + /* {663C8DFF-E8B3-4b82-AABF-19EA4D057A08} */ #define TBOOT_SHARED_UUID { 0x663c8dff, 0xe8b3, 0x4b82, 0xaabf, \ { 0x19, 0xea, 0x4d, 0x5, 0x7a, 0x8 } }; @@ -120,6 +130,7 @@ int tboot_parse_dmar_table(acpi_table_handler dmar_handler); int tboot_s3_resume(void); void tboot_s3_error(int error); +int tboot_wake_ap(int apicid, unsigned long sipi_vec); #endif /* __TBOOT_H__ */ |
From: Qiaowei R. <qia...@in...> - 2013-05-14 13:32:09
|
tboot provides a better AP wakeup mechanism based on cpu MWAIT feature for OS/VMM. With this mechanism, system will boot faster and will NOT require VT to be enabled. But it requires that OS/VMM must have support it, otherwise system can never boot up. Once this mechanism is enabled, tboot will put APs waiting in MWAIT loops before launching kernel. kernel can check the new flag field in v6 tboot shared page for the hint. If the bit TB_FLAG_AP_WAKE_SUPPORT in flag field is set, kernel BSP has to write the monitored memory (tboot->ap_wake_trigger) to bring APs out of MWAIT loops. The sipi vector should be written in tboot->ap_wake_addr before waking up APs. Signed-off-by: Qiaowei Ren <qia...@in...> Signed-off-by: Xiaoyan Zhang <xia...@in...> Signed-off-by: Gang Wei <gan...@in...> --- arch/x86/kernel/smpboot.c | 21 ++++++++++----------- arch/x86/kernel/tboot.c | 36 ++++++++++++++++++++++++++++++++++++ include/linux/tboot.h | 17 +++++++++++++++++ 3 files changed, 63 insertions(+), 11 deletions(-) diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index 9f190a2..07f979e 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -787,17 +787,16 @@ static int __cpuinit do_boot_cpu(int apicid, int cpu, struct task_struct *idle) } } - /* - * Wake up a CPU in difference cases: - * - Use the method in the APIC driver if it's defined - * Otherwise, - * - Use an INIT boot APIC message for APs or NMI for BSP. - */ - if (apic->wakeup_secondary_cpu) - boot_error = apic->wakeup_secondary_cpu(apicid, start_ip); - else - boot_error = wakeup_cpu_via_init_nmi(cpu, start_ip, apicid, - &cpu0_nmi_registered); + if (!tboot_wake_up(apicid, start_ip)) { + /* + * Kick the secondary CPU. Use the method in the APIC driver + * if it's defined - or use an INIT boot APIC message otherwise: + */ + if (apic->wakeup_secondary_cpu) + boot_error = apic->wakeup_secondary_cpu(apicid, start_ip); + else + boot_error = wakeup_secondary_cpu_via_init(apicid, start_ip); + } if (!boot_error) { /* diff --git a/arch/x86/kernel/tboot.c b/arch/x86/kernel/tboot.c index f84fe00..0ec1947 100644 --- a/arch/x86/kernel/tboot.c +++ b/arch/x86/kernel/tboot.c @@ -101,6 +101,8 @@ void __init tboot_probe(void) pr_debug("shutdown_entry: 0x%x\n", tboot->shutdown_entry); pr_debug("tboot_base: 0x%08x\n", tboot->tboot_base); pr_debug("tboot_size: 0x%x\n", tboot->tboot_size); + if (tboot->version >= 6) + pr_info("flags: 0x%08x\n", tboot->flags); } static pgd_t *tboot_pg_dir; @@ -453,3 +455,37 @@ int tboot_force_iommu(void) return 1; } + +/* + * tboot provides a better AP wakeup mechanism based on cpu MWAIT + * feature for OS/VMM. This mechanism is defaultly disabled, and + * could be enabled with tboot command line option: + * ap_wake_mwait=true|false + * + * With this mechanism, system will boot faster and will NOT require + * VT to be enabled. But it requires that OS/VMM must have support + * it, otherwise system can never boot up. + * + * Once this mechanism is enabled, tboot will put APs waiting in + * MWAIT loops before launching kernel. kernel can check the new + * flag field in v6 tboot shared page for the hint. If the bit + * TB_FLAG_AP_WAKE_SUPPORT in flag field is set, kernel BSP has + * to write the monitored memory (tboot->ap_wake_trigger) to bring + * APs out of MWAIT loops. The sipi vector should be written in + * tboot->ap_wake_addr before waking up APs. + */ +bool tboot_wake_up(int apicid, unsigned long sipi_vec) +{ + if (!tboot_enabled()) + return false; + + if ((tboot->version < 6) || + !(tboot->flags & TB_FLAG_AP_WAKE_SUPPORT)) + return false; + + tboot->ap_wake_addr = sipi_vec; + tboot->ap_wake_trigger = apicid; + + return true; +} + diff --git a/include/linux/tboot.h b/include/linux/tboot.h index c75128b..163d779 100644 --- a/include/linux/tboot.h +++ b/include/linux/tboot.h @@ -124,8 +124,23 @@ struct tboot { /* number of processors in wait-for-SIPI */ u32 num_in_wfs; + + /* + * version 6+ fields: + */ + + u32 flags; + + /* phys addr of kernel/VMM SIPI vector */ + u64 ap_wake_addr; + + /* kernel/VMM writes APIC ID to wake AP */ + u32 ap_wake_trigger; } __packed; +/* kernel/VMM use INIT-SIPI-SIPI if clear, ap_wake_* if set */ +#define TB_FLAG_AP_WAKE_SUPPORT 0X00000001 + /* * UUID for tboot data struct to facilitate matching * defined as {663C8DFF-E8B3-4b82-AABF-19EA4D057A08} by tboot, which is @@ -146,6 +161,7 @@ extern void tboot_shutdown(u32 shutdown_type); extern struct acpi_table_header *tboot_get_dmar_table( struct acpi_table_header *dmar_tbl); extern int tboot_force_iommu(void); +extern bool tboot_wake_up(int apicid, unsigned long sipi_vec); #else @@ -156,6 +172,7 @@ extern int tboot_force_iommu(void); do { } while (0) #define tboot_get_dmar_table(dmar_tbl) (dmar_tbl) #define tboot_force_iommu() 0 +#define tboot_wake_up(apicid, sipi_vec) 0 #endif /* !CONFIG_INTEL_TXT */ -- 1.7.9.5 |
From: Ren, Q. <qia...@in...> - 2013-05-14 14:56:25
|
On 2013-05-14, Borislav Petkov wrote: > On Wed, May 15, 2013 at 05:21:25AM +0800, Qiaowei Ren wrote: > > diff --git a/arch/x86/kernel/tboot.c b/arch/x86/kernel/tboot.c index > > f84fe00..0ec1947 100644 > > --- a/arch/x86/kernel/tboot.c > > +++ b/arch/x86/kernel/tboot.c > > @@ -101,6 +101,8 @@ void __init tboot_probe(void) > > pr_debug("shutdown_entry: 0x%x\n", tboot->shutdown_entry); > > pr_debug("tboot_base: 0x%08x\n", tboot->tboot_base); > > pr_debug("tboot_size: 0x%x\n", tboot->tboot_size); > > + if (tboot->version >= 6) > > + pr_info("flags: 0x%08x\n", tboot->flags); > > This should be pr_debug like the rest, no? Well. pr_debug is ok. Thanks, Qiaowei |
From: H. P. A. <hp...@zy...> - 2013-05-14 15:28:33
|
On 05/14/2013 02:21 PM, Qiaowei Ren wrote: > tboot provides a better AP wakeup mechanism based on cpu MWAIT > feature for OS/VMM. With this mechanism, system will boot faster > and will NOT require VT to be enabled. But it requires that > OS/VMM must have support it, otherwise system can never boot up. > > Once this mechanism is enabled, tboot will put APs waiting in > MWAIT loops before launching kernel. kernel can check the new > flag field in v6 tboot shared page for the hint. If the bit > TB_FLAG_AP_WAKE_SUPPORT in flag field is set, kernel BSP has > to write the monitored memory (tboot->ap_wake_trigger) to bring > APs out of MWAIT loops. The sipi vector should be written in > tboot->ap_wake_addr before waking up APs. > This really needs a *detailed* specification about the state the CPU is parked in. Most BIOSes do in fact park the CPUs in an mwait loop, but we can't use it because the CPU state they are parked in is ill-defined. This is a good idea, but please write (or point to) a spec about what the parked CPU state looks like and how the OS gets control. From the *looks* of the code I assume it is entered in 16-bit real mode but then it is important to know what parts of the register state are well-defined. > +#define TB_FLAG_AP_WAKE_SUPPORT 0X00000001 Minor nit: please lower-case the hexadecimal x. -hpa |
From: Ren, Q. <qia...@in...> - 2013-05-15 12:55:14
|
On 2013-05-14, H. Peter Anvin wrote: > On 05/14/2013 02:21 PM, Qiaowei Ren wrote: >> tboot provides a better AP wakeup mechanism based on cpu MWAIT >> feature for OS/VMM. With this mechanism, system will boot faster and >> will NOT require VT to be enabled. But it requires that OS/VMM must >> have support it, otherwise system can never boot up. >> >> Once this mechanism is enabled, tboot will put APs waiting in MWAIT >> loops before launching kernel. kernel can check the new flag field >> in >> v6 tboot shared page for the hint. If the bit >> TB_FLAG_AP_WAKE_SUPPORT in flag field is set, kernel BSP has to >> write the monitored memory >> (tboot->ap_wake_trigger) to bring APs out of MWAIT loops. The sipi >> vector should be written in >> tboot->ap_wake_addr before waking up APs. >> > > This really needs a *detailed* specification about the state the CPU is parked in. > Most BIOSes do in fact park the CPUs in an mwait loop, but we can't > use it because the CPU state they are parked in is ill-defined. > > This is a good idea, but please write (or point to) a spec about what > the parked CPU state looks like and how the OS gets control. From the > *looks* of the code I assume it is entered in 16-bit real mode but > then it is important to know what parts of the register state are well-defined. The following is how to do mwait for tboot & kernel: For bootstrap processor (BSP), "tboot TXT pre-launch" is executed after BIOS. In this stage, tboot will issue GETSEC[SENTER], which broadcasts messages to the chipset and other physical or logical processors in the platform. In response, other logical processors perform basic cleanup and other tasks, and then finally enter SENTER sleep state. Next, for BSP, SINIT will run and then enter "tboot post-launch", which will start all sleeping APs. If tboot command line option " ap_wake_mwait=true" is set, APs will do some work and then enter mwait loop. Kernel will be launched in BSP by tboot post-launch, and bring APs out of mwait loop. Tboot works in protected mode (but paging is disabled), and closes interrupt. For APs, MONITOR and MWAIT related code in tboot is as follows: while ( _tboot_shared.ap_wake_trigger != cpuid ) { cpu_monitor(&_tboot_shared.ap_wake_trigger, 0, 0); mb(); if ( _tboot_shared.ap_wake_trigger == cpuid ) break; cpu_mwait(0, 0); } Their extension and hint are all 0. According Intel manual: Extension=0: Treat interrupts as break events even if masked (e.g., even if EFLAGS.IF=0). Hint=0: the preferred optimized state the processor should enter is C0. So, when "tboot->ap_wake_trigger" is set by kernel, APs can exit from mwait loop. Peter, I don't know whether I explain your problem. What do you think about it? Thanks, Qiaowei |
From: H. P. A. <hp...@zy...> - 2013-05-15 14:45:26
|
No, this does not really answer the question of what the CPU state looks like. "Ren, Qiaowei" <qia...@in...> wrote: >On 2013-05-14, H. Peter Anvin wrote: >> On 05/14/2013 02:21 PM, Qiaowei Ren wrote: >>> tboot provides a better AP wakeup mechanism based on cpu MWAIT >>> feature for OS/VMM. With this mechanism, system will boot faster and >>> will NOT require VT to be enabled. But it requires that OS/VMM must >>> have support it, otherwise system can never boot up. >>> >>> Once this mechanism is enabled, tboot will put APs waiting in MWAIT >>> loops before launching kernel. kernel can check the new flag field >>> in >>> v6 tboot shared page for the hint. If the bit >>> TB_FLAG_AP_WAKE_SUPPORT in flag field is set, kernel BSP has to >>> write the monitored memory >>> (tboot->ap_wake_trigger) to bring APs out of MWAIT loops. The sipi >>> vector should be written in >>> tboot->ap_wake_addr before waking up APs. >>> >> >> This really needs a *detailed* specification about the state the CPU >is parked in. >> Most BIOSes do in fact park the CPUs in an mwait loop, but we can't >> use it because the CPU state they are parked in is ill-defined. >> >> This is a good idea, but please write (or point to) a spec about what >> the parked CPU state looks like and how the OS gets control. From >the >> *looks* of the code I assume it is entered in 16-bit real mode but >> then it is important to know what parts of the register state are >well-defined. > >The following is how to do mwait for tboot & kernel: > >For bootstrap processor (BSP), "tboot TXT pre-launch" is executed after >BIOS. In this stage, tboot will issue GETSEC[SENTER], which broadcasts >messages to the chipset and other physical or logical processors in the >platform. In response, other logical processors perform basic cleanup >and other tasks, and then finally enter SENTER sleep state. > >Next, for BSP, SINIT will run and then enter "tboot post-launch", which >will start all sleeping APs. If tboot command line option " >ap_wake_mwait=true" is set, APs will do some work and then enter mwait >loop. Kernel will be launched in BSP by tboot post-launch, and bring >APs out of mwait loop. > >Tboot works in protected mode (but paging is disabled), and closes >interrupt. For APs, MONITOR and MWAIT related code in tboot is as >follows: > while ( _tboot_shared.ap_wake_trigger != cpuid ) { > cpu_monitor(&_tboot_shared.ap_wake_trigger, 0, 0); > mb(); > if ( _tboot_shared.ap_wake_trigger == cpuid ) > break; > cpu_mwait(0, 0); > } >Their extension and hint are all 0. According Intel manual: > Extension=0: Treat interrupts as break events even if masked (e.g., >even if EFLAGS.IF=0). > Hint=0: the preferred optimized state the processor should enter is >C0. >So, when "tboot->ap_wake_trigger" is set by kernel, APs can exit from >mwait loop. > >Peter, I don't know whether I explain your problem. What do you think >about it? > >Thanks, >Qiaowei -- Sent from my mobile phone. Please excuse brevity and lack of formatting. |
From: Wei, G. <gan...@in...> - 2013-05-15 16:11:36
Attachments:
smime.p7s
|
Oh, the question should be in what states all the AP CPU registers are. Do you think we need to define the case like what Intel SDM defined for getsec[SENTER]? Jimmy H. Peter Anvin wrote on 2013-05-15: > No, this does not really answer the question of what the CPU state looks > like. > > "Ren, Qiaowei" <qia...@in...> wrote: > >> On 2013-05-14, H. Peter Anvin wrote: >>> On 05/14/2013 02:21 PM, Qiaowei Ren wrote: >>>> tboot provides a better AP wakeup mechanism based on cpu MWAIT >>>> feature for OS/VMM. With this mechanism, system will boot faster and >>>> will NOT require VT to be enabled. But it requires that OS/VMM must >>>> have support it, otherwise system can never boot up. >>>> >>>> Once this mechanism is enabled, tboot will put APs waiting in MWAIT >>>> loops before launching kernel. kernel can check the new flag field >>>> in >>>> v6 tboot shared page for the hint. If the bit >>>> TB_FLAG_AP_WAKE_SUPPORT in flag field is set, kernel BSP has to >>>> write the monitored memory >>>> (tboot->ap_wake_trigger) to bring APs out of MWAIT loops. The sipi >>>> vector should be written in >>>> tboot->ap_wake_addr before waking up APs. >>>> >>> >>> This really needs a *detailed* specification about the state the CPU >>> is parked in. Most BIOSes do in fact park the CPUs in an mwait loop, >>> but we can't use it because the CPU state they are parked in is >>> ill-defined. >>> >>> This is a good idea, but please write (or point to) a spec about what >>> the parked CPU state looks like and how the OS gets control. From the >>> *looks* of the code I assume it is entered in 16-bit real mode but >>> then it is important to know what parts of the register state are >> well-defined. >> >> The following is how to do mwait for tboot & kernel: >> >> For bootstrap processor (BSP), "tboot TXT pre-launch" is executed after >> BIOS. In this stage, tboot will issue GETSEC[SENTER], which broadcasts >> messages to the chipset and other physical or logical processors in the >> platform. In response, other logical processors perform basic cleanup >> and other tasks, and then finally enter SENTER sleep state. >> >> Next, for BSP, SINIT will run and then enter "tboot post-launch", which >> will start all sleeping APs. If tboot command line option " >> ap_wake_mwait=true" is set, APs will do some work and then enter mwait >> loop. Kernel will be launched in BSP by tboot post-launch, and bring >> APs out of mwait loop. >> >> Tboot works in protected mode (but paging is disabled), and closes >> interrupt. For APs, MONITOR and MWAIT related code in tboot is as >> follows: >> while ( _tboot_shared.ap_wake_trigger != cpuid ) { >> cpu_monitor(&_tboot_shared.ap_wake_trigger, 0, 0); >> mb(); >> if ( _tboot_shared.ap_wake_trigger == cpuid ) >> break; >> cpu_mwait(0, 0); >> } >> Their extension and hint are all 0. According Intel manual: >> Extension=0: Treat interrupts as break events even if masked (e.g., >> even if EFLAGS.IF=0). >> Hint=0: the preferred optimized state the processor should enter is >> C0. >> So, when "tboot->ap_wake_trigger" is set by kernel, APs can exit from >> mwait loop. >> >> Peter, I don't know whether I explain your problem. What do you think >> about it? >> >> Thanks, >> Qiaowei > Jimmy |
From: H. P. A. <hp...@zy...> - 2013-05-15 16:25:13
|
On 05/15/2013 09:10 AM, Wei, Gang wrote: > Oh, the question should be in what states all the AP CPU registers are. Do you > think we need to define the case like what Intel SDM defined for > getsec[SENTER]? > > Jimmy I'll take a look later today. -hpa |
From: Jan B. <JBe...@su...> - 2012-01-05 15:13:14
|
>>> On 05.01.12 at 15:53, "Wei, Gang" <gan...@in...> wrote: > tboot may be trying to put APs waiting in MWAIT loops before launching Xen. > Xen could check the new flag field in v6 tboot shared page for the hint. If > TB_FLAG_AP_WAKE_SUPPORT bit in flag field is set, Xen BSP have to write the > monitored memory(g_tboot_shared->ap_wake_trigger) to bring APs out of MWAIT > loops. The sipi vector should be written in g_tboot_shared->ap_wake_addr before > waking up APs. > > Signed-off-by: Joseph Cihula <jos...@in...> > Signed-off-by: Shane Wang <sha...@in...> > Signed-off-by: Gang Wei <gan...@in...> > > diff -r cbf1ce3afd74 xen/arch/x86/smpboot.c > --- a/xen/arch/x86/smpboot.c Sat Dec 31 16:18:55 2011 +0800 > +++ b/xen/arch/x86/smpboot.c Sat Dec 31 18:50:14 2011 +0800 > @@ -586,7 +586,9 @@ > smpboot_setup_warm_reset_vector(start_eip); > > /* Starting actual IPI sequence... */ > - boot_error = wakeup_secondary_cpu(apicid, start_eip); > + boot_error = tboot_wake_ap(apicid, start_eip); As tboot.h is being included here anyway, it would seem more clean to me to guard the call with tboot_in_measured_env() here rather than in the called function. > + if ( boot_error ) > + boot_error = wakeup_secondary_cpu(apicid, start_eip); > > if ( !boot_error ) > { > diff -r cbf1ce3afd74 xen/arch/x86/tboot.c > --- a/xen/arch/x86/tboot.c Sat Dec 31 16:18:55 2011 +0800 > +++ b/xen/arch/x86/tboot.c Sat Dec 31 18:50:14 2011 +0800 > @@ -123,6 +123,10 @@ > printk(" shutdown_entry: 0x%08x\n", tboot_shared->shutdown_entry); > printk(" tboot_base: 0x%08x\n", tboot_shared->tboot_base); > printk(" tboot_size: 0x%x\n", tboot_shared->tboot_size); > + if ( tboot_shared->version >= 5 ) > + printk(" num_in_wfs: %u\n", tboot_shared->num_in_wfs); You're printing this field, but aren't making any other use of it? > + if ( tboot_shared->version >= 6 ) > + printk(" flags: 0x%08x\n", tboot_shared->flags); > > /* these will be needed by tboot_protect_mem_regions() and/or > tboot_parse_dmar_table(), so get them now */ > @@ -529,6 +533,19 @@ > panic("Memory integrity was lost on resume (%d)\n", error); > } > > +int tboot_wake_ap(int apicid, unsigned long sipi_vec) > +{ > + if ( tboot_in_measured_env() && g_tboot_shared->version >= 6 && > + (g_tboot_shared->flags & TB_FLAG_AP_WAKE_SUPPORT) ) > + { > + printk("waking AP %d w/ monitor write\n", apicid); Please, no once-per-CPU printk()-s, especially if they don't depend on per-CPU properties. > + g_tboot_shared->ap_wake_addr = sipi_vec; > + g_tboot_shared->ap_wake_trigger = apicid; > + return 0; > + } > + return -1; > +} > + > /* > * Local variables: > * mode: C > diff -r cbf1ce3afd74 xen/include/asm-x86/tboot.h > --- a/xen/include/asm-x86/tboot.h Sat Dec 31 16:18:55 2011 +0800 > +++ b/xen/include/asm-x86/tboot.h Sat Dec 31 18:50:14 2011 +0800 > @@ -85,7 +85,7 @@ > typedef struct __packed { > /* version 3+ fields: */ > uuid_t uuid; /* {663C8DFF-E8B3-4b82-AABF-19EA4D057A08} */ > - uint32_t version; /* Version number; currently supports 0.4 > */ > + uint32_t version; /* Version number; currently supports 0.6 > */ > uint32_t log_addr; /* physical addr of tb_log_t log */ > uint32_t shutdown_entry; /* entry point for tboot shutdown */ > uint32_t shutdown_type; /* type of shutdown (TB_SHUTDOWN_*) */ > @@ -99,6 +99,13 @@ > /* version 4+ fields: */ > /* populated by tboot; will be encrypted > */ > uint8_t s3_key[TB_KEY_SIZE]; > + /* version 5+ fields: */ > + uint8_t reserved_align[3]; /* used to 4byte-align num_in_wfs */ > + uint32_t num_in_wfs; /* number of processors in wait-for-SIPI */ > + /* version 6+ fields: */ > + uint32_t flags; > + uint64_t ap_wake_addr; /* phys addr of kernel/VMM SIPI vector */ > + uint32_t ap_wake_trigger; /* kernel/VMM writes APIC ID to wake AP */ > } tboot_shared_t; > > #define TB_SHUTDOWN_REBOOT 0 > @@ -107,6 +114,9 @@ > #define TB_SHUTDOWN_S3 3 > #define TB_SHUTDOWN_HALT 4 > > +#define TB_FLAG_AP_WAKE_SUPPORT 0x00000001 /* kernel/VMM use > INIT-SIPI-SIPI > + if clear, ap_wake_* if set > */ > + > /* {663C8DFF-E8B3-4b82-AABF-19EA4D057A08} */ > #define TBOOT_SHARED_UUID { 0x663c8dff, 0xe8b3, 0x4b82, 0xaabf, \ > { 0x19, 0xea, 0x4d, 0x5, 0x7a, 0x8 } }; > @@ -120,6 +130,7 @@ > int tboot_parse_dmar_table(acpi_table_handler dmar_handler); > int tboot_s3_resume(void); > void tboot_s3_error(int error); > +int tboot_wake_ap(int apicid, unsigned long sipi_vec); > > #endif /* __TBOOT_H__ */ |
From: Wei, G. <gan...@in...> - 2012-01-09 15:44:30
|
Jan Beulich wrote on 2012-01-05: >>>> On 05.01.12 at 15:53, "Wei, Gang" <gan...@in...> wrote: >> tboot may be trying to put APs waiting in MWAIT loops before launching Xen. >> Xen could check the new flag field in v6 tboot shared page for the >> hint. If TB_FLAG_AP_WAKE_SUPPORT bit in flag field is set, Xen BSP >> have to write the monitored memory(g_tboot_shared->ap_wake_trigger) >> to bring APs out of MWAIT loops. The sipi vector should be written >> in g_tboot_shared->ap_wake_addr before waking up APs. >> >> Signed-off-by: Joseph Cihula <jos...@in...> >> Signed-off-by: Shane Wang <sha...@in...> >> Signed-off-by: Gang Wei <gan...@in...> >> >> diff -r cbf1ce3afd74 xen/arch/x86/smpboot.c >> --- a/xen/arch/x86/smpboot.c Sat Dec 31 16:18:55 2011 +0800 >> +++ b/xen/arch/x86/smpboot.c Sat Dec 31 18:50:14 2011 +0800 >> @@ -586,7 +586,9 @@ >> smpboot_setup_warm_reset_vector(start_eip); >> >> /* Starting actual IPI sequence... */ >> - boot_error = wakeup_secondary_cpu(apicid, start_eip); >> + boot_error = tboot_wake_ap(apicid, start_eip); > > As tboot.h is being included here anyway, it would seem more clean to > me to guard the call with tboot_in_measured_env() here rather than in > the called function. Ok, to be updated in next version. >> + if ( boot_error ) >> + boot_error = wakeup_secondary_cpu(apicid, start_eip); >> >> if ( !boot_error ) >> { >> diff -r cbf1ce3afd74 xen/arch/x86/tboot.c >> --- a/xen/arch/x86/tboot.c Sat Dec 31 16:18:55 2011 +0800 >> +++ b/xen/arch/x86/tboot.c Sat Dec 31 18:50:14 2011 +0800 >> @@ -123,6 +123,10 @@ >> printk(" shutdown_entry: 0x%08x\n", >> tboot_shared->shutdown_entry); printk(" tboot_base: 0x%08x\n", >> tboot_shared->tboot_base); printk(" tboot_size: 0x%x\n", >> tboot_shared->tboot_size); >> + if ( tboot_shared->version >= 5 ) >> + printk(" num_in_wfs: %u\n", tboot_shared->num_in_wfs); > > You're printing this field, but aren't making any other use of it? This field is just used by Linux kernel. I will remove this printk. >> + if ( tboot_shared->version >= 6 ) >> + printk(" flags: 0x%08x\n", tboot_shared->flags); >> >> /* these will be needed by tboot_protect_mem_regions() and/or >> tboot_parse_dmar_table(), so get them now */ @@ -529,6 > +533,19 >> @@ >> panic("Memory integrity was lost on resume (%d)\n", error); } >> +int tboot_wake_ap(int apicid, unsigned long sipi_vec) { >> + if ( tboot_in_measured_env() && g_tboot_shared->version >= 6 && >> + (g_tboot_shared->flags & TB_FLAG_AP_WAKE_SUPPORT) ) >> + { >> + printk("waking AP %d w/ monitor write\n", apicid); > > Please, no once-per-CPU printk()-s, especially if they don't depend on > per-CPU properties. Ok, remove it in next version. >> + g_tboot_shared->ap_wake_addr = sipi_vec; >> + g_tboot_shared->ap_wake_trigger = apicid; >> + return 0; >> + } >> + return -1; >> +} >> + >> /* >> * Local variables: >> * mode: C Thanks for comments. Jimmy |
From: Wei, G. <gan...@in...> - 2012-01-09 16:02:13
Attachments:
mwait-smp-up-for-tboot-v2.patch
|
tboot may be trying to put APs waiting in MWAIT loops before launching Xen. Xen could check the new flag field in v6 tboot shared page for the hint. If TB_FLAG_AP_WAKE_SUPPORT bit in flag field is set, Xen BSP have to write the monitored memory(g_tboot_shared->ap_wake_trigger) to bring APs out of MWAIT loops. The sipi vector should be written in g_tboot_shared->ap_wake_addr before waking up APs. Signed-off-by: Joseph Cihula <jos...@in...> Signed-off-by: Shane Wang <sha...@in...> Signed-off-by: Gang Wei <gan...@in...> diff -r 8087674cceb9 xen/arch/x86/smpboot.c --- a/xen/arch/x86/smpboot.c Thu Jan 05 22:42:02 2012 +0800 +++ b/xen/arch/x86/smpboot.c Mon Jan 09 23:55:41 2012 +0800 @@ -586,7 +586,10 @@ smpboot_setup_warm_reset_vector(start_eip); /* Starting actual IPI sequence... */ - boot_error = wakeup_secondary_cpu(apicid, start_eip); + if ( tboot_in_measured_env() ) + boot_error = tboot_wake_ap(apicid, start_eip); + else + boot_error = wakeup_secondary_cpu(apicid, start_eip); if ( !boot_error ) { diff -r 8087674cceb9 xen/arch/x86/tboot.c --- a/xen/arch/x86/tboot.c Thu Jan 05 22:42:02 2012 +0800 +++ b/xen/arch/x86/tboot.c Mon Jan 09 23:55:41 2012 +0800 @@ -123,6 +123,8 @@ printk(" shutdown_entry: 0x%08x\n", tboot_shared->shutdown_entry); printk(" tboot_base: 0x%08x\n", tboot_shared->tboot_base); printk(" tboot_size: 0x%x\n", tboot_shared->tboot_size); + if ( tboot_shared->version >= 6 ) + printk(" flags: 0x%08x\n", tboot_shared->flags); /* these will be needed by tboot_protect_mem_regions() and/or tboot_parse_dmar_table(), so get them now */ @@ -529,6 +531,18 @@ panic("Memory integrity was lost on resume (%d)\n", error); } +int tboot_wake_ap(int apicid, unsigned long sipi_vec) +{ + if ( g_tboot_shared->version >= 6 && + (g_tboot_shared->flags & TB_FLAG_AP_WAKE_SUPPORT) ) + { + g_tboot_shared->ap_wake_addr = sipi_vec; + g_tboot_shared->ap_wake_trigger = apicid; + return 0; + } + return -1; +} + /* * Local variables: * mode: C diff -r 8087674cceb9 xen/include/asm-x86/tboot.h --- a/xen/include/asm-x86/tboot.h Thu Jan 05 22:42:02 2012 +0800 +++ b/xen/include/asm-x86/tboot.h Mon Jan 09 23:55:41 2012 +0800 @@ -85,7 +85,7 @@ typedef struct __packed { /* version 3+ fields: */ uuid_t uuid; /* {663C8DFF-E8B3-4b82-AABF-19EA4D057A08} */ - uint32_t version; /* Version number; currently supports 0.4 */ + uint32_t version; /* Version number; currently supports 0.6 */ uint32_t log_addr; /* physical addr of tb_log_t log */ uint32_t shutdown_entry; /* entry point for tboot shutdown */ uint32_t shutdown_type; /* type of shutdown (TB_SHUTDOWN_*) */ @@ -99,6 +99,13 @@ /* version 4+ fields: */ /* populated by tboot; will be encrypted */ uint8_t s3_key[TB_KEY_SIZE]; + /* version 5+ fields: */ + uint8_t reserved_align[3]; /* used to 4byte-align num_in_wfs */ + uint32_t num_in_wfs; /* number of processors in wait-for-SIPI */ + /* version 6+ fields: */ + uint32_t flags; + uint64_t ap_wake_addr; /* phys addr of kernel/VMM SIPI vector */ + uint32_t ap_wake_trigger; /* kernel/VMM writes APIC ID to wake AP */ } tboot_shared_t; #define TB_SHUTDOWN_REBOOT 0 @@ -107,6 +114,9 @@ #define TB_SHUTDOWN_S3 3 #define TB_SHUTDOWN_HALT 4 +#define TB_FLAG_AP_WAKE_SUPPORT 0x00000001 /* kernel/VMM use INIT-SIPI-SIPI + if clear, ap_wake_* if set */ + /* {663C8DFF-E8B3-4b82-AABF-19EA4D057A08} */ #define TBOOT_SHARED_UUID { 0x663c8dff, 0xe8b3, 0x4b82, 0xaabf, \ { 0x19, 0xea, 0x4d, 0x5, 0x7a, 0x8 } }; @@ -120,6 +130,7 @@ int tboot_parse_dmar_table(acpi_table_handler dmar_handler); int tboot_s3_resume(void); void tboot_s3_error(int error); +int tboot_wake_ap(int apicid, unsigned long sipi_vec); #endif /* __TBOOT_H__ */ |
From: Jan B. <JBe...@su...> - 2012-01-09 16:09:41
|
>>> On 09.01.12 at 17:01, "Wei, Gang" <gan...@in...> wrote: > tboot may be trying to put APs waiting in MWAIT loops before launching Xen. > Xen could check the new flag field in v6 tboot shared page for the hint. If > TB_FLAG_AP_WAKE_SUPPORT bit in flag field is set, Xen BSP have to write the > monitored memory(g_tboot_shared->ap_wake_trigger) to bring APs out of MWAIT > loops. The sipi vector should be written in g_tboot_shared->ap_wake_addr before > waking up APs. > > Signed-off-by: Joseph Cihula <jos...@in...> > Signed-off-by: Shane Wang <sha...@in...> > Signed-off-by: Gang Wei <gan...@in...> > > diff -r 8087674cceb9 xen/arch/x86/smpboot.c > --- a/xen/arch/x86/smpboot.c Thu Jan 05 22:42:02 2012 +0800 > +++ b/xen/arch/x86/smpboot.c Mon Jan 09 23:55:41 2012 +0800 > @@ -586,7 +586,10 @@ > smpboot_setup_warm_reset_vector(start_eip); > > /* Starting actual IPI sequence... */ > - boot_error = wakeup_secondary_cpu(apicid, start_eip); > + if ( tboot_in_measured_env() ) > + boot_error = tboot_wake_ap(apicid, start_eip); > + else > + boot_error = wakeup_secondary_cpu(apicid, start_eip); I'm afraid this is broken now for older tboot - you want to call wakeup_secondary_cpu() if tboot_wake_ap() failed. All I had asked in the first review round was that you don't call tboot_wake_ap() without tboot_in_measured_env() returning true. Jan > if ( !boot_error ) > { > diff -r 8087674cceb9 xen/arch/x86/tboot.c > --- a/xen/arch/x86/tboot.c Thu Jan 05 22:42:02 2012 +0800 > +++ b/xen/arch/x86/tboot.c Mon Jan 09 23:55:41 2012 +0800 > @@ -123,6 +123,8 @@ > printk(" shutdown_entry: 0x%08x\n", tboot_shared->shutdown_entry); > printk(" tboot_base: 0x%08x\n", tboot_shared->tboot_base); > printk(" tboot_size: 0x%x\n", tboot_shared->tboot_size); > + if ( tboot_shared->version >= 6 ) > + printk(" flags: 0x%08x\n", tboot_shared->flags); > > /* these will be needed by tboot_protect_mem_regions() and/or > tboot_parse_dmar_table(), so get them now */ > @@ -529,6 +531,18 @@ > panic("Memory integrity was lost on resume (%d)\n", error); > } > > +int tboot_wake_ap(int apicid, unsigned long sipi_vec) > +{ > + if ( g_tboot_shared->version >= 6 && > + (g_tboot_shared->flags & TB_FLAG_AP_WAKE_SUPPORT) ) > + { > + g_tboot_shared->ap_wake_addr = sipi_vec; > + g_tboot_shared->ap_wake_trigger = apicid; > + return 0; > + } > + return -1; > +} > + > /* > * Local variables: > * mode: C > diff -r 8087674cceb9 xen/include/asm-x86/tboot.h > --- a/xen/include/asm-x86/tboot.h Thu Jan 05 22:42:02 2012 +0800 > +++ b/xen/include/asm-x86/tboot.h Mon Jan 09 23:55:41 2012 +0800 > @@ -85,7 +85,7 @@ > typedef struct __packed { > /* version 3+ fields: */ > uuid_t uuid; /* {663C8DFF-E8B3-4b82-AABF-19EA4D057A08} */ > - uint32_t version; /* Version number; currently supports 0.4 > */ > + uint32_t version; /* Version number; currently supports 0.6 > */ > uint32_t log_addr; /* physical addr of tb_log_t log */ > uint32_t shutdown_entry; /* entry point for tboot shutdown */ > uint32_t shutdown_type; /* type of shutdown (TB_SHUTDOWN_*) */ > @@ -99,6 +99,13 @@ > /* version 4+ fields: */ > /* populated by tboot; will be encrypted > */ > uint8_t s3_key[TB_KEY_SIZE]; > + /* version 5+ fields: */ > + uint8_t reserved_align[3]; /* used to 4byte-align num_in_wfs */ > + uint32_t num_in_wfs; /* number of processors in wait-for-SIPI */ > + /* version 6+ fields: */ > + uint32_t flags; > + uint64_t ap_wake_addr; /* phys addr of kernel/VMM SIPI vector */ > + uint32_t ap_wake_trigger; /* kernel/VMM writes APIC ID to wake AP */ > } tboot_shared_t; > > #define TB_SHUTDOWN_REBOOT 0 > @@ -107,6 +114,9 @@ > #define TB_SHUTDOWN_S3 3 > #define TB_SHUTDOWN_HALT 4 > > +#define TB_FLAG_AP_WAKE_SUPPORT 0x00000001 /* kernel/VMM use > INIT-SIPI-SIPI > + if clear, ap_wake_* if set > */ > + > /* {663C8DFF-E8B3-4b82-AABF-19EA4D057A08} */ > #define TBOOT_SHARED_UUID { 0x663c8dff, 0xe8b3, 0x4b82, 0xaabf, \ > { 0x19, 0xea, 0x4d, 0x5, 0x7a, 0x8 } }; > @@ -120,6 +130,7 @@ > int tboot_parse_dmar_table(acpi_table_handler dmar_handler); > int tboot_s3_resume(void); > void tboot_s3_error(int error); > +int tboot_wake_ap(int apicid, unsigned long sipi_vec); > > #endif /* __TBOOT_H__ */ |
From: Wei, G. <gan...@in...> - 2012-01-10 00:41:53
|
Jan Beulich wrote on 2012-01-10: >>>> On 09.01.12 at 17:01, "Wei, Gang" <gan...@in...> wrote: >> tboot may be trying to put APs waiting in MWAIT loops before launching Xen. >> Xen could check the new flag field in v6 tboot shared page for the >> hint. If TB_FLAG_AP_WAKE_SUPPORT bit in flag field is set, Xen BSP >> have to write the monitored memory(g_tboot_shared->ap_wake_trigger) >> to bring APs out of MWAIT loops. The sipi vector should be written >> in g_tboot_shared->ap_wake_addr before waking up APs. >> >> Signed-off-by: Joseph Cihula <jos...@in...> >> Signed-off-by: Shane Wang <sha...@in...> >> Signed-off-by: Gang Wei <gan...@in...> >> >> diff -r 8087674cceb9 xen/arch/x86/smpboot.c >> --- a/xen/arch/x86/smpboot.c Thu Jan 05 22:42:02 2012 +0800 >> +++ b/xen/arch/x86/smpboot.c Mon Jan 09 23:55:41 2012 +0800 >> @@ -586,7 +586,10 @@ >> smpboot_setup_warm_reset_vector(start_eip); >> >> /* Starting actual IPI sequence... */ >> - boot_error = wakeup_secondary_cpu(apicid, start_eip); >> + if ( tboot_in_measured_env() ) >> + boot_error = tboot_wake_ap(apicid, start_eip); >> + else >> + boot_error = wakeup_secondary_cpu(apicid, start_eip); > > I'm afraid this is broken now for older tboot - you want to call > wakeup_secondary_cpu() if tboot_wake_ap() failed. All I had asked in > the first review round was that you don't call > tboot_wake_ap() without tboot_in_measured_env() returning true. Oh, yes. I should not do this in mid-night. V3 patch will fix it. Jimmy |
From: Wei, G. <gan...@in...> - 2012-01-10 01:08:58
Attachments:
mwait-smp-up-for-tboot-v3.patch
|
tboot may be trying to put APs waiting in MWAIT loops before launching Xen. Xen could check the new flag field in v6 tboot shared page for the hint. If TB_FLAG_AP_WAKE_SUPPORT bit in flag field is set, Xen BSP have to write the monitored memory(g_tboot_shared->ap_wake_trigger) to bring APs out of MWAIT loops. The sipi vector should be written in g_tboot_shared->ap_wake_addr before waking up APs. Signed-off-by: Joseph Cihula <jos...@in...> Signed-off-by: Shane Wang <sha...@in...> Signed-off-by: Gang Wei <gan...@in...> diff -r 8087674cceb9 xen/arch/x86/smpboot.c --- a/xen/arch/x86/smpboot.c Thu Jan 05 22:42:02 2012 +0800 +++ b/xen/arch/x86/smpboot.c Tue Jan 10 09:01:25 2012 +0800 @@ -586,7 +586,13 @@ smpboot_setup_warm_reset_vector(start_eip); /* Starting actual IPI sequence... */ - boot_error = wakeup_secondary_cpu(apicid, start_eip); + if ( tboot_in_measured_env() ) + boot_error = tboot_wake_ap(apicid, start_eip); + else + boot_error = 1; + + if ( boot_error ) + boot_error = wakeup_secondary_cpu(apicid, start_eip); if ( !boot_error ) { diff -r 8087674cceb9 xen/arch/x86/tboot.c --- a/xen/arch/x86/tboot.c Thu Jan 05 22:42:02 2012 +0800 +++ b/xen/arch/x86/tboot.c Tue Jan 10 09:01:25 2012 +0800 @@ -123,6 +123,8 @@ printk(" shutdown_entry: 0x%08x\n", tboot_shared->shutdown_entry); printk(" tboot_base: 0x%08x\n", tboot_shared->tboot_base); printk(" tboot_size: 0x%x\n", tboot_shared->tboot_size); + if ( tboot_shared->version >= 6 ) + printk(" flags: 0x%08x\n", tboot_shared->flags); /* these will be needed by tboot_protect_mem_regions() and/or tboot_parse_dmar_table(), so get them now */ @@ -529,6 +531,18 @@ panic("Memory integrity was lost on resume (%d)\n", error); } +int tboot_wake_ap(int apicid, unsigned long sipi_vec) +{ + if ( g_tboot_shared->version >= 6 && + (g_tboot_shared->flags & TB_FLAG_AP_WAKE_SUPPORT) ) + { + g_tboot_shared->ap_wake_addr = sipi_vec; + g_tboot_shared->ap_wake_trigger = apicid; + return 0; + } + return 1; +} + /* * Local variables: * mode: C diff -r 8087674cceb9 xen/include/asm-x86/tboot.h --- a/xen/include/asm-x86/tboot.h Thu Jan 05 22:42:02 2012 +0800 +++ b/xen/include/asm-x86/tboot.h Tue Jan 10 09:01:25 2012 +0800 @@ -85,7 +85,7 @@ typedef struct __packed { /* version 3+ fields: */ uuid_t uuid; /* {663C8DFF-E8B3-4b82-AABF-19EA4D057A08} */ - uint32_t version; /* Version number; currently supports 0.4 */ + uint32_t version; /* Version number; currently supports 0.6 */ uint32_t log_addr; /* physical addr of tb_log_t log */ uint32_t shutdown_entry; /* entry point for tboot shutdown */ uint32_t shutdown_type; /* type of shutdown (TB_SHUTDOWN_*) */ @@ -99,6 +99,13 @@ /* version 4+ fields: */ /* populated by tboot; will be encrypted */ uint8_t s3_key[TB_KEY_SIZE]; + /* version 5+ fields: */ + uint8_t reserved_align[3]; /* used to 4byte-align num_in_wfs */ + uint32_t num_in_wfs; /* number of processors in wait-for-SIPI */ + /* version 6+ fields: */ + uint32_t flags; + uint64_t ap_wake_addr; /* phys addr of kernel/VMM SIPI vector */ + uint32_t ap_wake_trigger; /* kernel/VMM writes APIC ID to wake AP */ } tboot_shared_t; #define TB_SHUTDOWN_REBOOT 0 @@ -107,6 +114,9 @@ #define TB_SHUTDOWN_S3 3 #define TB_SHUTDOWN_HALT 4 +#define TB_FLAG_AP_WAKE_SUPPORT 0x00000001 /* kernel/VMM use INIT-SIPI-SIPI + if clear, ap_wake_* if set */ + /* {663C8DFF-E8B3-4b82-AABF-19EA4D057A08} */ #define TBOOT_SHARED_UUID { 0x663c8dff, 0xe8b3, 0x4b82, 0xaabf, \ { 0x19, 0xea, 0x4d, 0x5, 0x7a, 0x8 } }; @@ -120,6 +130,7 @@ int tboot_parse_dmar_table(acpi_table_handler dmar_handler); int tboot_s3_resume(void); void tboot_s3_error(int error); +int tboot_wake_ap(int apicid, unsigned long sipi_vec); #endif /* __TBOOT_H__ */ |