/* Independently written AMD64 System V instruction shims. C# owns policy. */
.section .text
.global _start
.type _start,@function
_start:
cli
cld
xor %rbp,%rbp
and $-16,%rsp
mov %rsp,xfe_initial_stack_top(%rip)
xor %eax,%eax
mov %ax,%fs
xor %edx,%edx
mov $0xc0000100,%ecx /* IA32_FS_BASE starts empty; each runtime thread installs its own TLS */
wrmsr
mov %cr0,%rax
and $-13,%rax /* clear EM and TS */
or $0x10002,%rax /* MP and CR0.WP: supervisor writes obey RO PTEs */
mov %rax,%cr0
mov %cr4,%rax
and $-262145,%rax /* OSXSAVE off: supported context state is x87/SSE, no AVX/YMM use */
or $0x600,%rax /* OSFXSR, OSXMMEXCPT */
mov %rax,%cr4
sub $16,%rsp
movl $0x1f80,(%rsp)
ldmxcsr (%rsp)
add $16,%rsp
fninit
call xfe_boot_info
mov %rax,%rdi
call XfeKernelMain
1: cli
hlt
jmp 1b
.size _start,.-_start
.global xfe_load_gdt
.type xfe_load_gdt,@function
xfe_load_gdt:
lgdt (%rdi)
pushq $8
lea 1f(%rip),%rax
push %rax
lretq
1: mov $16,%eax
mov %ax,%ds
mov %ax,%es
mov %ax,%ss
ret
.size xfe_load_gdt,.-xfe_load_gdt
/* Intentional fault probes used only by the boot protection self-test. The C#
handler validates CR2, saved RIP and the error code before choosing recovery. */
.global xfe_probe_access
.type xfe_probe_access,@function
xfe_probe_access:
cmp $1,%esi
je xfe_probe_write_fault
cmp $2,%esi
je 1f
.global xfe_probe_read_fault
xfe_probe_read_fault:
mov (%rdi),%rax
.global xfe_probe_read_resume
xfe_probe_read_resume:
ret
.global xfe_probe_write_fault
xfe_probe_write_fault:
movb $0,(%rdi)
.global xfe_probe_write_resume
xfe_probe_write_resume:
ret
1: call *%rdi
ret
.global xfe_probe_execute_resume
xfe_probe_execute_resume:
add $8,%rsp /* discard the faulting indirect call's return */
ret
.size xfe_probe_access,.-xfe_probe_access
/* System V call context. IRQ frames on the outgoing stack retain all volatile
registers. FXSAVE64 and IA32_FS_BASE are switched for every kernel thread. */
.global xfe_context_switch
.type xfe_context_switch,@function
xfe_context_switch:
mov %rax,80(%rdi)
mov %rdx,88(%rdi)
mov %rbx,0(%rdi)
mov %rbp,8(%rdi)
mov %r12,16(%rdi)
mov %r13,24(%rdi)
mov %r14,32(%rdi)
mov %r15,40(%rdi)
mov %rsp,48(%rdi)
lea 1f(%rip),%rax
mov %rax,56(%rdi)
pushfq
popq 64(%rdi)
mov $0xc0000100,%ecx
rdmsr
shl $32,%rdx
or %rdx,%rax
mov %rax,72(%rdi)
fxsave64 96(%rdi)
mov %rsi,%r11
mov 72(%r11),%rax
mov %rax,%rdx
shr $32,%rdx
wrmsr
fxrstor64 96(%r11)
mov 0(%r11),%rbx
mov 8(%r11),%rbp
mov 16(%r11),%r12
mov 24(%r11),%r13
mov 32(%r11),%r14
mov 40(%r11),%r15
mov 48(%r11),%rsp
pushq 64(%r11)
popfq
jmp *56(%r11)
1: ret
.size xfe_context_switch,.-xfe_context_switch
.global xfe_thread_trampoline
.type xfe_thread_trampoline,@function
xfe_thread_trampoline:
sub $8,%rsp
mov %rbx,%rdi
call xfe_kernel_thread_entry
ud2
.size xfe_thread_trampoline,.-xfe_thread_trampoline
/* Real preemption probe: never yields, calls a helper or allocates the red zone.
Distinct per-thread values occupy all 128 bytes below the live RSP; they must
survive at least three real LAPIC ticks, with independent FS and XMM15 state. */
.global xfe_thread_probe
.type xfe_thread_probe,@function
xfe_thread_probe:
push %rbx
push %r12
mov %rdi,%rbx
mov 8(%rbx),%r12
movq %r12,%xmm15
.irp displacement,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128
mov %r12,%rax
xor $\displacement,%rax
mov %rax,-\displacement(%rsp)
.endr
1: incq 16(%rbx)
mov %fs:0,%rax
cmp %r12,%rax
jne 3f
movq %xmm15,%rax
cmp %r12,%rax
jne 3f
.irp displacement,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128
mov %r12,%rax
xor $\displacement,%rax
cmp %rax,-\displacement(%rsp)
jne 3f
.endr
mov 56(%rbx),%rax
mov (%rax),%rax
sub 64(%rbx),%rax
cmp $3,%rax
jb 2f
mov 32(%rbx),%rax
cmpq $100000,(%rax)
jb 2f
cmpq $100000,16(%rbx)
jae 4f
2: lfence
rdtsc
shl $32,%rdx
or %rdx,%rax
cmp 48(%rbx),%rax
ja 3f
pause
jmp 1b
3: incq 24(%rbx)
4: movq $1,40(%rbx)
mov $1,%eax
pop %r12
pop %rbx
ret
.size xfe_thread_probe,.-xfe_thread_probe
/* Keep all integer and SSE registers. IDT IST3 selects the current thread's
private interrupt stack, preserving both the interrupted red zone and a
suspended IRQ frame when the scheduler switches to another kernel thread. */
.type xfe_interrupt_common,@function
xfe_interrupt_common:
cld
push %rax
push %rbx
push %rcx
push %rdx
push %rbp
push %rdi
push %rsi
push %r8
push %r9
push %r10
push %r11
push %r12
push %r13
push %r14
push %r15
mov %rsp,%rdi
mov %rsp,%rbx
and $-16,%rsp
sub $512,%rsp
fxsave64 (%rsp)
call XfeHandleInterrupt
fxrstor64 (%rsp)
mov %rbx,%rsp
pop %r15
pop %r14
pop %r13
pop %r12
pop %r11
pop %r10
pop %r9
pop %r8
pop %rsi
pop %rdi
pop %rbp
pop %rdx
pop %rcx
pop %rbx
pop %rax
add $16,%rsp
iretq
.size xfe_interrupt_common,.-xfe_interrupt_common
.macro ISR number
.global xfe_isr_\number
.type xfe_isr_\number,@function
xfe_isr_\number:
.if (\number != 8) && (\number != 10) && (\number != 11) && (\number != 12) && (\number != 13) && (\number != 14) && (\number != 17) && (\number != 21) && (\number != 29) && (\number != 30)
pushq $0
.endif
pushq $\number
jmp xfe_interrupt_common
.size xfe_isr_\number,.-xfe_isr_\number
.endm
.altmacro
.set vector,0
.rept 256
ISR %vector
.set vector,vector+1
.endr
.section .rodata
.global xfe_isr_table
.type xfe_isr_table,@object
xfe_isr_table:
.macro ISR_POINTER number
.quad xfe_isr_\number
.endm
.set vector,0
.rept 256
ISR_POINTER %vector
.set vector,vector+1
.endr
.size xfe_isr_table,.-xfe_isr_table
.section .note.GNU-stack,"",@progbits
/* Independently written AMD64 System V instruction shims. C# owns policy. */
.section .text
.global _start
.type _start,@function
_start:
cli
cld
xor %rbp,%rbp
and $-16,%rsp
mov %rsp,xfe_initial_stack_top(%rip)
xor %eax,%eax
mov %ax,%fs
xor %edx,%edx
mov $0xc0000100,%ecx /* IA32_FS_BASE starts empty; each runtime thread installs its own TLS */
wrmsr
mov %cr0,%rax
and $-13,%rax /* clear EM and TS */
or $0x10002,%rax /* MP and CR0.WP: supervisor writes obey RO PTEs */
mov %rax,%cr0
mov %cr4,%rax
and $-262145,%rax /* OSXSAVE off: supported context state is x87/SSE, no AVX/YMM use */
or $0x600,%rax /* OSFXSR, OSXMMEXCPT */
mov %rax,%cr4
sub $16,%rsp
movl $0x1f80,(%rsp)
ldmxcsr (%rsp)
add $16,%rsp
fninit
call xfe_boot_info
mov %rax,%rdi
call XfeKernelMain
1: cli
hlt
jmp 1b
.size _start,.-_start
.global xfe_load_gdt
.type xfe_load_gdt,@function
xfe_load_gdt:
lgdt (%rdi)
pushq $8
lea 1f(%rip),%rax
push %rax
lretq
1: mov $16,%eax
mov %ax,%ds
mov %ax,%es
mov %ax,%ss
ret
.size xfe_load_gdt,.-xfe_load_gdt
/* Intentional fault probes used only by the boot protection self-test. The C#
handler validates CR2, saved RIP and the error code before choosing recovery. */
.global xfe_probe_access
.type xfe_probe_access,@function
xfe_probe_access:
cmp $1,%esi
je xfe_probe_write_fault
cmp $2,%esi
je 1f
.global xfe_probe_read_fault
xfe_probe_read_fault:
mov (%rdi),%rax
.global xfe_probe_read_resume
xfe_probe_read_resume:
ret
.global xfe_probe_write_fault
xfe_probe_write_fault:
movb $0,(%rdi)
.global xfe_probe_write_resume
xfe_probe_write_resume:
ret
1: call *%rdi
ret
.global xfe_probe_execute_resume
xfe_probe_execute_resume:
add $8,%rsp /* discard the faulting indirect call's return */
ret
.size xfe_probe_access,.-xfe_probe_access
/* System V call context. IRQ frames on the outgoing stack retain all volatile
registers. FXSAVE64 and IA32_FS_BASE are switched for every kernel thread. */
.global xfe_context_switch
.type xfe_context_switch,@function
xfe_context_switch:
mov %rax,80(%rdi)
mov %rdx,88(%rdi)
mov %rbx,0(%rdi)
mov %rbp,8(%rdi)
mov %r12,16(%rdi)
mov %r13,24(%rdi)
mov %r14,32(%rdi)
mov %r15,40(%rdi)
mov %rsp,48(%rdi)
lea 1f(%rip),%rax
mov %rax,56(%rdi)
pushfq
popq 64(%rdi)
mov $0xc0000100,%ecx
rdmsr
shl $32,%rdx
or %rdx,%rax
mov %rax,72(%rdi)
fxsave64 96(%rdi)
mov %rsi,%r11
mov 72(%r11),%rax
mov %rax,%rdx
shr $32,%rdx
wrmsr
fxrstor64 96(%r11)
mov 0(%r11),%rbx
mov 8(%r11),%rbp
mov 16(%r11),%r12
mov 24(%r11),%r13
mov 32(%r11),%r14
mov 40(%r11),%r15
mov 48(%r11),%rsp
pushq 64(%r11)
popfq
jmp *56(%r11)
1: ret
.size xfe_context_switch,.-xfe_context_switch
.global xfe_thread_trampoline
.type xfe_thread_trampoline,@function
xfe_thread_trampoline:
sub $8,%rsp
mov %rbx,%rdi
call xfe_kernel_thread_entry
ud2
.size xfe_thread_trampoline,.-xfe_thread_trampoline
/* Real preemption probe: never yields, calls a helper or allocates the red zone.
Distinct per-thread values occupy all 128 bytes below the live RSP; they must
survive at least three real LAPIC ticks, with independent FS and XMM15 state. */
.global xfe_thread_probe
.type xfe_thread_probe,@function
xfe_thread_probe:
push %rbx
push %r12
mov %rdi,%rbx
mov 8(%rbx),%r12
movq %r12,%xmm15
.irp displacement,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128
mov %r12,%rax
xor $\displacement,%rax
mov %rax,-\displacement(%rsp)
.endr
1: incq 16(%rbx)
mov %fs:0,%rax
cmp %r12,%rax
jne 3f
movq %xmm15,%rax
cmp %r12,%rax
jne 3f
.irp displacement,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128
mov %r12,%rax
xor $\displacement,%rax
cmp %rax,-\displacement(%rsp)
jne 3f
.endr
mov 56(%rbx),%rax
mov (%rax),%rax
sub 64(%rbx),%rax
cmp $3,%rax
jb 2f
mov 32(%rbx),%rax
cmpq $100000,(%rax)
jb 2f
cmpq $100000,16(%rbx)
jae 4f
2: lfence
rdtsc
shl $32,%rdx
or %rdx,%rax
cmp 48(%rbx),%rax
ja 3f
pause
jmp 1b
3: incq 24(%rbx)
4: movq $1,40(%rbx)
mov $1,%eax
pop %r12
pop %rbx
ret
.size xfe_thread_probe,.-xfe_thread_probe
/* Keep all integer and SSE registers. IDT IST3 selects the current thread's
private interrupt stack, preserving both the interrupted red zone and a
suspended IRQ frame when the scheduler switches to another kernel thread. */
.type xfe_interrupt_common,@function
xfe_interrupt_common:
cld
push %rax
push %rbx
push %rcx
push %rdx
push %rbp
push %rdi
push %rsi
push %r8
push %r9
push %r10
push %r11
push %r12
push %r13
push %r14
push %r15
mov %rsp,%rdi
mov %rsp,%rbx
and $-16,%rsp
sub $512,%rsp
fxsave64 (%rsp)
call XfeHandleInterrupt
fxrstor64 (%rsp)
mov %rbx,%rsp
pop %r15
pop %r14
pop %r13
pop %r12
pop %r11
pop %r10
pop %r9
pop %r8
pop %rsi
pop %rdi
pop %rbp
pop %rdx
pop %rcx
pop %rbx
pop %rax
add $16,%rsp
iretq
.size xfe_interrupt_common,.-xfe_interrupt_common
.macro ISR number
.global xfe_isr_\number
.type xfe_isr_\number,@function
xfe_isr_\number:
.if (\number != 8) && (\number != 10) && (\number != 11) && (\number != 12) && (\number != 13) && (\number != 14) && (\number != 17) && (\number != 21) && (\number != 29) && (\number != 30)
pushq $0
.endif
pushq $\number
jmp xfe_interrupt_common
.size xfe_isr_\number,.-xfe_isr_\number
.endm
.altmacro
.set vector,0
.rept 256
ISR %vector
.set vector,vector+1
.endr
.section .rodata
.global xfe_isr_table
.type xfe_isr_table,@object
xfe_isr_table:
.macro ISR_POINTER number
.quad xfe_isr_\number
.endm
.set vector,0
.rept 256
ISR_POINTER %vector
.set vector,vector+1
.endr
.size xfe_isr_table,.-xfe_isr_table
.section .note.GNU-stack,"",@progbits
