123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627 |
- use core::{
- arch::asm,
- intrinsics::unlikely,
- mem::ManuallyDrop,
- sync::atomic::{compiler_fence, Ordering},
- };
- use alloc::{
- string::String,
- sync::{Arc, Weak},
- vec::Vec,
- };
- use kdepends::memoffset::offset_of;
- use system_error::SystemError;
- use x86::{controlregs::Cr4, segmentation::SegmentSelector};
- use crate::{
- arch::process::table::TSSManager,
- exception::InterruptArch,
- kerror, kwarn,
- libs::spinlock::SpinLockGuard,
- mm::VirtAddr,
- process::{
- fork::{CloneFlags, KernelCloneArgs},
- KernelStack, ProcessControlBlock, ProcessFlags, ProcessManager, PROCESS_SWITCH_RESULT,
- },
- syscall::Syscall,
- };
- use self::{
- kthread::kernel_thread_bootstrap_stage1,
- syscall::ARCH_SET_FS,
- table::{switch_fs_and_gs, KERNEL_DS, USER_DS},
- };
- use super::{fpu::FpState, interrupt::TrapFrame, syscall::X86_64GSData, CurrentIrqArch};
- pub mod idle;
- pub mod kthread;
- pub mod syscall;
- pub mod table;
- extern "C" {
-
- fn ret_from_intr();
- }
- #[allow(dead_code)]
- #[repr(align(32768))]
- union InitProcUnion {
-
- idle_stack: [u8; 32768],
- }
- #[link_section = ".data.init_proc_union"]
- #[no_mangle]
- static BSP_IDLE_STACK_SPACE: InitProcUnion = InitProcUnion {
- idle_stack: [0; 32768],
- };
- #[derive(Debug)]
- #[allow(dead_code)]
- pub struct ArchPCBInfo {
- rflags: usize,
- rbx: usize,
- r12: usize,
- r13: usize,
- r14: usize,
- r15: usize,
- rbp: usize,
- rsp: usize,
- rip: usize,
- cr2: usize,
- fsbase: usize,
- gsbase: usize,
- fs: SegmentSelector,
- gs: SegmentSelector,
-
- gsdata: X86_64GSData,
-
- fp_state: Option<FpState>,
- }
- #[allow(dead_code)]
- impl ArchPCBInfo {
-
-
-
-
-
-
-
-
-
- #[inline(never)]
- pub fn new(kstack: &KernelStack) -> Self {
- let mut r = Self {
- rflags: 0,
- rbx: 0,
- r12: 0,
- r13: 0,
- r14: 0,
- r15: 0,
- rbp: 0,
- rsp: 0,
- rip: 0,
- cr2: 0,
- fsbase: 0,
- gsbase: 0,
- gsdata: X86_64GSData {
- kaddr: VirtAddr::new(0),
- uaddr: VirtAddr::new(0),
- },
- fs: KERNEL_DS,
- gs: KERNEL_DS,
- fp_state: None,
- };
- r.rsp = kstack.stack_max_address().data() - 8;
- r.rbp = kstack.stack_max_address().data();
- return r;
- }
- pub fn set_stack(&mut self, stack: VirtAddr) {
- self.rsp = stack.data();
- }
- pub fn set_stack_base(&mut self, stack_base: VirtAddr) {
- self.rbp = stack_base.data();
- }
- pub fn rbp(&self) -> usize {
- self.rbp
- }
- pub unsafe fn push_to_stack(&mut self, value: usize) {
- self.rsp -= core::mem::size_of::<usize>();
- *(self.rsp as *mut usize) = value;
- }
- pub unsafe fn pop_from_stack(&mut self) -> usize {
- let value = *(self.rsp as *const usize);
- self.rsp += core::mem::size_of::<usize>();
- value
- }
- pub fn save_fp_state(&mut self) {
- if self.fp_state.is_none() {
- self.fp_state = Some(FpState::new());
- }
- self.fp_state.as_mut().unwrap().save();
- }
- pub fn restore_fp_state(&mut self) {
- if unlikely(self.fp_state.is_none()) {
- return;
- }
- self.fp_state.as_mut().unwrap().restore();
- }
-
- pub fn fp_state(&self) -> &Option<FpState> {
- &self.fp_state
- }
-
- pub fn clear_fp_state(&mut self) {
- if unlikely(self.fp_state.is_none()) {
- kwarn!("fp_state is none");
- return;
- }
- self.fp_state.as_mut().unwrap().clear();
- }
- pub unsafe fn save_fsbase(&mut self) {
- if x86::controlregs::cr4().contains(Cr4::CR4_ENABLE_FSGSBASE) {
- self.fsbase = x86::current::segmentation::rdfsbase() as usize;
- } else {
- self.fsbase = x86::msr::rdmsr(x86::msr::IA32_FS_BASE) as usize;
- }
- }
- pub unsafe fn save_gsbase(&mut self) {
- if x86::controlregs::cr4().contains(Cr4::CR4_ENABLE_FSGSBASE) {
- self.gsbase = x86::current::segmentation::rdgsbase() as usize;
- } else {
- self.gsbase = x86::msr::rdmsr(x86::msr::IA32_GS_BASE) as usize;
- }
- }
- pub unsafe fn restore_fsbase(&mut self) {
- if x86::controlregs::cr4().contains(Cr4::CR4_ENABLE_FSGSBASE) {
- x86::current::segmentation::wrfsbase(self.fsbase as u64);
- } else {
- x86::msr::wrmsr(x86::msr::IA32_FS_BASE, self.fsbase as u64);
- }
- }
- pub unsafe fn restore_gsbase(&mut self) {
- if x86::controlregs::cr4().contains(Cr4::CR4_ENABLE_FSGSBASE) {
- x86::current::segmentation::wrgsbase(self.gsbase as u64);
- } else {
- x86::msr::wrmsr(x86::msr::IA32_GS_BASE, self.gsbase as u64);
- }
- }
-
- pub unsafe fn store_kernel_gsbase(&self) {
- x86::msr::wrmsr(
- x86::msr::IA32_KERNEL_GSBASE,
- &self.gsdata as *const X86_64GSData as u64,
- );
- }
-
- pub fn init_syscall_stack(&mut self, stack: &KernelStack) {
- self.gsdata.set_kstack(stack.stack_max_address() - 8);
- }
- pub fn fsbase(&self) -> usize {
- self.fsbase
- }
- pub fn gsbase(&self) -> usize {
- self.gsbase
- }
- pub fn cr2_mut(&mut self) -> &mut usize {
- &mut self.cr2
- }
- pub fn fp_state_mut(&mut self) -> &mut Option<FpState> {
- &mut self.fp_state
- }
-
- pub fn clone_all(&self) -> Self {
- Self {
- rflags: self.rflags,
- rbx: self.rbx,
- r12: self.r12,
- r13: self.r13,
- r14: self.r14,
- r15: self.r15,
- rbp: self.rbp,
- rsp: self.rsp,
- rip: self.rip,
- cr2: self.cr2,
- fsbase: self.fsbase,
- gsbase: self.gsbase,
- fs: self.fs,
- gs: self.gs,
- gsdata: self.gsdata.clone(),
- fp_state: self.fp_state,
- }
- }
-
- pub fn clone_from(&mut self, from: &Self) {
- let gsdata = self.gsdata.clone();
- *self = from.clone_all();
- self.gsdata = gsdata;
- }
- }
- impl ProcessControlBlock {
-
- pub fn arch_current_pcb() -> Arc<Self> {
-
- let ptr = VirtAddr::new(x86::current::registers::rsp() as usize);
- let stack_base = VirtAddr::new(ptr.data() & (!(KernelStack::ALIGN - 1)));
-
- let p = stack_base.data() as *const *const ProcessControlBlock;
- if unlikely((unsafe { *p }).is_null()) {
- kerror!("p={:p}", p);
- panic!("current_pcb is null");
- }
- unsafe {
-
- let weak_wrapper: ManuallyDrop<Weak<ProcessControlBlock>> =
- ManuallyDrop::new(Weak::from_raw(*p));
- let new_arc: Arc<ProcessControlBlock> = weak_wrapper.upgrade().unwrap();
- return new_arc;
- }
- }
- }
- impl ProcessManager {
- pub fn arch_init() {
-
- }
-
-
-
- pub fn copy_thread(
- current_pcb: &Arc<ProcessControlBlock>,
- new_pcb: &Arc<ProcessControlBlock>,
- clone_args: KernelCloneArgs,
- current_trapframe: &TrapFrame,
- ) -> Result<(), SystemError> {
- let clone_flags = clone_args.flags;
- let mut child_trapframe = *current_trapframe;
-
- child_trapframe.set_return_value(0);
-
- let mut new_arch_guard = unsafe { new_pcb.arch_info() };
- let kernel_stack_guard = new_pcb.kernel_stack();
-
- new_arch_guard.set_stack_base(kernel_stack_guard.stack_max_address());
- let trap_frame_vaddr: VirtAddr =
- kernel_stack_guard.stack_max_address() - core::mem::size_of::<TrapFrame>();
- new_arch_guard.set_stack(trap_frame_vaddr);
-
- unsafe {
- let usp = clone_args.stack;
- if usp != 0 {
- child_trapframe.rsp = usp as u64;
- }
- let trap_frame_ptr = trap_frame_vaddr.data() as *mut TrapFrame;
- *trap_frame_ptr = child_trapframe;
- }
- let current_arch_guard = current_pcb.arch_info_irqsave();
- new_arch_guard.fsbase = current_arch_guard.fsbase;
- new_arch_guard.gsbase = current_arch_guard.gsbase;
- new_arch_guard.fs = current_arch_guard.fs;
- new_arch_guard.gs = current_arch_guard.gs;
- new_arch_guard.fp_state = current_arch_guard.fp_state;
-
- if let Some(fp_state) = current_arch_guard.fp_state.as_ref() {
- new_arch_guard.fp_state = Some(*fp_state);
- }
- drop(current_arch_guard);
-
- if new_pcb.flags().contains(ProcessFlags::KTHREAD) {
- let kthread_bootstrap_stage1_func_addr = kernel_thread_bootstrap_stage1 as usize;
- new_arch_guard.rip = kthread_bootstrap_stage1_func_addr;
- } else {
- new_arch_guard.rip = ret_from_intr as usize;
- }
-
- if clone_flags.contains(CloneFlags::CLONE_SETTLS) {
- drop(new_arch_guard);
- Syscall::do_arch_prctl_64(new_pcb, ARCH_SET_FS, clone_args.tls, true)?;
- }
- return Ok(());
- }
-
-
-
-
-
-
- pub unsafe fn switch_process(prev: Arc<ProcessControlBlock>, next: Arc<ProcessControlBlock>) {
- assert!(!CurrentIrqArch::is_irq_enabled());
-
- prev.arch_info_irqsave().save_fp_state();
-
- next.arch_info_irqsave().restore_fp_state();
-
- prev.arch_info_irqsave().save_fsbase();
- next.arch_info_irqsave().restore_fsbase();
-
- Self::switch_gsbase(&prev, &next);
-
- let next_addr_space = next.basic().user_vm().as_ref().unwrap().clone();
- compiler_fence(Ordering::SeqCst);
- next_addr_space.read().user_mapper.utable.make_current();
- drop(next_addr_space);
- compiler_fence(Ordering::SeqCst);
-
-
- let next_arch = SpinLockGuard::leak(next.arch_info_irqsave()) as *mut ArchPCBInfo;
- let prev_arch = SpinLockGuard::leak(prev.arch_info_irqsave()) as *mut ArchPCBInfo;
- (*prev_arch).rip = switch_back as usize;
-
- ProcessManager::current_pcb().preempt_enable();
- ProcessManager::current_pcb().preempt_enable();
-
- TSSManager::current_tss().set_rsp(
- x86::Ring::Ring0,
- next.kernel_stack().stack_max_address().data() as u64,
- );
- PROCESS_SWITCH_RESULT.as_mut().unwrap().get_mut().prev_pcb = Some(prev);
- PROCESS_SWITCH_RESULT.as_mut().unwrap().get_mut().next_pcb = Some(next);
-
- compiler_fence(Ordering::SeqCst);
-
- switch_to_inner(prev_arch, next_arch);
- }
- unsafe fn switch_gsbase(prev: &Arc<ProcessControlBlock>, next: &Arc<ProcessControlBlock>) {
- asm!("swapgs", options(nostack, preserves_flags));
- prev.arch_info_irqsave().save_gsbase();
- next.arch_info_irqsave().restore_gsbase();
-
- next.arch_info_irqsave().store_kernel_gsbase();
- asm!("swapgs", options(nostack, preserves_flags));
- }
- }
- #[naked]
- unsafe extern "sysv64" fn switch_to_inner(prev: *mut ArchPCBInfo, next: *mut ArchPCBInfo) {
- asm!(
-
-
-
-
-
-
- concat!("
- // Save old registers, and load new ones
- mov [rdi + {off_rbx}], rbx
- mov rbx, [rsi + {off_rbx}]
- mov [rdi + {off_r12}], r12
- mov r12, [rsi + {off_r12}]
- mov [rdi + {off_r13}], r13
- mov r13, [rsi + {off_r13}]
- mov [rdi + {off_r14}], r14
- mov r14, [rsi + {off_r14}]
- mov [rdi + {off_r15}], r15
- mov r15, [rsi + {off_r15}]
- // switch segment registers (这些寄存器只能通过接下来的switch_hook的return来切换)
- mov [rdi + {off_fs}], fs
- mov [rdi + {off_gs}], gs
- // mov fs, [rsi + {off_fs}]
- // mov gs, [rsi + {off_gs}]
- mov [rdi + {off_rbp}], rbp
- mov rbp, [rsi + {off_rbp}]
- mov [rdi + {off_rsp}], rsp
- mov rsp, [rsi + {off_rsp}]
- // // push RFLAGS (can only be modified via stack)
- pushfq
- // // pop RFLAGS into `self.rflags`
- pop QWORD PTR [rdi + {off_rflags}]
- // // push `next.rflags`
- push QWORD PTR [rsi + {off_rflags}]
- // // pop into RFLAGS
- popfq
- // push next rip to stack
- push QWORD PTR [rsi + {off_rip}]
- // When we return, we cannot even guarantee that the return address on the stack, points to
- // the calling function. Thus, we have to execute this Rust hook by
- // ourselves, which will unlock the contexts before the later switch.
- // Note that switch_finish_hook will be responsible for executing `ret`.
- jmp {switch_hook}
- "),
- off_rflags = const(offset_of!(ArchPCBInfo, rflags)),
- off_rbx = const(offset_of!(ArchPCBInfo, rbx)),
- off_r12 = const(offset_of!(ArchPCBInfo, r12)),
- off_r13 = const(offset_of!(ArchPCBInfo, r13)),
- off_r14 = const(offset_of!(ArchPCBInfo, r14)),
- off_rbp = const(offset_of!(ArchPCBInfo, rbp)),
- off_rsp = const(offset_of!(ArchPCBInfo, rsp)),
- off_r15 = const(offset_of!(ArchPCBInfo, r15)),
- off_rip = const(offset_of!(ArchPCBInfo, rip)),
- off_fs = const(offset_of!(ArchPCBInfo, fs)),
- off_gs = const(offset_of!(ArchPCBInfo, gs)),
- switch_hook = sym crate::process::switch_finish_hook,
- options(noreturn),
- );
- }
- #[naked]
- unsafe extern "sysv64" fn switch_back() -> ! {
- asm!("ret", options(noreturn));
- }
- pub unsafe fn arch_switch_to_user(path: String, argv: Vec<String>, envp: Vec<String>) -> ! {
-
- CurrentIrqArch::interrupt_disable();
- let current_pcb = ProcessManager::current_pcb();
- let trap_frame_vaddr = VirtAddr::new(
- current_pcb.kernel_stack().stack_max_address().data() - core::mem::size_of::<TrapFrame>(),
- );
-
- let new_rip = VirtAddr::new(ret_from_intr as usize);
- assert!(
- (x86::current::registers::rsp() as usize) < trap_frame_vaddr.data(),
- "arch_switch_to_user(): current_rsp >= fake trap
- frame vaddr, this may cause some illegal access to memory!
- rsp: {:#x}, trap_frame_vaddr: {:#x}",
- x86::current::registers::rsp() as usize,
- trap_frame_vaddr.data()
- );
- let mut arch_guard = current_pcb.arch_info_irqsave();
- arch_guard.rsp = trap_frame_vaddr.data();
- arch_guard.fs = USER_DS;
- arch_guard.gs = USER_DS;
-
- arch_guard.store_kernel_gsbase();
- switch_fs_and_gs(
- SegmentSelector::from_bits_truncate(arch_guard.fs.bits()),
- SegmentSelector::from_bits_truncate(arch_guard.gs.bits()),
- );
- arch_guard.rip = new_rip.data();
- drop(arch_guard);
-
- current_pcb.flags().remove(ProcessFlags::KTHREAD);
- current_pcb.worker_private().take();
- *current_pcb.sched_info().sched_policy.write_irqsave() = crate::sched::SchedPolicy::CFS;
- let mut trap_frame = TrapFrame::new();
- compiler_fence(Ordering::SeqCst);
- Syscall::do_execve(path, argv, envp, &mut trap_frame).unwrap_or_else(|e| {
- panic!(
- "arch_switch_to_user(): pid: {pid:?}, Failed to execve: , error: {e:?}",
- pid = current_pcb.pid(),
- e = e
- );
- });
- compiler_fence(Ordering::SeqCst);
-
- drop(current_pcb);
- compiler_fence(Ordering::SeqCst);
- ready_to_switch_to_user(trap_frame, trap_frame_vaddr.data(), new_rip.data());
- }
- #[inline(never)]
- unsafe extern "sysv64" fn ready_to_switch_to_user(
- trap_frame: TrapFrame,
- trapframe_vaddr: usize,
- new_rip: usize,
- ) -> ! {
- *(trapframe_vaddr as *mut TrapFrame) = trap_frame;
- compiler_fence(Ordering::SeqCst);
- asm!(
- "swapgs",
- "mov rsp, {trapframe_vaddr}",
- "push {new_rip}",
- "ret",
- trapframe_vaddr = in(reg) trapframe_vaddr,
- new_rip = in(reg) new_rip
- );
- unreachable!()
- }
|