mod.rs 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746
  1. pub mod barrier;
  2. pub mod bump;
  3. pub mod fault;
  4. pub mod pkru;
  5. use alloc::sync::Arc;
  6. use alloc::vec::Vec;
  7. use hashbrown::HashSet;
  8. use log::{debug, info};
  9. use x86::time::rdtsc;
  10. use x86_64::registers::model_specific::EferFlags;
  11. use crate::driver::serial::serial8250::send_to_default_serial8250_port;
  12. use crate::init::boot::boot_callbacks;
  13. use crate::libs::align::page_align_up;
  14. use crate::libs::lib_ui::screen_manager::scm_disable_put_to_window;
  15. use crate::libs::spinlock::SpinLock;
  16. use crate::mm::allocator::page_frame::{FrameAllocator, PageFrameCount, PageFrameUsage};
  17. use crate::mm::memblock::mem_block_manager;
  18. use crate::mm::ucontext::LockedVMA;
  19. use crate::{
  20. arch::MMArch,
  21. mm::allocator::{buddy::BuddyAllocator, bump::BumpAllocator},
  22. };
  23. use crate::mm::kernel_mapper::KernelMapper;
  24. use crate::mm::page::{EntryFlags, PageEntry, PAGE_1G_SHIFT};
  25. use crate::mm::{MemoryManagementArch, PageTableKind, PhysAddr, VirtAddr, VmFlags};
  26. use system_error::SystemError;
  27. use core::arch::asm;
  28. use core::fmt::Debug;
  29. use core::sync::atomic::{compiler_fence, AtomicBool, Ordering};
  30. use super::kvm::vmx::vmcs::VmcsFields;
  31. use super::kvm::vmx::vmx_asm_wrapper::vmx_vmread;
  32. pub type PageMapper =
  33. crate::mm::page::PageMapper<crate::arch::x86_64::mm::X86_64MMArch, LockedFrameAllocator>;
  34. /// 初始的CR3寄存器的值,用于内存管理初始化时,创建的第一个内核页表的位置
  35. static mut INITIAL_CR3_VALUE: PhysAddr = PhysAddr::new(0);
  36. static INNER_ALLOCATOR: SpinLock<Option<BuddyAllocator<MMArch>>> = SpinLock::new(None);
  37. #[derive(Clone, Copy, Debug)]
  38. pub struct X86_64MMBootstrapInfo {
  39. kernel_load_base_paddr: usize,
  40. kernel_code_start: usize,
  41. kernel_code_end: usize,
  42. kernel_data_end: usize,
  43. kernel_rodata_end: usize,
  44. start_brk: usize,
  45. }
  46. pub(super) static mut BOOTSTRAP_MM_INFO: Option<X86_64MMBootstrapInfo> = None;
  47. pub(super) fn x86_64_set_kernel_load_base_paddr(paddr: PhysAddr) {
  48. unsafe {
  49. BOOTSTRAP_MM_INFO.as_mut().unwrap().kernel_load_base_paddr = paddr.data();
  50. }
  51. }
  52. /// @brief X86_64的内存管理架构结构体
  53. #[derive(Debug, Clone, Copy, Hash)]
  54. pub struct X86_64MMArch;
  55. /// XD标志位是否被保留
  56. static XD_RESERVED: AtomicBool = AtomicBool::new(false);
  57. impl MemoryManagementArch for X86_64MMArch {
  58. /// X86目前支持缺页中断
  59. const PAGE_FAULT_ENABLED: bool = true;
  60. /// 4K页
  61. const PAGE_SHIFT: usize = 12;
  62. /// 每个页表项占8字节,总共有512个页表项
  63. const PAGE_ENTRY_SHIFT: usize = 9;
  64. /// 四级页表(PML4T、PDPT、PDT、PT)
  65. const PAGE_LEVELS: usize = 4;
  66. /// 页表项的有效位的index。在x86_64中,页表项的第[0, 47]位表示地址和flag,
  67. /// 第[48, 51]位表示保留。因此,有效位的index为52。
  68. /// 请注意,第63位是XD位,表示是否允许执行。
  69. const ENTRY_ADDRESS_SHIFT: usize = 52;
  70. const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT;
  71. const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT;
  72. const ENTRY_FLAG_PRESENT: usize = 1 << 0;
  73. const ENTRY_FLAG_READONLY: usize = 0;
  74. const ENTRY_FLAG_WRITEABLE: usize = 1 << 1;
  75. const ENTRY_FLAG_READWRITE: usize = 1 << 1;
  76. const ENTRY_FLAG_USER: usize = 1 << 2;
  77. const ENTRY_FLAG_WRITE_THROUGH: usize = 1 << 3;
  78. const ENTRY_FLAG_CACHE_DISABLE: usize = 1 << 4;
  79. const ENTRY_FLAG_NO_EXEC: usize = 1 << 63;
  80. /// x86_64不存在EXEC标志位,只有NO_EXEC(XD)标志位
  81. const ENTRY_FLAG_EXEC: usize = 0;
  82. const ENTRY_FLAG_ACCESSED: usize = 1 << 5;
  83. const ENTRY_FLAG_DIRTY: usize = 1 << 6;
  84. const ENTRY_FLAG_HUGE_PAGE: usize = 1 << 7;
  85. const ENTRY_FLAG_GLOBAL: usize = 1 << 8;
  86. /// 物理地址与虚拟地址的偏移量
  87. /// 0xffff_8000_0000_0000
  88. const PHYS_OFFSET: usize = Self::PAGE_NEGATIVE_MASK + (Self::PAGE_ADDRESS_SIZE >> 1);
  89. const KERNEL_LINK_OFFSET: usize = 0x100000;
  90. // 参考 https://code.dragonos.org.cn/xref/linux-6.1.9/arch/x86/include/asm/page_64_types.h#75
  91. const USER_END_VADDR: VirtAddr =
  92. VirtAddr::new((Self::PAGE_ADDRESS_SIZE >> 1) - Self::PAGE_SIZE);
  93. const USER_BRK_START: VirtAddr = VirtAddr::new(0x700000000000);
  94. const USER_STACK_START: VirtAddr = VirtAddr::new(0x6ffff0a00000);
  95. const FIXMAP_START_VADDR: VirtAddr = VirtAddr::new(0xffffb00000000000);
  96. /// 设置FIXMAP区域大小为16M
  97. const FIXMAP_SIZE: usize = 256 * 4096 * 16;
  98. const MMIO_BASE: VirtAddr = VirtAddr::new(0xffffa10000000000);
  99. const MMIO_SIZE: usize = 1 << PAGE_1G_SHIFT;
  100. /// @brief 获取物理内存区域
  101. unsafe fn init() {
  102. extern "C" {
  103. fn _text();
  104. fn _etext();
  105. fn _edata();
  106. fn _erodata();
  107. fn _end();
  108. fn _default_kernel_load_base();
  109. }
  110. Self::init_xd_rsvd();
  111. let bootstrap_info = X86_64MMBootstrapInfo {
  112. kernel_load_base_paddr: _default_kernel_load_base as usize,
  113. kernel_code_start: _text as usize,
  114. kernel_code_end: _etext as usize,
  115. kernel_data_end: _edata as usize,
  116. kernel_rodata_end: _erodata as usize,
  117. start_brk: _end as usize,
  118. };
  119. unsafe {
  120. BOOTSTRAP_MM_INFO = Some(bootstrap_info);
  121. }
  122. // 初始化物理内存区域
  123. boot_callbacks()
  124. .early_init_memory_blocks()
  125. .expect("init memory area failed");
  126. debug!("bootstrap info: {:?}", unsafe { BOOTSTRAP_MM_INFO });
  127. debug!("phys[0]=virt[0x{:x}]", unsafe {
  128. MMArch::phys_2_virt(PhysAddr::new(0)).unwrap().data()
  129. });
  130. // 初始化内存管理器
  131. unsafe { allocator_init() };
  132. send_to_default_serial8250_port("x86 64 mm init done\n\0".as_bytes());
  133. }
  134. /// @brief 刷新TLB中,关于指定虚拟地址的条目
  135. unsafe fn invalidate_page(address: VirtAddr) {
  136. compiler_fence(Ordering::SeqCst);
  137. asm!("invlpg [{0}]", in(reg) address.data(), options(nostack, preserves_flags));
  138. compiler_fence(Ordering::SeqCst);
  139. }
  140. /// @brief 刷新TLB中,所有的条目
  141. unsafe fn invalidate_all() {
  142. compiler_fence(Ordering::SeqCst);
  143. // 通过设置cr3寄存器,来刷新整个TLB
  144. Self::set_table(PageTableKind::User, Self::table(PageTableKind::User));
  145. compiler_fence(Ordering::SeqCst);
  146. }
  147. /// @brief 获取顶级页表的物理地址
  148. unsafe fn table(table_kind: PageTableKind) -> PhysAddr {
  149. match table_kind {
  150. PageTableKind::Kernel | PageTableKind::User => {
  151. compiler_fence(Ordering::SeqCst);
  152. let cr3 = x86::controlregs::cr3() as usize;
  153. compiler_fence(Ordering::SeqCst);
  154. return PhysAddr::new(cr3);
  155. }
  156. PageTableKind::EPT => {
  157. let eptp =
  158. vmx_vmread(VmcsFields::CTRL_EPTP_PTR as u32).expect("Failed to read eptp");
  159. return PhysAddr::new(eptp as usize);
  160. }
  161. }
  162. }
  163. /// @brief 设置顶级页表的物理地址到处理器中
  164. unsafe fn set_table(_table_kind: PageTableKind, table: PhysAddr) {
  165. compiler_fence(Ordering::SeqCst);
  166. asm!("mov cr3, {}", in(reg) table.data(), options(nostack, preserves_flags));
  167. compiler_fence(Ordering::SeqCst);
  168. }
  169. /// @brief 判断虚拟地址是否合法
  170. fn virt_is_valid(virt: VirtAddr) -> bool {
  171. return virt.is_canonical();
  172. }
  173. /// 获取内存管理初始化时,创建的第一个内核页表的地址
  174. fn initial_page_table() -> PhysAddr {
  175. unsafe {
  176. return INITIAL_CR3_VALUE;
  177. }
  178. }
  179. /// @brief 创建新的顶层页表
  180. ///
  181. /// 该函数会创建页表并复制内核的映射到新的页表中
  182. ///
  183. /// @return 新的页表
  184. fn setup_new_usermapper() -> Result<crate::mm::ucontext::UserMapper, SystemError> {
  185. let new_umapper: crate::mm::page::PageMapper<X86_64MMArch, LockedFrameAllocator> = unsafe {
  186. PageMapper::create(PageTableKind::User, LockedFrameAllocator)
  187. .ok_or(SystemError::ENOMEM)?
  188. };
  189. let current_ktable: KernelMapper = KernelMapper::lock();
  190. let copy_mapping = |pml4_entry_no| unsafe {
  191. let entry: PageEntry<X86_64MMArch> = current_ktable
  192. .table()
  193. .entry(pml4_entry_no)
  194. .unwrap_or_else(|| panic!("entry {} not found", pml4_entry_no));
  195. new_umapper.table().set_entry(pml4_entry_no, entry)
  196. };
  197. // 复制内核的映射
  198. for pml4_entry_no in MMArch::PAGE_KERNEL_INDEX..MMArch::PAGE_ENTRY_NUM {
  199. copy_mapping(pml4_entry_no);
  200. }
  201. return Ok(crate::mm::ucontext::UserMapper::new(new_umapper));
  202. }
  203. const PAGE_SIZE: usize = 1 << Self::PAGE_SHIFT;
  204. const PAGE_OFFSET_MASK: usize = Self::PAGE_SIZE - 1;
  205. const PAGE_MASK: usize = !(Self::PAGE_OFFSET_MASK);
  206. const PAGE_ADDRESS_SHIFT: usize = Self::PAGE_LEVELS * Self::PAGE_ENTRY_SHIFT + Self::PAGE_SHIFT;
  207. const PAGE_ADDRESS_SIZE: usize = 1 << Self::PAGE_ADDRESS_SHIFT;
  208. const PAGE_ADDRESS_MASK: usize = Self::PAGE_ADDRESS_SIZE - Self::PAGE_SIZE;
  209. const PAGE_ENTRY_SIZE: usize = 1 << (Self::PAGE_SHIFT - Self::PAGE_ENTRY_SHIFT);
  210. const PAGE_ENTRY_NUM: usize = 1 << Self::PAGE_ENTRY_SHIFT;
  211. const PAGE_ENTRY_MASK: usize = Self::PAGE_ENTRY_NUM - 1;
  212. const PAGE_KERNEL_INDEX: usize = (Self::PHYS_OFFSET & Self::PAGE_ADDRESS_MASK)
  213. >> (Self::PAGE_ADDRESS_SHIFT - Self::PAGE_ENTRY_SHIFT);
  214. const PAGE_NEGATIVE_MASK: usize = !((Self::PAGE_ADDRESS_SIZE) - 1);
  215. const ENTRY_ADDRESS_SIZE: usize = 1 << Self::ENTRY_ADDRESS_SHIFT;
  216. const ENTRY_ADDRESS_MASK: usize = Self::ENTRY_ADDRESS_SIZE - Self::PAGE_SIZE;
  217. const ENTRY_FLAGS_MASK: usize = !Self::ENTRY_ADDRESS_MASK;
  218. unsafe fn read<T>(address: VirtAddr) -> T {
  219. return core::ptr::read(address.data() as *const T);
  220. }
  221. unsafe fn write<T>(address: VirtAddr, value: T) {
  222. core::ptr::write(address.data() as *mut T, value);
  223. }
  224. unsafe fn write_bytes(address: VirtAddr, value: u8, count: usize) {
  225. core::ptr::write_bytes(address.data() as *mut u8, value, count);
  226. }
  227. unsafe fn phys_2_virt(phys: PhysAddr) -> Option<VirtAddr> {
  228. if let Some(vaddr) = phys.data().checked_add(Self::PHYS_OFFSET) {
  229. return Some(VirtAddr::new(vaddr));
  230. } else {
  231. return None;
  232. }
  233. }
  234. unsafe fn virt_2_phys(virt: VirtAddr) -> Option<PhysAddr> {
  235. if let Some(paddr) = virt.data().checked_sub(Self::PHYS_OFFSET) {
  236. return Some(PhysAddr::new(paddr));
  237. } else {
  238. return None;
  239. }
  240. }
  241. #[inline(always)]
  242. fn make_entry(paddr: PhysAddr, page_flags: usize) -> usize {
  243. return paddr.data() | page_flags;
  244. }
  245. fn vma_access_permitted(
  246. vma: Arc<LockedVMA>,
  247. write: bool,
  248. execute: bool,
  249. foreign: bool,
  250. ) -> bool {
  251. if execute {
  252. return true;
  253. }
  254. if foreign | vma.is_foreign() {
  255. return true;
  256. }
  257. pkru::pkru_allows_pkey(pkru::vma_pkey(vma), write)
  258. }
  259. const PROTECTION_MAP: [EntryFlags<MMArch>; 16] = protection_map();
  260. const PAGE_NONE: usize =
  261. Self::ENTRY_FLAG_PRESENT | Self::ENTRY_FLAG_ACCESSED | Self::ENTRY_FLAG_GLOBAL;
  262. const PAGE_SHARED: usize = Self::ENTRY_FLAG_PRESENT
  263. | Self::ENTRY_FLAG_READWRITE
  264. | Self::ENTRY_FLAG_USER
  265. | Self::ENTRY_FLAG_ACCESSED
  266. | Self::ENTRY_FLAG_NO_EXEC;
  267. const PAGE_SHARED_EXEC: usize = Self::ENTRY_FLAG_PRESENT
  268. | Self::ENTRY_FLAG_READWRITE
  269. | Self::ENTRY_FLAG_USER
  270. | Self::ENTRY_FLAG_ACCESSED;
  271. const PAGE_COPY_NOEXEC: usize = Self::ENTRY_FLAG_PRESENT
  272. | Self::ENTRY_FLAG_USER
  273. | Self::ENTRY_FLAG_ACCESSED
  274. | Self::ENTRY_FLAG_NO_EXEC;
  275. const PAGE_COPY_EXEC: usize =
  276. Self::ENTRY_FLAG_PRESENT | Self::ENTRY_FLAG_USER | Self::ENTRY_FLAG_ACCESSED;
  277. const PAGE_COPY: usize = Self::ENTRY_FLAG_PRESENT
  278. | Self::ENTRY_FLAG_USER
  279. | Self::ENTRY_FLAG_ACCESSED
  280. | Self::ENTRY_FLAG_NO_EXEC;
  281. const PAGE_READONLY: usize = Self::ENTRY_FLAG_PRESENT
  282. | Self::ENTRY_FLAG_USER
  283. | Self::ENTRY_FLAG_ACCESSED
  284. | Self::ENTRY_FLAG_NO_EXEC;
  285. const PAGE_READONLY_EXEC: usize =
  286. Self::ENTRY_FLAG_PRESENT | Self::ENTRY_FLAG_USER | Self::ENTRY_FLAG_ACCESSED;
  287. const PAGE_READ: usize = 0;
  288. const PAGE_READ_EXEC: usize = 0;
  289. const PAGE_WRITE: usize = 0;
  290. const PAGE_WRITE_EXEC: usize = 0;
  291. const PAGE_EXEC: usize = 0;
  292. }
  293. /// 获取保护标志的映射表
  294. ///
  295. ///
  296. /// ## 返回值
  297. /// - `[usize; 16]`: 长度为16的映射表
  298. const fn protection_map() -> [EntryFlags<MMArch>; 16] {
  299. let mut map = [unsafe { EntryFlags::from_data(0) }; 16];
  300. unsafe {
  301. map[VmFlags::VM_NONE.bits()] = EntryFlags::from_data(MMArch::PAGE_NONE);
  302. map[VmFlags::VM_READ.bits()] = EntryFlags::from_data(MMArch::PAGE_READONLY);
  303. map[VmFlags::VM_WRITE.bits()] = EntryFlags::from_data(MMArch::PAGE_COPY);
  304. map[VmFlags::VM_WRITE.bits() | VmFlags::VM_READ.bits()] =
  305. EntryFlags::from_data(MMArch::PAGE_COPY);
  306. map[VmFlags::VM_EXEC.bits()] = EntryFlags::from_data(MMArch::PAGE_READONLY_EXEC);
  307. map[VmFlags::VM_EXEC.bits() | VmFlags::VM_READ.bits()] =
  308. EntryFlags::from_data(MMArch::PAGE_READONLY_EXEC);
  309. map[VmFlags::VM_EXEC.bits() | VmFlags::VM_WRITE.bits()] =
  310. EntryFlags::from_data(MMArch::PAGE_COPY_EXEC);
  311. map[VmFlags::VM_EXEC.bits() | VmFlags::VM_WRITE.bits() | VmFlags::VM_READ.bits()] =
  312. EntryFlags::from_data(MMArch::PAGE_COPY_EXEC);
  313. map[VmFlags::VM_SHARED.bits()] = EntryFlags::from_data(MMArch::PAGE_NONE);
  314. map[VmFlags::VM_SHARED.bits() | VmFlags::VM_READ.bits()] =
  315. EntryFlags::from_data(MMArch::PAGE_READONLY);
  316. map[VmFlags::VM_SHARED.bits() | VmFlags::VM_WRITE.bits()] =
  317. EntryFlags::from_data(MMArch::PAGE_SHARED);
  318. map[VmFlags::VM_SHARED.bits() | VmFlags::VM_WRITE.bits() | VmFlags::VM_READ.bits()] =
  319. EntryFlags::from_data(MMArch::PAGE_SHARED);
  320. map[VmFlags::VM_SHARED.bits() | VmFlags::VM_EXEC.bits()] =
  321. EntryFlags::from_data(MMArch::PAGE_READONLY_EXEC);
  322. map[VmFlags::VM_SHARED.bits() | VmFlags::VM_EXEC.bits() | VmFlags::VM_READ.bits()] =
  323. EntryFlags::from_data(MMArch::PAGE_READONLY_EXEC);
  324. map[VmFlags::VM_SHARED.bits() | VmFlags::VM_EXEC.bits() | VmFlags::VM_WRITE.bits()] =
  325. EntryFlags::from_data(MMArch::PAGE_SHARED_EXEC);
  326. map[VmFlags::VM_SHARED.bits()
  327. | VmFlags::VM_EXEC.bits()
  328. | VmFlags::VM_WRITE.bits()
  329. | VmFlags::VM_READ.bits()] = EntryFlags::from_data(MMArch::PAGE_SHARED_EXEC);
  330. }
  331. // if X86_64MMArch::is_xd_reserved() {
  332. // map.iter_mut().for_each(|x| *x &= !Self::ENTRY_FLAG_NO_EXEC)
  333. // }
  334. map
  335. }
  336. impl X86_64MMArch {
  337. fn init_xd_rsvd() {
  338. // 读取ia32-EFER寄存器的值
  339. let efer: EferFlags = x86_64::registers::model_specific::Efer::read();
  340. if !efer.contains(EferFlags::NO_EXECUTE_ENABLE) {
  341. // NO_EXECUTE_ENABLE是false,那么就设置xd_reserved为true
  342. debug!("NO_EXECUTE_ENABLE is false, set XD_RESERVED to true");
  343. XD_RESERVED.store(true, Ordering::Relaxed);
  344. }
  345. compiler_fence(Ordering::SeqCst);
  346. }
  347. /// 判断XD标志位是否被保留
  348. pub fn is_xd_reserved() -> bool {
  349. // return XD_RESERVED.load(Ordering::Relaxed);
  350. // 由于暂时不支持execute disable,因此直接返回true
  351. // 不支持的原因是,目前好像没有能正确的设置page-level的xd位,会触发page fault
  352. return true;
  353. }
  354. pub unsafe fn read_array<T>(addr: VirtAddr, count: usize) -> Vec<T> {
  355. // 实现读取数组逻辑
  356. let mut vec = Vec::with_capacity(count);
  357. for i in 0..count {
  358. vec.push(Self::read(addr + i * core::mem::size_of::<T>()));
  359. }
  360. vec
  361. }
  362. }
  363. impl VirtAddr {
  364. /// @brief 判断虚拟地址是否合法
  365. #[inline(always)]
  366. pub fn is_canonical(self) -> bool {
  367. let x = self.data() & X86_64MMArch::PHYS_OFFSET;
  368. // 如果x为0,说明虚拟地址的高位为0,是合法的用户地址
  369. // 如果x为PHYS_OFFSET,说明虚拟地址的高位全为1,是合法的内核地址
  370. return x == 0 || x == X86_64MMArch::PHYS_OFFSET;
  371. }
  372. }
  373. unsafe fn allocator_init() {
  374. let virt_offset = VirtAddr::new(page_align_up(BOOTSTRAP_MM_INFO.unwrap().start_brk));
  375. let phy_offset = unsafe { MMArch::virt_2_phys(virt_offset) }.unwrap();
  376. mem_block_manager()
  377. .reserve_block(PhysAddr::new(0), phy_offset.data())
  378. .expect("Failed to reserve block");
  379. let mut bump_allocator = BumpAllocator::<X86_64MMArch>::new(phy_offset.data());
  380. debug!(
  381. "BumpAllocator created, offset={:?}",
  382. bump_allocator.offset()
  383. );
  384. // 暂存初始在head.S中指定的页表的地址,后面再考虑是否需要把它加到buddy的可用空间里面!
  385. // 现在不加的原因是,我担心会有安全漏洞问题:这些初始的页表,位于内核的数据段。如果归还到buddy,
  386. // 可能会产生一定的安全风险(有的代码可能根据虚拟地址来进行安全校验)
  387. let _old_page_table = MMArch::table(PageTableKind::Kernel);
  388. let new_page_table: PhysAddr;
  389. // 使用bump分配器,把所有的内存页都映射到页表
  390. {
  391. // 用bump allocator创建新的页表
  392. let mut mapper: crate::mm::page::PageMapper<MMArch, &mut BumpAllocator<MMArch>> =
  393. crate::mm::page::PageMapper::<MMArch, _>::create(
  394. PageTableKind::Kernel,
  395. &mut bump_allocator,
  396. )
  397. .expect("Failed to create page mapper");
  398. new_page_table = mapper.table().phys();
  399. debug!("PageMapper created");
  400. // 取消最开始时候,在head.S中指定的映射(暂时不刷新TLB)
  401. {
  402. let table = mapper.table();
  403. let empty_entry = PageEntry::<MMArch>::from_usize(0);
  404. for i in 0..MMArch::PAGE_ENTRY_NUM {
  405. table
  406. .set_entry(i, empty_entry)
  407. .expect("Failed to empty page table entry");
  408. }
  409. }
  410. debug!("Successfully emptied page table");
  411. let total_num = mem_block_manager().total_initial_memory_regions();
  412. for i in 0..total_num {
  413. let area = mem_block_manager().get_initial_memory_region(i).unwrap();
  414. // debug!("area: base={:?}, size={:#x}, end={:?}", area.base, area.size, area.base + area.size);
  415. for i in 0..area.size.div_ceil(MMArch::PAGE_SIZE) {
  416. let paddr = area.base.add(i * MMArch::PAGE_SIZE);
  417. let vaddr = unsafe { MMArch::phys_2_virt(paddr) }.unwrap();
  418. let flags = kernel_page_flags::<MMArch>(vaddr);
  419. let flusher = mapper
  420. .map_phys(vaddr, paddr, flags)
  421. .expect("Failed to map frame");
  422. // 暂时不刷新TLB
  423. flusher.ignore();
  424. }
  425. }
  426. }
  427. unsafe {
  428. INITIAL_CR3_VALUE = new_page_table;
  429. }
  430. debug!(
  431. "After mapping all physical memory, DragonOS used: {} KB",
  432. bump_allocator.offset() / 1024
  433. );
  434. // 初始化buddy_allocator
  435. let buddy_allocator = unsafe { BuddyAllocator::<X86_64MMArch>::new(bump_allocator).unwrap() };
  436. // 设置全局的页帧分配器
  437. unsafe { set_inner_allocator(buddy_allocator) };
  438. info!("Successfully initialized buddy allocator");
  439. // 关闭显示输出
  440. scm_disable_put_to_window();
  441. // make the new page table current
  442. {
  443. let mut binding = INNER_ALLOCATOR.lock();
  444. let mut allocator_guard = binding.as_mut().unwrap();
  445. debug!("To enable new page table.");
  446. compiler_fence(Ordering::SeqCst);
  447. let mapper = crate::mm::page::PageMapper::<MMArch, _>::new(
  448. PageTableKind::Kernel,
  449. new_page_table,
  450. &mut allocator_guard,
  451. );
  452. compiler_fence(Ordering::SeqCst);
  453. mapper.make_current();
  454. compiler_fence(Ordering::SeqCst);
  455. debug!("New page table enabled");
  456. }
  457. debug!("Successfully enabled new page table");
  458. }
  459. #[no_mangle]
  460. pub extern "C" fn rs_test_buddy() {
  461. test_buddy();
  462. }
  463. pub fn test_buddy() {
  464. // 申请内存然后写入数据然后free掉
  465. // 总共申请200MB内存
  466. const TOTAL_SIZE: usize = 200 * 1024 * 1024;
  467. for i in 0..10 {
  468. debug!("Test buddy, round: {i}");
  469. // 存放申请的内存块
  470. let mut v: Vec<(PhysAddr, PageFrameCount)> = Vec::with_capacity(60 * 1024);
  471. // 存放已经申请的内存块的地址(用于检查重复)
  472. let mut addr_set: HashSet<PhysAddr> = HashSet::new();
  473. let mut allocated = 0usize;
  474. let mut free_count = 0usize;
  475. while allocated < TOTAL_SIZE {
  476. let mut random_size = 0u64;
  477. unsafe { x86::random::rdrand64(&mut random_size) };
  478. // 一次最多申请4M
  479. random_size %= 1024 * 4096;
  480. if random_size == 0 {
  481. continue;
  482. }
  483. let random_size =
  484. core::cmp::min(page_align_up(random_size as usize), TOTAL_SIZE - allocated);
  485. let random_size = PageFrameCount::from_bytes(random_size.next_power_of_two()).unwrap();
  486. // 获取帧
  487. let (paddr, allocated_frame_count) =
  488. unsafe { LockedFrameAllocator.allocate(random_size).unwrap() };
  489. assert!(allocated_frame_count.data().is_power_of_two());
  490. assert!(paddr.data() % MMArch::PAGE_SIZE == 0);
  491. unsafe {
  492. assert!(MMArch::phys_2_virt(paddr)
  493. .as_ref()
  494. .unwrap()
  495. .check_aligned(allocated_frame_count.data() * MMArch::PAGE_SIZE));
  496. }
  497. allocated += allocated_frame_count.data() * MMArch::PAGE_SIZE;
  498. v.push((paddr, allocated_frame_count));
  499. assert!(addr_set.insert(paddr), "duplicate address: {:?}", paddr);
  500. // 写入数据
  501. let vaddr = unsafe { MMArch::phys_2_virt(paddr).unwrap() };
  502. let slice = unsafe {
  503. core::slice::from_raw_parts_mut(
  504. vaddr.data() as *mut u8,
  505. allocated_frame_count.data() * MMArch::PAGE_SIZE,
  506. )
  507. };
  508. for (i, item) in slice.iter_mut().enumerate() {
  509. *item = ((i + unsafe { rdtsc() } as usize) % 256) as u8;
  510. }
  511. // 随机释放一个内存块
  512. if !v.is_empty() {
  513. let mut random_index = 0u64;
  514. unsafe { x86::random::rdrand64(&mut random_index) };
  515. // 70%概率释放
  516. if random_index % 10 > 7 {
  517. continue;
  518. }
  519. random_index %= v.len() as u64;
  520. let random_index = random_index as usize;
  521. let (paddr, allocated_frame_count) = v.remove(random_index);
  522. assert!(addr_set.remove(&paddr));
  523. unsafe { LockedFrameAllocator.free(paddr, allocated_frame_count) };
  524. free_count += allocated_frame_count.data() * MMArch::PAGE_SIZE;
  525. }
  526. }
  527. debug!(
  528. "Allocated {} MB memory, release: {} MB, no release: {} bytes",
  529. allocated / 1024 / 1024,
  530. free_count / 1024 / 1024,
  531. (allocated - free_count)
  532. );
  533. debug!("Now, to release buddy memory");
  534. // 释放所有的内存
  535. for (paddr, allocated_frame_count) in v {
  536. unsafe { LockedFrameAllocator.free(paddr, allocated_frame_count) };
  537. assert!(addr_set.remove(&paddr));
  538. free_count += allocated_frame_count.data() * MMArch::PAGE_SIZE;
  539. }
  540. debug!("release done!, allocated: {allocated}, free_count: {free_count}");
  541. }
  542. }
  543. /// 全局的页帧分配器
  544. #[derive(Debug, Clone, Copy, Hash)]
  545. pub struct LockedFrameAllocator;
  546. impl FrameAllocator for LockedFrameAllocator {
  547. unsafe fn allocate(&mut self, mut count: PageFrameCount) -> Option<(PhysAddr, PageFrameCount)> {
  548. count = count.next_power_of_two();
  549. if let Some(ref mut allocator) = *INNER_ALLOCATOR.lock_irqsave() {
  550. return allocator.allocate(count);
  551. } else {
  552. return None;
  553. }
  554. }
  555. unsafe fn free(&mut self, address: crate::mm::PhysAddr, count: PageFrameCount) {
  556. assert!(count.data().is_power_of_two());
  557. if let Some(ref mut allocator) = *INNER_ALLOCATOR.lock_irqsave() {
  558. return allocator.free(address, count);
  559. }
  560. }
  561. unsafe fn usage(&self) -> PageFrameUsage {
  562. if let Some(ref mut allocator) = *INNER_ALLOCATOR.lock_irqsave() {
  563. return allocator.usage();
  564. } else {
  565. panic!("usage error");
  566. }
  567. }
  568. }
  569. /// 获取内核地址默认的页面标志
  570. pub unsafe fn kernel_page_flags<A: MemoryManagementArch>(virt: VirtAddr) -> EntryFlags<A> {
  571. let info: X86_64MMBootstrapInfo = BOOTSTRAP_MM_INFO.unwrap();
  572. if virt.data() >= info.kernel_code_start && virt.data() < info.kernel_code_end {
  573. // Remap kernel code execute
  574. return EntryFlags::new().set_execute(true).set_write(true);
  575. } else if virt.data() >= info.kernel_data_end && virt.data() < info.kernel_rodata_end {
  576. // Remap kernel rodata read only
  577. return EntryFlags::new().set_execute(true);
  578. } else {
  579. return EntryFlags::new().set_write(true).set_execute(true);
  580. }
  581. }
  582. unsafe fn set_inner_allocator(allocator: BuddyAllocator<MMArch>) {
  583. static FLAG: AtomicBool = AtomicBool::new(false);
  584. if FLAG
  585. .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
  586. .is_err()
  587. {
  588. panic!("Cannot set inner allocator twice!");
  589. }
  590. *INNER_ALLOCATOR.lock() = Some(allocator);
  591. }
  592. /// 低地址重映射的管理器
  593. ///
  594. /// 低地址重映射的管理器,在smp初始化完成之前,需要使用低地址的映射,因此需要在smp初始化完成之后,取消这一段映射
  595. pub struct LowAddressRemapping;
  596. impl LowAddressRemapping {
  597. // 映射64M
  598. const REMAP_SIZE: usize = 64 * 1024 * 1024;
  599. pub unsafe fn remap_at_low_address(mapper: &mut PageMapper) {
  600. for i in 0..(Self::REMAP_SIZE / MMArch::PAGE_SIZE) {
  601. let paddr = PhysAddr::new(i * MMArch::PAGE_SIZE);
  602. let vaddr = VirtAddr::new(i * MMArch::PAGE_SIZE);
  603. let flags = kernel_page_flags::<MMArch>(vaddr);
  604. let flusher = mapper
  605. .map_phys(vaddr, paddr, flags)
  606. .expect("Failed to map frame");
  607. // 暂时不刷新TLB
  608. flusher.ignore();
  609. }
  610. }
  611. /// 取消低地址的映射
  612. pub unsafe fn unmap_at_low_address(mapper: &mut PageMapper, flush: bool) {
  613. for i in 0..(Self::REMAP_SIZE / MMArch::PAGE_SIZE) {
  614. let vaddr = VirtAddr::new(i * MMArch::PAGE_SIZE);
  615. let (_, _, flusher) = mapper
  616. .unmap_phys(vaddr, true)
  617. .expect("Failed to unmap frame");
  618. if !flush {
  619. flusher.ignore();
  620. }
  621. }
  622. }
  623. }