strat9_kernel/memory/
paging.rs

1//! Virtual Memory Management (Paging) for Strat9-OS
2//!
3//! Uses the `x86_64` crate's `OffsetPageTable` which is designed for the HHDM
4//! (Higher Half Direct Map) pattern : exactly what Limine provides.
5//!
6//! Provides map/unmap/translate operations on the active page table.
7
8use x86_64::{
9    registers::control::Cr3,
10    structures::paging::{
11        FrameAllocator as X86FrameAllocator, Mapper, OffsetPageTable, Page, PageTable,
12        PageTableFlags, PhysFrame as X86PhysFrame, Size4KiB, Translate,
13    },
14    PhysAddr, VirtAddr,
15};
16
17use crate::{
18    memory::frame::{FrameAllocOptions, FramePurpose},
19    sync::SpinLock,
20};
21
22/// Wrapper around the buddy allocator implementing the x86_64 crate's `FrameAllocator` trait.
23///
24/// Used by `OffsetPageTable` when it needs a new intermediate page-table node
25/// (PML4 / PDPT / PD / PT).
26///
27/// # Safety invariant: page-table frames MUST be zeroed
28///
29/// The x86_64 CPU page-table walker reads all 512 entries of every
30/// intermediate node it traverses, regardless of which entries are "in use".
31/// If a newly allocated page-table frame contains stale bytes (left behind by
32/// the slab allocator or a previous allocation), any non-zero entry is decoded
33/// as a valid PTE pointing to an arbitrary physical address.  The first fetch
34/// from such an address becomes the new RIP after the Ring 3 transition :
35/// explaining why RIP is non-deterministic across boots.
36///
37/// `BuddyFrameAllocator` enforces zeroing via `FrameAllocOptions::new()
38///  .purpose(FramePurpose::PageTable)` which:
39///
40///  1. Calls the buddy allocator for a raw order-0 frame.
41///  2. CAS-claims the frame via the [`MetaSlot`](crate::memory::MetaSlot) refcount field
42///     (`REFCOUNT_UNUSED` → `1`).
43///  3. Zeros the 4 KiB with a single `ptr::write_bytes` through the HHDM.
44///  4. Sets purpose flags on the [`MetaSlot`](crate::memory::MetaSlot) with `Release` ordering.
45///  5. Stores `refcount = 1` with `Release` ordering so any future reader
46///     that loads the refcount with `Acquire` observes a fully-initialised frame.
47///
48/// This matches the Asterinas OSTD pattern (`FrameAllocOptions` + per-frame
49/// [`MetaSlot`](crate::memory::MetaSlot) with refcount CAS). Metadata lives in
50/// dedicated slots (not in mapped page bytes); see [`get_meta_slot`](crate::memory::get_meta_slot).
51pub struct BuddyFrameAllocator;
52
53// SAFETY: `BuddyFrameAllocator::allocate_frame` returns 4KiB-aligned,
54// exclusively-owned physical frames.  Exclusive ownership is guaranteed by
55// the buddy's own bitmap + free-list discipline.  Frames allocated with
56// `FramePurpose::PageTable` are always fully zeroed before being returned.
57unsafe impl X86FrameAllocator<Size4KiB> for BuddyFrameAllocator {
58    fn allocate_frame(&mut self) -> Option<X86PhysFrame<Size4KiB>> {
59        // SAFETY: `BuddyFrameAllocator` is only ever called from within
60        // `OffsetPageTable` during page-table operations.  Those occur either
61        // during single-threaded early boot, or while the caller holds a lock
62        // that disables IRQs (e.g. the scheduler SpinLock, the AddressSpace
63        // lock).  IRQs are therefore guaranteed to be disabled.
64        let token = unsafe { crate::sync::IrqDisabledToken::token_from_trusted_context() };
65
66        // `PageTable` purpose enforces:
67        //  - `zeroed = true` unconditionally (cannot be overridden by callers).
68        //  - `FrameMeta::flags` stamped with `KERNEL | ALLOCATED`.
69        //  - `FrameMeta::refcount` set to 1 with `Release` ordering after
70        //    zeroing, so any `Acquire` load of the refcount observes a clean
71        //    frame.
72        let frame = FrameAllocOptions::new()
73            .purpose(FramePurpose::PageTable)
74            .allocate(&token)
75            .ok()?;
76
77        X86PhysFrame::from_start_address(frame.start_address).ok()
78    }
79}
80
81/// Paging initialization flag.
82static mut PAGING_READY: bool = false;
83
84/// Physical address of the kernel's level-4 page table (set at init, never changes).
85static mut KERNEL_CR3: PhysAddr = PhysAddr::new_truncate(0);
86
87/// Serializes mutations of the canonical kernel page tables.
88///
89/// The active-CR3 mapping helpers are still caller-synchronized by their own
90/// higher-level address-space locks, but kernel-global mappings such as vmalloc
91/// must not race while allocating or wiring intermediate page-table levels.
92static KERNEL_PT_LOCK: SpinLock<()> = SpinLock::new(());
93
94/// Returns whether initialized.
95pub fn is_initialized() -> bool {
96    unsafe { *(&raw const PAGING_READY) }
97}
98
99/// Initialize the paging subsystem.
100///
101/// Reads the active CR3 (level-4 page table) and creates an `OffsetPageTable`
102/// mapper using the HHDM offset for physical-to-virtual translation.
103///
104/// Must be called after the buddy allocator and HHDM offset are initialized.
105pub fn init(hhdm_offset: u64) {
106    let phys_offset = VirtAddr::new(hhdm_offset);
107    let (level_4_frame, _flags) = Cr3::read();
108    let level_4_phys = level_4_frame.start_address().as_u64();
109    let level_4_virt = phys_offset + level_4_phys;
110
111    // SAFETY: called once during single-threaded init. The HHDM offset correctly
112    // maps all physical RAM to virtual addresses. CR3 points to a valid page table
113    // set up by Limine.
114    unsafe {
115        let kcr3 = &raw mut KERNEL_CR3;
116        *kcr3 = level_4_frame.start_address();
117        let ready = &raw mut PAGING_READY;
118        *ready = true;
119    }
120
121    log::info!(
122        "Paging initialized: CR3={:#x}, HHDM={:#x}, L4 table @ {:#x}",
123        level_4_phys,
124        hhdm_offset,
125        level_4_virt.as_u64(),
126    );
127}
128
129/// Map all RAM regions from the memory map into the HHDM.
130///
131/// This ensures that every byte of physical RAM is accessible through the
132/// higher-half direct map. Should be called after paging::init.
133/// Fix for VMWare Workstation which doesn't identity-map all RAM by default, causing
134/// the kernel to crash when it tries to access unmapped RAM (e.g. for the buddy allocator's
135/// metadata array). Limine's initial map only covers the first 1GB of RAM, which is not enough
136/// for our 2GB test VM. This function lazily maps any missing RAM regions on
137/// demand using `ensure_identity_map_range()`, which checks if the region is already mapped
138/// before mapping it. This allows the kernel to boot successfully on VMWare Workstation without
139/// requiring changes to the bootloader or Limine configuration.
140///
141pub fn map_all_ram(memory_regions: &[crate::boot::entry::MemoryRegion]) {
142    use crate::boot::entry::MemoryKind;
143
144    for region in memory_regions {
145        if matches!(region.kind, MemoryKind::Free | MemoryKind::Reclaim) {
146            log::debug!(
147                "Mapping RAM region to HHDM: phys=0x{:x}..0x{:x}",
148                region.base,
149                region.base + region.size
150            );
151            ensure_identity_map_range(region.base, region.size);
152        }
153    }
154}
155
156/// Map a virtual page to a physical frame with the given flags.
157///
158/// Intermediate page tables are allocated from the buddy allocator as needed.
159pub fn map_page(
160    page: Page<Size4KiB>,
161    frame: X86PhysFrame<Size4KiB>,
162    flags: PageTableFlags,
163) -> Result<(), &'static str> {
164    if !is_initialized() {
165        return Err("Paging not initialized");
166    }
167    let phys_offset = VirtAddr::new(crate::memory::hhdm_offset());
168    let (level_4_frame, _) = Cr3::read();
169    let level_4_virt = phys_offset + level_4_frame.start_address().as_u64();
170    // SAFETY: level_4_virt points to the active CR3 PML4 via HHDM.
171    let mapper = unsafe { &mut *level_4_virt.as_mut_ptr::<PageTable>() };
172    let mut mapper = unsafe { OffsetPageTable::new(mapper, phys_offset) };
173    let mut allocator = BuddyFrameAllocator;
174
175    unsafe {
176        mapper
177            .map_to(page, frame, flags, &mut allocator)
178            .map_err(|_| "Failed to map page")?
179            .flush();
180    }
181    Ok(())
182}
183
184/// Map a page into the kernel's canonical page tables (not the active CR3).
185///
186/// This ensures that the mapping is visible from all address spaces, because
187/// every user address space clones the kernel half (PML4[256..512]) from the
188/// kernel's L4 table at creation time.
189///
190/// Used by vmalloc so that heap allocations are kernel-global.
191/// Intermediate page tables are allocated from the buddy allocator as needed.
192pub fn map_page_kernel(
193    page: Page<Size4KiB>,
194    frame: X86PhysFrame<Size4KiB>,
195    flags: PageTableFlags,
196) -> Result<(), &'static str> {
197    if !is_initialized() {
198        return Err("Paging not initialized");
199    }
200    let _guard = KERNEL_PT_LOCK.lock();
201    // SAFETY: KERNEL_CR3 is set once during init and never changes.
202    let kernel_cr3 = unsafe { *(&raw const KERNEL_CR3) };
203    let phys_offset = VirtAddr::new(crate::memory::hhdm_offset());
204    let level_4_virt = phys_offset + kernel_cr3.as_u64();
205    // SAFETY: level_4_virt points to the kernel's L4 table via HHDM.
206    let mapper = unsafe { &mut *level_4_virt.as_mut_ptr::<PageTable>() };
207    let mut mapper = unsafe { OffsetPageTable::new(mapper, phys_offset) };
208    let mut allocator = BuddyFrameAllocator;
209
210    unsafe {
211        mapper
212            .map_to(page, frame, flags, &mut allocator)
213            .map_err(|_| "Failed to map page (kernel)")?
214            .flush();
215    }
216    Ok(())
217}
218
219/// Unmap a page from the active CR3, returning the physical frame.
220pub fn unmap_page(page: Page<Size4KiB>) -> Result<X86PhysFrame<Size4KiB>, &'static str> {
221    if !is_initialized() {
222        return Err("Paging not initialized");
223    }
224    let phys_offset = VirtAddr::new(crate::memory::hhdm_offset());
225    let (level_4_frame, _) = Cr3::read();
226    let level_4_virt = phys_offset + level_4_frame.start_address().as_u64();
227    // SAFETY: level_4_virt points to the active CR3 PML4 via HHDM.
228    let mapper = unsafe { &mut *level_4_virt.as_mut_ptr::<PageTable>() };
229    let mut mapper = unsafe { OffsetPageTable::new(mapper, phys_offset) };
230    let (frame, flush) = mapper.unmap(page).map_err(|_| "Failed to unmap page")?;
231    flush.flush();
232    Ok(frame)
233}
234
235/// Unmap a page from the kernel's canonical page tables.
236///
237/// This is the counterpart to `map_page_kernel`. It removes the mapping from
238/// the kernel's L4 table so that the page is no longer visible in any address
239/// space.
240pub fn unmap_page_kernel(page: Page<Size4KiB>) -> Result<X86PhysFrame<Size4KiB>, &'static str> {
241    if !is_initialized() {
242        return Err("Paging not initialized");
243    }
244    let _guard = KERNEL_PT_LOCK.lock();
245    // SAFETY: KERNEL_CR3 is set once during init and never changes.
246    let kernel_cr3 = unsafe { *(&raw const KERNEL_CR3) };
247    let phys_offset = VirtAddr::new(crate::memory::hhdm_offset());
248    let level_4_virt = phys_offset + kernel_cr3.as_u64();
249    // SAFETY: level_4_virt points to the kernel's L4 table via HHDM.
250    let mapper = unsafe { &mut *level_4_virt.as_mut_ptr::<PageTable>() };
251    let mut mapper = unsafe { OffsetPageTable::new(mapper, phys_offset) };
252    let (frame, flush) = mapper
253        .unmap(page)
254        .map_err(|_| "Failed to unmap page (kernel)")?;
255    flush.flush();
256    Ok(frame)
257}
258
259/// Translate a virtual address to its mapped physical address.
260///
261/// Returns `None` if the address is not mapped.
262pub fn translate(addr: VirtAddr) -> Option<PhysAddr> {
263    if !is_initialized() {
264        return None;
265    }
266    let phys_offset = VirtAddr::new(crate::memory::hhdm_offset());
267    let (level_4_frame, _) = Cr3::read();
268    let level_4_virt = phys_offset + level_4_frame.start_address().as_u64();
269    // SAFETY: level_4_virt points to the active CR3 PML4 via HHDM.
270    let mapper = unsafe { &mut *level_4_virt.as_mut_ptr::<PageTable>() };
271    let mapper = unsafe { OffsetPageTable::new(mapper, phys_offset) };
272    mapper.translate_addr(addr)
273}
274
275fn translate_via_active_page_tables(addr: VirtAddr) -> Option<PhysAddr> {
276    let hhdm = crate::memory::hhdm_offset();
277    let (level_4_frame, _) = Cr3::read();
278
279    unsafe {
280        let l4_ptr = (level_4_frame.start_address().as_u64() + hhdm) as *const u64;
281        let l4e = *l4_ptr.add(((addr.as_u64() >> 39) & 0x1FF) as usize);
282        if l4e & 1 == 0 {
283            return None;
284        }
285
286        let l3_ptr = ((l4e & 0x000F_FFFF_FFFF_F000) + hhdm) as *const u64;
287        let l3e = *l3_ptr.add(((addr.as_u64() >> 30) & 0x1FF) as usize);
288        if l3e & 1 == 0 {
289            return None;
290        }
291        if l3e & 0x80 != 0 {
292            return Some(PhysAddr::new(
293                (l3e & 0x000F_FFFF_C000_0000) + (addr.as_u64() & 0x3FFF_FFFF),
294            ));
295        }
296
297        let l2_ptr = ((l3e & 0x000F_FFFF_FFFF_F000) + hhdm) as *const u64;
298        let l2e = *l2_ptr.add(((addr.as_u64() >> 21) & 0x1FF) as usize);
299        if l2e & 1 == 0 {
300            return None;
301        }
302        if l2e & 0x80 != 0 {
303            return Some(PhysAddr::new(
304                (l2e & 0x000F_FFFF_FFE0_0000) + (addr.as_u64() & 0x1F_FFFF),
305            ));
306        }
307
308        let l1_ptr = ((l2e & 0x000F_FFFF_FFFF_F000) + hhdm) as *const u64;
309        let l1e = *l1_ptr.add(((addr.as_u64() >> 12) & 0x1FF) as usize);
310        if l1e & 1 == 0 {
311            return None;
312        }
313
314        Some(PhysAddr::new(
315            (l1e & 0x000F_FFFF_FFFF_F000) + (addr.as_u64() & 0xFFF),
316        ))
317    }
318}
319
320/// Returns whether the current page tables map the HHDM view of the whole range.
321///
322/// This helper is safe before `paging::init()` and is intended for early boot
323/// allocators that must only touch memory already reachable through the current
324/// firmware-provided direct map.
325pub fn is_hhdm_range_mapped_now(phys_base: u64, size: u64) -> bool {
326    if size == 0 {
327        return true;
328    }
329
330    let start = phys_base & !0xFFF;
331    let end = phys_base.saturating_add(size).saturating_add(0xFFF) & !0xFFF;
332
333    let mut phys = start;
334    while phys < end {
335        let virt = VirtAddr::new(crate::memory::phys_to_virt(phys));
336        let Some(mapped) = translate_via_active_page_tables(virt) else {
337            return false;
338        };
339        if mapped.as_u64() != phys {
340            return false;
341        }
342        phys = phys.saturating_add(4096);
343    }
344    true
345}
346
347/// Read the current CR3 value (physical address of the active level-4 page table).
348pub fn active_page_table() -> PhysAddr {
349    let (frame, _) = Cr3::read();
350    frame.start_address()
351}
352
353/// Return the physical address of the kernel's level-4 page table.
354///
355/// This is the CR3 value captured at init time : used by `AddressSpace::new_user()`
356/// to clone kernel mappings (PML4 entries 256..512) into new address spaces.
357pub fn kernel_l4_phys() -> PhysAddr {
358    // SAFETY: Written once during single-threaded init, read-only after that.
359    unsafe { *(&raw const KERNEL_CR3) }
360}
361
362/// Ensure a physical address is identity-mapped in the HHDM region.
363///
364/// If the page is not present, it is mapped with Read/Write permissions.
365/// This is used to lazily map MMIO or legacy BIOS regions (like ACPI tables)
366/// that might have been skipped by the bootloader's initial map.
367pub fn ensure_identity_map(phys_addr: u64) {
368    let virt_addr = crate::memory::phys_to_virt(phys_addr);
369    let page = Page::<Size4KiB>::containing_address(VirtAddr::new(virt_addr));
370    let frame = X86PhysFrame::containing_address(PhysAddr::new(phys_addr));
371
372    if translate(VirtAddr::new(virt_addr)).is_none() {
373        log::debug!(
374            "Identity mapping missing page: {:#x} -> {:#x}",
375            phys_addr,
376            virt_addr
377        );
378        // Map as Present | Writable (generic safe default for MMIO/BIOS)
379        let flags = PageTableFlags::PRESENT | PageTableFlags::WRITABLE;
380        if let Err(e) = map_page(page, frame, flags) {
381            log::error!("Failed to identity map {:#x}: {}", phys_addr, e);
382        }
383    }
384}
385
386/// Ensure a physical range is mapped in the HHDM region.
387///
388/// Builds a single `OffsetPageTable` for the entire range instead of
389/// one per page, and emits a single summary log instead of per-page noise.
390pub fn ensure_identity_map_range(phys_base: u64, size: u64) {
391    if size == 0 || !is_initialized() {
392        return;
393    }
394
395    let page_size = 4096u64;
396    let start = phys_base & !(page_size - 1);
397    let end = (phys_base.saturating_add(size).saturating_add(page_size - 1)) & !(page_size - 1);
398    if start >= end {
399        return;
400    }
401
402    let phys_offset = VirtAddr::new(crate::memory::hhdm_offset());
403    let (level_4_frame, _) = Cr3::read();
404    let level_4_virt = phys_offset + level_4_frame.start_address().as_u64();
405
406    // SAFETY: level_4_virt points to the active CR3 PML4 via HHDM.
407    let l4_table = unsafe { &mut *level_4_virt.as_mut_ptr::<PageTable>() };
408    let mut mapper = unsafe { OffsetPageTable::new(l4_table, phys_offset) };
409    let mut allocator = BuddyFrameAllocator;
410    let flags = PageTableFlags::PRESENT | PageTableFlags::WRITABLE;
411
412    let mut mapped_count: u64 = 0;
413    let mut p = start;
414    while p < end {
415        let virt = VirtAddr::new(crate::memory::phys_to_virt(p));
416        // Only map if not already present.
417        if mapper.translate_addr(virt).is_none() {
418            let page = Page::<Size4KiB>::containing_address(virt);
419            let frame = X86PhysFrame::containing_address(PhysAddr::new(p));
420            // SAFETY: frame is a valid physical page; mapper uses HHDM offset.
421            match unsafe { mapper.map_to(page, frame, flags, &mut allocator) } {
422                Ok(flush) => {
423                    flush.flush();
424                    mapped_count += 1;
425                }
426                Err(_) => {
427                    log::error!("ensure_identity_map_range: failed to map {:#x}", p);
428                }
429            }
430        }
431        p = p.saturating_add(page_size);
432    }
433
434    if mapped_count > 0 {
435        log::debug!(
436            "Identity mapped {} pages: phys {:#x}..{:#x}",
437            mapped_count,
438            start,
439            end,
440        );
441    }
442}
strat9_kernel/memory/paging.rs

strat9_kernel/memory/
paging.rs
