strat9_kernel/syscall/
fork.rs

1//! `fork()` syscall implementation with copy-on-write (COW).
2//!
3//! This module implements the `fork()` syscall, which creates a new child process by cloning the
4//! calling process. The child gets a copy of the parent's address space, but the actual physical
5//! memory is shared between parent and child until either of them writes to it, at which point
6//! the kernel transparently creates a private copy for the writing process (copy-on-write).
7//!
8//! The main entry point is `sys_fork`, which performs the necessary checks, clones the address space
9//! with COW semantics, and creates a new `Task` for the child process. The
10//! `handle_cow_fault` function is called from the page fault handler when a write fault occurs on a COW page, and it resolves the fault by either making the page writable (if the faulting process is the sole owner) or by copying the page to a new frame and updating the mapping.
11//!
12//! Source : https://man7.org/linux/man2/fork.2.html
13//!          https://man7.org/linux/man2/vfork.2.html
14//!          https://man7.org/linux/man2/clone.2.html
15//!
16//! TODO: implement `vfork()` and `clone()` with more fine-grained control over sharing.
17//!
18//! COW :
19//!   https://en.wikipedia.org/wiki/Copy-on-write
20//!   https://lwn.net/Articles/531114/
21//!   
22//!   
23//!   
24//!
25
26use crate::{
27    arch::xshim::TranslateResult,
28    memory::{resolve_handle, AddressSpace, EffectiveMapping, VmaPageSize},
29    process::{
30        current_task_clone,
31        scheduler::add_task_with_parent,
32        signal::{SigActionData, SigStack, SignalSet},
33        task::{CpuContext, KernelStack, Pid, SyncUnsafeCell, Task},
34        TaskId, TaskState,
35    },
36    syscall::{error::SyscallError, SyscallFrame},
37};
38use alloc::{boxed::Box, sync::Arc};
39use core::{
40    mem::offset_of,
41    sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering},
42};
43/// Result returned by [`sys_fork`].
44pub struct ForkResult {
45    pub child_pid: Pid,
46}
47
48/// Performs the local invlpg operation.
49#[inline]
50fn local_invlpg(vaddr: u64) {
51    // Local TLB invalidation is sufficient here: this kernel currently runs
52    // one task per user address space (no shared user CR3 across CPUs).
53    crate::arch::tlb::local_page(crate::arch::xshim::VirtAddr::new(vaddr));
54}
55
56#[repr(C)]
57#[derive(Clone, Copy)]
58struct ForkUserContext {
59    r15: u64,
60    r14: u64,
61    r13: u64,
62    r12: u64,
63    rbp: u64,
64    rbx: u64,
65    r11: u64,
66    r10: u64,
67    r9: u64,
68    r8: u64,
69    rsi: u64,
70    rdi: u64,
71    rdx: u64,
72    rcx: u64,
73    user_rip: u64,
74    user_cs: u64,
75    user_rflags: u64,
76    user_rsp: u64,
77    user_ss: u64,
78}
79
80const OFF_R15: usize = offset_of!(ForkUserContext, r15);
81const OFF_R14: usize = offset_of!(ForkUserContext, r14);
82const OFF_R13: usize = offset_of!(ForkUserContext, r13);
83const OFF_R12: usize = offset_of!(ForkUserContext, r12);
84const OFF_RBP: usize = offset_of!(ForkUserContext, rbp);
85const OFF_RBX: usize = offset_of!(ForkUserContext, rbx);
86const OFF_R11: usize = offset_of!(ForkUserContext, r11);
87const OFF_R10: usize = offset_of!(ForkUserContext, r10);
88const OFF_R9: usize = offset_of!(ForkUserContext, r9);
89const OFF_R8: usize = offset_of!(ForkUserContext, r8);
90const OFF_RSI: usize = offset_of!(ForkUserContext, rsi);
91const OFF_RDI: usize = offset_of!(ForkUserContext, rdi);
92const OFF_RDX: usize = offset_of!(ForkUserContext, rdx);
93const OFF_RCX: usize = offset_of!(ForkUserContext, rcx);
94const OFF_USER_RIP: usize = offset_of!(ForkUserContext, user_rip);
95const OFF_USER_CS: usize = offset_of!(ForkUserContext, user_cs);
96const OFF_USER_RFLAGS: usize = offset_of!(ForkUserContext, user_rflags);
97const OFF_USER_RSP: usize = offset_of!(ForkUserContext, user_rsp);
98const OFF_USER_SS: usize = offset_of!(ForkUserContext, user_ss);
99
100/// Child bootstrap: restore user register snapshot and enter Ring 3.
101extern "C" fn fork_child_start(ctx_ptr: u64) -> ! {
102    let boxed = unsafe { Box::from_raw(ctx_ptr as *mut ForkUserContext) };
103    let ctx = *boxed;
104    unsafe { fork_iret_from_ctx(&ctx as *const ForkUserContext) }
105}
106
107/// Performs the fork iret from ctx operation.
108#[unsafe(naked)]
109unsafe extern "C" fn fork_iret_from_ctx(_ctx: *const ForkUserContext) -> ! {
110    core::arch::naked_asm!(
111        // Mask IRQs before touching GS. The user RFLAGS frame re-enables IF.
112        "cli",
113        "mov rsi, rdi",
114
115        // ===== Build IRET frame FIRST, using r8 as scratch ===========
116        // (r8 has not been restored yet, so we can clobber it safely)
117        "mov r8, [rsi + {off_user_ss}]",
118        "push r8",                            // SS
119        "mov r8, [rsi + {off_user_rsp}]",
120        "push r8",                            // user RSP
121        "mov r8, [rsi + {off_user_rflags}]",
122        "push r8",                            // user RFLAGS
123        "mov r8, [rsi + {off_user_cs}]",
124        "push r8",                            // CS
125        "mov r8, [rsi + {off_user_rip}]",
126        "push r8",                            // user RIP
127
128        // ===== Now restore ALL general-purpose registers============
129        "mov r15, [rsi + {off_r15}]",
130        "mov r14, [rsi + {off_r14}]",
131        "mov r13, [rsi + {off_r13}]",
132        "mov r12, [rsi + {off_r12}]",
133        "mov rbp, [rsi + {off_rbp}]",
134        "mov rbx, [rsi + {off_rbx}]",
135        "mov r11, [rsi + {off_r11}]",
136        "mov r10, [rsi + {off_r10}]",
137        "mov r9,  [rsi + {off_r9}]",
138        "mov r8,  [rsi + {off_r8}]",          // r8 now gets its correct value
139        "mov rdx, [rsi + {off_rdx}]",
140        "mov rcx, [rsi + {off_rcx}]",
141        "mov rdi, [rsi + {off_rdi}]",
142        "mov rax, 0",                         // child fork() returns 0
143        "mov rsi, [rsi + {off_rsi}]",         // rsi restored last
144        "swapgs",
145        "iretq",
146        off_r15 = const OFF_R15,
147        off_r14 = const OFF_R14,
148        off_r13 = const OFF_R13,
149        off_r12 = const OFF_R12,
150        off_rbp = const OFF_RBP,
151        off_rbx = const OFF_RBX,
152        off_r11 = const OFF_R11,
153        off_r10 = const OFF_R10,
154        off_r9 = const OFF_R9,
155        off_r8 = const OFF_R8,
156        off_rsi = const OFF_RSI,
157        off_rdi = const OFF_RDI,
158        off_rdx = const OFF_RDX,
159        off_rcx = const OFF_RCX,
160        off_user_rip = const OFF_USER_RIP,
161        off_user_cs = const OFF_USER_CS,
162        off_user_rflags = const OFF_USER_RFLAGS,
163        off_user_rsp = const OFF_USER_RSP,
164        off_user_ss = const OFF_USER_SS,
165    );
166}
167
168/// Performs the build child task operation.
169fn build_child_task(
170    parent: &Arc<Task>,
171    child_as: Arc<AddressSpace>,
172    bootstrap_ctx: Box<ForkUserContext>,
173) -> Result<Arc<Task>, SyscallError> {
174    let kernel_stack =
175        KernelStack::allocate(Task::DEFAULT_STACK_SIZE).map_err(|_| SyscallError::OutOfMemory)?;
176    let context = CpuContext::new(fork_child_start as *const () as u64, &kernel_stack);
177
178    let parent_caps = unsafe { (&*parent.process.capabilities.get()).clone() };
179    let parent_fd = unsafe { (&*parent.process.fd_table.get()).clone_for_fork() };
180    let parent_blocked = parent.blocked_signals.clone();
181    let parent_actions: [SigActionData; 64] = unsafe { *parent.process.signal_actions.get() };
182    let parent_sigstack: Option<SigStack> = unsafe { *parent.signal_stack.get() };
183    let interrupt_frame = crate::syscall::SyscallFrame {
184        r15: bootstrap_ctx.r15,
185        r14: bootstrap_ctx.r14,
186        r13: bootstrap_ctx.r13,
187        r12: bootstrap_ctx.r12,
188        rbp: bootstrap_ctx.rbp,
189        rbx: bootstrap_ctx.rbx,
190        r11: bootstrap_ctx.r11,
191        r10: bootstrap_ctx.r10,
192        r9: bootstrap_ctx.r9,
193        r8: bootstrap_ctx.r8,
194        rsi: bootstrap_ctx.rsi,
195        rdi: bootstrap_ctx.rdi,
196        rdx: bootstrap_ctx.rdx,
197        rcx: bootstrap_ctx.rcx,
198        rax: 0,
199        iret_rip: bootstrap_ctx.user_rip,
200        iret_cs: bootstrap_ctx.user_cs,
201        iret_rflags: bootstrap_ctx.user_rflags,
202        iret_rsp: bootstrap_ctx.user_rsp,
203        iret_ss: bootstrap_ctx.user_ss,
204    };
205
206    let (pid, tid, tgid) = Task::allocate_process_ids();
207    child_as.set_owner_pid(pid);
208    let task = Arc::new(Task {
209        id: TaskId::new(),
210        pid,
211        tid,
212        tgid,
213        pgid: AtomicU32::new(parent.pgid.load(Ordering::Relaxed)),
214        sid: AtomicU32::new(parent.sid.load(Ordering::Relaxed)),
215        uid: AtomicU32::new(parent.uid.load(Ordering::Relaxed)),
216        euid: AtomicU32::new(parent.euid.load(Ordering::Relaxed)),
217        gid: AtomicU32::new(parent.gid.load(Ordering::Relaxed)),
218        egid: AtomicU32::new(parent.egid.load(Ordering::Relaxed)),
219        state: core::sync::atomic::AtomicU8::new(TaskState::Ready as u8),
220        priority: parent.priority,
221        context: SyncUnsafeCell::new(context),
222        resume_kind: SyncUnsafeCell::new(crate::process::task::ResumeKind::RetFrame),
223        interrupt_rsp: AtomicU64::new(0),
224        kernel_stack,
225        user_stack: None,
226        stack_canary: core::sync::atomic::AtomicU64::new(0),
227        stack_canary_addr: core::sync::atomic::AtomicU64::new(0),
228        kernel_stack_user: SyncUnsafeCell::new(None),
229
230        name: "fork-child",
231        process: alloc::sync::Arc::new(crate::process::process::Process {
232            pid,
233            address_space: crate::process::task::SyncUnsafeCell::new(child_as),
234            address_space_lock: crate::sync::SpinLock::new(()),
235            fd_table: crate::process::task::SyncUnsafeCell::new(parent_fd),
236            capabilities: crate::process::task::SyncUnsafeCell::new(parent_caps),
237            signal_actions: crate::process::task::SyncUnsafeCell::new(parent_actions),
238            brk: core::sync::atomic::AtomicU64::new(
239                parent
240                    .process
241                    .brk
242                    .load(core::sync::atomic::Ordering::Relaxed),
243            ),
244            mmap_hint: core::sync::atomic::AtomicU64::new(
245                parent
246                    .process
247                    .mmap_hint
248                    .load(core::sync::atomic::Ordering::Relaxed),
249            ),
250            cwd: crate::process::task::SyncUnsafeCell::new(
251                unsafe { &*parent.process.cwd.get() }.clone(),
252            ),
253            // POSIX: cwd_fd IS inherited (capability to the CWD directory).
254            cwd_fd: core::sync::atomic::AtomicU64::new(
255                parent
256                    .process
257                    .cwd_fd
258                    .load(core::sync::atomic::Ordering::Relaxed),
259            ),
260            umask: core::sync::atomic::AtomicU32::new(
261                parent
262                    .process
263                    .umask
264                    .load(core::sync::atomic::Ordering::Relaxed),
265            ),
266            ipc_quota: crate::ipc::quota::IpcQuota::new(),
267        }),
268        // POSIX: pending signals are NOT inherited by the child.
269        pending_signals: SignalSet::new(),
270        // POSIX: signal mask IS inherited.
271        blocked_signals: parent_blocked,
272        irq_signal_delivery_blocked: AtomicBool::new(false),
273        signal_stack: SyncUnsafeCell::new(parent_sigstack),
274        itimers: crate::process::timer::ITimers::new(),
275        wake_pending: AtomicBool::new(false),
276        wake_deadline_ns: AtomicU64::new(0),
277        trampoline_entry: AtomicU64::new(0),
278        trampoline_stack_top: AtomicU64::new(0),
279        trampoline_arg0: AtomicU64::new(0),
280        ticks: AtomicU64::new(0),
281        sched_policy: SyncUnsafeCell::new(parent.sched_policy()),
282        home_cpu: AtomicUsize::new(usize::MAX),
283        last_cpu: AtomicUsize::new(usize::MAX),
284        affinity_mask: AtomicU64::new(0),
285        vruntime: AtomicU64::new(parent.vruntime()),
286        fair_rq_generation: AtomicU64::new(0),
287        fair_on_rq: AtomicBool::new(false),
288        // POSIX: clear_child_tid is NOT inherited : child starts with 0.
289        clear_child_tid: AtomicU64::new(0),
290        robust_list_head: AtomicU64::new(0),
291        robust_list_len: AtomicUsize::new(0),
292        // POSIX: cwd IS inherited.
293        // POSIX: umask IS inherited.
294        // FS.base: child starts with 0 (its own TLS not yet set up).
295        user_fs_base: AtomicU64::new(0),
296        fpu_state: {
297            let parent_fpu = unsafe { &*parent.fpu_state.get() };
298            let mut child_fpu = crate::process::task::ExtendedState::new();
299            child_fpu.copy_from(parent_fpu);
300            SyncUnsafeCell::new(child_fpu)
301        },
302        xcr0_mask: AtomicU64::new(parent.xcr0_mask.load(core::sync::atomic::Ordering::Relaxed)),
303        rt_link: intrusive_collections::LinkedListLink::new(),
304        rt_budget_remaining: AtomicU64::new(
305            parent
306                .rt_budget_remaining
307                .load(core::sync::atomic::Ordering::Relaxed),
308        ),
309        rt_budget_period_start: AtomicU64::new(
310            parent
311                .rt_budget_period_start
312                .load(core::sync::atomic::Ordering::Relaxed),
313        ),
314        rt_degraded: AtomicBool::new(
315            parent
316                .rt_degraded
317                .load(core::sync::atomic::Ordering::Relaxed),
318        ),
319        fair_wait_ticks: AtomicU64::new(0),
320    });
321
322    // CpuContext initial stack layout: r15, r14, r13(arg), r12(entry), rbp, rbx, ret
323    unsafe {
324        let ctx = &mut *task.context.get();
325        let frame = ctx.saved_rsp as *mut u64;
326        *frame.add(2) = Box::into_raw(bootstrap_ctx) as u64;
327    }
328
329    task.seed_interrupt_frame(interrupt_frame);
330
331    Ok(task)
332}
333
334/// SYS_PROC_FORK (302): fork with copy-on-write address-space cloning.
335pub fn sys_fork(frame: &SyscallFrame) -> Result<ForkResult, SyscallError> {
336    let parent = current_task_clone().ok_or(SyscallError::PermissionDenied)?;
337
338    // 1. Sanity check: cannot fork a kernel thread.
339    if parent.is_kernel() {
340        log::warn!("fork: attempt to fork kernel thread '{}'", parent.name);
341        return Err(SyscallError::PermissionDenied);
342    }
343
344    // 2. Capability check: check if task is restricted from forking.
345    // For now, we allow fork for all user processes unless restricted.
346    // TODO: implement ResourceType::Process/Task restricted capabilities.
347
348    let parent_as = parent.process.address_space_arc();
349
350    // 3. Memory check: ensure parent has actual user-space mappings.
351    if !parent_as.has_user_mappings() {
352        log::warn!(
353            "fork: attempt to fork task '{}' with no user mappings",
354            parent.name
355        );
356        return Err(SyscallError::InvalidArgument);
357    }
358
359    let child_as = parent_as
360        .clone_cow()
361        .map_err(|_| SyscallError::OutOfMemory)?;
362
363    let child_user_ctx = Box::new(ForkUserContext {
364        r15: frame.r15,
365        r14: frame.r14,
366        r13: frame.r13,
367        r12: frame.r12,
368        rbp: frame.rbp,
369        rbx: frame.rbx,
370        r11: frame.r11,
371        r10: frame.r10,
372        r9: frame.r9,
373        r8: frame.r8,
374        rsi: frame.rsi,
375        rdi: frame.rdi,
376        rdx: frame.rdx,
377        rcx: frame.rcx,
378        user_rip: frame.iret_rip,
379        user_cs: frame.iret_cs,
380        user_rflags: frame.iret_rflags,
381        user_rsp: frame.iret_rsp,
382        user_ss: frame.iret_ss,
383    });
384
385    let child_task = build_child_task(&parent, child_as, child_user_ctx)?;
386    let child_pid = child_task.pid;
387    add_task_with_parent(child_task, parent.id);
388
389    Ok(ForkResult { child_pid })
390}
391
392/// Called from the page fault handler when a write fault occurs on a present page.
393/// Returns Ok(()) if the fault was successfully handled (COW resolution),
394/// or Err if it wasn't a COW fault (real access violation).
395pub fn handle_cow_fault(virt_addr: u64, address_space: &AddressSpace) -> Result<(), &'static str> {
396    use crate::{
397        arch::xshim::{PageTableFlags, Size2MiB, Size4KiB, VirtAddr},
398        memory::paging::BuddyFrameAllocator,
399        x86_crate_shim::structures::paging::{Mapper, Page, Translate},
400    };
401
402    let mapping = address_space
403        .effective_mapping_containing(virt_addr)
404        .ok_or("Page not mapped")?;
405    let page_start = mapping.start;
406    let page = Page::<Size4KiB>::containing_address(VirtAddr::new(page_start));
407
408    // SAFETY: we are in an exception handler, address space is active.
409    let mut mapper = unsafe { address_space.mapper() };
410
411    // Check if page is mapped and has COW flag.
412    let (phys_frame_addr, flags) = match mapper.translate(VirtAddr::new(page_start)) {
413        TranslateResult::Mapped {
414            frame,
415            offset: _,
416            flags,
417        } => (frame.start_address(), flags),
418        _ => return Err("Page not mapped"),
419    };
420
421    // We use BIT_9 as software COW flag
422    const COW_BIT: PageTableFlags = PageTableFlags::BIT_9;
423
424    if !flags.contains(COW_BIT) {
425        return Err("Not a COW page");
426    }
427
428    let old_handle = mapping.handle;
429    let refcount = crate::memory::cow::handle_get_refcount(old_handle);
430
431    if refcount == 1 {
432        // Case 1: we are the sole owner. Just make it writable.
433        let new_flags = (flags | PageTableFlags::WRITABLE) & !COW_BIT;
434
435        unsafe {
436            match mapping.page_size {
437                VmaPageSize::Small => mapper
438                    .update_flags(page, new_flags)
439                    .map_err(|_| "Failed to update 4K flags")?
440                    .flush(),
441                VmaPageSize::Huge => mapper
442                    .update_flags(
443                        Page::<Size2MiB>::containing_address(VirtAddr::new(page_start)),
444                        new_flags | PageTableFlags::HUGE_PAGE,
445                    )
446                    .map_err(|_| "Failed to update 2M flags")?
447                    .flush(),
448            }
449        }
450        let tracked_flags = match mapping.page_size {
451            VmaPageSize::Small => new_flags,
452            VmaPageSize::Huge => new_flags | PageTableFlags::HUGE_PAGE,
453        };
454        let _ = address_space.update_effective_mapping_flags(page_start, tracked_flags);
455        // Only the current CPU can hold this CR3 in the current design.
456        local_invlpg(virt_addr);
457        return Ok(());
458    }
459
460    // Case 2: shared page. Copy to new frame.
461    let mut frame_allocator = BuddyFrameAllocator;
462    let order = match mapping.page_size {
463        VmaPageSize::Small => 0,
464        VmaPageSize::Huge => 9,
465    };
466    let copy_bytes = mapping.page_size.bytes() as usize;
467    let new_frame = crate::sync::with_irqs_disabled(|token| {
468        if order == 0 {
469            crate::memory::allocate_frame(token)
470        } else {
471            crate::memory::allocate_phys_contiguous(token, order)
472        }
473    })
474    .map_err(|_| "OOM during COW copy")?;
475
476    // Copy content
477    unsafe {
478        let src = crate::memory::phys_to_virt(phys_frame_addr.as_u64()) as *const u8;
479        let dst = crate::memory::phys_to_virt(new_frame.start_address.as_u64()) as *mut u8;
480        core::ptr::copy_nonoverlapping(src, dst, copy_bytes);
481    }
482
483    // Update mapping to new frame, Writable, no COW
484    let new_flags = (flags | PageTableFlags::WRITABLE) & !COW_BIT;
485    let tracked_flags = match mapping.page_size {
486        VmaPageSize::Small => new_flags,
487        VmaPageSize::Huge => new_flags | PageTableFlags::HUGE_PAGE,
488    };
489    let new_handle = resolve_handle(new_frame.start_address);
490
491    // Replace existing mapping (present+COW) by the private writable mapping.
492    let remap_res: Result<(), &'static str> = match mapping.page_size {
493        VmaPageSize::Small => {
494            let old_unmapped = mapper
495                .unmap(page)
496                .map_err(|_| "Failed to unmap old 4K COW frame")?
497                .0;
498            debug_assert_eq!(old_unmapped.start_address(), phys_frame_addr);
499            unsafe {
500                mapper.map_to(
501                    page,
502                    crate::arch::xshim::PhysFrame::<Size4KiB>::containing_address(
503                        new_frame.start_address,
504                    ),
505                    new_flags,
506                    &mut frame_allocator,
507                )
508            }
509            .map(|flush| flush.flush())
510            .map_err(|_| "Failed to map new 4K COW frame")
511        }
512        VmaPageSize::Huge => {
513            let huge_page = Page::<Size2MiB>::containing_address(VirtAddr::new(page_start));
514            let old_unmapped = mapper
515                .unmap(huge_page)
516                .map_err(|_| "Failed to unmap old 2M COW frame")?
517                .0;
518            debug_assert_eq!(old_unmapped.start_address(), phys_frame_addr);
519            unsafe {
520                mapper.map_to(
521                    huge_page,
522                    crate::arch::xshim::PhysFrame::<Size2MiB>::containing_address(
523                        new_frame.start_address,
524                    ),
525                    tracked_flags,
526                    &mut frame_allocator,
527                )
528            }
529            .map(|flush| flush.flush())
530            .map_err(|_| "Failed to map new 2M COW frame")
531        }
532    };
533    if remap_res.is_err() {
534        match mapping.page_size {
535            VmaPageSize::Small => unsafe {
536                let _ = mapper.map_to(
537                    page,
538                    crate::arch::xshim::PhysFrame::<Size4KiB>::containing_address(phys_frame_addr),
539                    flags,
540                    &mut frame_allocator,
541                );
542            },
543            VmaPageSize::Huge => unsafe {
544                let huge_page = Page::<Size2MiB>::containing_address(VirtAddr::new(page_start));
545                let _ = mapper.map_to(
546                    huge_page,
547                    crate::arch::xshim::PhysFrame::<Size2MiB>::containing_address(phys_frame_addr),
548                    flags,
549                    &mut frame_allocator,
550                );
551            },
552        }
553        crate::sync::with_irqs_disabled(|token| {
554            if order == 0 {
555                crate::memory::free_frame(token, new_frame);
556            } else {
557                crate::memory::free_phys_contiguous(token, new_frame, order);
558            }
559        });
560        return Err(remap_res.err().unwrap_or("Failed to map new COW frame"));
561    }
562
563    // The new private frame is the sole owner; set refcount=1 directly.
564    // BuddyFrameAllocator returns a raw frame (refcount still REFCOUNT_UNUSED).
565    // frame_inc_ref would wrap REFCOUNT_UNUSED to 0 : use set_refcount instead.
566    crate::memory::cow::handle_init_ref(new_handle);
567
568    if address_space
569        .register_effective_mapping(EffectiveMapping {
570            start: page_start,
571            cap_id: mapping.cap_id,
572            handle: new_handle,
573            flags: tracked_flags,
574            page_size: mapping.page_size,
575        })
576        .is_err()
577    {
578        match mapping.page_size {
579            VmaPageSize::Small => {
580                let _ = mapper.unmap(page);
581                let _ = unsafe {
582                    mapper.map_to(
583                        page,
584                        crate::arch::xshim::PhysFrame::<Size4KiB>::containing_address(
585                            phys_frame_addr,
586                        ),
587                        flags,
588                        &mut frame_allocator,
589                    )
590                }
591                .map(|flush| flush.flush());
592            }
593            VmaPageSize::Huge => {
594                let huge_page = Page::<Size2MiB>::containing_address(VirtAddr::new(page_start));
595                let _ = mapper.unmap(huge_page);
596                let _ = unsafe {
597                    mapper.map_to(
598                        huge_page,
599                        crate::arch::xshim::PhysFrame::<Size2MiB>::containing_address(
600                            phys_frame_addr,
601                        ),
602                        flags,
603                        &mut frame_allocator,
604                    )
605                }
606                .map(|flush| flush.flush());
607            }
608        }
609        crate::sync::with_irqs_disabled(|token| {
610            if order == 0 {
611                crate::memory::free_frame(token, new_frame);
612            } else {
613                crate::memory::free_phys_contiguous(token, new_frame, order);
614            }
615        });
616        return Err("Failed to track new COW mapping");
617    }
618
619    // Only the current CPU can hold this CR3 in the current design.
620    local_invlpg(virt_addr);
621
622    // Replacing the effective mapping at the same address already unregisters
623    // the previous mapping identity for old_handle. There is no transient pin
624    // to drop in this path.
625
626    Ok(())
627}
strat9_kernel/syscall/fork.rs

strat9_kernel/syscall/
fork.rs
