strat9_kernel/process/
task.rs

1//! Task Management
2//!
3//! Defines the Task structure and related types for the Strat9-OS scheduler.
4
5use crate::memory::AddressSpace;
6use alloc::sync::Arc;
7use core::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicU8, AtomicUsize, Ordering};
8use intrusive_collections::LinkedListLink;
9use x86_64::{PhysAddr, VirtAddr};
10
11/// POSIX process ID.
12pub type Pid = u32;
13/// POSIX thread ID.
14pub type Tid = u32;
15
16/// Performs the next pid operation.
17#[inline]
18fn next_pid() -> Pid {
19    static NEXT_PID: AtomicU32 = AtomicU32::new(1);
20    NEXT_PID.fetch_add(1, Ordering::SeqCst)
21}
22
23/// Performs the next tid operation.
24#[inline]
25fn next_tid() -> Tid {
26    static NEXT_TID: AtomicU32 = AtomicU32::new(1);
27    NEXT_TID.fetch_add(1, Ordering::SeqCst)
28}
29
30/// Unique identifier for a task
31#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
32pub struct TaskId(pub u64);
33
34impl TaskId {
35    /// Generate a new unique task ID
36    pub fn new() -> Self {
37        static NEXT_ID: AtomicU64 = AtomicU64::new(0);
38        TaskId(NEXT_ID.fetch_add(1, Ordering::SeqCst))
39    }
40
41    /// Get the raw u64 value
42    pub fn as_u64(self) -> u64 {
43        self.0
44    }
45
46    /// Create a TaskId from a raw u64 (for IPC reply routing).
47    pub fn from_u64(raw: u64) -> Self {
48        TaskId(raw)
49    }
50}
51
52impl core::fmt::Display for TaskId {
53    /// Performs the fmt operation.
54    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
55        write!(f, "{}", self.0)
56    }
57}
58
59/// Priority levels for tasks
60#[derive(Debug, Clone, Copy, PartialEq, Eq)]
61pub enum TaskPriority {
62    Idle = 0,
63    Low = 1,
64    Normal = 2,
65    High = 3,
66    Realtime = 4,
67}
68
69/// State of a task in the scheduler
70#[repr(u8)]
71#[derive(Debug, Clone, Copy, PartialEq, Eq)]
72pub enum TaskState {
73    /// Task is ready to be scheduled
74    Ready = 0,
75    /// Task is currently running
76    Running = 1,
77    /// Task is blocked waiting for an event
78    Blocked = 2,
79    /// Task has exited
80    Dead = 3,
81}
82
83/// How this task must be resumed the next time the scheduler selects it.
84///
85/// - `RetFrame`: legacy kernel-only context switch using `ret`
86/// - `IretFrame`: interrupt/syscall-like frame restored with `iretq`
87#[derive(Debug, Clone, Copy, PartialEq, Eq)]
88pub enum ResumeKind {
89    RetFrame,
90    IretFrame,
91}
92
93use core::cell::UnsafeCell;
94
95/// A wrapper around UnsafeCell that implements Sync for TaskState
96pub struct SyncUnsafeCell<T> {
97    inner: UnsafeCell<T>,
98}
99
100unsafe impl<T> Sync for SyncUnsafeCell<T> {}
101
102impl<T> SyncUnsafeCell<T> {
103    /// Creates a new instance.
104    pub const fn new(value: T) -> Self {
105        Self {
106            inner: UnsafeCell::new(value),
107        }
108    }
109
110    /// Performs the get operation.
111    pub fn get(&self) -> *mut T {
112        self.inner.get()
113    }
114}
115
116/// FPU/SSE/AVX extended state, saved and restored on context switch.
117///
118/// When XSAVE is available, uses `xsave`/`xrstor` with a variable-size area.
119/// Falls back to `fxsave`/`fxrstor` (512 bytes) on older CPUs.
120#[repr(C, align(64))]
121pub struct ExtendedState {
122    pub data: [u8; Self::MAX_XSAVE_SIZE],
123    pub size: usize,
124    pub uses_xsave: bool,
125    pub xcr0_mask: u64,
126}
127
128impl core::fmt::Debug for ExtendedState {
129    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
130        f.debug_struct("ExtendedState")
131            .field("size", &self.size)
132            .field("uses_xsave", &self.uses_xsave)
133            .field("xcr0_mask", &self.xcr0_mask)
134            .finish()
135    }
136}
137
138impl ExtendedState {
139    pub const FXSAVE_SIZE: usize = 512;
140    pub const MAX_XSAVE_SIZE: usize = 2688;
141
142    /// Create a new default state using the host's maximum capabilities.
143    pub fn new() -> Self {
144        crate::serial_println!("[trace][fpu] ExtendedState::new enter");
145        let (uses_xsave, size, default_xcr0) = if crate::arch::x86_64::cpuid::host_uses_xsave() {
146            crate::serial_println!("[trace][fpu] ExtendedState::new host_uses_xsave=true");
147            let xcr0 = crate::arch::x86_64::cpuid::host_default_xcr0();
148            crate::serial_println!(
149                "[trace][fpu] ExtendedState::new host_default_xcr0={:#x}",
150                xcr0
151            );
152            let sz =
153                crate::arch::x86_64::cpuid::xsave_size_for_xcr0(xcr0).min(Self::MAX_XSAVE_SIZE);
154            crate::serial_println!("[trace][fpu] ExtendedState::new xsave_size={}", sz);
155            (true, sz, xcr0)
156        } else {
157            crate::serial_println!("[trace][fpu] ExtendedState::new host_uses_xsave=false");
158            (false, Self::FXSAVE_SIZE, 0x3)
159        };
160
161        crate::serial_println!(
162            "[trace][fpu] ExtendedState::new build state uses_xsave={} size={} xcr0={:#x}",
163            uses_xsave,
164            size,
165            default_xcr0
166        );
167        let mut state = Self {
168            data: [0u8; Self::MAX_XSAVE_SIZE],
169            size,
170            uses_xsave,
171            xcr0_mask: default_xcr0,
172        };
173        crate::serial_println!("[trace][fpu] ExtendedState::new state allocated");
174        state.set_defaults();
175        crate::serial_println!("[trace][fpu] ExtendedState::new defaults set");
176        state
177    }
178
179    /// Create a state for a specific XCR0 mask (per-silo feature restriction).
180    pub fn for_xcr0(xcr0: u64) -> Self {
181        let uses_xsave = crate::arch::x86_64::cpuid::host_uses_xsave();
182        let size = if uses_xsave {
183            crate::arch::x86_64::cpuid::xsave_size_for_xcr0(xcr0).min(Self::MAX_XSAVE_SIZE)
184        } else {
185            Self::FXSAVE_SIZE
186        };
187
188        let mut state = Self {
189            data: [0u8; Self::MAX_XSAVE_SIZE],
190            size,
191            uses_xsave,
192            xcr0_mask: xcr0,
193        };
194        state.set_defaults();
195        state
196    }
197
198    fn set_defaults(&mut self) {
199        // Set x87 FCW = 0x037F and MXCSR = 0x1F80 directly in the buffer.
200        // These match the x87/SSE INIT values, so even though the XSAVE
201        // header's XSTATE_BV field (bytes 512-519) remains 0, the first
202        // XRSTOR will restore the same values from INIT defaults.
203        // This is correct and safe by design.
204        self.data[0] = 0x7F;
205        self.data[1] = 0x03;
206        self.data[24] = 0x80;
207        self.data[25] = 0x1F;
208    }
209
210    /// Copy the state from another `ExtendedState`.
211    pub fn copy_from(&mut self, other: &ExtendedState) {
212        let len = other.size.min(self.size);
213        self.data[..len].copy_from_slice(&other.data[..len]);
214    }
215}
216
217#[inline]
218fn normalized_xcr0(xcr0: u64) -> u64 {
219    if !crate::arch::x86_64::cpuid::host_uses_xsave() {
220        return 0x3;
221    }
222
223    // Use the lock-free cache to avoid acquiring the HOST_CPU spinlock
224    // with interrupts disabled in the context-switch hot path.
225    let host_xcr0 = crate::arch::x86_64::cpuid::host_default_xcr0_fast();
226    (xcr0 & host_xcr0).max(0x3)
227}
228
229/// Represents a single task/thread in the system
230pub struct Task {
231    /// Unique identifier for this task
232    pub id: TaskId,
233    /// Process identifier visible to userspace.
234    pub pid: Pid,
235    /// Thread identifier visible to userspace.
236    pub tid: Tid,
237    /// Thread-group identifier (equals process leader PID).
238    pub tgid: Pid,
239    /// Process group id (job-control group).
240    pub pgid: AtomicU32,
241    /// Session id.
242    pub sid: AtomicU32,
243    /// real user id.
244    pub uid: AtomicU32,
245    /// effective user id.
246    pub euid: AtomicU32,
247    /// real group id.
248    pub gid: AtomicU32,
249    /// effective group id.
250    pub egid: AtomicU32,
251    /// Current state of the task. Stored as AtomicU8 for lock-free cross-CPU visibility.
252    /// Use `get_state()` / `set_state()` for typed access.
253    pub state: AtomicU8,
254    /// Priority level of the task
255    pub priority: TaskPriority,
256    /// Saved CPU context for this task (just the stack pointer)
257    pub context: SyncUnsafeCell<CpuContext>,
258    /// Resume convention for this task's saved kernel stack frame.
259    pub resume_kind: SyncUnsafeCell<ResumeKind>,
260    /// Saved interrupt/syscall-compatible frame pointer for `iretq`-based resume.
261    pub interrupt_rsp: AtomicU64,
262    /// Kernel stack for this task
263    pub kernel_stack: KernelStack,
264    /// User stack for this task (if applicable)
265    pub user_stack: Option<UserStack>,
266    /// Task name for debugging purposes
267    pub name: &'static str,
268    /// Capabilities granted to this task
269    /// Address space for this task (kernel tasks share the kernel AS)
270    pub process: Arc<crate::process::process::Process>,
271    /// File descriptor table for this task
272    /// Pending signals for this task
273    pub pending_signals: super::signal::SignalSet,
274    /// Blocked signals mask for this task
275    pub blocked_signals: super::signal::SignalSet,
276    /// Suppress repeated IRQ-return delivery attempts until a normal delivery
277    /// path runs again.
278    pub irq_signal_delivery_blocked: AtomicBool,
279    /// Signal actions (handlers) for this task
280    /// Signal alternate stack for this task
281    pub signal_stack: SyncUnsafeCell<Option<super::signal::SigStack>>,
282    /// Interval timers (ITIMER_REAL, ITIMER_VIRTUAL, ITIMER_PROF)
283    pub itimers: super::timer::ITimers,
284    /// Pending wakeup flag: set by `wake_task()` when the task is not yet
285    /// in `blocked_tasks` (it is still transitioning to Blocked state).
286    /// Checked by `block_current_task()` : if set, the task skips blocking
287    /// and continues execution, preventing a lost-wakeup race.
288    pub wake_pending: AtomicBool,
289    /// Sleep deadline in nanoseconds (monotonic). If non-zero, the task
290    /// is sleeping until this time. Checked by the scheduler to auto-wake.
291    pub wake_deadline_ns: AtomicU64,
292    /// Program break (end of heap), in bytes. 0 = not yet initialised.
293    /// Lazily set to `BRK_BASE` on the first `sys_brk` call.
294    /// mmap_hint: next candidate virtual address for anonymous mmap allocations
295    /// User-space entry point for ring3 trampoline (ELF tasks only, 0 otherwise).
296    pub trampoline_entry: AtomicU64,
297    /// User-space stack top for ring3 trampoline (ELF tasks only, 0 otherwise).
298    pub trampoline_stack_top: AtomicU64,
299    /// First argument (RDI) passed to the user process on entry (e.g. bootstrap cap handle).
300    pub trampoline_arg0: AtomicU64,
301    /// Total CPU ticks consumed by this task
302    pub ticks: AtomicU64,
303    /// Scheduling policy (Fair, RealTime, Idle)
304    pub sched_policy: SyncUnsafeCell<crate::process::sched::SchedPolicy>,
305    /// Home CPU index for this task. Set when the task is first scheduled
306    /// or explicitly assigned. Used by `wake_task()` to route to the correct
307    /// per-CPU runqueue without acquiring `GLOBAL_SCHED_STATE`.
308    pub home_cpu: AtomicUsize,
309    /// Virtual runtime for CFS
310    pub vruntime: AtomicU64,
311    /// Monotonic token identifying the currently valid FAIR runqueue entry.
312    pub fair_rq_generation: AtomicU64,
313    /// Whether this task is logically present in the FAIR runqueue.
314    pub fair_on_rq: AtomicBool,
315    /// TID address for futex-based thread join (set_tid_address).
316    /// The kernel writes 0 here when the thread exits, then futex_wake.
317    pub clear_child_tid: AtomicU64,
318    /// Robust list head pointer (set by set_robust_list).
319    /// Points to a userspace robust_list_head used for dead-thread mutex cleanup.
320    pub robust_list_head: AtomicU64,
321    /// Length of the robust list head structure (as passed to set_robust_list).
322    pub robust_list_len: AtomicUsize,
323    /// Current working directory (POSIX, inherited by children).
324    /// File creation mask (inherited by children, NOT reset by exec).
325    /// User-space FS.base (TLS on x86_64, set via arch_prctl ARCH_SET_FS).
326    /// Saved/restored across context switches.
327    pub user_fs_base: AtomicU64,
328    /// FPU/SSE/AVX extended state saved during context switch.
329    pub fpu_state: SyncUnsafeCell<ExtendedState>,
330    /// XCR0 mask for this task (inherited from its silo).
331    pub xcr0_mask: AtomicU64,
332    /// Intrusive linked-list link for the RT run queue.
333    ///
334    /// Only touched while holding the per-CPU scheduler spinlock.
335    pub rt_link: LinkedListLink,
336}
337
338// SAFETY: `LinkedListLink` uses `UnsafeCell` internally and is therefore
339// `!Sync` by default, but all mutations to `rt_link` are performed under the
340// per-CPU scheduler spinlock.  Every other non-atomic field in `Task` is
341// similarly protected by the appropriate lock or by the task's own atomics.
342unsafe impl Sync for Task {}
343
344impl Task {
345    /// Leave this much headroom above the synthetic `SyscallFrame`.
346    ///
347    /// The raw IRQ switch path does `mov rsp, next_rsp` and then `call
348    /// finish_interrupt_switch`, so `next_rsp` must be close to the top of the
349    /// kernel stack to preserve downward growth room for the call chain.
350    const BOOTSTRAP_INTERRUPT_FRAME_TOP_HEADROOM: usize = 0x1000;
351
352    /// Canary placed below the interrupt frame to detect stack underflow
353    /// (interrupt handler overflowing downward past the frame)
354    const STACK_UNDERFLOW_CANARY_OFFSET: usize = 0x100; // 256 bytes from base
355
356    /// Performs the default sched policy operation.
357    pub fn default_sched_policy(priority: TaskPriority) -> crate::process::sched::SchedPolicy {
358        use crate::process::sched::{nice::Nice, real_time::RealTimePriority, SchedPolicy};
359        match priority {
360            TaskPriority::Idle => SchedPolicy::Idle,
361            TaskPriority::Realtime => SchedPolicy::RealTimeRR {
362                prio: RealTimePriority::new(50),
363            },
364            TaskPriority::High => SchedPolicy::Fair(Nice::new(-10)),
365            TaskPriority::Low => SchedPolicy::Fair(Nice::new(10)),
366            TaskPriority::Normal => SchedPolicy::Fair(Nice::default()),
367        }
368    }
369
370    /// Get the current scheduling policy of the task
371    pub fn sched_policy(&self) -> crate::process::sched::SchedPolicy {
372        unsafe { *self.sched_policy.get() }
373    }
374
375    /// Set the scheduling policy of the task
376    pub fn set_sched_policy(&self, policy: crate::process::sched::SchedPolicy) {
377        unsafe {
378            *self.sched_policy.get() = policy;
379        }
380    }
381
382    /// Returns the current resume convention for this task.
383    pub fn resume_kind(&self) -> ResumeKind {
384        unsafe { *self.resume_kind.get() }
385    }
386
387    /// Sets the resume convention for this task.
388    pub fn set_resume_kind(&self, kind: ResumeKind) {
389        unsafe {
390            *self.resume_kind.get() = kind;
391        }
392    }
393
394    /// Returns the saved `iretq`-compatible frame pointer for this task.
395    pub fn interrupt_rsp(&self) -> u64 {
396        self.interrupt_rsp.load(Ordering::Acquire)
397    }
398
399    /// Updates the saved `iretq`-compatible frame pointer for this task.
400    pub fn set_interrupt_rsp(&self, rsp: u64) {
401        self.interrupt_rsp.store(rsp, Ordering::Release);
402    }
403
404    /// Seed a synthetic interrupt frame for tasks that have not yet been
405    /// preempted from an IRQ path but must still be resumable via `iretq`.
406    pub fn seed_interrupt_frame(&self, frame: crate::syscall::SyscallFrame) {
407        let stack_base = self.kernel_stack.virt_base.as_u64();
408        let stack_top = stack_base + self.kernel_stack.size as u64;
409        let frame_size = core::mem::size_of::<crate::syscall::SyscallFrame>() as u64;
410        let raw_frame_addr = stack_top
411            .saturating_sub(Self::BOOTSTRAP_INTERRUPT_FRAME_TOP_HEADROOM as u64)
412            .saturating_sub(frame_size);
413        let frame_addr = raw_frame_addr & !0xF;
414        let frame_end = frame_addr + core::mem::size_of::<crate::syscall::SyscallFrame>() as u64;
415        assert!(
416            frame_addr >= stack_base && frame_end <= stack_top,
417            "kernel stack too small for bootstrap interrupt frame"
418        );
419        unsafe {
420            (frame_addr as *mut crate::syscall::SyscallFrame).write(frame);
421
422            // Place underflow canary below the frame (at lower address)
423            // This detects if interrupt handler overflows downward past expected range
424            let canary_addr = stack_base + Self::STACK_UNDERFLOW_CANARY_OFFSET as u64;
425            *(canary_addr as *mut u64) = 0xBAD57ACBAD57AC;
426        }
427        self.set_interrupt_rsp(frame_addr);
428    }
429
430    /// Seed an `iretq`-compatible frame from the legacy `CpuContext` bootstrap
431    /// layout used by kernel tasks (`ret` into `task_entry_trampoline`).
432    ///
433    /// The synthesised frame always sets IF=1 so that IRQ-driven resumes keep
434    /// receiving timer interrupts. First-launch tasks still enter through the
435    /// legacy `ret` trampoline and must explicitly re-enable interrupts in
436    /// `task_post_switch_enter`.
437    pub fn seed_kernel_interrupt_frame_from_context(&self) {
438        let stack_base = self.kernel_stack.virt_base.as_u64();
439        let stack_top = stack_base + self.kernel_stack.size as u64;
440        let saved_rsp = unsafe { (*self.context.get()).saved_rsp as *const u64 };
441        let saved_rsp_val = saved_rsp as u64;
442        debug_assert!(
443            saved_rsp_val >= stack_base && saved_rsp_val.saturating_add(7 * 8) <= stack_top,
444            "saved_rsp outside kernel stack while seeding interrupt frame"
445        );
446        let ret_target = unsafe { *saved_rsp.add(6) };
447        // Always set IF=1 (bit 9) so IRQ-driven resumes keep interrupts enabled.
448        // First-launch tasks still need an explicit sti() in
449        // task_post_switch_enter because the legacy bootstrap path reaches the
450        // entry point through a plain ret, not an iretq restoring RFLAGS.
451        let rflags = 0x202u64; // bit 9 = IF, bit 1 = reserved (always 1)
452        let frame = unsafe {
453            crate::syscall::SyscallFrame {
454                r15: *saved_rsp.add(0),
455                r14: *saved_rsp.add(1),
456                r13: *saved_rsp.add(2),
457                r12: *saved_rsp.add(3),
458                rbp: *saved_rsp.add(4),
459                rbx: *saved_rsp.add(5),
460                r11: 0,
461                r10: 0,
462                r9: 0,
463                r8: 0,
464                rsi: 0,
465                rdi: 0,
466                rdx: 0,
467                rcx: 0,
468                rax: 0,
469                iret_rip: ret_target,
470                iret_cs: crate::arch::x86_64::gdt::kernel_code_selector().0 as u64,
471                iret_rflags: rflags,
472                iret_rsp: self.kernel_stack.virt_base.as_u64() + self.kernel_stack.size as u64,
473                iret_ss: crate::arch::x86_64::gdt::kernel_data_selector().0 as u64,
474            }
475        };
476        self.seed_interrupt_frame(frame);
477    }
478
479    /// Get virtual runtime
480    pub fn vruntime(&self) -> u64 {
481        self.vruntime.load(Ordering::Relaxed)
482    }
483
484    /// Set virtual runtime
485    pub fn set_vruntime(&self, vruntime: u64) {
486        self.vruntime.store(vruntime, Ordering::Relaxed);
487    }
488
489    /// Prepare a new FAIR runqueue entry and return its generation token.
490    pub fn fair_prepare_enqueue(&self) -> (u64, bool) {
491        let was_queued = self.fair_on_rq.swap(true, Ordering::AcqRel);
492        let generation = self.fair_rq_generation.fetch_add(1, Ordering::Relaxed) + 1;
493        (generation, was_queued)
494    }
495
496    /// Returns the generation of the currently valid FAIR entry.
497    pub fn fair_generation(&self) -> u64 {
498        self.fair_rq_generation.load(Ordering::Relaxed)
499    }
500
501    /// Returns whether the task is logically queued in FAIR.
502    pub fn fair_is_on_rq(&self) -> bool {
503        self.fair_on_rq.load(Ordering::Acquire)
504    }
505
506    /// Marks the task as dequeued from FAIR.
507    pub fn fair_mark_dequeued(&self) -> bool {
508        self.fair_on_rq.swap(false, Ordering::AcqRel)
509    }
510
511    /// Invalidates the current FAIR entry so stale heap nodes can be skipped lazily.
512    pub fn fair_invalidate_rq_entry(&self) -> bool {
513        let was_queued = self.fair_on_rq.swap(false, Ordering::AcqRel);
514        self.fair_rq_generation.fetch_add(1, Ordering::Relaxed);
515        was_queued
516    }
517
518    /// Read the current task state atomically.
519    #[inline]
520    pub fn get_state(&self) -> TaskState {
521        let raw = self.state.load(Ordering::Acquire);
522        debug_assert!(
523            raw <= TaskState::Dead as u8,
524            "get_state: invalid TaskState discriminant {:#x}",
525            raw
526        );
527        // SAFETY: `raw` is always one of the four valid `#[repr(u8)]`
528        // discriminants (0..=3); the only writer is `set_state` which stores
529        // a cast from the same enum.
530        unsafe { core::mem::transmute(raw) }
531    }
532
533    /// Write the task state atomically. Uses Release ordering so the new state
534    /// is visible to any CPU that subsequently does an Acquire load.
535    #[inline]
536    pub fn set_state(&self, new_state: TaskState) {
537        self.state.store(new_state as u8, Ordering::Release);
538    }
539}
540
541/// CPU context saved/restored during context switches.
542///
543/// Only stores the saved RSP. All callee-saved registers (rbx, rbp, r12-r15)
544/// are pushed onto the task's kernel stack by `switch_context()`.
545#[repr(C)]
546pub struct CpuContext {
547    /// Saved stack pointer (points into the task's kernel stack)
548    pub saved_rsp: u64,
549}
550
551impl CpuContext {
552    /// Create a new CPU context for a task starting at the given entry point.
553    ///
554    /// Sets up a fake stack frame on the kernel stack that looks like
555    /// `switch_context()` just pushed callee-saved registers. When
556    /// `switch_context()` or `restore_first_task()` pops them and does `ret`,
557    /// it will jump to `task_entry_trampoline`, which enables interrupts
558    /// and jumps to the real entry point (stored in r12).
559    ///
560    /// Stack layout (growing downward):
561    /// ```text
562    /// [stack_top]
563    ///   0xDEADBEEFCAFEBABE      <- stack canary
564    ///   task_entry_trampoline   <- ret target
565    ///   0  (r15)
566    ///   0  (r14)
567    ///   0  (r13)
568    ///   entry_point (r12)      <- trampoline reads this
569    ///   0  (rbp)
570    ///   0  (rbx)
571    ///   <- saved_rsp points here
572    /// ```
573    pub fn new(entry_point: u64, kernel_stack: &KernelStack) -> Self {
574        let stack_top = kernel_stack.virt_base.as_u64() + kernel_stack.size as u64;
575
576        // Reserve space for the stack canary before building the fake frame.
577        const STACK_CANARY: u64 = 0xDEADBEEFCAFEBABE;
578        let canary_addr = stack_top - 8;
579        let initial_rsp = canary_addr - 7 * 8;
580
581        // SAFETY: We own this stack memory and it's properly allocated and zeroed.
582        // The stack region [virt_base, virt_base + size) is valid.
583        unsafe {
584            let stack = initial_rsp as *mut u64;
585            // Push order must match switch_context pops (LIFO, but we write linearly from RSP up):
586            // [RSP+0]  = r15
587            // [RSP+8]  = r14
588            // [RSP+16] = r13
589            // [RSP+24] = r12 (entry point)
590            // [RSP+32] = rbp
591            // [RSP+40] = rbx
592            // [RSP+48] = ret (trampoline)
593            *stack.add(0) = 0; // r15
594            *stack.add(1) = 0; // r14
595            *stack.add(2) = 0; // r13
596            *stack.add(3) = entry_point; // r12 (trampoline target)
597            *stack.add(4) = 0; // rbp
598            *stack.add(5) = 0; // rbx
599            *stack.add(6) = task_entry_trampoline as *const () as u64; // ret address
600        }
601
602        // Add stack canary at the very top (leave the frame below it so `ret` still points
603        // to `task_entry_trampoline`). The canary slot must be reserved before writing the
604        // frame to avoid overwriting the trampoline address.
605        unsafe {
606            let canary_ptr = canary_addr as *mut u64;
607            *canary_ptr = STACK_CANARY;
608        }
609
610        // Verify canary is still intact
611        unsafe {
612            let canary_ptr = canary_addr as *const u64;
613            let canary = *canary_ptr;
614            if canary != STACK_CANARY {
615                crate::serial_force_println!(
616                    "[PANIC] Stack canary corrupted at setup! entry_point={:#x} canary={:#x}",
617                    entry_point,
618                    canary
619                );
620            }
621        }
622
623        // Debug: verify entire stack frame
624        unsafe {
625            let stack = initial_rsp as *const u64;
626            crate::serial_println!(
627                "[CpuContext] frame verify: r15={:#x} r14={:#x} r13={:#x} r12={:#x} rbp={:#x} rbx={:#x} ret={:#x}",
628                *stack.add(0),
629                *stack.add(1),
630                *stack.add(2),
631                *stack.add(3),
632                *stack.add(4),
633                *stack.add(5),
634                *stack.add(6)
635            );
636            // Verify canary one more time
637            let canary_ptr = canary_addr as *const u64;
638            let canary = *canary_ptr;
639            if canary != STACK_CANARY {
640                crate::serial_force_println!(
641                    "[CpuContext] CANARY CORRUPTED AFTER FRAME SETUP! canary={:#x}",
642                    canary
643                );
644            }
645
646            // Debug: check if stack memory overlaps with another task
647            crate::serial_println!(
648                "[CpuContext] stack range: base={:#x} top={:#x} initial_rsp={:#x}",
649                kernel_stack.virt_base.as_u64(),
650                stack_top,
651                initial_rsp
652            );
653        }
654
655        CpuContext {
656            saved_rsp: initial_rsp,
657        }
658    }
659}
660
661/// Trampoline for newly created tasks.
662///
663/// When a new task is first scheduled, `switch_context()` pops the fake
664/// callee-saved registers and `ret`s here, then tail-jumps into the actual
665/// post-switch entry helper.
666#[unsafe(naked)]
667pub unsafe extern "C" fn task_entry_trampoline() -> ! {
668    core::arch::naked_asm!(
669        "mov al, 'T'",
670        "out 0xe9, al",
671        "call {finish_switch}",
672        "mov al, '1'",
673        "out 0xe9, al",
674        "mov rdi, r12", // entry_point
675        "mov rsi, r13", // arg0
676        "and rsp, -16",
677        "sub rsp, 8",
678        "jmp {post_switch_enter}",
679        finish_switch = sym crate::process::scheduler::finish_switch,
680        post_switch_enter = sym task_post_switch_enter,
681    );
682}
683
684fn task_post_switch_enter(entry: u64, arg0: u64) -> ! {
685    // E9 breadcrumb: 'P' = reached post_switch_enter (no serial lock needed).
686    unsafe {
687        core::arch::asm!("out 0xe9, al", in("al") b'P', options(nomem, nostack));
688    }
689
690    crate::arch::x86_64::percpu::mark_tlb_ready_current();
691
692    let cpu = crate::arch::x86_64::percpu::current_cpu_index();
693
694    let is_user_entry = crate::process::scheduler::current_task_clone_try()
695        .map(|task| task.trampoline_entry.load(Ordering::Relaxed) != 0)
696        .unwrap_or(false);
697
698    // Single diagnostic print (IF may be 0 or 1 depending on RFLAGS seed; either
699    // way E9 is IRQ-safe and this is the LAST trace call before entry_fn).
700    if let Some(task) = crate::process::scheduler::current_task_clone_try() {
701        crate::e9_println!(
702            "[pse] cpu={} tid={} user={} entry={:#x}",
703            cpu,
704            task.id.as_u64(),
705            is_user_entry,
706            entry
707        );
708        if is_user_entry {
709            crate::serial_println!(
710                "[trace][task] post_switch_enter cpu={} tid={} entry={:#x}",
711                cpu,
712                task.id.as_u64(),
713                entry
714            );
715        }
716    }
717
718    // First-launch tasks arrive here via the legacy `ret` bootstrap path, which
719    // does not restore RFLAGS. Re-enable interrupts now that `finish_switch()`
720    // has completed and the task is running on its own stack.
721    crate::arch::x86_64::sti();
722
723    // User tasks still transition to Ring 3 via iretq later and will restore
724    // their own RFLAGS there.
725
726    let entry_fn: extern "C" fn(u64) -> ! = unsafe { core::mem::transmute(entry as usize) };
727    entry_fn(arg0)
728}
729
730/// Kernel stack for a task
731pub struct KernelStack {
732    /// Physical address of the stack
733    pub base: PhysAddr,
734    /// Virtual address of the stack
735    pub virt_base: VirtAddr,
736    /// Size of the stack
737    pub size: usize,
738}
739
740impl KernelStack {
741    /// Allocate a new kernel stack using the buddy allocator
742    pub fn allocate(size: usize) -> Result<Self, &'static str> {
743        // Calculate number of pages needed (round up)
744        let pages = (size + 4095) / 4096;
745        let order = pages.next_power_of_two().trailing_zeros() as u8;
746
747        crate::serial_println!("[trace][task] kstack allocate begin size={}", size);
748        crate::serial_println!(
749            "[trace][task] kstack allocate pages={} order={}",
750            pages,
751            order
752        );
753
754        crate::serial_println!(
755            "[trace][task] kstack allocate calling allocate_frames order={}",
756            order
757        );
758        let frame = crate::sync::with_irqs_disabled(|token| {
759            crate::memory::allocate_kernel_stack_frames(token, order)
760        })
761        .map_err(|_| "Failed to allocate kernel stack")?;
762        crate::serial_println!(
763            "[trace][task] kstack allocate frame phys={:#x}",
764            frame.start_address.as_u64()
765        );
766
767        let phys_base = frame.start_address;
768        let virt_base = VirtAddr::new(crate::memory::phys_to_virt(phys_base.as_u64()));
769        crate::serial_println!(
770            "[trace][task] kstack allocate virt_base={:#x}",
771            virt_base.as_u64()
772        );
773
774        // Zero out the stack for safety
775        unsafe {
776            core::ptr::write_bytes(virt_base.as_mut_ptr::<u8>(), 0, size);
777        }
778        crate::serial_println!("[trace][task] kstack allocate memset done");
779
780        // Debug: verify zeroing worked
781        unsafe {
782            let first_word = *(virt_base.as_ptr::<u64>());
783            let mid_offset = size / 2;
784            let mid_word = *((virt_base.as_u64() + mid_offset as u64) as *const u64);
785            let last_offset = size - 8;
786            let last_word = *((virt_base.as_u64() + last_offset as u64) as *const u64);
787            if first_word != 0 || mid_word != 0 || last_word != 0 {
788                crate::serial_force_println!(
789                    "[WARN] kstack zeroing failed! first={:#x} mid={:#x} last={:#x}",
790                    first_word,
791                    mid_word,
792                    last_word
793                );
794            }
795        }
796
797        Ok(KernelStack {
798            base: phys_base,
799            virt_base,
800            size,
801        })
802    }
803
804    /// Debug: check if this stack overlaps with another range
805    pub fn overlaps(&self, other_base: u64, other_size: usize) -> bool {
806        let self_end = self.virt_base.as_u64() + self.size as u64;
807        let other_end = other_base + other_size as u64;
808        !(self_end <= other_base || other_end <= self.virt_base.as_u64())
809    }
810}
811
812impl Drop for KernelStack {
813    /// Performs the drop operation.
814    fn drop(&mut self) {
815        use crate::memory::frame::PhysFrame;
816
817        let pages = (self.size + 4095) / 4096;
818        let order = pages.next_power_of_two().trailing_zeros() as u8;
819        let frame = PhysFrame {
820            start_address: self.base,
821        };
822
823        crate::sync::with_irqs_disabled(|token| {
824            crate::memory::free_kernel_stack_frames(token, frame, order);
825        });
826    }
827}
828
829/// User stack for a task (when running in userspace)
830pub struct UserStack {
831    /// Virtual address of the user stack
832    pub virt_base: VirtAddr,
833    /// Size of the stack
834    pub size: usize,
835}
836
837impl Task {
838    /// Default kernel stack size (64 KB - increased from 16KB due to overflow)
839    pub const DEFAULT_STACK_SIZE: usize = 65536;
840
841    /// Create a new kernel task with a real allocated stack
842    pub fn new_kernel_task(
843        entry_point: extern "C" fn() -> !,
844        name: &'static str,
845        priority: TaskPriority,
846    ) -> Result<Arc<Self>, &'static str> {
847        Self::new_kernel_task_with_stack(entry_point, name, priority, Self::DEFAULT_STACK_SIZE)
848    }
849
850    /// Create a new kernel task with a custom kernel stack size.
851    pub fn new_kernel_task_with_stack(
852        entry_point: extern "C" fn() -> !,
853        name: &'static str,
854        priority: TaskPriority,
855        stack_size: usize,
856    ) -> Result<Arc<Self>, &'static str> {
857        crate::serial_println!(
858            "[trace][task] new_kernel_task_with_stack begin name={} stack_size={}",
859            name,
860            stack_size
861        );
862        // Allocate a real kernel stack
863        let kernel_stack = KernelStack::allocate(stack_size)?;
864        crate::serial_println!("[trace][task] new_kernel_task_with_stack kstack done");
865
866        // Create CPU context with the allocated stack
867        let context = CpuContext::new(entry_point as *const () as u64, &kernel_stack);
868        crate::serial_println!("[trace][task] new_kernel_task_with_stack context done");
869        let id = TaskId::new();
870        let (pid, tid, tgid) = Self::allocate_process_ids();
871        crate::serial_println!(
872            "[trace][task] new_kernel_task_with_stack ids done id={} pid={} tid={} tgid={}",
873            id.as_u64(),
874            pid,
875            tid,
876            tgid
877        );
878        let fpu_state = ExtendedState::new();
879        let xcr0_mask = fpu_state.xcr0_mask;
880
881        let process = Arc::new(crate::process::process::Process::new(
882            pid,
883            crate::memory::kernel_address_space().clone(),
884        ));
885        crate::serial_println!("[trace][task] new_kernel_task_with_stack process done");
886
887        log::debug!(
888            "[task][create] name={} id={} pid={} tid={} kstack={:?} kstack_kib={}",
889            name,
890            id.as_u64(),
891            pid,
892            tid,
893            kernel_stack.virt_base,
894            kernel_stack.size / 1024
895        );
896
897        let task = Arc::new(Task {
898            id,
899            pid,
900            tid,
901            tgid,
902            pgid: AtomicU32::new(pid),
903            sid: AtomicU32::new(pid),
904            uid: AtomicU32::new(0),
905            euid: AtomicU32::new(0),
906            gid: AtomicU32::new(0),
907            egid: AtomicU32::new(0),
908            state: AtomicU8::new(TaskState::Ready as u8),
909            priority,
910            context: SyncUnsafeCell::new(context),
911            resume_kind: SyncUnsafeCell::new(ResumeKind::RetFrame),
912            interrupt_rsp: AtomicU64::new(0),
913            kernel_stack,
914            user_stack: None,
915            name,
916            process,
917            pending_signals: super::signal::SignalSet::new(),
918            blocked_signals: super::signal::SignalSet::new(),
919            irq_signal_delivery_blocked: AtomicBool::new(false),
920            signal_stack: SyncUnsafeCell::new(None),
921            itimers: super::timer::ITimers::new(),
922            wake_pending: AtomicBool::new(false),
923            wake_deadline_ns: AtomicU64::new(0),
924            trampoline_entry: AtomicU64::new(0),
925            trampoline_stack_top: AtomicU64::new(0),
926            trampoline_arg0: AtomicU64::new(0),
927            ticks: AtomicU64::new(0),
928            sched_policy: SyncUnsafeCell::new(Self::default_sched_policy(priority)),
929            home_cpu: AtomicUsize::new(usize::MAX),
930            vruntime: AtomicU64::new(0),
931            fair_rq_generation: AtomicU64::new(0),
932            fair_on_rq: AtomicBool::new(false),
933            clear_child_tid: AtomicU64::new(0),
934            robust_list_head: AtomicU64::new(0),
935            robust_list_len: AtomicUsize::new(0),
936            user_fs_base: AtomicU64::new(0),
937            fpu_state: SyncUnsafeCell::new(fpu_state),
938            xcr0_mask: AtomicU64::new(xcr0_mask),
939            rt_link: LinkedListLink::new(),
940        });
941        task.seed_kernel_interrupt_frame_from_context();
942        Ok(task)
943    }
944
945    /// Create a new user task with its own address space (stub for future use).
946    ///
947    /// The entry point and user stack must already be mapped in the given address space.
948    pub fn new_user_task(
949        entry_point: u64,
950        address_space: Arc<AddressSpace>,
951        name: &'static str,
952        priority: TaskPriority,
953    ) -> Result<Arc<Self>, &'static str> {
954        let kernel_stack = KernelStack::allocate(Self::DEFAULT_STACK_SIZE)?;
955        let context = CpuContext::new(entry_point, &kernel_stack);
956        let id = TaskId::new();
957        let (pid, tid, tgid) = Self::allocate_process_ids();
958        let fpu_state = ExtendedState::new();
959        let xcr0_mask = fpu_state.xcr0_mask;
960
961        log::debug!(
962            "[task][create] name={} id={} pid={} tid={} user_as_cr3={:#x}",
963            name,
964            id.as_u64(),
965            pid,
966            tid,
967            address_space.cr3().as_u64()
968        );
969
970        Ok(Arc::new(Task {
971            id,
972            pid,
973            tid,
974            tgid,
975            pgid: AtomicU32::new(pid),
976            sid: AtomicU32::new(pid),
977            uid: AtomicU32::new(0),
978            euid: AtomicU32::new(0),
979            gid: AtomicU32::new(0),
980            egid: AtomicU32::new(0),
981            state: AtomicU8::new(TaskState::Ready as u8),
982            priority,
983            context: SyncUnsafeCell::new(context),
984            resume_kind: SyncUnsafeCell::new(ResumeKind::RetFrame),
985            interrupt_rsp: AtomicU64::new(0),
986            kernel_stack,
987            user_stack: None,
988            name,
989            process: Arc::new(crate::process::process::Process::new(pid, address_space)),
990            pending_signals: super::signal::SignalSet::new(),
991            blocked_signals: super::signal::SignalSet::new(),
992            irq_signal_delivery_blocked: AtomicBool::new(false),
993            signal_stack: SyncUnsafeCell::new(None),
994            itimers: super::timer::ITimers::new(),
995            wake_pending: AtomicBool::new(false),
996            wake_deadline_ns: AtomicU64::new(0),
997            trampoline_entry: AtomicU64::new(0),
998            trampoline_stack_top: AtomicU64::new(0),
999            trampoline_arg0: AtomicU64::new(0),
1000            ticks: AtomicU64::new(0),
1001            sched_policy: SyncUnsafeCell::new(Self::default_sched_policy(priority)),
1002            home_cpu: AtomicUsize::new(usize::MAX),
1003            vruntime: AtomicU64::new(0),
1004            fair_rq_generation: AtomicU64::new(0),
1005            fair_on_rq: AtomicBool::new(false),
1006            clear_child_tid: AtomicU64::new(0),
1007            robust_list_head: AtomicU64::new(0),
1008            robust_list_len: AtomicUsize::new(0),
1009            user_fs_base: AtomicU64::new(0),
1010            fpu_state: SyncUnsafeCell::new(fpu_state),
1011            xcr0_mask: AtomicU64::new(xcr0_mask),
1012            rt_link: LinkedListLink::new(),
1013        }))
1014    }
1015
1016    /// Reset signal handlers during execve.
1017    ///
1018    /// POSIX requires handlers installed by userspace to revert to SIG_DFL on
1019    /// exec, while dispositions already set to SIG_IGN remain ignored.
1020    pub fn reset_signals(&self) {
1021        // SAFETY: We have a valid reference to the task.
1022        unsafe {
1023            let actions = &mut *self.process.signal_actions.get();
1024            for action in actions.iter_mut() {
1025                if !action.is_ignore() {
1026                    *action = super::signal::SigActionData::default();
1027                }
1028            }
1029        }
1030    }
1031
1032    /// Returns true if this is a kernel task (shares the kernel address space).
1033    pub fn is_kernel(&self) -> bool {
1034        self.process.address_space_arc().is_kernel()
1035    }
1036
1037    /// Allocate POSIX identifiers for a new process leader.
1038    pub fn allocate_process_ids() -> (Pid, Tid, Pid) {
1039        let pid = next_pid();
1040        let tid = next_tid();
1041        (pid, tid, pid)
1042    }
1043
1044    /// Print the memory layout of Task and Process structs for debugging.
1045    ///
1046    /// Computes field offsets at runtime using addr_of! so the output is
1047    /// accurate regardless of Rust's struct reordering decisions.
1048    /// Call this early in kernel init to validate the crash-site offset analysis.
1049    pub fn debug_print_layout() {
1050        use core::mem;
1051        crate::serial_println!("[layout] === Struct Layout Debug ===");
1052        crate::serial_println!(
1053            "[layout] sizeof(Task)          = {}",
1054            mem::size_of::<Task>()
1055        );
1056        crate::serial_println!(
1057            "[layout] sizeof(ExtendedState) = {}",
1058            mem::size_of::<ExtendedState>()
1059        );
1060        crate::serial_println!(
1061            "[layout] alignof(ExtendedState)= {}",
1062            mem::align_of::<ExtendedState>()
1063        );
1064        crate::serial_println!(
1065            "[layout] sizeof(CpuContext)    = {}",
1066            mem::size_of::<CpuContext>()
1067        );
1068        crate::serial_println!(
1069            "[layout] sizeof(KernelStack)   = {}",
1070            mem::size_of::<KernelStack>()
1071        );
1072        crate::serial_println!(
1073            "[layout] sizeof(Process)       = {}",
1074            mem::size_of::<crate::process::process::Process>()
1075        );
1076        crate::serial_println!(
1077            "[layout] sizeof(FileDescriptorTable) = {}",
1078            mem::size_of::<crate::vfs::fd::FileDescriptorTable>()
1079        );
1080        crate::serial_println!(
1081            "[layout] sizeof(CapabilityTable)     = {}",
1082            mem::size_of::<crate::capability::CapabilityTable>()
1083        );
1084        crate::serial_println!(
1085            "[layout] sizeof(SigActionData)       = {}",
1086            mem::size_of::<crate::process::signal::SigActionData>()
1087        );
1088
1089        // Use heap-allocated MaybeUninit to avoid stack overflow from the ~3 KiB
1090        // ExtendedState embedded in Task. We only take *addresses* (addr_of!),
1091        // never read the uninitialized data itself, so this is sound.
1092        let task_box: alloc::boxed::Box<core::mem::MaybeUninit<Task>> =
1093            alloc::boxed::Box::new_uninit();
1094        // Cast to *const Task : we never read Task data, only compute field addresses.
1095        let task_ptr = task_box.as_ptr() as *const Task;
1096        let base = task_ptr as u64;
1097        // SAFETY: We only take addresses via addr_of!, no uninitialized reads.
1098        unsafe {
1099            let off_id = core::ptr::addr_of!((*task_ptr).id) as u64 - base;
1100            let off_pid = core::ptr::addr_of!((*task_ptr).pid) as u64 - base;
1101            let off_context = core::ptr::addr_of!((*task_ptr).context) as u64 - base;
1102            let off_kstack = core::ptr::addr_of!((*task_ptr).kernel_stack) as u64 - base;
1103            let off_process = core::ptr::addr_of!((*task_ptr).process) as u64 - base;
1104            let off_fpu = core::ptr::addr_of!((*task_ptr).fpu_state) as u64 - base;
1105            let off_xcr0 = core::ptr::addr_of!((*task_ptr).xcr0_mask) as u64 - base;
1106            let off_ticks = core::ptr::addr_of!((*task_ptr).ticks) as u64 - base;
1107            let off_name = core::ptr::addr_of!((*task_ptr).name) as u64 - base;
1108            let off_vruntime = core::ptr::addr_of!((*task_ptr).vruntime) as u64 - base;
1109            crate::serial_println!("[layout] Task field offsets (byte offset from Task data ptr):");
1110            crate::serial_println!("[layout]   id           @ +{:#x}", off_id);
1111            crate::serial_println!("[layout]   pid          @ +{:#x}", off_pid);
1112            crate::serial_println!("[layout]   context      @ +{:#x}", off_context);
1113            crate::serial_println!("[layout]   kernel_stack @ +{:#x}", off_kstack);
1114            crate::serial_println!("[layout]   process      @ +{:#x}", off_process);
1115            crate::serial_println!("[layout]   fpu_state    @ +{:#x}", off_fpu);
1116            crate::serial_println!("[layout]   xcr0_mask    @ +{:#x}", off_xcr0);
1117            crate::serial_println!("[layout]   ticks        @ +{:#x}", off_ticks);
1118            crate::serial_println!("[layout]   name         @ +{:#x}", off_name);
1119            crate::serial_println!("[layout]   vruntime     @ +{:#x}", off_vruntime);
1120        }
1121        // Arc<T> ArcInner overhead: strong(8)+weak(8)+data = data at offset 16.
1122        // So the crash at [ArcInner<Task>+0xbf8] means Task.process is at offset
1123        // 0xbf8 - 16 = 0xbe8 inside Task data. Check against off_process above.
1124        crate::serial_println!(
1125            "[layout] Expected task.process crash offset from Task data: {:#x}",
1126            0xbf8u64.saturating_sub(16)
1127        );
1128
1129        // Process field offsets
1130        let proc_box: alloc::boxed::Box<core::mem::MaybeUninit<crate::process::process::Process>> =
1131            alloc::boxed::Box::new_uninit();
1132        #[allow(unused_variables)]
1133        let proc_ptr = proc_box.as_ptr() as *const crate::process::process::Process;
1134        let proc_base = proc_ptr as u64;
1135        unsafe {
1136            let off_pid = core::ptr::addr_of!((*proc_ptr).pid) as u64 - proc_base;
1137            let off_as = core::ptr::addr_of!((*proc_ptr).address_space) as u64 - proc_base;
1138            let off_fd = core::ptr::addr_of!((*proc_ptr).fd_table) as u64 - proc_base;
1139            let off_caps = core::ptr::addr_of!((*proc_ptr).capabilities) as u64 - proc_base;
1140            let off_sigs = core::ptr::addr_of!((*proc_ptr).signal_actions) as u64 - proc_base;
1141            let off_brk = core::ptr::addr_of!((*proc_ptr).brk) as u64 - proc_base;
1142            crate::serial_println!(
1143                "[layout] Process field offsets (byte offset from Process data ptr):"
1144            );
1145            crate::serial_println!("[layout]   pid            @ +{:#x}", off_pid);
1146            crate::serial_println!("[layout]   address_space  @ +{:#x}", off_as);
1147            crate::serial_println!("[layout]   fd_table       @ +{:#x}", off_fd);
1148            crate::serial_println!("[layout]   capabilities   @ +{:#x}", off_caps);
1149            crate::serial_println!("[layout]   signal_actions @ +{:#x}", off_sigs);
1150            crate::serial_println!("[layout]   brk            @ +{:#x}", off_brk);
1151        }
1152        // The crash reads [ArcInner<Process>+0x830].
1153        // ArcInner<Process>.data is at ArcInner+16, so Process offset is 0x830-16 = 0x820.
1154        crate::serial_println!(
1155            "[layout] Expected process field crash offset from Process data: {:#x}",
1156            0x830u64.saturating_sub(16)
1157        );
1158        crate::serial_println!("[layout] ===========================");
1159    }
1160}
1161
1162/// Context switch dispatcher. Picks the xsave or fxsave path based on host
1163/// capabilities, then performs the full save/swap/restore sequence.
1164///
1165/// # Safety
1166/// Caller must ensure all pointers in `target` are valid and interrupts are disabled.
1167pub(super) unsafe fn do_switch_context(target: &super::scheduler::SwitchTarget) {
1168    if crate::arch::x86_64::cpuid::host_uses_xsave() {
1169        let old_xcr0 = normalized_xcr0(target.old_xcr0);
1170        let new_xcr0 = normalized_xcr0(target.new_xcr0);
1171        switch_context_xsave(
1172            target.old_rsp_ptr,
1173            target.new_rsp_ptr,
1174            target.old_fpu_ptr,
1175            target.new_fpu_ptr,
1176            new_xcr0,
1177            old_xcr0,
1178        );
1179    } else {
1180        switch_context_fxsave(
1181            target.old_rsp_ptr,
1182            target.new_rsp_ptr,
1183            target.old_fpu_ptr,
1184            target.new_fpu_ptr,
1185        );
1186    }
1187}
1188
1189/// First-task restore dispatcher. Like `do_switch_context` but without
1190/// saving old state (there is no previous task).
1191///
1192/// # Safety
1193/// Caller must ensure pointers are valid and interrupts are disabled. Never returns.
1194pub(super) unsafe fn do_restore_first_task(
1195    frame_ptr: *const u64, // Points to the stack frame (r15, r14, r13, r12, rbp, rbx, ret)
1196    fpu_ptr: *const u8,
1197    xcr0: u64,
1198) -> ! {
1199    // Debug: verify frame pointer
1200    crate::serial_force_println!(
1201        "[task] do_restore_first_task frame_ptr={:#x} fpu_ptr={:#x}",
1202        frame_ptr as u64,
1203        fpu_ptr as u64
1204    );
1205
1206    // Verify the stack frame contains expected values
1207    crate::serial_force_println!(
1208        "[task] do_restore_first_task stack frame: r15={:#x} r14={:#x} r13={:#x} r12={:#x} rbp={:#x} rbx={:#x} ret={:#x}",
1209        *frame_ptr.add(0),
1210        *frame_ptr.add(1),
1211        *frame_ptr.add(2),
1212        *frame_ptr.add(3),
1213        *frame_ptr.add(4),
1214        *frame_ptr.add(5),
1215        *frame_ptr.add(6)
1216    );
1217
1218    // Verify canary immediately above the fake frame (frame is 7 words long).
1219    let canary_addr = frame_ptr as u64 + 56;
1220    let canary = *(canary_addr as *const u64);
1221    crate::serial_force_println!(
1222        "[task] do_restore_first_task canary at {:#x} = {:#x} (expected 0xdeadbeefcafebabe)",
1223        canary_addr,
1224        canary
1225    );
1226
1227    if crate::arch::x86_64::cpuid::host_uses_xsave() {
1228        restore_first_task_xsave(frame_ptr, fpu_ptr, normalized_xcr0(xcr0));
1229    } else {
1230        let _ = xcr0;
1231        restore_first_task_fxsave(frame_ptr, fpu_ptr);
1232    }
1233}
1234
1235//  FXSAVE path (legacy, no XSAVE support)
1236
1237/// rdi=old_rsp, rsi=new_rsp, rdx=old_fpu, rcx=new_fpu
1238#[unsafe(naked)]
1239unsafe extern "C" fn switch_context_fxsave(
1240    _old_rsp_ptr: *mut u64,
1241    _new_rsp_ptr: *const u64,
1242    _old_fpu_ptr: *mut u8,
1243    _new_fpu_ptr: *const u8,
1244) {
1245    core::arch::naked_asm!(
1246        "fxsave [rdx]",
1247        "push rbx",
1248        "push rbp",
1249        "push r12",
1250        "push r13",
1251        "push r14",
1252        "push r15",
1253        "mov [rdi], rsp",
1254        "mov rsp, [rsi]",
1255        "pop r15",
1256        "pop r14",
1257        "pop r13",
1258        "pop r12",
1259        "pop rbp",
1260        "pop rbx",
1261        "fxrstor [rcx]",
1262        "ret",
1263    );
1264}
1265
1266/// rdi=frame_ptr, rsi=fpu_ptr
1267#[unsafe(naked)]
1268unsafe extern "C" fn restore_first_task_fxsave(_rsp_ptr: *const u64, _fpu_ptr: *const u8) -> ! {
1269    // Debug: output pointers before restore (will be last serial output)
1270    // We can't use serial_println in naked functions, so this is just a marker
1271    // The actual debug output is in do_restore_first_task
1272    core::arch::naked_asm!(
1273        // `do_restore_first_task` passes the frame address directly.
1274        "mov rsp, rdi",
1275        "pop r15",
1276        "pop r14",
1277        "pop r13",
1278        "pop r12",
1279        "pop rbp",
1280        "pop rbx",
1281        "fxrstor [rsi]",
1282        "ret",
1283    );
1284}
1285
1286//  XSAVE path (with XCR0 switching per-silo)
1287
1288/// rdi=old_rsp, rsi=new_rsp, rdx=old_fpu, rcx=new_fpu, r8=new_xcr0, r9=old_xcr0
1289#[unsafe(naked)]
1290unsafe extern "C" fn switch_context_xsave(
1291    _old_rsp_ptr: *mut u64,
1292    _new_rsp_ptr: *const u64,
1293    _old_fpu_ptr: *mut u8,
1294    _new_fpu_ptr: *const u8,
1295    _new_xcr0: u64,
1296    _old_xcr0: u64,
1297) {
1298    core::arch::naked_asm!(
1299        "mov r10, rdx",
1300        "mov r11, r8",
1301        "test r11, r11",
1302        "jnz 10f",
1303        "mov r11, 3",
1304        "10:",
1305        "test r9, r9",
1306        "jnz 11f",
1307        "mov r9, 3",
1308        "11:",
1309        "mov r8, r9", // Store normalized old_xcr0 in r8 before it is modified
1310        "mov eax, r9d",
1311        "shr r9, 32",
1312        "mov edx, r9d",
1313        "xsave [r10]",
1314        "push rbx",
1315        "push rbp",
1316        "push r12",
1317        "push r13",
1318        "push r14",
1319        "push r15",
1320        "mov [rdi], rsp",
1321        "mov rsp, [rsi]",
1322        "pop r15",
1323        "pop r14",
1324        "pop r13",
1325        "pop r12",
1326        "pop rbp",
1327        "pop rbx",
1328        "cmp r11, r8", // Check if new_xcr0 == old_xcr0
1329        "je 20f",      // Skip xsetbv if they are equal
1330        "push rcx",
1331        "mov ecx, 0",
1332        "mov eax, r11d",
1333        "mov r8, r11",
1334        "shr r8, 32",
1335        "mov edx, r8d",
1336        "xsetbv",
1337        "pop rcx",
1338        "20:",
1339        "mov eax, r11d",
1340        "mov r8, r11",
1341        "shr r8, 32",
1342        "mov edx, r8d",
1343        "xrstor [rcx]",
1344        "ret",
1345    );
1346}
1347
1348/// rdi=frame_ptr, rsi=fpu_ptr, rdx=xcr0
1349#[unsafe(naked)]
1350unsafe extern "C" fn restore_first_task_xsave(
1351    _rsp_ptr: *const u64,
1352    _fpu_ptr: *const u8,
1353    _xcr0: u64,
1354) -> ! {
1355    core::arch::naked_asm!(
1356        // `do_restore_first_task` passes the frame address directly.
1357        "mov rsp, rdi",
1358        "pop r15",
1359        "pop r14",
1360        "pop r13",
1361        "pop r12",
1362        "pop rbp",
1363        "pop rbx",
1364        "mov r8, rdx",
1365        "test r8, r8",
1366        "jnz 10f",
1367        "mov r8, 3",
1368        "10:",
1369        "mov r9, r8",
1370        "push rsi",
1371        "mov ecx, 0",
1372        "mov eax, r8d",
1373        "shr r8, 32",
1374        "mov edx, r8d",
1375        "xsetbv",
1376        "pop rsi",
1377        "mov eax, r9d",
1378        "shr r9, 32",
1379        "mov edx, r9d",
1380        "xrstor [rsi]",
1381        "ret",
1382    );
1383}
strat9_kernel/process/task.rs

strat9_kernel/process/
task.rs
