strat9_kernel/hardware/virtio/
common.rs

1//! Common VirtIO infrastructure
2//!
3//! Provides virtqueue management and device initialization logic
4//! shared across all VirtIO drivers.
5//!
6//! Reference: VirtIO spec v1.2, Section 2 (Basic Facilities of a Virtio Device)
7//! https://docs.oasis-open.org/virtio/virtio/v1.2/os/virtio-v1.2-os.html#_basic-facilities-of-a-virtio-device
8
9use super::{vring_flags, VirtqDesc};
10use crate::{
11    arch::pci::{Bar, PciDevice},
12    memory::{self, PhysFrame},
13};
14use core::{
15    ptr::{read_volatile, write_volatile},
16    sync::atomic::{fence, AtomicU16, Ordering},
17};
18use endian_num::Le;
19
20/// VirtIO device features
21pub mod features {
22    pub const VIRTIO_F_RING_INDIRECT_DESC: u64 = 1 << 28;
23    pub const VIRTIO_F_RING_EVENT_IDX: u64 = 1 << 29;
24    pub const VIRTIO_F_VERSION_1: u64 = 1 << 32;
25    pub const VIRTIO_F_ACCESS_PLATFORM: u64 = 1 << 33;
26    pub const VIRTIO_F_RING_PACKED: u64 = 1 << 34;
27    pub const VIRTIO_F_IN_ORDER: u64 = 1 << 35;
28    pub const VIRTIO_F_ORDER_PLATFORM: u64 = 1 << 36;
29    pub const VIRTIO_F_SR_IOV: u64 = 1 << 37;
30    pub const VIRTIO_F_NOTIFICATION_DATA: u64 = 1 << 38;
31}
32
33/// Available ring structure (device -> driver notifications)
34#[repr(C)]
35pub struct VirtqAvail {
36    pub flags: AtomicU16,
37    pub idx: AtomicU16,
38    // ring follows (variable length)
39    // used_event follows ring (if VIRTIO_F_RING_EVENT_IDX)
40}
41
42/// Used ring element
43#[repr(C)]
44#[derive(Debug, Clone, Copy)]
45pub struct VirtqUsedElem {
46    /// Index of start of used descriptor chain
47    pub id: u32,
48    /// Total length of the descriptor chain
49    pub len: u32,
50}
51
52/// Used ring structure (driver -> device notifications)
53#[repr(C)]
54pub struct VirtqUsed {
55    pub flags: AtomicU16,
56    pub idx: AtomicU16,
57    // ring follows (variable length)
58    // avail_event follows ring (if VIRTIO_F_RING_EVENT_IDX)
59}
60
61/// A VirtIO virtqueue
62///
63/// This structure manages the split virtqueue format as described in
64/// VirtIO spec section 2.6 (Split Virtqueues).
65pub struct Virtqueue {
66    /// Queue size (must be power of 2)
67    queue_size: u16,
68
69    /// Contiguous legacy vring allocation backing desc+avail+used
70    _ring_area: PhysFrame,
71
72    /// Physical address of descriptor table
73    desc_area: u64,
74
75    /// Physical address of available ring
76    avail_area: u64,
77
78    /// Physical address of used ring
79    used_area: u64,
80
81    /// Virtual address of descriptor table
82    desc_ptr: *mut VirtqDesc,
83
84    /// Virtual address of available ring
85    avail_ptr: *mut VirtqAvail,
86
87    /// Virtual address of available ring entries
88    avail_ring_ptr: *mut u16,
89
90    /// Virtual address of used ring
91    used_ptr: *mut VirtqUsed,
92
93    /// Virtual address of used ring entries
94    used_ring_ptr: *mut VirtqUsedElem,
95
96    /// Head of the free-descriptor linked list (stored in desc[i].next).
97    /// When queue_size ≤ INLINE_FREE_SIZE we use the inline array instead,
98    /// keeping the hot path entirely in-stack / in-struct (no Vec overhead).
99    first_free: u16,
100    /// Number of free descriptors.
101    free_count: u16,
102
103    /// Last seen used index
104    last_used_idx: u16,
105
106    /// Next available index
107    next_avail_idx: u16,
108}
109
110// Send is safe because we manage synchronization via SpinLocks in usage
111unsafe impl Send for Virtqueue {}
112
113impl Virtqueue {
114    #[inline]
115    fn align_up(value: usize, align: usize) -> usize {
116        debug_assert!(align.is_power_of_two());
117        (value + align - 1) & !(align - 1)
118    }
119
120    /// Create a new virtqueue with the specified size
121    ///
122    /// # Safety
123    /// The caller must ensure that the allocated memory is properly mapped
124    /// and accessible.
125    pub unsafe fn new(queue_size: u16) -> Result<Self, &'static str> {
126        if !queue_size.is_power_of_two() {
127            return Err("Queue size must be power of 2");
128        }
129
130        let desc_size = queue_size as usize * core::mem::size_of::<VirtqDesc>();
131        let avail_size = 6 + queue_size as usize * 2;
132        let used_size = 6 + queue_size as usize * core::mem::size_of::<VirtqUsedElem>();
133        // Legacy virtio layout (Section 2.5.1): the device derives ring offsets
134        // from the descriptor-table PFN using PAGE_ALIGN.  The driver MUST
135        // match that same alignment or device and driver will read/write
136        // different memory locations.
137        let avail_offset = Self::align_up(desc_size, 4096); // PAGE_ALIGN(desc_size)
138        let used_offset = Self::align_up(avail_offset + avail_size, 4096); // PAGE_ALIGN(avail_end)
139        let total_size = used_offset + used_size;
140
141        // Critical: legacy QUEUE_PFN describes one contiguous vring region.
142        let ring_pages = (total_size + 4095) / 4096;
143        let ring_order = ring_pages.next_power_of_two().trailing_zeros() as u8;
144        let ring_area = crate::sync::with_irqs_disabled(|token| {
145            memory::allocate_phys_contiguous(token, ring_order)
146        })
147        .map_err(|_| "Failed to allocate virtqueue ring")?;
148        let ring_phys = ring_area.start_address.as_u64();
149        let desc_phys = ring_phys;
150        let avail_phys = ring_phys + avail_offset as u64;
151        let used_phys = ring_phys + used_offset as u64;
152
153        // SAFETY: we just allocated these frames; convert phys => virt via HHDM
154        // With UEFI bootloader HHDM, all physical memory is already mapped, so we can
155        // directly use phys_to_virt without additional page table modifications.
156        // DO NOT call ensure_identity_map here - it can corrupt active page tables!
157
158        let desc_virt = crate::memory::phys_to_virt(desc_phys);
159        let avail_virt = crate::memory::phys_to_virt(avail_phys);
160        let used_virt = crate::memory::phys_to_virt(used_phys);
161
162        let desc_ptr = desc_virt as *mut VirtqDesc;
163        let avail_ptr = avail_virt as *mut VirtqAvail;
164        let avail_ring_ptr = (avail_virt + 4) as *mut u16;
165        let used_ptr = used_virt as *mut VirtqUsed;
166        let used_ring_ptr = (used_virt + 4) as *mut VirtqUsedElem;
167
168        // Zero out the memory
169        // SAFETY: we allocated these pages and they're mapped via HHDM
170        core::ptr::write_bytes(desc_ptr as *mut u8, 0, total_size);
171
172        // Initialize free-descriptor linked list through desc[i].next.
173        // Descriptor i points to i+1; the last has flags=0 and next=0.
174        for i in 0..queue_size - 1 {
175            let desc = &mut *desc_ptr.add(i as usize);
176            desc.next = Le::<u16>::from_ne(i + 1);
177        }
178        // last descriptor does not chain further
179        let last_desc = &mut *desc_ptr.add((queue_size - 1) as usize);
180        last_desc.next = Le::<u16>::from_ne(0);
181        last_desc.flags = Le::<u16>::from_ne(0);
182
183        Ok(Self {
184            queue_size,
185            _ring_area: ring_area,
186            desc_area: desc_phys,
187            avail_area: avail_phys,
188            used_area: used_phys,
189            desc_ptr,
190            avail_ptr,
191            avail_ring_ptr,
192            used_ptr,
193            used_ring_ptr,
194            first_free: 0,
195            free_count: queue_size,
196            last_used_idx: 0,
197            next_avail_idx: 0,
198        })
199    }
200
201    /// Get the physical address of the descriptor table
202    pub fn desc_area(&self) -> u64 {
203        self.desc_area
204    }
205
206    /// Get the physical address of the available ring
207    pub fn avail_area(&self) -> u64 {
208        self.avail_area
209    }
210
211    /// Get the physical address of the used ring
212    pub fn used_area(&self) -> u64 {
213        self.used_area
214    }
215
216    /// Get the queue size (number of descriptors)
217    pub fn queue_size(&self) -> usize {
218        self.queue_size as usize
219    }
220
221    /// Allocate a descriptor from the free list.
222    ///
223    /// Returns the head descriptor index, or `None` if none are free.
224    pub fn alloc_descriptor(&mut self) -> Option<u16> {
225        if self.free_count == 0 {
226            return None;
227        }
228        let idx = self.first_free;
229        // SAFETY: idx is a valid free descriptor (free_count > 0).
230        let desc = unsafe { &*self.desc_ptr.add(idx as usize) };
231        self.first_free = desc.next.to_ne(); // chain to next free
232        self.free_count -= 1;
233        Some(idx)
234    }
235
236    /// Free a descriptor chain back into the free list.
237    ///
238    /// Walks the chain via NEXT flags and re-links each descriptor
239    /// at the head of the free list.
240    pub fn free_descriptor(&mut self, head: u16) {
241        let mut current = head;
242
243        loop {
244            // SAFETY: current is a valid descriptor index (belongs to this queue).
245            let desc = unsafe { &*self.desc_ptr.add(current as usize) };
246            let has_next = desc.flags.to_ne() & vring_flags::NEXT != 0;
247            let next = desc.next.to_ne();
248
249            // Re-link this descriptor at the head of the free list.
250            let desc_mut = unsafe { &mut *self.desc_ptr.add(current as usize) };
251            desc_mut.next = Le::<u16>::from_ne(self.first_free);
252            self.first_free = current;
253            self.free_count += 1;
254
255            if !has_next {
256                break;
257            }
258            current = next;
259        }
260    }
261
262    /// Add a buffer to the virtqueue
263    ///
264    /// Returns the descriptor index (token) that can be used to track completion
265    ///
266    /// # Arguments
267    /// * `buffers`: A list of (physical_address, length, is_write_only)
268    pub fn add_buffer(&mut self, buffers: &[(u64, u32, bool)]) -> Result<u16, &'static str> {
269        if buffers.is_empty() {
270            return Err("Empty buffer list");
271        }
272
273        if (buffers.len() as u16) > self.free_count {
274            return Err("Not enough free descriptors");
275        }
276
277        // Allocate descriptor chain
278        let head = self.alloc_descriptor().ok_or("No free descriptors")?;
279        let mut current = head;
280
281        for (i, &(addr, len, write)) in buffers.iter().enumerate() {
282            let is_last = i == buffers.len() - 1;
283
284            // SAFETY: current is a valid index regulated by alloc_descriptor
285            let desc = unsafe { &mut *self.desc_ptr.add(current as usize) };
286            desc.addr = Le::<u64>::from_ne(addr);
287            desc.len = Le::<u32>::from_ne(len);
288            desc.flags = Le::<u16>::from_ne(if write { vring_flags::WRITE } else { 0 });
289
290            if !is_last {
291                let next = self.alloc_descriptor().ok_or("No free descriptors")?;
292                desc.flags = Le::<u16>::from_ne(desc.flags.to_ne() | vring_flags::NEXT);
293                desc.next = Le::<u16>::from_ne(next);
294                current = next;
295            }
296        }
297
298        // Add to available ring
299        // SAFETY: Atomic load
300        let avail_idx = unsafe { (*self.avail_ptr).idx.load(Ordering::Acquire) };
301        let ring_idx = (avail_idx % self.queue_size) as usize;
302
303        // SAFETY: ring_idx is bounded by queue_size
304        unsafe {
305            write_volatile(self.avail_ring_ptr.add(ring_idx), head);
306        }
307
308        // Memory barrier before updating index to ensure device sees the descriptor table updates
309        fence(Ordering::Release);
310
311        // Update available index
312        // SAFETY: Atomic store
313        unsafe {
314            (*self.avail_ptr)
315                .idx
316                .store(avail_idx.wrapping_add(1), Ordering::Release);
317        }
318
319        self.next_avail_idx = avail_idx.wrapping_add(1);
320
321        Ok(head)
322    }
323
324    /// Check if there are any used buffers
325    pub fn has_used(&self) -> bool {
326        // SAFETY: Atomic load
327        let used_idx = unsafe { (*self.used_ptr).idx.load(Ordering::Acquire) };
328        self.last_used_idx != used_idx
329    }
330
331    /// Return a snapshot of `(device_used_idx, driver_last_used_idx)` for diagnostics.
332    pub fn used_indices(&self) -> (u16, u16) {
333        // SAFETY: Atomic load from the used ring header.
334        let used_idx = unsafe { (*self.used_ptr).idx.load(Ordering::Acquire) };
335        (used_idx, self.last_used_idx)
336    }
337
338    /// Peek at the next used buffer without consuming it.
339    ///
340    /// Returns (descriptor_index, length_written) but does NOT advance
341    /// `last_used_idx` or free descriptors.  The caller must call
342    /// `get_used` afterwards to actually consume the entry.
343    /// Returns `None` if the used ring is empty.
344    pub fn peek_used(&self) -> Option<(u16, u32)> {
345        let used_idx = unsafe { (*self.used_ptr).idx.load(Ordering::Acquire) };
346        if self.last_used_idx == used_idx {
347            return None;
348        }
349        let ring_idx = (self.last_used_idx % self.queue_size) as usize;
350        let elem = unsafe { read_volatile(self.used_ring_ptr.add(ring_idx)) };
351        Some((elem.id as u16, elem.len))
352    }
353
354    /// Get the next used buffer
355    ///
356    /// Returns (descriptor_index, length_written)
357    pub fn get_used(&mut self) -> Option<(u16, u32)> {
358        // SAFETY: Atomic load
359        let used_idx = unsafe { (*self.used_ptr).idx.load(Ordering::Acquire) };
360
361        if self.last_used_idx == used_idx {
362            return None;
363        }
364
365        let ring_idx = (self.last_used_idx % self.queue_size) as usize;
366
367        // SAFETY: ring_idx is bounded by queue_size
368        let elem = unsafe { read_volatile(self.used_ring_ptr.add(ring_idx)) };
369
370        self.last_used_idx = self.last_used_idx.wrapping_add(1);
371
372        self.free_descriptor(elem.id as u16);
373
374        Some((elem.id as u16, elem.len))
375    }
376
377    /// Notify the device (should write to queue_notify register)
378    ///
379    /// When `VIRTIO_F_RING_EVENT_IDX` is negotiated, the device sets the
380    /// `VRING_USED_F_NO_NOTIFY` flag in the used ring to suppress
381    /// driver notifications.  Without event_idx we always notify.
382    pub fn should_notify(&self) -> bool {
383        // SAFETY: used_ptr points to the valid used ring header.
384        let used_flags = unsafe { (*self.used_ptr).flags.load(Ordering::Acquire) };
385        // VRING_USED_F_NO_NOTIFY = 1
386        (used_flags & 1) == 0
387    }
388}
389
390/// VirtIO device base
391///
392/// Common functionality for all VirtIO devices
393pub struct VirtioDevice {
394    /// PCI device
395    pub pci_dev: PciDevice,
396
397    /// I/O base address (BAR0 for legacy devices)
398    pub io_base: u16,
399}
400
401impl VirtioDevice {
402    /// Create a new VirtIO device from a PCI device
403    ///
404    /// # Safety
405    /// The PCI device must be a valid VirtIO device
406    pub unsafe fn new(pci_dev: PciDevice) -> Result<Self, &'static str> {
407        // Read BAR0 (I/O space for legacy VirtIO devices)
408        let bar0 = pci_dev.read_bar(0).ok_or("BAR0 not present")?;
409
410        let io_base = match bar0 {
411            Bar::Io { port } => port,
412            _ => return Err("BAR0 is not I/O space (legacy VirtIO required)"),
413        };
414
415        // Enable I/O space and bus mastering
416        pci_dev.enable_io_space();
417        pci_dev.enable_bus_master();
418
419        Ok(Self { pci_dev, io_base })
420    }
421
422    /// Read an 8-bit value from a device register
423    pub fn read_reg_u8(&self, offset: u16) -> u8 {
424        // SAFETY: I/O port access to VirtIO device registers
425        unsafe { crate::arch::io::inb(self.io_base + offset) }
426    }
427
428    /// Read a 16-bit value from a device register
429    pub fn read_reg_u16(&self, offset: u16) -> u16 {
430        // SAFETY: I/O port access to VirtIO device registers
431        unsafe { crate::arch::io::inw(self.io_base + offset) }
432    }
433
434    /// Read a 32-bit value from a device register
435    pub fn read_reg_u32(&self, offset: u16) -> u32 {
436        // SAFETY: I/O port access to VirtIO device registers
437        unsafe { crate::arch::io::inl(self.io_base + offset) }
438    }
439
440    /// Write an 8-bit value to a device register
441    pub fn write_reg_u8(&self, offset: u16, value: u8) {
442        // SAFETY: I/O port access to VirtIO device registers
443        unsafe { crate::arch::io::outb(self.io_base + offset, value) }
444    }
445
446    /// Write a 16-bit value to a device register
447    pub fn write_reg_u16(&self, offset: u16, value: u16) {
448        // SAFETY: I/O port access to VirtIO device registers
449        unsafe { crate::arch::io::outw(self.io_base + offset, value) }
450    }
451
452    /// Write a 32-bit value to a device register
453    pub fn write_reg_u32(&self, offset: u16, value: u32) {
454        // SAFETY: I/O port access to VirtIO device registers
455        unsafe { crate::arch::io::outl(self.io_base + offset, value) }
456    }
457
458    /// Read device features
459    pub fn read_device_features(&self) -> u32 {
460        self.read_reg_u32(0) // VIRTIO_PCI_HOST_FEATURES
461    }
462
463    /// Write guest features
464    pub fn write_guest_features(&self, features: u32) {
465        self.write_reg_u32(4, features); // VIRTIO_PCI_GUEST_FEATURES
466    }
467
468    /// Get device status
469    pub fn get_status(&self) -> u8 {
470        self.read_reg_u8(18) // VIRTIO_PCI_STATUS
471    }
472
473    /// Set device status
474    pub fn set_status(&self, status: u8) {
475        self.write_reg_u8(18, status); // VIRTIO_PCI_STATUS
476    }
477
478    /// Add status flags
479    pub fn add_status(&self, status: u8) {
480        let current = self.get_status();
481        self.set_status(current | status);
482    }
483
484    /// Reset the device
485    pub fn reset(&self) {
486        self.set_status(0);
487    }
488
489    /// Read ISR status (clears interrupt)
490    pub fn read_isr_status(&self) -> u8 {
491        self.read_reg_u8(19) // VIRTIO_PCI_ISR
492    }
493
494    /// Acknowledge interrupt (write 0 to ISR)
495    pub fn ack_interrupt(&self) {
496        // Reading ISR already clears it, but we can also write to acknowledge
497        let _ = self.read_reg_u8(19); // VIRTIO_PCI_ISR
498    }
499
500    /// Setup a virtqueue
501    pub fn setup_queue(&self, queue_index: u16, queue: &Virtqueue) {
502        // Select queue
503        self.write_reg_u16(14, queue_index); // VIRTIO_PCI_QUEUE_SEL
504
505        // Read max queue size; warn if our size exceeds it
506        let max = self.read_reg_u16(12); // VIRTIO_PCI_QUEUE_NUM
507        if max != 0 && (queue.queue_size() as u16) > max {
508            log::warn!(
509                "virtio: queue {} size {} > device max {}",
510                queue_index,
511                queue.queue_size(),
512                max,
513            );
514        }
515
516        // Set queue addresses (page-aligned physical addresses >> 12)
517        let desc_pfn = (queue.desc_area() >> 12) as u32;
518        self.write_reg_u32(8, desc_pfn); // VIRTIO_PCI_QUEUE_PFN
519
520        log::info!(
521            "virtio: queue {} set up (size={}, pfn={:#x})",
522            queue_index,
523            queue.queue_size(),
524            desc_pfn,
525        );
526    }
527
528    /// Read the queue size exposed by the selected legacy PCI queue.
529    pub fn queue_max_size(&self, queue_index: u16) -> u16 {
530        self.write_reg_u16(14, queue_index); // VIRTIO_PCI_QUEUE_SEL
531        self.read_reg_u16(12) // VIRTIO_PCI_QUEUE_NUM
532    }
533
534    /// Notify a queue
535    pub fn notify_queue(&self, queue_index: u16) {
536        // Write as 32-bit: some QEMU/config combos ignore 16-bit writes
537        // to the QueueNotify register.
538        self.write_reg_u32(16, queue_index as u32); // VIRTIO_PCI_QUEUE_NOTIFY
539    }
540}
strat9_kernel/hardware/virtio/common.rs

strat9_kernel/hardware/virtio/
common.rs
