1#![allow(dead_code)]
13
14use crate::{
15 hardware::pci_client::{self as pci, Bar, ProbeCriteria},
16 memory::{allocate_zeroed_frame, paging, phys_to_virt},
17};
18use alloc::{sync::Arc, vec::Vec};
19use core::{
20 ptr::{read_volatile, write_volatile},
21 sync::atomic::{AtomicBool, AtomicU8, AtomicUsize, Ordering},
22};
23use spin::Mutex;
24
25const XHCI_MMIO_SIZE: usize = 0x10000;
26const XHCI_PORT_REG_BASE: usize = 0x400;
27const XHCI_PORT_REG_STRIDE: usize = 0x10;
28const XHCI_RING_TRBS: usize = 64;
29const MAX_ENDPOINTS: usize = 31;
30
31const USBCMD_RUN_STOP: u32 = 1 << 0;
32const USBCMD_HCRST: u32 = 1 << 1;
33const USBCMD_INTE: u32 = 1 << 2;
34
35const USBSTS_HCH: u32 = 1 << 0;
36const USBSTS_CNR: u32 = 1 << 11;
37
38const PORTSC_CCS: u32 = 1 << 0;
39const PORTSC_PED: u32 = 1 << 1;
40const PORTSC_PR: u32 = 1 << 4;
41const PORTSC_PP: u32 = 1 << 9;
42const PORTSC_SPEED_SHIFT: u32 = 10;
43const PORTSC_W1C_MASK: u32 = 0xFE0000;
44
45const TRB_TYPE_NORMAL: u32 = 1;
46const TRB_TYPE_SETUP_STAGE: u32 = 2;
47const TRB_TYPE_DATA_STAGE: u32 = 3;
48const TRB_TYPE_STATUS_STAGE: u32 = 4;
49const TRB_TYPE_ENABLE_SLOT: u32 = 9;
50const TRB_TYPE_ADDRESS_DEVICE: u32 = 11;
51const TRB_TYPE_CONFIGURE_ENDPOINT: u32 = 12;
52const TRB_TYPE_TRANSFER_EVENT: u32 = 32;
53const TRB_TYPE_LINK: u32 = 6;
54
55const TRB_CYCLE: u32 = 1 << 0;
56const TRB_IOC: u32 = 1 << 5;
57const TRB_DIR_IN: u32 = 1 << 16;
58const TRB_DIR_OUT: u32 = 0;
59const TRB_TC: u32 = 1 << 1;
60
61const TRB_TYPE_SHIFT: u32 = 10;
62const TRB_IDT: u32 = 1 << 6;
63const TRB_TD_SIZE_SHIFT: u32 = 17;
64const TRB_TD_SIZE_MASK: u32 = 0x1F;
65
66const EP_TYPE_CONTROL: u32 = 4;
67const EP_TYPE_INTR_IN: u32 = 7;
68
69const fn TRB_GET_TYPE(d3: u32) -> u32 {
70 (d3 >> TRB_TYPE_SHIFT) & 0xFF
71}
72
73#[repr(C)]
74struct CapRegisters {
75 caplength: u8,
76 _reserved: u8,
77 _hciversion: u16,
78 hcsparams1: u32,
79 _hcsparams2: u32,
80 _hcsparams3: u32,
81 _hccparams1: u32,
82 dboff: u32,
83 rtsoff: u32,
84 _hccparams2: u32,
85}
86
87#[repr(C)]
88struct OpRegisters {
89 usbcmd: u32,
90 usbsts: u32,
91 _pagesize: u32,
92 _reserved0: [u32; 2],
93 _dnctrl: u32,
94 crcr: u64,
95 _reserved1: [u32; 4],
96 dcbaap: u64,
97 config: u32,
98}
99
100#[repr(C)]
101struct RuntimeRegisters {
102 _mfindex: u32,
103 _reserved: [u32; 7],
104 ir: [InterrupterRegisters; 1],
105}
106
107#[repr(C)]
108struct InterrupterRegisters {
109 iman: u32,
110 _imod: u32,
111 erstsz: u32,
112 _reserved: u32,
113 erstba: u64,
114 erdp: u64,
115}
116
117#[repr(C)]
118#[derive(Clone, Copy)]
119struct Trb {
120 d0: u32,
121 d1: u32,
122 d2: u32,
123 d3: u32,
124}
125
126impl Trb {
127 fn link(addr: u64, toggle_cycle: bool) -> Self {
128 Self {
129 d0: (addr & 0xFFFFFFFF) as u32,
130 d1: ((addr >> 32) & 0xFFFFFFFF) as u32,
131 d2: 0,
132 d3: ((TRB_TYPE_LINK << TRB_TYPE_SHIFT) as u32)
133 | TRB_CYCLE
134 | (if toggle_cycle { TRB_TC } else { 0 }),
135 }
136 }
137
138 fn normal(addr: u64, len: u32, cycle: bool, ioc: bool) -> Self {
139 let mut d3 = (TRB_TYPE_NORMAL << TRB_TYPE_SHIFT) as u32 | if cycle { TRB_CYCLE } else { 0 };
140 if ioc {
141 d3 |= TRB_IOC;
142 }
143 Self {
144 d0: (addr & 0xFFFFFFFF) as u32,
145 d1: ((addr >> 32) & 0xFFFFFFFF) as u32,
146 d2: len,
147 d3,
148 }
149 }
150
151 fn setup_stage(addr: u64, cycle: bool) -> Self {
152 let mut d3 =
153 (TRB_TYPE_SETUP_STAGE << TRB_TYPE_SHIFT) as u32 | if cycle { TRB_CYCLE } else { 0 };
154 d3 |= TRB_IDT;
155 Self {
156 d0: (addr & 0xFFFFFFFF) as u32,
157 d1: ((addr >> 32) & 0xFFFFFFFF) as u32,
158 d2: 8,
159 d3,
160 }
161 }
162
163 fn data_stage(addr: u64, len: u32, dir_in: bool, cycle: bool, ioc: bool) -> Self {
164 let mut d3 =
165 (TRB_TYPE_DATA_STAGE << TRB_TYPE_SHIFT) as u32 | if cycle { TRB_CYCLE } else { 0 };
166 if dir_in {
167 d3 |= TRB_DIR_IN;
168 }
169 if ioc {
170 d3 |= TRB_IOC;
171 }
172 let td_size = ((len + TRB_TD_SIZE_MASK) / (TRB_TD_SIZE_MASK + 1)) & TRB_TD_SIZE_MASK;
173 let d2 = (td_size << TRB_TD_SIZE_SHIFT) | len;
174 Self {
175 d0: (addr & 0xFFFFFFFF) as u32,
176 d1: ((addr >> 32) & 0xFFFFFFFF) as u32,
177 d2,
178 d3,
179 }
180 }
181
182 fn status_stage(cycle: bool, dir_in: bool) -> Self {
183 let mut d3 =
184 (TRB_TYPE_STATUS_STAGE << TRB_TYPE_SHIFT) as u32 | if cycle { TRB_CYCLE } else { 0 };
185 if dir_in {
186 d3 |= TRB_DIR_IN;
187 }
188 d3 |= TRB_IOC;
189 Self {
190 d0: 0,
191 d1: 0,
192 d2: 0,
193 d3,
194 }
195 }
196}
197
198#[repr(C, packed)]
199struct SlotContext {
200 d0: u32,
201 d1: u32,
202 d2: u32,
203 d3: u32,
204 d4: u32,
205 d5: u32,
206 d6: u32,
207 d7: u32,
208}
209
210#[repr(C, packed)]
211struct EndpointContext {
212 d0: u32,
213 d1: u32,
214 d2: u32,
215 d3: u32,
216 d4: u32,
217 d5: u32,
218 d6: u32,
219 d7: u32,
220}
221
222#[repr(C, packed)]
223struct InputControlContext {
224 d0: u32,
225 d1: u32,
226 d2: [u32; 30],
227}
228
229#[repr(C, packed)]
230struct InputContext {
231 ctrl: InputControlContext,
232 slot: SlotContext,
233 eps: [EndpointContext; 31],
234}
235
236struct XhciPort {
237 port_num: usize,
238 enabled: bool,
239 connected: bool,
240 speed: u8,
241}
242
243struct DeviceSlot {
244 slot_id: u8,
245 usb_address: u8,
246 input_ctx: *mut InputContext,
247 input_ctx_phys: u64,
248 ep_transfer_rings: [*mut Trb; MAX_ENDPOINTS],
249 ep_transfer_ring_phys: [u64; MAX_ENDPOINTS],
250 ep_dequeue: [usize; MAX_ENDPOINTS],
251 ep_cycle: [bool; MAX_ENDPOINTS],
252 configured: bool,
253 ep_buf: [*mut u8; MAX_ENDPOINTS],
254 ep_buf_phys: [u64; MAX_ENDPOINTS],
255 ep_buf_len: [usize; MAX_ENDPOINTS],
256 ep_active: [bool; MAX_ENDPOINTS],
257}
258
259unsafe impl Send for DeviceSlot {}
260unsafe impl Sync for DeviceSlot {}
261
262impl DeviceSlot {
263 fn new(slot_id: u8) -> Self {
264 Self {
265 slot_id,
266 usb_address: 0,
267 input_ctx: core::ptr::null_mut(),
268 input_ctx_phys: 0,
269 ep_transfer_rings: [core::ptr::null_mut(); MAX_ENDPOINTS],
270 ep_transfer_ring_phys: [0; MAX_ENDPOINTS],
271 ep_dequeue: [0; MAX_ENDPOINTS],
272 ep_cycle: [true; MAX_ENDPOINTS],
273 configured: false,
274 ep_buf: [core::ptr::null_mut(); MAX_ENDPOINTS],
275 ep_buf_phys: [0; MAX_ENDPOINTS],
276 ep_buf_len: [0; MAX_ENDPOINTS],
277 ep_active: [false; MAX_ENDPOINTS],
278 }
279 }
280}
281
282pub struct XhciController {
283 mmio_base: usize,
284 cap_regs: *const CapRegisters,
285 op_regs: *mut OpRegisters,
286 rt_regs: *mut RuntimeRegisters,
287 db_regs: *mut u32,
288 caplength: u8,
289 max_ports: usize,
290 ports: Vec<XhciPort>,
291 device_ctx: *mut u8,
292 device_ctx_phys: u64,
293 cmd_ring: *mut Trb,
294 cmd_ring_phys: u64,
295 cmd_ring_deq: usize,
296 cmd_ring_cycle: bool,
297 event_ring: *mut Trb,
298 event_ring_phys: u64,
299 event_ring_deq: AtomicUsize,
300 event_ring_cycle: AtomicBool,
301 slot_id: AtomicU8,
302 ctrl_transfer_buf: *mut u8,
303 ctrl_transfer_buf_phys: u64,
304 device_slots: Vec<Option<DeviceSlot>>,
305}
306
307unsafe impl Send for XhciController {}
308unsafe impl Sync for XhciController {}
309
310impl XhciController {
311 pub unsafe fn new(pci_dev: pci::PciDevice) -> Result<Self, &'static str> {
312 let bar = match pci_dev.read_bar(0) {
313 Some(Bar::Memory64 { addr, .. }) => addr,
314 _ => return Err("Invalid BAR"),
315 };
316 paging::ensure_identity_map_range(bar, XHCI_MMIO_SIZE as u64);
317
318 let mmio_base = phys_to_virt(bar) as usize;
319 let cap_regs = mmio_base as *const CapRegisters;
320 let caplength = (*cap_regs).caplength;
321 let op_regs = (mmio_base + caplength as usize) as *mut OpRegisters;
322
323 let dboff = (*cap_regs).dboff;
324 let db_regs = (mmio_base + dboff as usize) as *mut u32;
325
326 let rtsoff = (*cap_regs).rtsoff;
327 let rt_regs = (mmio_base + rtsoff as usize) as *mut RuntimeRegisters;
328
329 let max_ports = (((*cap_regs).hcsparams1 >> 24) & 0xFF) as usize;
330
331 let mut controller = Self {
332 mmio_base,
333 cap_regs,
334 op_regs,
335 rt_regs,
336 db_regs,
337 caplength,
338 max_ports,
339 ports: Vec::new(),
340 device_ctx: core::ptr::null_mut(),
341 device_ctx_phys: 0,
342 cmd_ring: core::ptr::null_mut(),
343 cmd_ring_phys: 0,
344 cmd_ring_deq: 0,
345 cmd_ring_cycle: true,
346 event_ring: core::ptr::null_mut(),
347 event_ring_phys: 0,
348 event_ring_deq: AtomicUsize::new(0),
349 event_ring_cycle: AtomicBool::new(true),
350 slot_id: AtomicU8::new(0),
351 ctrl_transfer_buf: core::ptr::null_mut(),
352 ctrl_transfer_buf_phys: 0,
353 device_slots: Vec::new(),
354 };
355
356 controller.init()?;
357 Ok(controller)
358 }
359
360 fn init(&mut self) -> Result<(), &'static str> {
361 unsafe {
362 for _ in 0..100_000 {
363 if self.read_usbsts() & USBSTS_CNR == 0 {
364 break;
365 }
366 core::hint::spin_loop();
367 }
368 if self.read_usbsts() & USBSTS_CNR != 0 {
369 return Err("xHCI: controller not ready (CNR)");
370 }
371
372 let mut usbcmd = self.read_usbcmd();
373 usbcmd &= !USBCMD_RUN_STOP;
374 self.write_usbcmd(usbcmd);
375 for _ in 0..100_000 {
376 if self.read_usbsts() & USBSTS_HCH != 0 {
377 break;
378 }
379 core::hint::spin_loop();
380 }
381 if self.read_usbsts() & USBSTS_HCH == 0 {
382 return Err("xHCI: controller did not halt");
383 }
384
385 self.write_usbcmd(self.read_usbcmd() | USBCMD_HCRST);
386 for _ in 0..100_000 {
387 if self.read_usbcmd() & USBCMD_HCRST == 0 {
388 break;
389 }
390 core::hint::spin_loop();
391 }
392 if self.read_usbcmd() & USBCMD_HCRST != 0 {
393 return Err("xHCI: controller reset timed out");
394 }
395 let mut cnr_timeout = 1_000_000u32;
396 while self.read_usbsts() & USBSTS_CNR != 0 {
397 if cnr_timeout == 0 {
398 return Err("xHCI: CNR did not clear after reset");
399 }
400 cnr_timeout -= 1;
401 core::hint::spin_loop();
402 }
403
404 for i in 0..self.max_ports {
405 let portsc = self.read_portsc(i);
406 self.ports.push(XhciPort {
407 port_num: i,
408 enabled: (portsc & PORTSC_PED) != 0,
409 connected: (portsc & PORTSC_CCS) != 0,
410 speed: ((portsc >> PORTSC_SPEED_SHIFT) & 0xF) as u8,
411 });
412 }
413
414 self.init_rings()?;
415 self.init_interrupter()?;
416 self.init_ctrl_transfer_buf()?;
417
418 let max_slots = self.max_device_slots();
419 self.write_config(max_slots);
420 self.write_usbcmd(self.read_usbcmd() | USBCMD_RUN_STOP | USBCMD_INTE);
421
422 self.enumerate_all_ports();
423 }
424 Ok(())
425 }
426
427 unsafe fn init_rings(&mut self) -> Result<(), &'static str> {
428 let cmd_frame = allocate_zeroed_frame().ok_or("Failed to allocate cmd ring")?;
429 self.cmd_ring_phys = cmd_frame.start_address.as_u64();
430 self.cmd_ring = phys_to_virt(self.cmd_ring_phys) as *mut Trb;
431 core::ptr::write_bytes(self.cmd_ring as *mut u8, 0, 4096);
432 core::ptr::write(
433 self.cmd_ring.add(XHCI_RING_TRBS - 1),
434 Trb::link(self.cmd_ring_phys, true),
435 );
436 self.write_crcr(self.cmd_ring_phys | 1);
437
438 let event_frame = allocate_zeroed_frame().ok_or("Failed to allocate event ring")?;
439 self.event_ring_phys = event_frame.start_address.as_u64();
440 self.event_ring = phys_to_virt(self.event_ring_phys) as *mut Trb;
441 core::ptr::write_bytes(self.event_ring as *mut u8, 0, 4096);
442
443 let dev_frame = allocate_zeroed_frame().ok_or("Failed to allocate DCBAA")?;
444 self.device_ctx_phys = dev_frame.start_address.as_u64();
445 self.device_ctx = phys_to_virt(self.device_ctx_phys) as *mut u8;
446 core::ptr::write_bytes(self.device_ctx, 0, 4096);
447 self.write_dcbaap(self.device_ctx_phys);
448
449 Ok(())
450 }
451
452 unsafe fn init_interrupter(&mut self) -> Result<(), &'static str> {
453 let erst_frame = allocate_zeroed_frame().ok_or("Failed to allocate ERST")?;
454 let erst_phys = erst_frame.start_address.as_u64();
455 let erst_virt = phys_to_virt(erst_phys) as *mut u64;
456 core::ptr::write_bytes(erst_virt as *mut u8, 0, 4096);
457
458 let erst_entry = erst_virt as *mut u8;
459 let addr_bytes = self.event_ring_phys.to_le_bytes();
460 core::ptr::copy_nonoverlapping(addr_bytes.as_ptr(), erst_entry, 8);
461 let seg_size: u32 = XHCI_RING_TRBS as u32;
462 let size_bytes = seg_size.to_le_bytes();
463 core::ptr::copy_nonoverlapping(size_bytes.as_ptr(), erst_entry.add(8), 4);
464
465 let ir = &mut (*self.rt_regs).ir[0];
466 write_volatile(core::ptr::addr_of_mut!(ir.erstsz), 1);
467 core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst);
468 write_volatile(core::ptr::addr_of_mut!(ir.erstba), erst_phys);
469 write_volatile(core::ptr::addr_of_mut!(ir.erdp), self.event_ring_phys);
470 write_volatile(core::ptr::addr_of_mut!(ir.iman), 3);
471
472 Ok(())
473 }
474
475 unsafe fn init_ctrl_transfer_buf(&mut self) -> Result<(), &'static str> {
476 let buf_frame = allocate_zeroed_frame().ok_or("Failed to allocate ctrl buf")?;
477 self.ctrl_transfer_buf_phys = buf_frame.start_address.as_u64();
478 self.ctrl_transfer_buf = phys_to_virt(self.ctrl_transfer_buf_phys) as *mut u8;
479 core::ptr::write_bytes(self.ctrl_transfer_buf, 0, 4096);
480 Ok(())
481 }
482
483 unsafe fn read_portsc(&self, port: usize) -> u32 {
484 let port_offset = XHCI_PORT_REG_BASE + (port * XHCI_PORT_REG_STRIDE);
485 let portsc_ptr = (self.op_regs as *const u8).add(port_offset) as *const u32;
486 portsc_ptr.read_volatile()
487 }
488
489 unsafe fn write_portsc(&self, port: usize, val: u32) {
490 let port_offset = XHCI_PORT_REG_BASE + (port * XHCI_PORT_REG_STRIDE);
491 let portsc_ptr = (self.op_regs as *const u8).add(port_offset) as *mut u32;
492 portsc_ptr.write_volatile(val);
493 }
494
495 unsafe fn read_usbcmd(&self) -> u32 {
496 read_volatile(core::ptr::addr_of!((*self.op_regs).usbcmd))
497 }
498
499 unsafe fn write_usbcmd(&self, value: u32) {
500 write_volatile(core::ptr::addr_of_mut!((*self.op_regs).usbcmd), value);
501 }
502
503 unsafe fn read_usbsts(&self) -> u32 {
504 read_volatile(core::ptr::addr_of!((*self.op_regs).usbsts))
505 }
506
507 unsafe fn write_crcr(&self, value: u64) {
508 write_volatile(core::ptr::addr_of_mut!((*self.op_regs).crcr), value);
509 }
510
511 unsafe fn write_dcbaap(&self, value: u64) {
512 write_volatile(core::ptr::addr_of_mut!((*self.op_regs).dcbaap), value);
513 }
514
515 unsafe fn write_config(&self, value: u32) {
516 write_volatile(core::ptr::addr_of_mut!((*self.op_regs).config), value);
517 }
518
519 fn max_device_slots(&self) -> u32 {
520 unsafe { read_volatile(core::ptr::addr_of!((*self.cap_regs).hcsparams1)) & 0xFF }
521 }
522
523 fn max_ports_from_hw(&self) -> u32 {
524 unsafe { (read_volatile(core::ptr::addr_of!((*self.cap_regs).hcsparams1)) >> 24) & 0xFF }
525 }
526
527 unsafe fn cmd_ring_enqueue(&mut self, trb: Trb) {
528 let idx = self.cmd_ring_deq;
529 let mut trb = trb;
530 if self.cmd_ring_cycle {
531 trb.d3 |= TRB_CYCLE;
532 } else {
533 trb.d3 &= !TRB_CYCLE;
534 }
535 core::ptr::write_volatile(self.cmd_ring.add(idx), trb);
536 self.cmd_ring_deq = idx + 1;
537
538 if self.cmd_ring_deq >= 63 {
539 let link = Trb::link(self.cmd_ring_phys, true);
540 let mut link_trb = link;
541 if self.cmd_ring_cycle {
542 link_trb.d3 |= TRB_CYCLE;
543 } else {
544 link_trb.d3 &= !TRB_CYCLE;
545 }
546 core::ptr::write_volatile(self.cmd_ring.add(63), link_trb);
547 self.cmd_ring_deq = 0;
548 self.cmd_ring_cycle = !self.cmd_ring_cycle;
549 }
550
551 core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst);
552 core::ptr::write_volatile(self.db_regs.add(0), 0);
553 }
554
555 unsafe fn wait_for_event(&mut self) -> Result<Trb, &'static str> {
556 for _ in 0..1000000 {
557 let idx = self.event_ring_deq.load(Ordering::Acquire);
558 let trb = core::ptr::read_volatile(self.event_ring.add(idx));
559
560 let expected_c = if self.event_ring_cycle.load(Ordering::Acquire) {
561 TRB_CYCLE
562 } else {
563 0
564 };
565 if (trb.d3 & TRB_CYCLE) == expected_c {
566 let new_deq = (idx + 1) % 64;
567 self.event_ring_deq.store(new_deq, Ordering::Release);
568 if new_deq == 0 {
569 self.event_ring_cycle.store(
570 !self.event_ring_cycle.load(Ordering::Acquire),
571 Ordering::Release,
572 );
573 }
574 let ir = &mut (*self.rt_regs).ir[0];
575 ir.erdp = (self.event_ring_phys + (new_deq as u64) * 16) | (1 << 3);
576 return Ok(trb);
577 }
578 core::hint::spin_loop();
579 }
580 Err("Event timeout")
581 }
582
583 unsafe fn ring_doorbell(&self, slot_id: u8, endpoint: u8) {
584 let db_index = (slot_id as usize) * 32 + (endpoint as usize);
585 core::ptr::write_volatile(self.db_regs.add(db_index), 0);
586 }
587
588 unsafe fn alloc_input_context(&mut self, slot_id: u8) -> Result<(), &'static str> {
589 let frame = allocate_zeroed_frame().ok_or("Failed to allocate input context")?;
590 let phys = frame.start_address.as_u64();
591 let virt = phys_to_virt(phys) as *mut InputContext;
592
593 let idx = slot_id as usize;
594 if idx >= self.device_slots.len() {
595 self.device_slots.resize_with(idx + 1, || None);
596 }
597 let dev = self.device_slots[idx].as_mut().unwrap();
598 dev.input_ctx = virt;
599 dev.input_ctx_phys = phys;
600
601 let dcbaa = self.device_ctx as *mut u64;
602 dcbaa.add(idx as usize).write_volatile(phys);
603
604 Ok(())
605 }
606
607 unsafe fn alloc_transfer_ring(
608 &mut self,
609 slot_id: u8,
610 endpoint: u8,
611 ) -> Result<(), &'static str> {
612 let frame = allocate_zeroed_frame().ok_or("Failed to allocate transfer ring")?;
613 let phys = frame.start_address.as_u64();
614 let virt = phys_to_virt(phys) as *mut Trb;
615
616 core::ptr::write_bytes(virt as *mut u8, 0, 4096);
617 core::ptr::write(virt.add(XHCI_RING_TRBS - 1), Trb::link(phys, true));
618
619 let idx = slot_id as usize;
620 if idx < self.device_slots.len() {
621 if let Some(ref mut dev) = self.device_slots[idx] {
622 let ep = endpoint as usize;
623 if ep < MAX_ENDPOINTS {
624 dev.ep_transfer_rings[ep] = virt;
625 dev.ep_transfer_ring_phys[ep] = phys;
626 dev.ep_dequeue[ep] = 0;
627 dev.ep_cycle[ep] = true;
628 }
629 }
630 }
631 Ok(())
632 }
633
634 unsafe fn write_endpoint_context(
635 &self,
636 slot_id: u8,
637 endpoint: u8,
638 tr_phys: u64,
639 max_packet: u32,
640 ep_type: u32,
641 interval: u32,
642 ) {
643 let dcbaa = self.device_ctx as *mut u64;
644 let ctx_addr = dcbaa.add(slot_id as usize).read_volatile() as *mut u8;
645 if ctx_addr.is_null() {
646 return;
647 }
648
649 let ep_offset = 32 * ((endpoint - 1) as usize) + 32;
650 let ep_ctx = ctx_addr.add(ep_offset) as *mut EndpointContext;
651
652 let d0 = (ep_type & 0x7) << 3 | 0;
653 let d1 = (max_packet & 0x7FF) | (0 << 16);
654 let d2 = (tr_phys & 0xFFFFFFFF) as u32;
655 let d3 = ((tr_phys >> 32) & 0xFFFFFFFF) as u32;
656 let d4 = interval & 0xFF;
657
658 (*ep_ctx).d0 = d0;
659 (*ep_ctx).d1 = d1;
660 (*ep_ctx).d2 = d2;
661 (*ep_ctx).d3 = d3;
662 (*ep_ctx).d4 = d4;
663 (*ep_ctx).d5 = 0;
664 (*ep_ctx).d6 = 0;
665 (*ep_ctx).d7 = 0;
666 }
667
668 unsafe fn reset_port(&self, port: usize) -> bool {
669 let mut portsc = self.read_portsc(port);
670 if portsc & PORTSC_CCS == 0 {
671 return false;
672 }
673
674 portsc = self.read_portsc(port);
675 self.write_portsc(port, portsc | PORTSC_PR);
676
677 for _ in 0..500_000 {
678 portsc = self.read_portsc(port);
679 if portsc & PORTSC_PR == 0 {
680 break;
681 }
682 core::hint::spin_loop();
683 }
684
685 for _ in 0..500_000 {
686 portsc = self.read_portsc(port);
687 if portsc & PORTSC_PED != 0 {
688 return true;
689 }
690 if portsc & PORTSC_CCS == 0 {
691 return false;
692 }
693 core::hint::spin_loop();
694 }
695
696 self.read_portsc(port) & PORTSC_PED != 0
697 }
698
699 fn enable_slot(&mut self) -> Result<u8, &'static str> {
700 unsafe {
701 self.cmd_ring_enqueue(Trb {
702 d0: 0,
703 d1: 0,
704 d2: 0,
705 d3: (TRB_TYPE_ENABLE_SLOT << TRB_TYPE_SHIFT) as u32,
706 });
707
708 let event = self.wait_for_event()?;
709 let completion_code = (event.d2 >> 24) & 0xFF;
710 if completion_code != 1 {
711 log::warn!("[xHCI] Enable Slot failed: completion={}", completion_code);
712 return Err("Enable slot failed");
713 }
714 let slot_id = ((event.d3 >> 24) & 0xFF) as u8;
715 if slot_id == 0 {
716 return Err("No slot available");
717 }
718 self.slot_id.store(slot_id, Ordering::SeqCst);
719
720 let idx = slot_id as usize;
721 if idx >= self.device_slots.len() {
722 self.device_slots.resize_with(idx + 1, || None);
723 }
724 self.device_slots[idx] = Some(DeviceSlot::new(slot_id));
725
726 self.alloc_input_context(slot_id)?;
727 self.alloc_transfer_ring(slot_id, 1)?;
728
729 let slot_ctx = &mut (*self.device_slots[idx].as_ref().unwrap().input_ctx).slot;
730 slot_ctx.d0 = (1 << 27) | (slot_id as u32);
731 slot_ctx.d1 = 0;
732 slot_ctx.d2 = 0;
733
734 self.write_endpoint_context(
735 slot_id,
736 1,
737 self.device_slots[idx]
738 .as_ref()
739 .unwrap()
740 .ep_transfer_ring_phys[1],
741 8,
742 EP_TYPE_CONTROL,
743 0,
744 );
745
746 log::info!("[xHCI] Enable Slot: slot_id={}", slot_id);
747 Ok(slot_id)
748 }
749 }
750
751 fn set_address(&mut self, slot_id: u8, address: u8) -> Result<(), &'static str> {
752 unsafe {
753 let idx = slot_id as usize;
754 if idx >= self.device_slots.len() || self.device_slots[idx].is_none() {
755 return Err("Invalid slot for Address Device");
756 }
757
758 let input_ctx_phys = self.device_slots[idx].as_ref().unwrap().input_ctx_phys;
759
760 let slot_ctx = &mut (*self.device_slots[idx].as_ref().unwrap().input_ctx).slot;
761 slot_ctx.d0 = (1 << 27) | (address as u32);
762 slot_ctx.d1 = 0;
763 slot_ctx.d2 = 0;
764
765 self.cmd_ring_enqueue(Trb {
766 d0: (input_ctx_phys & 0xFFFFFFFF) as u32,
767 d1: ((input_ctx_phys >> 32) & 0xFFFFFFFF) as u32,
768 d2: (slot_id as u32) << 24,
769 d3: (TRB_TYPE_ADDRESS_DEVICE << TRB_TYPE_SHIFT) as u32,
770 });
771
772 let event = self.wait_for_event()?;
773 let completion = (event.d2 >> 24) & 0xFF;
774 if completion != 1 {
775 log::warn!(
776 "[xHCI] Address Device failed: slot={} completion={}",
777 slot_id,
778 completion
779 );
780 return Err("Set address failed");
781 }
782
783 self.device_slots[idx].as_mut().unwrap().usb_address = address;
784 log::info!("[xHCI] Address Device: slot={} addr={}", slot_id, address);
785 }
786 Ok(())
787 }
788
789 pub fn setup_endpoint(
790 &mut self,
791 slot_id: u8,
792 endpoint: u8,
793 max_packet: u32,
794 ep_type: u32,
795 interval: u32,
796 _max_burst: u32,
797 ) -> Result<(), &'static str> {
798 unsafe {
799 let idx = slot_id as usize;
800 if idx >= self.device_slots.len() || self.device_slots[idx].is_none() {
801 return Err("Invalid slot for Setup Endpoint");
802 }
803
804 self.alloc_transfer_ring(slot_id, endpoint)?;
805
806 self.write_endpoint_context(
807 slot_id,
808 endpoint,
809 self.device_slots[idx]
810 .as_ref()
811 .unwrap()
812 .ep_transfer_ring_phys[endpoint as usize],
813 max_packet,
814 ep_type,
815 interval,
816 );
817
818 let input_ctx_phys = self.device_slots[idx].as_ref().unwrap().input_ctx_phys;
819
820 self.cmd_ring_enqueue(Trb {
821 d0: (input_ctx_phys & 0xFFFFFFFF) as u32,
822 d1: ((input_ctx_phys >> 32) & 0xFFFFFFFF) as u32,
823 d2: (slot_id as u32) << 24,
824 d3: (TRB_TYPE_CONFIGURE_ENDPOINT << TRB_TYPE_SHIFT) as u32,
825 });
826
827 let event = self.wait_for_event()?;
828 let completion = (event.d2 >> 24) & 0xFF;
829 if completion != 1 {
830 log::warn!(
831 "[xHCI] Configure Endpoint failed: slot={} ep={} completion={}",
832 slot_id,
833 endpoint,
834 completion
835 );
836 return Err("Configure endpoint failed");
837 }
838
839 log::info!(
840 "[xHCI] Endpoint configured: slot={} ep={} type={}",
841 slot_id,
842 endpoint,
843 ep_type
844 );
845 }
846 Ok(())
847 }
848
849 fn enumerate_all_ports(&mut self) {
850 let mut usb_address: u8 = 1;
851
852 for port in 0..self.max_ports {
853 let portsc = unsafe { self.read_portsc(port) };
854 let connected = (portsc & PORTSC_CCS) != 0;
855 if !connected {
856 continue;
857 }
858
859 log::info!("[xHCI] Port {} connected, resetting...", port);
860
861 if !unsafe { self.reset_port(port) } {
862 log::warn!("[xHCI] Port {} reset failed", port);
863 continue;
864 }
865
866 let speed = unsafe { ((self.read_portsc(port) >> PORTSC_SPEED_SHIFT) & 0xF) as u8 };
867 log::info!("[xHCI] Port {} speed={}", port, speed);
868
869 match self.enable_slot() {
870 Ok(slot_id) => {
871 if self.set_address(slot_id, usb_address).is_err() {
872 log::warn!("[xHCI] Port {} address failed", port);
873 continue;
874 }
875 usb_address += 1;
876
877 let mut dev_desc = [0u8; 18];
878 if self.get_device_descriptor(slot_id, &mut dev_desc).is_ok() {
879 let vid = u16::from_le_bytes([dev_desc[2], dev_desc[3]]);
880 let pid = u16::from_le_bytes([dev_desc[4], dev_desc[5]]);
881 let dev_class = dev_desc[4];
882 let max_packet0 = u16::from_le_bytes([dev_desc[7], dev_desc[8]]);
883 log::info!(
884 "[xHCI] Device: VID={:04x} PID={:04x} class={:02x} max_pkt0={}",
885 vid,
886 pid,
887 dev_class,
888 max_packet0
889 );
890
891 if max_packet0 == 64
892 || max_packet0 == 32
893 || max_packet0 == 16
894 || max_packet0 == 8
895 {
896 let idx = slot_id as usize;
897 if idx < self.device_slots.len() {
898 if let Some(ref mut dev) = self.device_slots[idx] {
899 dev.configured = true;
900 }
901 }
902 }
903
904 crate::hardware::usb::hid::enumerate_device(port, slot_id, &dev_desc);
905 } else {
906 log::warn!("[xHCI] Port {} get device descriptor failed", port);
907 }
908 }
909 Err(e) => {
910 log::warn!("[xHCI] Port {} enable slot failed: {}", port, e);
911 }
912 }
913 }
914 }
915
916 pub fn alloc_interrupt_buffer(
917 &mut self,
918 slot_id: u8,
919 endpoint: u8,
920 len: usize,
921 ) -> Result<(*mut u8, u64), &'static str> {
922 let frame = allocate_zeroed_frame().ok_or("Failed to allocate interrupt buffer")?;
923 let phys = frame.start_address.as_u64();
924 let virt = phys_to_virt(phys) as *mut u8;
925
926 let idx = slot_id as usize;
927 if idx < self.device_slots.len() {
928 if let Some(ref mut dev) = self.device_slots[idx] {
929 let ep = endpoint as usize;
930 if ep < MAX_ENDPOINTS {
931 dev.ep_buf[ep] = virt;
932 dev.ep_buf_phys[ep] = phys;
933 dev.ep_buf_len[ep] = len;
934 }
935 }
936 }
937 Ok((virt, phys))
938 }
939
940 pub fn submit_interrupt_transfer(
941 &mut self,
942 slot_id: u8,
943 endpoint: u8,
944 ) -> Result<(), &'static str> {
945 let idx = slot_id as usize;
946 if idx >= self.device_slots.len() || self.device_slots[idx].is_none() {
947 return Err("Invalid slot for interrupt transfer");
948 }
949
950 let ep = endpoint as usize;
951 if ep >= MAX_ENDPOINTS {
952 return Err("Invalid endpoint");
953 }
954
955 let dev = self.device_slots[idx].as_ref().unwrap();
956 let tr_ring = dev.ep_transfer_rings[ep];
957 let tr_phys = dev.ep_transfer_ring_phys[ep];
958 let buf_phys = dev.ep_buf_phys[ep];
959 let buf_len = dev.ep_buf_len[ep];
960 if tr_ring.is_null() || buf_phys == 0 {
961 return Err("No transfer ring or buffer for endpoint");
962 }
963
964 let deq = dev.ep_dequeue[ep];
965 let cycle = dev.ep_cycle[ep];
966
967 let trb = Trb::normal(buf_phys, buf_len as u32, cycle, true);
968 unsafe {
969 core::ptr::write_volatile(tr_ring.add(deq), trb);
970 core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst);
971 self.ring_doorbell(slot_id, endpoint);
972 }
973
974 self.device_slots[idx].as_mut().unwrap().ep_active[ep] = true;
975
976 Ok(())
977 }
978
979 pub fn port_count(&self) -> usize {
980 self.max_ports
981 }
982
983 pub fn is_port_connected(&self, port: usize) -> bool {
984 if port >= self.ports.len() {
985 return false;
986 }
987 self.ports[port].connected
988 }
989
990 pub fn get_device_descriptor(
991 &mut self,
992 slot_id: u8,
993 buf: &mut [u8; 18],
994 ) -> Result<usize, &'static str> {
995 let setup = [0x80, 0x06, 0x00, 0x01, 0x00, 0x00, 18, 0x00];
996 unsafe { self.ctrl_transfer(slot_id, &setup, Some(buf), 18) }
997 }
998
999 pub fn get_configuration_descriptor(
1000 &mut self,
1001 slot_id: u8,
1002 config_idx: u8,
1003 buf: &mut [u8],
1004 len: usize,
1005 ) -> Result<usize, &'static str> {
1006 let setup = [
1007 0x80,
1008 0x06,
1009 config_idx,
1010 0x02,
1011 0x00,
1012 0x00,
1013 (len & 0xFF) as u8,
1014 ((len >> 8) & 0xFF) as u8,
1015 ];
1016 unsafe { self.ctrl_transfer(slot_id, &setup, Some(buf), len) }
1017 }
1018
1019 pub fn set_configuration(&mut self, slot_id: u8, config_value: u8) -> Result<(), &'static str> {
1020 let setup = [0x00, 0x09, config_value, 0x00, 0x00, 0x00, 0x00, 0x00];
1021 unsafe {
1022 self.ctrl_transfer(slot_id, &setup, None, 0)?;
1023 }
1024 Ok(())
1025 }
1026
1027 pub fn set_protocol(
1028 &mut self,
1029 slot_id: u8,
1030 interface: u8,
1031 protocol: u8,
1032 ) -> Result<(), &'static str> {
1033 let setup = [0x21, 0x0B, protocol, interface, 0x00, 0x00, 0x00, 0x00];
1034 unsafe {
1035 self.ctrl_transfer(slot_id, &setup, None, 0)?;
1036 }
1037 Ok(())
1038 }
1039
1040 pub fn get_port_speed(&self, port: usize) -> u8 {
1041 if port >= self.ports.len() {
1042 return 0;
1043 }
1044 self.ports[port].speed
1045 }
1046
1047 unsafe fn ctrl_transfer(
1048 &mut self,
1049 slot_id: u8,
1050 setup_data: &[u8; 8],
1051 data_buf: Option<&mut [u8]>,
1052 data_len: usize,
1053 ) -> Result<usize, &'static str> {
1054 let idx = slot_id as usize;
1055 if idx >= self.device_slots.len() || self.device_slots[idx].is_none() {
1056 return Err("Invalid slot for control transfer");
1057 }
1058
1059 let dev = self.device_slots[idx].as_ref().unwrap();
1060 let tr_ring = dev.ep_transfer_rings[1];
1061 let tr_phys = dev.ep_transfer_ring_phys[1];
1062 if tr_ring.is_null() {
1063 return Err("No transfer ring for EP0");
1064 }
1065
1066 let mut deq;
1067 let cycle;
1068
1069 core::ptr::write_bytes(tr_ring as *mut u8, 0, 4096);
1070 core::ptr::write(tr_ring.add(XHCI_RING_TRBS - 1), Trb::link(tr_phys, true));
1071 deq = 0;
1072 cycle = true;
1073
1074 let setup_phys = self.ctrl_transfer_buf_phys;
1075 let setup_virt = self.ctrl_transfer_buf;
1076 core::ptr::copy_nonoverlapping(setup_data.as_ptr(), setup_virt, 8);
1077
1078 let setup_trb = Trb::setup_stage(setup_phys, cycle);
1079 core::ptr::write_volatile(tr_ring.add(deq), setup_trb);
1080 deq += 1;
1081
1082 let has_data = data_buf.is_some();
1083 let dir_in = if has_data {
1084 (setup_data[0] & 0x80) != 0
1085 } else {
1086 false
1087 };
1088
1089 if let Some(buf) = &data_buf {
1090 let data_phys = self.ctrl_transfer_buf_phys + 8;
1091 let data_virt = self.ctrl_transfer_buf.add(8);
1092
1093 if dir_in && data_len > 0 {
1094 core::ptr::write_bytes(data_virt, 0, data_len);
1095 } else if !dir_in && data_len > 0 {
1096 core::ptr::copy_nonoverlapping(buf.as_ptr(), data_virt, data_len);
1097 }
1098
1099 let data_trb = Trb::data_stage(data_phys, data_len as u32, dir_in, cycle, false);
1100 core::ptr::write_volatile(tr_ring.add(deq), data_trb);
1101 deq += 1;
1102
1103 let status_trb = Trb::status_stage(cycle, !dir_in);
1104 core::ptr::write_volatile(tr_ring.add(deq), status_trb);
1105 deq += 1;
1106 } else {
1107 let status_trb = Trb::status_stage(cycle, true);
1108 core::ptr::write_volatile(tr_ring.add(deq), status_trb);
1109 deq += 1;
1110 }
1111
1112 core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst);
1113 self.ring_doorbell(slot_id, 1);
1114
1115 let mut transferred = 0;
1116 for _ in 0..3 {
1117 let event = self.wait_for_event()?;
1118 let trb_type = TRB_GET_TYPE(event.d3);
1119 let completion = (event.d2 >> 24) & 0xFF;
1120
1121 if completion != 1 {
1122 log::warn!(
1123 "[xHCI] ctrl_transfer event error: type={} completion={}",
1124 trb_type,
1125 completion
1126 );
1127 return Err("Control transfer event error");
1128 }
1129
1130 if trb_type == TRB_TYPE_STATUS_STAGE {
1131 if has_data && data_len > 0 && dir_in {
1132 let data_virt = self.ctrl_transfer_buf.add(8);
1133 if let Some(buf) = data_buf {
1134 core::ptr::copy_nonoverlapping(data_virt, buf.as_mut_ptr(), data_len);
1135 }
1136 transferred = data_len;
1137 }
1138 break;
1139 }
1140 }
1141
1142 self.device_slots[idx].as_mut().unwrap().ep_dequeue[1] = deq;
1143 self.device_slots[idx].as_mut().unwrap().ep_cycle[1] = cycle;
1144
1145 Ok(transferred)
1146 }
1147}
1148
1149static XHCI_CONTROLLERS: Mutex<Vec<Arc<Mutex<XhciController>>>> = Mutex::new(Vec::new());
1150static XHCI_INITIALIZED: AtomicBool = AtomicBool::new(false);
1151pub static XHCI_IRQ_LINE: AtomicU8 = AtomicU8::new(0);
1152
1153pub fn init() {
1154 log::info!("[xHCI] Scanning for xHCI controllers...");
1155
1156 let candidates = pci::probe_all(ProbeCriteria {
1157 vendor_id: None,
1158 device_id: None,
1159 class_code: Some(0x0C),
1160 subclass: Some(0x03),
1161 prog_if: Some(0x30),
1162 });
1163
1164 for pci_dev in candidates.into_iter() {
1165 log::info!(
1166 "xHCI: Found controller at {:?} (VEN:{:04x} DEV:{:04x})",
1167 pci_dev.address,
1168 pci_dev.vendor_id,
1169 pci_dev.device_id
1170 );
1171
1172 let irq = pci_dev.interrupt_line;
1173 pci_dev.enable_memory_space();
1174 pci_dev.enable_bus_master();
1175
1176 match unsafe { XhciController::new(pci_dev) } {
1177 Ok(controller) => {
1178 log::info!("[xHCI] Initialized with {} ports", controller.port_count());
1179 XHCI_IRQ_LINE.store(irq, Ordering::Relaxed);
1180 XHCI_CONTROLLERS
1181 .lock()
1182 .push(Arc::new(Mutex::new(controller)));
1183 crate::arch::x86_64::idt::register_xhci_irq(irq);
1184 }
1185 Err(e) => {
1186 log::warn!("xHCI: Failed to initialize controller: {}", e);
1187 }
1188 }
1189 }
1190
1191 XHCI_INITIALIZED.store(true, Ordering::SeqCst);
1192 log::info!(
1193 "[xHCI] Found {} controller(s)",
1194 XHCI_CONTROLLERS.lock().len()
1195 );
1196}
1197
1198pub fn get_controller(index: usize) -> Option<Arc<Mutex<XhciController>>> {
1199 XHCI_CONTROLLERS.lock().get(index).cloned()
1200}
1201
1202pub fn is_available() -> bool {
1203 XHCI_INITIALIZED.load(Ordering::Relaxed) && !XHCI_CONTROLLERS.lock().is_empty()
1204}
1205
1206pub fn handle_interrupt() {
1207 if let Some(controller_arc) = get_controller(0) {
1208 let mut controller = controller_arc.lock();
1209 unsafe {
1210 let ir = &mut (*controller.rt_regs).ir[0];
1211 if (ir.iman & 1) != 0 {
1212 let mut processed = 0;
1213 while processed < 16 {
1214 let idx = controller.event_ring_deq.load(Ordering::Acquire);
1215 let trb = core::ptr::read_volatile(controller.event_ring.add(idx));
1216
1217 let expected_c = if controller.event_ring_cycle.load(Ordering::Acquire) {
1218 TRB_CYCLE
1219 } else {
1220 0
1221 };
1222 if (trb.d3 & TRB_CYCLE) != expected_c {
1223 break;
1224 }
1225
1226 let trb_type = TRB_GET_TYPE(trb.d3);
1227 match trb_type {
1228 TRB_TYPE_TRANSFER_EVENT => {
1229 let slot_id = ((trb.d3 >> 24) & 0xFF) as u8;
1230 let ep_id = ((trb.d2 >> 16) & 0x1F) as u8;
1231 let completion = (trb.d2 >> 24) & 0xFF;
1232 let transferred = (trb.d2 & 0xFFFF) as usize;
1233
1234 if completion == 1 && ep_id >= 1 && (ep_id as usize) < MAX_ENDPOINTS {
1235 let idx = slot_id as usize;
1236 if idx < controller.device_slots.len() {
1237 if let Some(ref mut dev) = controller.device_slots[idx] {
1238 let ep = ep_id as usize;
1239 let buf = dev.ep_buf[ep];
1240 let buf_len = dev.ep_buf_len[ep];
1241 let actual_len = if transferred < buf_len {
1242 transferred
1243 } else {
1244 buf_len
1245 };
1246
1247 if !buf.is_null() && actual_len > 0 {
1248 crate::hardware::usb::hid::receive_interrupt_report(
1249 slot_id, ep_id, buf, actual_len,
1250 );
1251 }
1252
1253 dev.ep_dequeue[ep] =
1254 (dev.ep_dequeue[ep] + 1) % (XHCI_RING_TRBS - 1);
1255 if dev.ep_dequeue[ep] == 0 {
1256 dev.ep_cycle[ep] = !dev.ep_cycle[ep];
1257 }
1258 dev.ep_active[ep] = false;
1259 }
1260 }
1261 } else if completion != 1 && completion != 13 {
1262 let idx = slot_id as usize;
1263 if idx < controller.device_slots.len() {
1264 if let Some(ref mut dev) = controller.device_slots[idx] {
1265 let ep = ep_id as usize;
1266 if ep < MAX_ENDPOINTS {
1267 dev.ep_dequeue[ep] =
1268 (dev.ep_dequeue[ep] + 1) % (XHCI_RING_TRBS - 1);
1269 if dev.ep_dequeue[ep] == 0 {
1270 dev.ep_cycle[ep] = !dev.ep_cycle[ep];
1271 }
1272 dev.ep_active[ep] = false;
1273 }
1274 }
1275 }
1276 }
1277 }
1278 _ => {}
1279 }
1280
1281 let new_deq = (idx + 1) % 64;
1282 controller.event_ring_deq.store(new_deq, Ordering::Release);
1283 if new_deq == 0 {
1284 controller.event_ring_cycle.store(
1285 !controller.event_ring_cycle.load(Ordering::Acquire),
1286 Ordering::Release,
1287 );
1288 }
1289
1290 let new_erdp = controller.event_ring_phys + (new_deq as u64) * 16;
1291 ir.erdp = new_erdp | (1 << 3);
1292
1293 processed += 1;
1294 }
1295 }
1296
1297 let db_regs = controller.db_regs;
1298 for slot_idx in 0..controller.device_slots.len() {
1299 if let Some(ref mut dev) = controller.device_slots[slot_idx] {
1300 for ep in 1..MAX_ENDPOINTS {
1301 if dev.ep_active[ep]
1302 || dev.ep_buf[ep].is_null()
1303 || dev.ep_transfer_rings[ep].is_null()
1304 {
1305 continue;
1306 }
1307 let deq = dev.ep_dequeue[ep];
1308 let cycle = dev.ep_cycle[ep];
1309 let buf_phys = dev.ep_buf_phys[ep];
1310 let buf_len = dev.ep_buf_len[ep];
1311 let tr_ring = dev.ep_transfer_rings[ep];
1312 let slot = dev.slot_id;
1313
1314 let trb = Trb::normal(buf_phys, buf_len as u32, cycle, true);
1315 core::ptr::write_volatile(tr_ring.add(deq), trb);
1316
1317 core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst);
1318 let db_index = (slot as usize) * 32 + ep;
1319 core::ptr::write_volatile(db_regs.add(db_index), 0);
1320 dev.ep_active[ep] = true;
1321 }
1322 }
1323 }
1324 }
1325 }
1326}