1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771 |
- #include "xhci.h"
- #include <common/kprint.h>
- #include <debug/bug.h>
- #include <common/spinlock.h>
- #include <mm/mm.h>
- #include <mm/slab.h>
- #include <debug/traceback/traceback.h>
- #include <common/time.h>
- #include <exception/irq.h>
- #include <driver/interrupt/apic/apic.h>
- #pragma GCC optimize("O0")
- spinlock_t xhci_controller_init_lock = {0};
- static int xhci_ctrl_count = 0;
- static struct xhci_host_controller_t xhci_hc[XHCI_MAX_HOST_CONTROLLERS] = {0};
- void xhci_hc_irq_enable(uint64_t irq_num);
- void xhci_hc_irq_disable(uint64_t irq_num);
- uint64_t xhci_hc_irq_install(uint64_t irq_num, void *arg);
- void xhci_hc_irq_uninstall(uint64_t irq_num);
- static int xhci_hc_find_available_id();
- static int xhci_hc_stop(int id);
- static int xhci_hc_reset(int id);
- static int xhci_hc_stop_legacy(int id);
- static int xhci_hc_start_sched(int id);
- static int xhci_hc_stop_sched(int id);
- static uint32_t xhci_hc_get_protocol_offset(int id, uint32_t list_off, const int version, uint32_t *offset, uint32_t *count, uint16_t *protocol_flag);
- static int xhci_hc_pair_ports(int id);
- static uint64_t xhci_create_ring(int trbs);
- static uint64_t xhci_create_event_ring(int trbs, uint64_t *ret_ring_addr);
- void xhci_hc_irq_handler(uint64_t irq_num, uint64_t cid, struct pt_regs *regs);
- static int xhci_hc_init_intr(int id);
- static int xhci_hc_start_ports(int id);
- static int xhci_send_command(int id, struct xhci_TRB_t *trb, const bool do_ring);
- static uint64_t xhci_initialize_slot(const int id, const int slot_id, const int port, const int speed, const int max_packet);
- static void xhci_initialize_ep(const int id, const uint64_t slot_vaddr, const int slot_id, const int ep_num, const int max_packet, const int type, const int direction, const int speed, const int ep_interval);
- static int xhci_set_address(const int id, const uint64_t slot_vaddr, const int slot_id, const bool block);
- static int xhci_control_in(const int id, void *target, const int in_size, const int slot_id, const int max_packet);
- static int xhci_setup_stage(struct xhci_ep_ring_info_t *ep, const struct usb_request_packet_t *packet, const uint8_t direction);
- static int xhci_data_stage(struct xhci_ep_ring_info_t *ep, uint64_t buf_vaddr, uint8_t trb_type, const uint32_t size, uint8_t direction, const int max_packet, const uint64_t status_vaddr);
- static int xhci_status_stage(const int id, uint8_t direction, uint64_t status_buf_vaddr);
- static int xhci_wait_for_interrupt(const int id, uint64_t status_vaddr);
- hardware_intr_controller xhci_hc_intr_controller =
- {
- .enable = xhci_hc_irq_enable,
- .disable = xhci_hc_irq_disable,
- .install = xhci_hc_irq_install,
- .uninstall = xhci_hc_irq_uninstall,
- .ack = apic_local_apic_edge_ack,
- };
- #define xhci_read_cap_reg32(id, offset) (__read4b(xhci_hc[id].vbase + (offset)))
- #define xhci_get_ptr_cap_reg32(id, offset) ((uint32_t *)(xhci_hc[id].vbase + (offset)))
- #define xhci_write_cap_reg32(id, offset, value) (__write4b(xhci_hc[id].vbase + (offset), (value)))
- #define xhci_read_cap_reg64(id, offset) (__read8b(xhci_hc[id].vbase + (offset)))
- #define xhci_get_ptr_reg64(id, offset) ((uint64_t *)(xhci_hc[id].vbase + (offset)))
- #define xhci_write_cap_reg64(id, offset, value) (__write8b(xhci_hc[id].vbase + (offset), (value)))
- #define xhci_read_op_reg8(id, offset) (*(uint8_t *)(xhci_hc[id].vbase_op + (offset)))
- #define xhci_get_ptr_op_reg8(id, offset) ((uint8_t *)(xhci_hc[id].vbase_op + (offset)))
- #define xhci_write_op_reg8(id, offset, value) (*(uint8_t *)(xhci_hc[id].vbase_op + (offset)) = (uint8_t)(value))
- #define xhci_read_op_reg32(id, offset) (__read4b(xhci_hc[id].vbase_op + (offset)))
- #define xhci_get_ptr_op_reg32(id, offset) ((uint32_t *)(xhci_hc[id].vbase_op + (offset)))
- #define xhci_write_op_reg32(id, offset, value) (__write4b(xhci_hc[id].vbase_op + (offset), (value)))
- #define xhci_read_op_reg64(id, offset) (__read8b(xhci_hc[id].vbase_op + (offset)))
- #define xhci_get_ptr_op_reg64(id, offset) ((uint64_t *)(xhci_hc[id].vbase_op + (offset)))
- #define xhci_write_op_reg64(id, offset, value) (__write8b(xhci_hc[id].vbase_op + (offset), (value)))
- #define xhci_calc_intr_vaddr(id, num) (xhci_hc[id].vbase + xhci_hc[id].rts_offset + XHCI_RT_IR0 + (num)*XHCI_IR_SIZE)
- #define xhci_read_intr_reg32(id, num, intr_offset) (__read4b(xhci_calc_intr_vaddr(id, num) + (intr_offset)))
- #define xhci_write_intr_reg32(id, num, intr_offset, value) (__write4b(xhci_calc_intr_vaddr(id, num) + (intr_offset), (value)))
- #define xhci_read_intr_reg64(id, num, intr_offset) (__read8b(xhci_calc_intr_vaddr(id, num) + (intr_offset)))
- #define xhci_write_intr_reg64(id, num, intr_offset, value) (__write8b(xhci_calc_intr_vaddr(id, num) + (intr_offset), (value)))
- #define xhci_is_aligned64(addr) (((addr)&0x3f) == 0)
- #define XHCI_PORT_IS_USB2(cid, pid) ((xhci_hc[cid].ports[pid].flags & XHCI_PROTOCOL_INFO) == XHCI_PROTOCOL_USB2)
- #define XHCI_PORT_IS_USB3(cid, pid) ((xhci_hc[cid].ports[pid].flags & XHCI_PROTOCOL_INFO) == XHCI_PROTOCOL_USB3)
- #define XHCI_PORT_IS_USB2_HSO(cid, pid) ((xhci_hc[cid].ports[pid].flags & XHCI_PROTOCOL_HSO) == XHCI_PROTOCOL_HSO)
- #define XHCI_PORT_HAS_PAIR(cid, pid) ((xhci_hc[cid].ports[pid].flags & XHCI_PROTOCOL_HAS_PAIR) == XHCI_PROTOCOL_HAS_PAIR)
- #define XHCI_PORT_IS_ACTIVE(cid, pid) ((xhci_hc[cid].ports[pid].flags & XHCI_PROTOCOL_ACTIVE) == XHCI_PROTOCOL_ACTIVE)
- #define xhci_TRB_set_link_cmd(trb_vaddr) \
- do \
- { \
- struct xhci_TRB_normal_t *ptr = (struct xhci_TRB_normal_t *)(trb_vaddr); \
- ptr->TRB_type = TRB_TYPE_LINK; \
- ptr->ioc = 0; \
- ptr->chain = 0; \
- ptr->ent = 0; \
- ptr->cycle = 1; \
- } while (0)
- #define xhci_ep_set_dequeue_cycle_state(ep_ctx_ptr, state) ((ep_ctx_ptr)->tr_dequeue_ptr |= ((state)&1))
- #define xhci_ep_get_dequeue_cycle_state(ep_ctx_ptr) (((ep_ctx_ptr)->tr_dequeue_ptr) & 1)
- static int xhci_hc_find_available_id()
- {
- if (unlikely(xhci_ctrl_count >= XHCI_MAX_HOST_CONTROLLERS))
- return -1;
- for (int i = 0; i < XHCI_MAX_HOST_CONTROLLERS; ++i)
- {
- if (xhci_hc[i].pci_dev_hdr == NULL)
- return i;
- }
- return -1;
- }
- static __always_inline void xhci_get_trb(struct xhci_TRB_t *trb, const uint64_t address)
- {
- trb->param = __read8b(address);
- trb->status = __read4b(address + 8);
- trb->command = __read4b(address + 12);
- }
- static __always_inline void xhci_set_trb(struct xhci_TRB_t *trb, const uint64_t address)
- {
- __write8b(address, trb->param);
- __write4b(address + 8, trb->status);
- __write4b(address + 12, trb->command);
- }
- static __always_inline void __write_ep(int id, uint64_t slot_vaddr, int ep_num, struct xhci_ep_context_t *ep)
- {
- memcpy((void *)(slot_vaddr + ep_num * xhci_hc[id].context_size), ep, sizeof(struct xhci_ep_context_t));
- }
- static __always_inline void __read_from_ep(int id, uint64_t slot_vaddr, int ep_num, struct xhci_ep_context_t *ep)
- {
- memcpy(ep, (void *)(slot_vaddr + ep_num * xhci_hc[id].context_size), sizeof(struct xhci_ep_context_t));
- }
- static __always_inline void __write_slot(const uint64_t vaddr, struct xhci_slot_context_t *slot_ctx)
- {
- memcpy((void *)vaddr, slot_ctx, sizeof(struct xhci_slot_context_t));
- }
- static __always_inline void __read_from_slot(struct xhci_slot_context_t *slot_ctx, uint64_t slot_vaddr)
- {
- memcpy(slot_ctx, (void *)slot_vaddr, sizeof(struct xhci_slot_context_t));
- }
- static __always_inline void __xhci_write_doorbell(const int id, const uint16_t slot_id, const uint32_t value)
- {
-
- io_mfence();
- xhci_write_cap_reg32(id, xhci_hc[id].db_offset + slot_id * sizeof(uint32_t), value);
- io_mfence();
- }
- static __always_inline void __xhci_write_trb(struct xhci_ep_ring_info_t *ep_info, struct xhci_TRB_t *trb)
- {
- memcpy((void *)ep_info->current_ep_ring_vaddr, trb, sizeof(struct xhci_TRB_t));
- ep_info->current_ep_ring_vaddr += sizeof(struct xhci_TRB_t);
- struct xhci_TRB_normal_t *ptr = (struct xhci_TRB_normal_t *)(ep_info->current_ep_ring_vaddr);
-
- if (unlikely(ptr->TRB_type == TRB_TYPE_LINK))
- {
- ptr->cycle = ep_info->current_ep_ring_cycle;
- ep_info->current_ep_ring_vaddr = ep_info->ep_ring_vbase;
- ep_info->current_ep_ring_cycle ^= 1;
- }
- }
- static int xhci_hc_stop(int id)
- {
-
- if (unlikely((xhci_read_op_reg32(id, XHCI_OPS_USBSTS) & (1 << 0)) == 1))
- return 0;
- io_mfence();
- xhci_write_op_reg32(id, XHCI_OPS_USBCMD, 0x00000000);
- io_mfence();
- char timeout = 17;
- while ((xhci_read_op_reg32(id, XHCI_OPS_USBSTS) & (1 << 0)) == 0)
- {
- io_mfence();
- usleep(1000);
- if (--timeout == 0)
- return -ETIMEDOUT;
- }
- return 0;
- }
- static int xhci_hc_reset(int id)
- {
- int retval = 0;
- io_mfence();
-
- if ((xhci_read_op_reg32(id, XHCI_OPS_USBSTS) & (1 << 0)) == 0)
- {
- io_mfence();
- kdebug("stopping usb hc...");
-
- retval = xhci_hc_stop(id);
- if (unlikely(retval))
- return retval;
- }
- int timeout = 500;
-
- uint32_t cmd = xhci_read_op_reg32(id, XHCI_OPS_USBCMD);
- io_mfence();
- cmd |= (1 << 1);
- xhci_write_op_reg32(id, XHCI_OPS_USBCMD, cmd);
- io_mfence();
- io_mfence();
- while (xhci_read_op_reg32(id, XHCI_OPS_USBCMD) & (1 << 1))
- {
- io_mfence();
- usleep(1000);
- if (--timeout == 0)
- return -ETIMEDOUT;
- }
- return retval;
- }
- static int xhci_hc_stop_legacy(int id)
- {
- uint64_t current_offset = xhci_hc[id].ext_caps_off;
- do
- {
-
- if ((xhci_read_cap_reg32(id, current_offset) & 0xff) == XHCI_XECP_ID_LEGACY)
- {
- io_mfence();
-
- xhci_write_cap_reg32(id, current_offset, xhci_read_cap_reg32(id, current_offset) | XHCI_XECP_LEGACY_OS_OWNED);
- io_mfence();
-
- int timeout = XHCI_XECP_LEGACY_TIMEOUT;
- while ((xhci_read_cap_reg32(id, current_offset) & XHCI_XECP_LEGACY_OWNING_MASK) != XHCI_XECP_LEGACY_OS_OWNED)
- {
- io_mfence();
- usleep(1000);
- if (--timeout == 0)
- {
- kerror("The BIOS doesn't stop legacy support.");
- return -ETIMEDOUT;
- }
- }
-
- return 0;
- }
- io_mfence();
-
- int next_off = ((xhci_read_cap_reg32(id, current_offset) & 0xff00) >> 8) << 2;
- io_mfence();
-
- current_offset = next_off ? (current_offset + next_off) : 0;
- } while (current_offset);
-
- return 0;
- }
- static int xhci_hc_start_sched(int id)
- {
- io_mfence();
- xhci_write_op_reg32(id, XHCI_OPS_USBCMD, (1 << 0) | (1 << 2) | (1 << 3));
- io_mfence();
- usleep(100 * 1000);
- }
- static int xhci_hc_stop_sched(int id)
- {
- io_mfence();
- xhci_write_op_reg32(id, XHCI_OPS_USBCMD, 0x00);
- io_mfence();
- }
- static uint32_t xhci_hc_get_protocol_offset(int id, uint32_t list_off, const int version, uint32_t *offset, uint32_t *count, uint16_t *protocol_flag)
- {
- if (count)
- *count = 0;
- do
- {
- uint32_t dw0 = xhci_read_cap_reg32(id, list_off);
- io_mfence();
- uint32_t next_list_off = (dw0 >> 8) & 0xff;
- next_list_off = next_list_off ? (list_off + (next_list_off << 2)) : 0;
- if ((dw0 & 0xff) == XHCI_XECP_ID_PROTOCOL && ((dw0 & 0xff000000) >> 24) == version)
- {
- uint32_t dw2 = xhci_read_cap_reg32(id, list_off + 8);
- io_mfence();
- if (offset != NULL)
- *offset = (uint32_t)(dw2 & 0xff) - 1;
- if (count != NULL)
- *count = (uint32_t)((dw2 & 0xff00) >> 8);
- if (protocol_flag != NULL && version == 2)
- *protocol_flag = (uint16_t)((dw2 >> 16) & 0x0fff);
- return next_list_off;
- }
- list_off = next_list_off;
- } while (list_off);
- return 0;
- }
- static int xhci_hc_pair_ports(int id)
- {
- struct xhci_caps_HCSPARAMS1_reg_t hcs1;
- io_mfence();
- memcpy(&hcs1, xhci_get_ptr_cap_reg32(id, XHCI_CAPS_HCSPARAMS1), sizeof(struct xhci_caps_HCSPARAMS1_reg_t));
- io_mfence();
-
- xhci_hc[id].port_num = hcs1.max_ports;
-
- xhci_hc[id].port_num_u2 = 0;
- xhci_hc[id].port_num_u3 = 0;
- uint32_t next_off = xhci_hc[id].ext_caps_off;
- uint32_t offset, cnt;
- uint16_t protocol_flags = 0;
-
- while (next_off)
- {
- io_mfence();
- next_off = xhci_hc_get_protocol_offset(id, next_off, 2, &offset, &cnt, &protocol_flags);
- io_mfence();
- if (cnt)
- {
- for (int i = 0; i < cnt; ++i)
- {
- io_mfence();
- xhci_hc[id].ports[offset + i].offset = xhci_hc[id].port_num_u2++;
- xhci_hc[id].ports[offset + i].flags = XHCI_PROTOCOL_USB2;
- io_mfence();
-
- if (protocol_flags & 2)
- xhci_hc[id].ports[offset + i].flags |= XHCI_PROTOCOL_HSO;
- }
- }
- }
-
- next_off = xhci_hc[id].ext_caps_off;
- while (next_off)
- {
- io_mfence();
- next_off = xhci_hc_get_protocol_offset(id, next_off, 3, &offset, &cnt, &protocol_flags);
- io_mfence();
- if (cnt)
- {
- for (int i = 0; i < cnt; ++i)
- {
- io_mfence();
- xhci_hc[id].ports[offset + i].offset = xhci_hc[id].port_num_u3++;
- xhci_hc[id].ports[offset + i].flags = XHCI_PROTOCOL_USB3;
- }
- }
- }
-
- for (int i = 0; i < xhci_hc[id].port_num; ++i)
- {
- for (int j = 0; j < xhci_hc[id].port_num; ++j)
- {
- if (unlikely(i == j))
- continue;
- io_mfence();
- if ((xhci_hc[id].ports[i].offset == xhci_hc[id].ports[j].offset) &&
- ((xhci_hc[id].ports[i].flags & XHCI_PROTOCOL_INFO) != (xhci_hc[id].ports[j].flags & XHCI_PROTOCOL_INFO)))
- {
- xhci_hc[id].ports[i].paired_port_num = j;
- xhci_hc[id].ports[i].flags |= XHCI_PROTOCOL_HAS_PAIR;
- io_mfence();
- xhci_hc[id].ports[j].paired_port_num = i;
- xhci_hc[id].ports[j].flags |= XHCI_PROTOCOL_HAS_PAIR;
- }
- }
- }
-
- for (int i = 0; i < xhci_hc[id].port_num; ++i)
- {
- io_mfence();
- if (XHCI_PORT_IS_USB3(id, i) ||
- (XHCI_PORT_IS_USB2(id, i) && (!XHCI_PORT_HAS_PAIR(id, i))))
- xhci_hc[id].ports[i].flags |= XHCI_PROTOCOL_ACTIVE;
- }
- kinfo("Found %d ports on root hub, usb2 ports:%d, usb3 ports:%d", xhci_hc[id].port_num, xhci_hc[id].port_num_u2, xhci_hc[id].port_num_u3);
-
- return 0;
- }
- static uint64_t xhci_create_ring(int trbs)
- {
- int total_size = trbs * sizeof(struct xhci_TRB_t);
- const uint64_t vaddr = (uint64_t)kmalloc(total_size, 0);
- io_mfence();
- memset((void *)vaddr, 0, total_size);
- io_mfence();
-
- xhci_TRB_set_link_cmd(vaddr + total_size - sizeof(struct xhci_TRB_t));
- io_mfence();
- return vaddr;
- }
- static uint64_t xhci_create_event_ring(int trbs, uint64_t *ret_ring_addr)
- {
- const uint64_t table_vaddr = (const uint64_t)kmalloc(64, 0);
- io_mfence();
- if (unlikely(table_vaddr == NULL))
- return -ENOMEM;
- memset((void *)table_vaddr, 0, 64);
-
- const uint64_t seg_vaddr = (const uint64_t)kmalloc(trbs * sizeof(struct xhci_TRB_t), 0);
- io_mfence();
- if (unlikely(seg_vaddr == NULL))
- return -ENOMEM;
- memset((void *)seg_vaddr, 0, trbs * sizeof(struct xhci_TRB_t));
- io_mfence();
-
- *(uint64_t *)(table_vaddr) = virt_2_phys(seg_vaddr);
- *(uint64_t *)(table_vaddr + 8) = trbs;
- *ret_ring_addr = seg_vaddr;
- return table_vaddr;
- }
- void xhci_hc_irq_enable(uint64_t irq_num)
- {
- int cid = xhci_find_hcid_by_irq_num(irq_num);
- io_mfence();
- if (WARN_ON(cid == -1))
- return;
- io_mfence();
- pci_start_msi(xhci_hc[cid].pci_dev_hdr);
- io_mfence();
- xhci_hc_start_sched(cid);
- io_mfence();
- xhci_hc_start_ports(cid);
- }
- void xhci_hc_irq_disable(uint64_t irq_num)
- {
- int cid = xhci_find_hcid_by_irq_num(irq_num);
- io_mfence();
- if (WARN_ON(cid == -1))
- return;
- xhci_hc_stop_sched(cid);
- io_mfence();
- pci_disable_msi(xhci_hc[cid].pci_dev_hdr);
- io_mfence();
- }
- uint64_t xhci_hc_irq_install(uint64_t irq_num, void *arg)
- {
- int cid = xhci_find_hcid_by_irq_num(irq_num);
- io_mfence();
- if (WARN_ON(cid == -1))
- return -EINVAL;
- struct xhci_hc_irq_install_info_t *info = (struct xhci_hc_irq_install_info_t *)arg;
- struct msi_desc_t msi_desc;
- memset(&msi_desc, 0, sizeof(struct msi_desc_t));
- io_mfence();
- msi_desc.irq_num = irq_num;
- msi_desc.msi_index = 0;
- msi_desc.pci_dev = (struct pci_device_structure_header_t *)xhci_hc[cid].pci_dev_hdr;
- msi_desc.assert = info->assert;
- msi_desc.edge_trigger = info->edge_trigger;
- msi_desc.processor = info->processor;
- msi_desc.pci.msi_attribute.is_64 = 1;
- msi_desc.pci.msi_attribute.is_msix = 1;
- io_mfence();
- int retval = pci_enable_msi(&msi_desc);
- return 0;
- }
- void xhci_hc_irq_uninstall(uint64_t irq_num)
- {
-
- int cid = xhci_find_hcid_by_irq_num(irq_num);
- io_mfence();
- if (WARN_ON(cid == -1))
- return;
- xhci_hc_stop(cid);
- io_mfence();
- }
- void xhci_hc_irq_handler(uint64_t irq_num, uint64_t cid, struct pt_regs *regs)
- {
-
-
- xhci_write_op_reg32(cid, XHCI_OPS_USBSTS, xhci_read_op_reg32(cid, XHCI_OPS_USBSTS));
-
- const uint32_t iman0 = xhci_read_intr_reg32(cid, 0, XHCI_IR_MAN);
- uint64_t dequeue_reg = xhci_read_intr_reg64(cid, 0, XHCI_IR_DEQUEUE);
- if (((iman0 & 3) == 3) || (dequeue_reg & 8))
- {
-
- xhci_write_intr_reg32(cid, 0, XHCI_IR_MAN, iman0 | 3);
- io_mfence();
- struct xhci_TRB_t event_trb, origin_trb;
- uint64_t origin_vaddr;
-
- uint64_t last_event_ring_vaddr = xhci_hc[cid].current_event_ring_vaddr;
- xhci_get_trb(&event_trb, xhci_hc[cid].current_event_ring_vaddr);
- while ((event_trb.command & 1) == xhci_hc[cid].current_event_ring_cycle)
- {
- struct xhci_TRB_cmd_complete_t *event_trb_ptr = (struct xhci_TRB_cmd_complete_t *)&event_trb;
- if ((event_trb.command & (1 << 2)) == 0)
- {
-
-
- switch (event_trb_ptr->code)
- {
- case TRB_COMP_TRB_SUCCESS:
- switch (event_trb_ptr->TRB_type)
- {
- case TRB_TYPE_COMMAND_COMPLETION:
- origin_vaddr = (uint64_t)phys_2_virt(event_trb.param);
-
- xhci_get_trb(&origin_trb, origin_vaddr);
- switch (((struct xhci_TRB_normal_t *)&origin_trb)->TRB_type)
- {
- case TRB_TYPE_ENABLE_SLOT:
-
- origin_trb.command &= 0x00ffffff;
- origin_trb.command |= (event_trb.command & 0xff000000);
- origin_trb.status = event_trb.status;
- break;
- default:
- origin_trb.status = event_trb.status;
- break;
- }
-
- origin_trb.status |= XHCI_IRQ_DONE;
-
- xhci_set_trb(&origin_trb, origin_vaddr);
-
- break;
- }
- break;
- default:
- break;
- }
- }
- else
- {
- switch (event_trb_ptr->TRB_type)
- {
- case TRB_TYPE_TRANS_EVENT:
-
- __write4b((uint64_t)phys_2_virt(event_trb.param), (event_trb.status | XHCI_IRQ_DONE));
- break;
- default:
- break;
- }
- }
-
- last_event_ring_vaddr = xhci_hc[cid].current_event_ring_vaddr;
- xhci_hc[cid].current_event_ring_vaddr += sizeof(struct xhci_TRB_t);
- xhci_get_trb(&event_trb, xhci_hc[cid].current_event_ring_vaddr);
- if (((struct xhci_TRB_normal_t *)&event_trb)->TRB_type == TRB_TYPE_LINK)
- {
- xhci_hc[cid].current_event_ring_vaddr = xhci_hc[cid].event_ring_vaddr;
- xhci_get_trb(&event_trb, xhci_hc[cid].current_event_ring_vaddr);
- }
- }
-
-
- xhci_write_intr_reg64(cid, 0, XHCI_IR_DEQUEUE, virt_2_phys(last_event_ring_vaddr) | (1 << 3));
- io_mfence();
- }
- }
- static int xhci_reset_port(const int id, const int port)
- {
- int retval = 0;
-
- uint64_t port_status_offset = XHCI_OPS_PRS + port * 16;
- io_mfence();
-
- if ((xhci_read_op_reg32(id, port_status_offset + XHCI_PORT_PORTSC) & (1 << 9)) == 0)
- {
- kdebug("port is power off, starting...");
- io_mfence();
- xhci_write_cap_reg32(id, port_status_offset + XHCI_PORT_PORTSC, (1 << 9));
- io_mfence();
- usleep(2000);
-
- if ((xhci_read_op_reg32(id, port_status_offset + XHCI_PORT_PORTSC) & (1 << 9)) == 0)
- {
- kdebug("cannot power on %d", port);
- return -EAGAIN;
- }
- }
-
- io_mfence();
-
- xhci_write_op_reg32(id, port_status_offset + XHCI_PORT_PORTSC, (1 << 9) | XHCI_PORTUSB_CHANGE_BITS);
-
- io_mfence();
-
- if (XHCI_PORT_IS_USB3(id, port))
- xhci_write_op_reg32(id, port_status_offset + XHCI_PORT_PORTSC, (1 << 9) | (1 << 31));
- else
- xhci_write_op_reg32(id, port_status_offset + XHCI_PORT_PORTSC, (1 << 9) | (1 << 4));
- retval = -ETIMEDOUT;
-
-
- int timeout = 100;
- while (timeout)
- {
- io_mfence();
- uint32_t val = xhci_read_op_reg32(id, port_status_offset + XHCI_PORT_PORTSC);
- io_mfence();
- if (val & (1 << 21))
- break;
-
- #ifdef __QEMU_EMULATION__
- if (XHCI_PORT_IS_USB3(id, port) && (val & (1 << 31)) == 0)
- break;
- else if (XHCI_PORT_IS_USB2(id, port) && (val & (1 << 4)) == 0)
- break;
- #endif
- --timeout;
- usleep(500);
- }
-
- if (timeout > 0)
- {
-
- usleep(USB_TIME_RST_REC * 100);
- uint32_t val = xhci_read_op_reg32(id, port_status_offset + XHCI_PORT_PORTSC);
- io_mfence();
-
-
- if (val & (1 << 1))
- {
-
- retval = 0;
- io_mfence();
-
- xhci_write_op_reg32(id, port_status_offset + XHCI_PORT_PORTSC, (1 << 9) | XHCI_PORTUSB_CHANGE_BITS);
- io_mfence();
- }
- else
- retval = -1;
- }
-
-
- if (retval == 0 && XHCI_PORT_IS_USB2(id, port))
- {
- xhci_hc[id].ports[port].flags |= XHCI_PROTOCOL_ACTIVE;
- if (XHCI_PORT_HAS_PAIR(id, port))
- xhci_hc[id].ports[xhci_hc[id].ports[port].paired_port_num].flags &= ~(XHCI_PROTOCOL_ACTIVE);
- }
-
- if (retval != 0 && XHCI_PORT_IS_USB3(id, port))
- {
- xhci_hc[id].ports[port].flags &= ~XHCI_PROTOCOL_ACTIVE;
- xhci_hc[id].ports[xhci_hc[id].ports[port].paired_port_num].flags |= XHCI_PROTOCOL_ACTIVE;
- }
- return retval;
- }
- static uint64_t xhci_initialize_slot(const int id, const int slot_id, const int port, const int speed, const int max_packet)
- {
-
-
- uint64_t device_context_vaddr = (uint64_t)kzalloc(xhci_hc[id].context_size * 2, 0);
-
-
- __write8b(xhci_hc[id].dcbaap_vaddr + (slot_id * sizeof(uint64_t)), virt_2_phys(device_context_vaddr));
- struct xhci_slot_context_t slot_ctx = {0};
- slot_ctx.entries = 1;
- slot_ctx.speed = speed;
- slot_ctx.route_string = 0;
- slot_ctx.rh_port_num = port + 1;
- slot_ctx.max_exit_latency = 0;
- slot_ctx.int_target = 0;
- slot_ctx.slot_state = XHCI_SLOT_STATE_DISABLED_OR_ENABLED;
- slot_ctx.device_address = 0;
-
- __write_slot(device_context_vaddr, &slot_ctx);
- xhci_initialize_ep(id, device_context_vaddr, slot_id, XHCI_EP_CONTROL, max_packet, USB_EP_CONTROL, 0, speed, 0);
- return device_context_vaddr;
- }
- static void xhci_initialize_ep(const int id, const uint64_t slot_vaddr, const int slot_id, const int ep_num, const int max_packet, const int type, const int direction, const int speed, const int ep_interval)
- {
-
- if (type != USB_EP_CONTROL)
- return;
- struct xhci_ep_context_t ep_ctx = {0};
- memset(&ep_ctx, 0, sizeof(struct xhci_ep_context_t));
- xhci_hc[id].control_ep_info.ep_ring_vbase = xhci_create_ring(XHCI_TRBS_PER_RING);
-
- ep_ctx.tr_dequeue_ptr = virt_2_phys(xhci_hc[id].control_ep_info.ep_ring_vbase);
- xhci_ep_set_dequeue_cycle_state(&ep_ctx, XHCI_TRB_CYCLE_ON);
- xhci_hc[id].control_ep_info.current_ep_ring_vaddr = xhci_hc[id].control_ep_info.ep_ring_vbase;
- xhci_hc[id].control_ep_info.current_ep_ring_cycle = xhci_ep_get_dequeue_cycle_state(&ep_ctx);
-
-
-
- ep_ctx.max_packet_size = max_packet;
- ep_ctx.linear_stream_array = 0;
- ep_ctx.max_primary_streams = 0;
- ep_ctx.mult = 0;
- ep_ctx.ep_state = XHCI_EP_STATE_DISABLED;
- ep_ctx.hid = 0;
- ep_ctx.ep_type = 4;
- ep_ctx.average_trb_len = 8;
- ep_ctx.err_cnt = 3;
- ep_ctx.max_burst_size = 0;
- ep_ctx.interval = ep_interval;
-
- __write_ep(id, slot_vaddr, ep_num, &ep_ctx);
- }
- static int xhci_set_address(const int id, const uint64_t slot_vaddr, const int slot_id, const bool block)
- {
- int retval = 0;
- struct xhci_slot_context_t slot;
- struct xhci_ep_context_t ep;
-
- uint64_t input_ctx_buffer = (uint64_t)kzalloc(xhci_hc[id].context_size * 32, 0);
-
- __write4b(input_ctx_buffer + 4, 0x3);
-
-
- __read_from_slot(&slot, slot_vaddr);
- __read_from_ep(id, slot_vaddr, 1, &ep);
- ep.err_cnt = 3;
- kdebug("slot.slot_state=%d, speed=%d, root hub port num=%d", slot.slot_state, slot.speed, slot.rh_port_num);
- kdebug("ep.type=%d, max_packet=%d, dequeue_ptr=%#018lx", ep.ep_type, ep.max_packet_size, ep.tr_dequeue_ptr);
- __write_slot(input_ctx_buffer + xhci_hc[id].context_size, &slot);
- __write_ep(id, input_ctx_buffer, 2, &ep);
- struct xhci_TRB_normal_t trb = {0};
- trb.buf_paddr = virt_2_phys(input_ctx_buffer);
- trb.bei = (block ? 1 : 0);
- trb.TRB_type = TRB_TYPE_ADDRESS_DEVICE;
- trb.intr_target = 0;
- trb.cycle = xhci_hc[id].cmd_trb_cycle;
- trb.Reserved |= ((slot_id << 8) & 0xffff);
- retval = xhci_send_command(id, (struct xhci_TRB_t *)&trb, true);
- if (unlikely(retval != 0))
- {
- kerror("slotid:%d, address device failed", slot_id);
- goto failed;
- }
- struct xhci_TRB_cmd_complete_t *trb_done = (struct xhci_TRB_cmd_complete_t *)&trb;
- if (trb_done->code == TRB_COMP_TRB_SUCCESS)
- {
-
- ksuccess("slot %d successfully addressed.", slot_id);
- retval = 0;
- }
- else
- retval = -EAGAIN;
- done:;
- failed:;
- kfree((void *)input_ctx_buffer);
- return retval;
- }
- static int xhci_setup_stage(struct xhci_ep_ring_info_t *ep, const struct usb_request_packet_t *packet, const uint8_t direction)
- {
-
- struct xhci_TRB_setup_stage_t trb = {0};
- trb.bmRequestType = packet->request_type;
- trb.bRequest = packet->request;
- trb.wValue = packet->value;
- trb.wIndex = packet->index;
- trb.wLength = packet->length;
- trb.transfer_legth = 8;
- trb.intr_target = 0;
- trb.cycle = ep->current_ep_ring_cycle;
- trb.ioc = 0;
- trb.idt = 1;
- trb.TRB_type = TRB_TYPE_SETUP_STAGE;
- trb.trt = direction;
-
- __xhci_write_trb(ep, (struct xhci_TRB_t *)&trb);
- return 1;
- }
- static int xhci_data_stage(struct xhci_ep_ring_info_t *ep, uint64_t buf_vaddr, uint8_t trb_type, const uint32_t size, uint8_t direction, const int max_packet, const uint64_t status_vaddr)
- {
- if (size == 0)
- return 0;
- int64_t remain_bytes = size;
- uint32_t remain_packets = (size + max_packet - 1) / max_packet;
- struct xhci_TRB_data_stage_t trb = {0};
- int count_packets = 0;
-
- while (remain_bytes > 0)
- {
- --remain_packets;
- trb.buf_paddr = virt_2_phys(buf_vaddr);
- trb.intr_target = 0;
- trb.TD_size = remain_packets;
- trb.transfer_length = (remain_bytes < max_packet ? size : max_packet);
- trb.dir = direction;
- trb.TRB_type = trb_type;
- trb.chain = 1;
- trb.ent = (remain_packets == 0);
- trb.cycle = ep->current_ep_ring_cycle;
- trb.ioc = 0;
-
- __xhci_write_trb(ep, (struct xhci_TRB_t *)&trb);
- buf_vaddr += max_packet;
- remain_bytes -= max_packet;
- ++count_packets;
-
- trb_type = TRB_TYPE_NORMAL;
- direction = 0;
- }
-
- memset(&trb, 0, sizeof(struct xhci_TRB_data_stage_t *));
- trb.buf_paddr = virt_2_phys(status_vaddr);
- trb.intr_target = 0;
- trb.cycle = ep->current_ep_ring_cycle;
- trb.ioc = 1;
- trb.TRB_type = TRB_TYPE_EVENT_DATA;
- __xhci_write_trb(ep, (struct xhci_TRB_t *)&trb);
- return count_packets + 1;
- }
- static int xhci_status_stage(const int id, uint8_t direction, uint64_t status_buf_vaddr)
- {
-
- {
- struct xhci_TRB_status_stage_t trb = {0};
-
- trb.intr_target = 0;
- trb.cycle = xhci_hc[id].control_ep_info.current_ep_ring_cycle;
- trb.ent = 0;
- trb.ioc = 1;
- trb.TRB_type = TRB_TYPE_STATUS_STAGE;
- trb.dir = direction;
- __xhci_write_trb(&xhci_hc[id].control_ep_info, (struct xhci_TRB_t *)&trb);
- }
- {
-
- struct xhci_TRB_data_stage_t trb = {0};
- trb.buf_paddr = virt_2_phys(status_buf_vaddr);
- trb.intr_target = 0;
- trb.TRB_type = TRB_TYPE_EVENT_DATA;
- trb.ioc = 1;
- trb.cycle = xhci_hc[id].control_ep_info.current_ep_ring_cycle;
- __xhci_write_trb(&xhci_hc[id].control_ep_info, (struct xhci_TRB_t *)&trb);
- }
- return 2;
- }
- static int xhci_wait_for_interrupt(const int id, uint64_t status_vaddr)
- {
- int timer = 500;
- while (timer)
- {
- if (__read4b(status_vaddr) & XHCI_IRQ_DONE)
- {
- uint32_t status = __read4b(status_vaddr);
-
- switch (xhci_get_comp_code(status))
- {
- case TRB_COMP_TRB_SUCCESS:
- case TRB_COMP_SHORT_PACKET:
- return 0;
- break;
- case TRB_COMP_STALL_ERROR:
- case TRB_COMP_DATA_BUFFER_ERROR:
- case TRB_COMP_BABBLE_DETECTION:
- return -EINVAL;
- default:
- kerror("xhci wait interrupt: status=%#010x, complete_code=%d", status, xhci_get_comp_code(status));
- return -EIO;
- }
- }
- --timer;
- usleep(1000);
- }
- kerror(" USB xHCI Interrupt wait timed out.");
- return -ETIMEDOUT;
- }
- static int xhci_control_in(const int id, void *target, const int in_size, const int slot_id, const int max_packet)
- {
- uint64_t status_buf_vaddr = (uint64_t)kzalloc(16, 0);
- uint64_t data_buf_vaddr = (uint64_t)kzalloc(256, 0);
- int retval = 0;
- struct usb_request_packet_t packet = {0};
- packet.request_type = USB_REQ_TYPE_GET_REQUEST;
- packet.request = USB_REQ_GET_DESCRIPTOR;
- packet.value = (USB_DT_DEVICE << 8);
- packet.length = in_size;
-
- xhci_setup_stage(&xhci_hc[id].control_ep_info, &packet, XHCI_DIR_IN);
- xhci_data_stage(&xhci_hc[id].control_ep_info, data_buf_vaddr, TRB_TYPE_DATA_STAGE, in_size, XHCI_DIR_IN_BIT, max_packet, status_buf_vaddr);
- #ifndef __QEMU_EMULATION__
-
-
- __xhci_write_doorbell(id, slot_id, XHCI_EP_CONTROL);
- retval = xhci_wait_for_interrupt(id, status_buf_vaddr);
- if (unlikely(retval != 0))
- goto failed;
- #endif
- memset((void *)status_buf_vaddr, 0, 16);
- xhci_status_stage(id, XHCI_DIR_OUT_BIT, status_buf_vaddr);
- __xhci_write_doorbell(id, slot_id, XHCI_EP_CONTROL);
- retval = xhci_wait_for_interrupt(id, status_buf_vaddr);
- if (unlikely(retval != 0))
- goto failed;
-
- memcpy(target, (void *)data_buf_vaddr, in_size);
- retval = in_size;
- goto done;
- failed:;
- kdebug("wait 4 interrupt failed");
- retval = 0;
- done:;
-
- kfree((void *)status_buf_vaddr);
- kfree((void *)data_buf_vaddr);
- return retval;
- }
- static int xhci_get_descriptor(const int id, const int port_id)
- {
- int retval = 0;
- int count = 0;
- struct usb_device_desc dev_desc = {0};
- uint32_t dword;
-
- uint32_t port_register_offset = XHCI_OPS_PRS + 16 * port_id;
-
- dword = xhci_read_op_reg32(id, port_register_offset + XHCI_PORT_PORTSC);
-
- uint32_t speed = ((dword >> 10) & 0xf);
-
- struct xhci_TRB_normal_t trb = {0};
- trb.TRB_type = TRB_TYPE_ENABLE_SLOT;
-
- if (xhci_send_command(id, (struct xhci_TRB_t *)&trb, true) != 0)
- {
- kerror("portid:%d: send enable slot failed", port_id);
- return -ETIMEDOUT;
- }
-
- uint32_t slot_id = ((struct xhci_TRB_cmd_complete_t *)&trb)->slot_id;
- int16_t max_packet;
- if (slot_id != 0)
- {
-
- switch (speed)
- {
- case XHCI_PORT_SPEED_LOW:
- max_packet = 8;
- break;
- case XHCI_PORT_SPEED_FULL:
- case XHCI_PORT_SPEED_HI:
- max_packet = 64;
- break;
- case XHCI_PORT_SPEED_SUPER:
- max_packet = 512;
- break;
- }
- }
-
-
- uint64_t slot_vaddr = xhci_initialize_slot(id, slot_id, port_id, speed, max_packet);
-
-
-
- retval = xhci_set_address(id, slot_vaddr, slot_id, false);
- if (retval != 0)
- return retval;
-
- count = xhci_control_in(id, &dev_desc, 18, slot_id, max_packet);
- if (unlikely(count == 0))
- return -EAGAIN;
-
-
- printk(" Found USB Device:\n"
- " port: %i\n"
- " len: %i\n"
- " type: %i\n"
- " version: %01X.%02X\n"
- " class: %i\n"
- " subclass: %i\n"
- " protocol: %i\n"
- " max packet size: %i\n"
- " vendor id: 0x%04X\n"
- " product id: 0x%04X\n"
- " release ver: %i%i.%i%i\n"
- " manufacture index: %i (index to a string)\n"
- " product index: %i\n"
- " serial index: %i\n"
- " number of configs: %i\n",
- port_id, dev_desc.len, dev_desc.type, dev_desc.usb_version >> 8, dev_desc.usb_version & 0xFF, dev_desc._class, dev_desc.subclass,
- dev_desc.protocol, dev_desc.max_packet_size, dev_desc.vendor_id, dev_desc.product_id,
- (dev_desc.device_rel & 0xF000) >> 12, (dev_desc.device_rel & 0x0F00) >> 8,
- (dev_desc.device_rel & 0x00F0) >> 4, (dev_desc.device_rel & 0x000F) >> 0,
- dev_desc.manufacturer_index, dev_desc.procuct_index, dev_desc.serial_index, dev_desc.config);
- return 0;
- }
- static int xhci_hc_start_ports(int id)
- {
- int cnt = 0;
-
-
- for (int i = 0; i < xhci_hc[id].port_num; ++i)
- {
- if (XHCI_PORT_IS_USB3(id, i) && XHCI_PORT_IS_ACTIVE(id, i))
- {
- io_mfence();
-
- int rst_ret = xhci_reset_port(id, i);
-
-
- if (likely(rst_ret == 0))
-
- {
-
- if (xhci_get_descriptor(id, i) == 0)
- ++cnt;
- kdebug("usb3 port %d get desc ok", i);
- }
- }
- }
- kdebug("Active usb3 ports:%d", cnt);
-
- for (int i = 0; i < xhci_hc[id].port_num; ++i)
- {
- if (XHCI_PORT_IS_USB2(id, i) && XHCI_PORT_IS_ACTIVE(id, i))
- {
-
-
-
- if (likely(xhci_reset_port(id, i) == 0))
-
- {
-
- if (xhci_get_descriptor(id, i) == 0)
- ++cnt;
- kdebug("USB2 port %d get desc ok", i);
- }
- }
- }
- kinfo("xHCI controller %d: Started %d ports.", id, cnt);
- return 0;
- }
- static int xhci_hc_init_intr(int id)
- {
- uint64_t retval = 0;
- struct xhci_caps_HCSPARAMS1_reg_t hcs1;
- struct xhci_caps_HCSPARAMS2_reg_t hcs2;
- io_mfence();
- memcpy(&hcs1, xhci_get_ptr_cap_reg32(id, XHCI_CAPS_HCSPARAMS1), sizeof(struct xhci_caps_HCSPARAMS1_reg_t));
- io_mfence();
- memcpy(&hcs2, xhci_get_ptr_cap_reg32(id, XHCI_CAPS_HCSPARAMS2), sizeof(struct xhci_caps_HCSPARAMS2_reg_t));
- io_mfence();
- uint32_t max_segs = (1 << (uint32_t)(hcs2.ERST_Max));
- uint32_t max_interrupters = hcs1.max_intrs;
-
- retval = xhci_create_event_ring(4096, &xhci_hc[id].event_ring_vaddr);
- io_mfence();
- if (unlikely((int64_t)(retval) == -ENOMEM))
- return -ENOMEM;
- xhci_hc[id].event_ring_table_vaddr = retval;
- xhci_hc[id].current_event_ring_vaddr = xhci_hc[id].event_ring_vaddr;
- retval = 0;
- xhci_hc[id].current_event_ring_cycle = 1;
-
- io_mfence();
- xhci_write_intr_reg32(id, 0, XHCI_IR_MAN, 0x3);
- io_mfence();
- xhci_write_intr_reg32(id, 0, XHCI_IR_MOD, 0);
- io_mfence();
- xhci_write_intr_reg32(id, 0, XHCI_IR_TABLE_SIZE, 1);
- io_mfence();
- xhci_write_intr_reg64(id, 0, XHCI_IR_DEQUEUE, virt_2_phys(xhci_hc[id].current_event_ring_vaddr) | (1 << 3));
- io_mfence();
- xhci_write_intr_reg64(id, 0, XHCI_IR_TABLE_ADDR, virt_2_phys(xhci_hc[id].event_ring_table_vaddr));
- io_mfence();
-
- xhci_write_op_reg32(id, XHCI_OPS_USBSTS, (1 << 10) | (1 << 4) | (1 << 3) | (1 << 2));
- io_mfence();
-
-
- struct xhci_hc_irq_install_info_t install_info;
- install_info.assert = 1;
- install_info.edge_trigger = 1;
- install_info.processor = 0;
- char *buf = (char *)kmalloc(16, 0);
- memset(buf, 0, 16);
- sprintk(buf, "xHCI HC%d", id);
- io_mfence();
- irq_register(xhci_controller_irq_num[id], &install_info, &xhci_hc_irq_handler, id, &xhci_hc_intr_controller, buf);
- io_mfence();
- kfree(buf);
- kdebug("xhci host controller %d: interrupt registered. irq num=%d", id, xhci_controller_irq_num[id]);
- return 0;
- }
- static int xhci_send_command(int id, struct xhci_TRB_t *trb, const bool do_ring)
- {
- uint64_t origin_trb_vaddr = xhci_hc[id].cmd_trb_vaddr;
-
- __write8b(xhci_hc[id].cmd_trb_vaddr, trb->param);
- __write4b(xhci_hc[id].cmd_trb_vaddr + 8, trb->status);
- __write4b(xhci_hc[id].cmd_trb_vaddr + 12, trb->command | xhci_hc[id].cmd_trb_cycle);
- xhci_hc[id].cmd_trb_vaddr += sizeof(struct xhci_TRB_t);
- {
-
- struct xhci_TRB_normal_t *ptr = (struct xhci_TRB_normal_t *)xhci_hc[id].cmd_trb_vaddr;
- if (ptr->TRB_type == TRB_TYPE_LINK)
- {
- ptr->cycle = xhci_hc[id].cmd_trb_cycle;
- xhci_hc[id].cmd_trb_vaddr = xhci_hc[id].cmd_ring_vaddr;
- xhci_hc[id].cmd_trb_cycle ^= 1;
- }
- }
- if (do_ring)
- {
- __xhci_write_doorbell(id, 0, 0);
-
- int timer = 400;
- const uint32_t iman0 = xhci_read_intr_reg32(id, 0, XHCI_IR_MAN);
-
-
- while (timer && ((__read4b(origin_trb_vaddr + 8) & XHCI_IRQ_DONE) == 0))
- {
- usleep(1000);
- --timer;
- }
- uint32_t x = xhci_read_cap_reg32(id, xhci_hc[id].rts_offset + 0x20);
- if (timer == 0)
- return -ETIMEDOUT;
- else
- {
- xhci_get_trb(trb, origin_trb_vaddr);
- trb->status &= (~XHCI_IRQ_DONE);
- }
- }
- return 0;
- }
- void xhci_init(struct pci_device_structure_general_device_t *dev_hdr)
- {
- if (xhci_ctrl_count >= XHCI_MAX_HOST_CONTROLLERS)
- {
- kerror("Initialize xhci controller failed: exceed the limit of max controllers.");
- return;
- }
- spin_lock(&xhci_controller_init_lock);
- kinfo("Initializing xhci host controller: bus=%#02x, device=%#02x, func=%#02x, VendorID=%#04x, irq_line=%d, irq_pin=%d", dev_hdr->header.bus, dev_hdr->header.device, dev_hdr->header.func, dev_hdr->header.Vendor_ID, dev_hdr->Interrupt_Line, dev_hdr->Interrupt_PIN);
- io_mfence();
- int cid = xhci_hc_find_available_id();
- if (cid < 0)
- {
- kerror("Initialize xhci controller failed: exceed the limit of max controllers.");
- goto failed_exceed_max;
- }
- memset(&xhci_hc[cid], 0, sizeof(struct xhci_host_controller_t));
- xhci_hc[cid].controller_id = cid;
- xhci_hc[cid].pci_dev_hdr = dev_hdr;
- io_mfence();
- {
- uint32_t tmp = pci_read_config(dev_hdr->header.bus, dev_hdr->header.device, dev_hdr->header.func, 0x4);
- tmp |= 0x6;
-
- pci_write_config(dev_hdr->header.bus, dev_hdr->header.device, dev_hdr->header.func, 0x4, tmp);
- }
- io_mfence();
-
- xhci_hc[cid].vbase = SPECIAL_MEMOEY_MAPPING_VIRT_ADDR_BASE + XHCI_MAPPING_OFFSET + 65536 * xhci_hc[cid].controller_id;
-
- mm_map_phys_addr(xhci_hc[cid].vbase, dev_hdr->BAR0 & (~0xf), 65536, PAGE_KERNEL_PAGE | PAGE_PWT | PAGE_PCD, true);
- io_mfence();
-
- xhci_hc[cid].vbase_op = xhci_hc[cid].vbase + (xhci_read_cap_reg32(cid, XHCI_CAPS_CAPLENGTH) & 0xff);
- io_mfence();
-
- FAIL_ON_TO(xhci_hc_reset(cid), failed);
- io_mfence();
-
- uint16_t iversion = *(uint16_t *)(xhci_hc[cid].vbase + XHCI_CAPS_HCIVERSION);
- struct xhci_caps_HCCPARAMS1_reg_t hcc1;
- struct xhci_caps_HCCPARAMS2_reg_t hcc2;
- struct xhci_caps_HCSPARAMS1_reg_t hcs1;
- struct xhci_caps_HCSPARAMS2_reg_t hcs2;
- memcpy(&hcc1, xhci_get_ptr_cap_reg32(cid, XHCI_CAPS_HCCPARAMS1), sizeof(struct xhci_caps_HCCPARAMS1_reg_t));
- memcpy(&hcc2, xhci_get_ptr_cap_reg32(cid, XHCI_CAPS_HCCPARAMS2), sizeof(struct xhci_caps_HCCPARAMS2_reg_t));
- memcpy(&hcs1, xhci_get_ptr_cap_reg32(cid, XHCI_CAPS_HCSPARAMS1), sizeof(struct xhci_caps_HCSPARAMS1_reg_t));
- memcpy(&hcs2, xhci_get_ptr_cap_reg32(cid, XHCI_CAPS_HCSPARAMS2), sizeof(struct xhci_caps_HCSPARAMS2_reg_t));
- xhci_hc[cid].db_offset = xhci_read_cap_reg32(cid, XHCI_CAPS_DBOFF) & (~0x3);
- io_mfence();
- xhci_hc[cid].rts_offset = xhci_read_cap_reg32(cid, XHCI_CAPS_RTSOFF) & (~0x1f);
- io_mfence();
- xhci_hc[cid].ext_caps_off = 1UL * (hcc1.xECP) * 4;
- xhci_hc[cid].context_size = (hcc1.csz) ? 64 : 32;
- if (iversion < 0x95)
- kwarn("Unsupported/Unknowned xHCI controller version: %#06x. This may cause unexpected behavior.", iversion);
- {
-
- uint32_t tmp = pci_read_config(dev_hdr->header.bus, dev_hdr->header.device, dev_hdr->header.func, 0x60);
- tmp |= (0x20 << 8);
- pci_write_config(dev_hdr->header.bus, dev_hdr->header.device, dev_hdr->header.func, 0x60, tmp);
- }
-
- if (((pci_read_config(dev_hdr->header.bus, dev_hdr->header.device, dev_hdr->header.func, 0) & 0xffff) == 0x8086) &&
- (((pci_read_config(dev_hdr->header.bus, dev_hdr->header.device, dev_hdr->header.func, 0) >> 16) & 0xffff) == 0x1E31) &&
- ((pci_read_config(dev_hdr->header.bus, dev_hdr->header.device, dev_hdr->header.func, 8) & 0xff) == 4))
- {
- kdebug("Is a Panther Point device");
- pci_write_config(dev_hdr->header.bus, dev_hdr->header.device, dev_hdr->header.func, 0xd8, 0xffffffff);
- pci_write_config(dev_hdr->header.bus, dev_hdr->header.device, dev_hdr->header.func, 0xd0, 0xffffffff);
- }
- io_mfence();
-
- FAIL_ON_TO(xhci_hc_stop_legacy(cid), failed);
- io_mfence();
-
- FAIL_ON_TO(xhci_hc_pair_ports(cid), failed);
- io_mfence();
-
-
- xhci_hc[cid].page_size = (xhci_read_op_reg32(cid, XHCI_OPS_PAGESIZE) & 0xffff) << 12;
- io_mfence();
-
- xhci_hc[cid].dcbaap_vaddr = (uint64_t)kzalloc(2048, 0);
- io_mfence();
-
- if (unlikely(!xhci_is_aligned64(xhci_hc[cid].dcbaap_vaddr)))
- {
- kerror("dcbaap isn't 64 byte aligned.");
- goto failed_free_dyn;
- }
-
- xhci_write_op_reg64(cid, XHCI_OPS_DCBAAP, virt_2_phys(xhci_hc[cid].dcbaap_vaddr));
- io_mfence();
-
- uint32_t max_scratchpad_buf = (((uint32_t)hcs2.max_scratchpad_buf_HI5) << 5) | hcs2.max_scratchpad_buf_LO5;
- kdebug("max scratchpad buffer=%d", max_scratchpad_buf);
- if (max_scratchpad_buf > 0)
- {
- xhci_hc[cid].scratchpad_buf_array_vaddr = (uint64_t)kzalloc(sizeof(uint64_t) * max_scratchpad_buf, 0);
- __write8b(xhci_hc[cid].dcbaap_vaddr, virt_2_phys(xhci_hc[cid].scratchpad_buf_array_vaddr));
-
- for (int i = 0; i < max_scratchpad_buf; ++i)
- {
- uint64_t buf_vaddr = (uint64_t)kzalloc(xhci_hc[cid].page_size, 0);
- __write8b(xhci_hc[cid].scratchpad_buf_array_vaddr, virt_2_phys(buf_vaddr));
- }
- }
-
- xhci_hc[cid].cmd_ring_vaddr = xhci_create_ring(XHCI_CMND_RING_TRBS);
- xhci_hc[cid].cmd_trb_vaddr = xhci_hc[cid].cmd_ring_vaddr;
- if (unlikely(!xhci_is_aligned64(xhci_hc[cid].cmd_ring_vaddr)))
- {
- kerror("cmd ring isn't 64 byte aligned.");
- goto failed_free_dyn;
- }
-
- xhci_hc[cid].cmd_trb_cycle = XHCI_TRB_CYCLE_ON;
- io_mfence();
-
- xhci_write_op_reg64(cid, XHCI_OPS_CRCR, virt_2_phys(xhci_hc[cid].cmd_ring_vaddr) | xhci_hc[cid].cmd_trb_cycle);
-
- uint32_t max_slots = hcs1.max_slots;
-
- io_mfence();
- xhci_write_op_reg32(cid, XHCI_OPS_CONFIG, max_slots);
- io_mfence();
-
- xhci_write_op_reg32(cid, XHCI_OPS_DNCTRL, (1 << 1));
- io_mfence();
- FAIL_ON_TO(xhci_hc_init_intr(cid), failed_free_dyn);
- io_mfence();
- ++xhci_ctrl_count;
- io_mfence();
- spin_unlock(&xhci_controller_init_lock);
- io_mfence();
- return;
- failed_free_dyn:;
- if (xhci_hc[cid].dcbaap_vaddr)
- kfree((void *)xhci_hc[cid].dcbaap_vaddr);
- if (xhci_hc[cid].cmd_ring_vaddr)
- kfree((void *)xhci_hc[cid].cmd_ring_vaddr);
- if (xhci_hc[cid].event_ring_table_vaddr)
- kfree((void *)xhci_hc[cid].event_ring_table_vaddr);
- if (xhci_hc[cid].event_ring_vaddr)
- kfree((void *)xhci_hc[cid].event_ring_vaddr);
- failed:;
- io_mfence();
-
- mm_unmap_addr(xhci_hc[cid].vbase, 65536);
- io_mfence();
-
- memset((void *)&xhci_hc[cid], 0, sizeof(struct xhci_host_controller_t));
- failed_exceed_max:;
- kerror("Failed to initialize controller: bus=%d, dev=%d, func=%d", dev_hdr->header.bus, dev_hdr->header.device, dev_hdr->header.func);
- spin_unlock(&xhci_controller_init_lock);
- }
|