diff options
Diffstat (limited to 'drivers/usb/host/xhci-mem.c')
| -rw-r--r-- | drivers/usb/host/xhci-mem.c | 1337 |
1 files changed, 895 insertions, 442 deletions
diff --git a/drivers/usb/host/xhci-mem.c b/drivers/usb/host/xhci-mem.c index 627f3438028..8056d90690e 100644 --- a/drivers/usb/host/xhci-mem.c +++ b/drivers/usb/host/xhci-mem.c @@ -24,8 +24,10 @@ #include <linux/pci.h> #include <linux/slab.h> #include <linux/dmapool.h> +#include <linux/dma-mapping.h> #include "xhci.h" +#include "xhci-trace.h" /* * Allocates a generic ring segment from the ring pool, sets the dma address, @@ -34,25 +36,29 @@ * Section 4.11.1.1: * "All components of all Command and Transfer TRBs shall be initialized to '0'" */ -static struct xhci_segment *xhci_segment_alloc(struct xhci_hcd *xhci, gfp_t flags) +static struct xhci_segment *xhci_segment_alloc(struct xhci_hcd *xhci, + unsigned int cycle_state, gfp_t flags) { struct xhci_segment *seg; dma_addr_t dma; + int i; seg = kzalloc(sizeof *seg, flags); if (!seg) return NULL; - xhci_dbg(xhci, "Allocating priv segment structure at %p\n", seg); seg->trbs = dma_pool_alloc(xhci->segment_pool, flags, &dma); if (!seg->trbs) { kfree(seg); return NULL; } - xhci_dbg(xhci, "// Allocating segment at %p (virtual) 0x%llx (DMA)\n", - seg->trbs, (unsigned long long)dma); - memset(seg->trbs, 0, SEGMENT_SIZE); + memset(seg->trbs, 0, TRB_SEGMENT_SIZE); + /* If the cycle state is 0, set the cycle bit to 1 for all the TRBs */ + if (cycle_state == 0) { + for (i = 0; i < TRBS_PER_SEGMENT; i++) + seg->trbs[i].link.control |= cpu_to_le32(TRB_CYCLE); + } seg->dma = dma; seg->next = NULL; @@ -61,18 +67,27 @@ static struct xhci_segment *xhci_segment_alloc(struct xhci_hcd *xhci, gfp_t flag static void xhci_segment_free(struct xhci_hcd *xhci, struct xhci_segment *seg) { - if (!seg) - return; if (seg->trbs) { - xhci_dbg(xhci, "Freeing DMA segment at %p (virtual) 0x%llx (DMA)\n", - seg->trbs, (unsigned long long)seg->dma); dma_pool_free(xhci->segment_pool, seg->trbs, seg->dma); seg->trbs = NULL; } - xhci_dbg(xhci, "Freeing priv segment structure at %p\n", seg); kfree(seg); } +static void xhci_free_segments_for_ring(struct xhci_hcd *xhci, + struct xhci_segment *first) +{ + struct xhci_segment *seg; + + seg = first->next; + while (seg != first) { + struct xhci_segment *next = seg->next; + xhci_segment_free(xhci, seg); + seg = next; + } + xhci_segment_free(xhci, first); +} + /* * Make the prev segment point to the next segment. * @@ -81,52 +96,199 @@ static void xhci_segment_free(struct xhci_hcd *xhci, struct xhci_segment *seg) * related flags, such as End TRB, Toggle Cycle, and no snoop. */ static void xhci_link_segments(struct xhci_hcd *xhci, struct xhci_segment *prev, - struct xhci_segment *next, bool link_trbs) + struct xhci_segment *next, enum xhci_ring_type type) { u32 val; if (!prev || !next) return; prev->next = next; - if (link_trbs) { - prev->trbs[TRBS_PER_SEGMENT-1].link.segment_ptr = next->dma; + if (type != TYPE_EVENT) { + prev->trbs[TRBS_PER_SEGMENT-1].link.segment_ptr = + cpu_to_le64(next->dma); /* Set the last TRB in the segment to have a TRB type ID of Link TRB */ - val = prev->trbs[TRBS_PER_SEGMENT-1].link.control; + val = le32_to_cpu(prev->trbs[TRBS_PER_SEGMENT-1].link.control); val &= ~TRB_TYPE_BITMASK; val |= TRB_TYPE(TRB_LINK); /* Always set the chain bit with 0.95 hardware */ - if (xhci_link_trb_quirk(xhci)) + /* Set chain bit for isoc rings on AMD 0.96 host */ + if (xhci_link_trb_quirk(xhci) || + (type == TYPE_ISOC && + (xhci->quirks & XHCI_AMD_0x96_HOST))) val |= TRB_CHAIN; - prev->trbs[TRBS_PER_SEGMENT-1].link.control = val; + prev->trbs[TRBS_PER_SEGMENT-1].link.control = cpu_to_le32(val); } - xhci_dbg(xhci, "Linking segment 0x%llx to segment 0x%llx (DMA)\n", - (unsigned long long)prev->dma, - (unsigned long long)next->dma); } -/* XXX: Do we need the hcd structure in all these functions? */ -void xhci_ring_free(struct xhci_hcd *xhci, struct xhci_ring *ring) +/* + * Link the ring to the new segments. + * Set Toggle Cycle for the new ring if needed. + */ +static void xhci_link_rings(struct xhci_hcd *xhci, struct xhci_ring *ring, + struct xhci_segment *first, struct xhci_segment *last, + unsigned int num_segs) +{ + struct xhci_segment *next; + + if (!ring || !first || !last) + return; + + next = ring->enq_seg->next; + xhci_link_segments(xhci, ring->enq_seg, first, ring->type); + xhci_link_segments(xhci, last, next, ring->type); + ring->num_segs += num_segs; + ring->num_trbs_free += (TRBS_PER_SEGMENT - 1) * num_segs; + + if (ring->type != TYPE_EVENT && ring->enq_seg == ring->last_seg) { + ring->last_seg->trbs[TRBS_PER_SEGMENT-1].link.control + &= ~cpu_to_le32(LINK_TOGGLE); + last->trbs[TRBS_PER_SEGMENT-1].link.control + |= cpu_to_le32(LINK_TOGGLE); + ring->last_seg = last; + } +} + +/* + * We need a radix tree for mapping physical addresses of TRBs to which stream + * ID they belong to. We need to do this because the host controller won't tell + * us which stream ring the TRB came from. We could store the stream ID in an + * event data TRB, but that doesn't help us for the cancellation case, since the + * endpoint may stop before it reaches that event data TRB. + * + * The radix tree maps the upper portion of the TRB DMA address to a ring + * segment that has the same upper portion of DMA addresses. For example, say I + * have segments of size 1KB, that are always 1KB aligned. A segment may + * start at 0x10c91000 and end at 0x10c913f0. If I use the upper 10 bits, the + * key to the stream ID is 0x43244. I can use the DMA address of the TRB to + * pass the radix tree a key to get the right stream ID: + * + * 0x10c90fff >> 10 = 0x43243 + * 0x10c912c0 >> 10 = 0x43244 + * 0x10c91400 >> 10 = 0x43245 + * + * Obviously, only those TRBs with DMA addresses that are within the segment + * will make the radix tree return the stream ID for that ring. + * + * Caveats for the radix tree: + * + * The radix tree uses an unsigned long as a key pair. On 32-bit systems, an + * unsigned long will be 32-bits; on a 64-bit system an unsigned long will be + * 64-bits. Since we only request 32-bit DMA addresses, we can use that as the + * key on 32-bit or 64-bit systems (it would also be fine if we asked for 64-bit + * PCI DMA addresses on a 64-bit system). There might be a problem on 32-bit + * extended systems (where the DMA address can be bigger than 32-bits), + * if we allow the PCI dma mask to be bigger than 32-bits. So don't do that. + */ +static int xhci_insert_segment_mapping(struct radix_tree_root *trb_address_map, + struct xhci_ring *ring, + struct xhci_segment *seg, + gfp_t mem_flags) +{ + unsigned long key; + int ret; + + key = (unsigned long)(seg->dma >> TRB_SEGMENT_SHIFT); + /* Skip any segments that were already added. */ + if (radix_tree_lookup(trb_address_map, key)) + return 0; + + ret = radix_tree_maybe_preload(mem_flags); + if (ret) + return ret; + ret = radix_tree_insert(trb_address_map, + key, ring); + radix_tree_preload_end(); + return ret; +} + +static void xhci_remove_segment_mapping(struct radix_tree_root *trb_address_map, + struct xhci_segment *seg) +{ + unsigned long key; + + key = (unsigned long)(seg->dma >> TRB_SEGMENT_SHIFT); + if (radix_tree_lookup(trb_address_map, key)) + radix_tree_delete(trb_address_map, key); +} + +static int xhci_update_stream_segment_mapping( + struct radix_tree_root *trb_address_map, + struct xhci_ring *ring, + struct xhci_segment *first_seg, + struct xhci_segment *last_seg, + gfp_t mem_flags) { struct xhci_segment *seg; - struct xhci_segment *first_seg; + struct xhci_segment *failed_seg; + int ret; - if (!ring || !ring->first_seg) + if (WARN_ON_ONCE(trb_address_map == NULL)) + return 0; + + seg = first_seg; + do { + ret = xhci_insert_segment_mapping(trb_address_map, + ring, seg, mem_flags); + if (ret) + goto remove_streams; + if (seg == last_seg) + return 0; + seg = seg->next; + } while (seg != first_seg); + + return 0; + +remove_streams: + failed_seg = seg; + seg = first_seg; + do { + xhci_remove_segment_mapping(trb_address_map, seg); + if (seg == failed_seg) + return ret; + seg = seg->next; + } while (seg != first_seg); + + return ret; +} + +static void xhci_remove_stream_mapping(struct xhci_ring *ring) +{ + struct xhci_segment *seg; + + if (WARN_ON_ONCE(ring->trb_address_map == NULL)) return; - first_seg = ring->first_seg; - seg = first_seg->next; - xhci_dbg(xhci, "Freeing ring at %p\n", ring); - while (seg != first_seg) { - struct xhci_segment *next = seg->next; - xhci_segment_free(xhci, seg); - seg = next; + + seg = ring->first_seg; + do { + xhci_remove_segment_mapping(ring->trb_address_map, seg); + seg = seg->next; + } while (seg != ring->first_seg); +} + +static int xhci_update_stream_mapping(struct xhci_ring *ring, gfp_t mem_flags) +{ + return xhci_update_stream_segment_mapping(ring->trb_address_map, ring, + ring->first_seg, ring->last_seg, mem_flags); +} + +/* XXX: Do we need the hcd structure in all these functions? */ +void xhci_ring_free(struct xhci_hcd *xhci, struct xhci_ring *ring) +{ + if (!ring) + return; + + if (ring->first_seg) { + if (ring->type == TYPE_STREAM) + xhci_remove_stream_mapping(ring); + xhci_free_segments_for_ring(xhci, ring->first_seg); } - xhci_segment_free(xhci, first_seg); - ring->first_seg = NULL; + kfree(ring); } -static void xhci_initialize_ring_info(struct xhci_ring *ring) +static void xhci_initialize_ring_info(struct xhci_ring *ring, + unsigned int cycle_state) { /* The ring is empty, so the enqueue pointer == dequeue pointer */ ring->enqueue = ring->first_seg->trbs; @@ -136,11 +298,58 @@ static void xhci_initialize_ring_info(struct xhci_ring *ring) /* The ring is initialized to 0. The producer must write 1 to the cycle * bit to handover ownership of the TRB, so PCS = 1. The consumer must * compare CCS to the cycle bit to check ownership, so CCS = 1. + * + * New rings are initialized with cycle state equal to 1; if we are + * handling ring expansion, set the cycle state equal to the old ring. */ - ring->cycle_state = 1; + ring->cycle_state = cycle_state; /* Not necessary for new rings, but needed for re-initialized rings */ ring->enq_updates = 0; ring->deq_updates = 0; + + /* + * Each segment has a link TRB, and leave an extra TRB for SW + * accounting purpose + */ + ring->num_trbs_free = ring->num_segs * (TRBS_PER_SEGMENT - 1) - 1; +} + +/* Allocate segments and link them for a ring */ +static int xhci_alloc_segments_for_ring(struct xhci_hcd *xhci, + struct xhci_segment **first, struct xhci_segment **last, + unsigned int num_segs, unsigned int cycle_state, + enum xhci_ring_type type, gfp_t flags) +{ + struct xhci_segment *prev; + + prev = xhci_segment_alloc(xhci, cycle_state, flags); + if (!prev) + return -ENOMEM; + num_segs--; + + *first = prev; + while (num_segs > 0) { + struct xhci_segment *next; + + next = xhci_segment_alloc(xhci, cycle_state, flags); + if (!next) { + prev = *first; + while (prev) { + next = prev->next; + xhci_segment_free(xhci, prev); + prev = next; + } + return -ENOMEM; + } + xhci_link_segments(xhci, prev, next, type); + + prev = next; + num_segs--; + } + xhci_link_segments(xhci, prev, *first, type); + *last = prev; + + return 0; } /** @@ -151,51 +360,38 @@ static void xhci_initialize_ring_info(struct xhci_ring *ring) * See section 4.9.1 and figures 15 and 16. */ static struct xhci_ring *xhci_ring_alloc(struct xhci_hcd *xhci, - unsigned int num_segs, bool link_trbs, gfp_t flags) + unsigned int num_segs, unsigned int cycle_state, + enum xhci_ring_type type, gfp_t flags) { struct xhci_ring *ring; - struct xhci_segment *prev; + int ret; ring = kzalloc(sizeof *(ring), flags); - xhci_dbg(xhci, "Allocating ring at %p\n", ring); if (!ring) return NULL; + ring->num_segs = num_segs; INIT_LIST_HEAD(&ring->td_list); + ring->type = type; if (num_segs == 0) return ring; - ring->first_seg = xhci_segment_alloc(xhci, flags); - if (!ring->first_seg) + ret = xhci_alloc_segments_for_ring(xhci, &ring->first_seg, + &ring->last_seg, num_segs, cycle_state, type, flags); + if (ret) goto fail; - num_segs--; - - prev = ring->first_seg; - while (num_segs > 0) { - struct xhci_segment *next; - - next = xhci_segment_alloc(xhci, flags); - if (!next) - goto fail; - xhci_link_segments(xhci, prev, next, link_trbs); - - prev = next; - num_segs--; - } - xhci_link_segments(xhci, prev, ring->first_seg, link_trbs); - if (link_trbs) { + /* Only event ring does not use link TRB */ + if (type != TYPE_EVENT) { /* See section 4.9.2.1 and 6.4.4.1 */ - prev->trbs[TRBS_PER_SEGMENT-1].link.control |= (LINK_TOGGLE); - xhci_dbg(xhci, "Wrote link toggle flag to" - " segment %p (virtual), 0x%llx (DMA)\n", - prev, (unsigned long long)prev->dma); + ring->last_seg->trbs[TRBS_PER_SEGMENT - 1].link.control |= + cpu_to_le32(LINK_TOGGLE); } - xhci_initialize_ring_info(ring); + xhci_initialize_ring_info(ring, cycle_state); return ring; fail: - xhci_ring_free(xhci, ring); + kfree(ring); return NULL; } @@ -207,14 +403,13 @@ void xhci_free_or_cache_endpoint_ring(struct xhci_hcd *xhci, rings_cached = virt_dev->num_rings_cached; if (rings_cached < XHCI_MAX_RINGS_CACHED) { - virt_dev->num_rings_cached++; - rings_cached = virt_dev->num_rings_cached; virt_dev->ring_cache[rings_cached] = virt_dev->eps[ep_index].ring; + virt_dev->num_rings_cached++; xhci_dbg(xhci, "Cached old ring, " "%d ring%s cached\n", - rings_cached, - (rings_cached > 1) ? "s" : ""); + virt_dev->num_rings_cached, + (virt_dev->num_rings_cached > 1) ? "s" : ""); } else { xhci_ring_free(xhci, virt_dev->eps[ep_index].ring); xhci_dbg(xhci, "Ring cache full (%d rings), " @@ -228,39 +423,105 @@ void xhci_free_or_cache_endpoint_ring(struct xhci_hcd *xhci, * pointers to the beginning of the ring. */ static void xhci_reinit_cached_ring(struct xhci_hcd *xhci, - struct xhci_ring *ring) + struct xhci_ring *ring, unsigned int cycle_state, + enum xhci_ring_type type) { struct xhci_segment *seg = ring->first_seg; + int i; + do { memset(seg->trbs, 0, sizeof(union xhci_trb)*TRBS_PER_SEGMENT); + if (cycle_state == 0) { + for (i = 0; i < TRBS_PER_SEGMENT; i++) + seg->trbs[i].link.control |= + cpu_to_le32(TRB_CYCLE); + } /* All endpoint rings have link TRBs */ - xhci_link_segments(xhci, seg, seg->next, 1); + xhci_link_segments(xhci, seg, seg->next, type); seg = seg->next; } while (seg != ring->first_seg); - xhci_initialize_ring_info(ring); + ring->type = type; + xhci_initialize_ring_info(ring, cycle_state); /* td list should be empty since all URBs have been cancelled, * but just in case... */ INIT_LIST_HEAD(&ring->td_list); } +/* + * Expand an existing ring. + * Look for a cached ring or allocate a new ring which has same segment numbers + * and link the two rings. + */ +int xhci_ring_expansion(struct xhci_hcd *xhci, struct xhci_ring *ring, + unsigned int num_trbs, gfp_t flags) +{ + struct xhci_segment *first; + struct xhci_segment *last; + unsigned int num_segs; + unsigned int num_segs_needed; + int ret; + + num_segs_needed = (num_trbs + (TRBS_PER_SEGMENT - 1) - 1) / + (TRBS_PER_SEGMENT - 1); + + /* Allocate number of segments we needed, or double the ring size */ + num_segs = ring->num_segs > num_segs_needed ? + ring->num_segs : num_segs_needed; + + ret = xhci_alloc_segments_for_ring(xhci, &first, &last, + num_segs, ring->cycle_state, ring->type, flags); + if (ret) + return -ENOMEM; + + if (ring->type == TYPE_STREAM) + ret = xhci_update_stream_segment_mapping(ring->trb_address_map, + ring, first, last, flags); + if (ret) { + struct xhci_segment *next; + do { + next = first->next; + xhci_segment_free(xhci, first); + if (first == last) + break; + first = next; + } while (true); + return ret; + } + + xhci_link_rings(xhci, ring, first, last, num_segs); + xhci_dbg_trace(xhci, trace_xhci_dbg_ring_expansion, + "ring expansion succeed, now has %d segments", + ring->num_segs); + + return 0; +} + #define CTX_SIZE(_hcc) (HCC_64BYTE_CONTEXT(_hcc) ? 64 : 32) static struct xhci_container_ctx *xhci_alloc_container_ctx(struct xhci_hcd *xhci, int type, gfp_t flags) { - struct xhci_container_ctx *ctx = kzalloc(sizeof(*ctx), flags); + struct xhci_container_ctx *ctx; + + if ((type != XHCI_CTX_TYPE_DEVICE) && (type != XHCI_CTX_TYPE_INPUT)) + return NULL; + + ctx = kzalloc(sizeof(*ctx), flags); if (!ctx) return NULL; - BUG_ON((type != XHCI_CTX_TYPE_DEVICE) && (type != XHCI_CTX_TYPE_INPUT)); ctx->type = type; ctx->size = HCC_64BYTE_CONTEXT(xhci->hcc_params) ? 2048 : 1024; if (type == XHCI_CTX_TYPE_INPUT) ctx->size += CTX_SIZE(xhci->hcc_params); ctx->bytes = dma_pool_alloc(xhci->device_pool, flags, &ctx->dma); + if (!ctx->bytes) { + kfree(ctx); + return NULL; + } memset(ctx->bytes, 0, ctx->size); return ctx; } @@ -277,7 +538,9 @@ static void xhci_free_container_ctx(struct xhci_hcd *xhci, struct xhci_input_control_ctx *xhci_get_input_control_ctx(struct xhci_hcd *xhci, struct xhci_container_ctx *ctx) { - BUG_ON(ctx->type != XHCI_CTX_TYPE_INPUT); + if (ctx->type != XHCI_CTX_TYPE_INPUT) + return NULL; + return (struct xhci_input_control_ctx *)ctx->bytes; } @@ -311,13 +574,13 @@ static void xhci_free_stream_ctx(struct xhci_hcd *xhci, unsigned int num_stream_ctxs, struct xhci_stream_ctx *stream_ctx, dma_addr_t dma) { - struct pci_dev *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller); + struct device *dev = xhci_to_hcd(xhci)->self.controller; + size_t size = sizeof(struct xhci_stream_ctx) * num_stream_ctxs; - if (num_stream_ctxs > MEDIUM_STREAM_ARRAY_SIZE) - pci_free_consistent(pdev, - sizeof(struct xhci_stream_ctx)*num_stream_ctxs, + if (size > MEDIUM_STREAM_ARRAY_SIZE) + dma_free_coherent(dev, size, stream_ctx, dma); - else if (num_stream_ctxs <= SMALL_STREAM_ARRAY_SIZE) + else if (size <= SMALL_STREAM_ARRAY_SIZE) return dma_pool_free(xhci->small_streams_pool, stream_ctx, dma); else @@ -339,13 +602,13 @@ static struct xhci_stream_ctx *xhci_alloc_stream_ctx(struct xhci_hcd *xhci, unsigned int num_stream_ctxs, dma_addr_t *dma, gfp_t mem_flags) { - struct pci_dev *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller); + struct device *dev = xhci_to_hcd(xhci)->self.controller; + size_t size = sizeof(struct xhci_stream_ctx) * num_stream_ctxs; - if (num_stream_ctxs > MEDIUM_STREAM_ARRAY_SIZE) - return pci_alloc_consistent(pdev, - sizeof(struct xhci_stream_ctx)*num_stream_ctxs, - dma); - else if (num_stream_ctxs <= SMALL_STREAM_ARRAY_SIZE) + if (size > MEDIUM_STREAM_ARRAY_SIZE) + return dma_alloc_coherent(dev, size, + dma, mem_flags); + else if (size <= SMALL_STREAM_ARRAY_SIZE) return dma_pool_alloc(xhci->small_streams_pool, mem_flags, dma); else @@ -359,21 +622,10 @@ struct xhci_ring *xhci_dma_to_transfer_ring( { if (ep->ep_state & EP_HAS_STREAMS) return radix_tree_lookup(&ep->stream_info->trb_address_map, - address >> SEGMENT_SHIFT); + address >> TRB_SEGMENT_SHIFT); return ep->ring; } -/* Only use this when you know stream_info is valid */ -#ifdef CONFIG_USB_XHCI_HCD_DEBUGGING -static struct xhci_ring *dma_to_stream_ring( - struct xhci_stream_info *stream_info, - u64 address) -{ - return radix_tree_lookup(&stream_info->trb_address_map, - address >> SEGMENT_SHIFT); -} -#endif /* CONFIG_USB_XHCI_HCD_DEBUGGING */ - struct xhci_ring *xhci_stream_id_to_ring( struct xhci_virt_device *dev, unsigned int ep_index, @@ -391,58 +643,6 @@ struct xhci_ring *xhci_stream_id_to_ring( return ep->stream_info->stream_rings[stream_id]; } -#ifdef CONFIG_USB_XHCI_HCD_DEBUGGING -static int xhci_test_radix_tree(struct xhci_hcd *xhci, - unsigned int num_streams, - struct xhci_stream_info *stream_info) -{ - u32 cur_stream; - struct xhci_ring *cur_ring; - u64 addr; - - for (cur_stream = 1; cur_stream < num_streams; cur_stream++) { - struct xhci_ring *mapped_ring; - int trb_size = sizeof(union xhci_trb); - - cur_ring = stream_info->stream_rings[cur_stream]; - for (addr = cur_ring->first_seg->dma; - addr < cur_ring->first_seg->dma + SEGMENT_SIZE; - addr += trb_size) { - mapped_ring = dma_to_stream_ring(stream_info, addr); - if (cur_ring != mapped_ring) { - xhci_warn(xhci, "WARN: DMA address 0x%08llx " - "didn't map to stream ID %u; " - "mapped to ring %p\n", - (unsigned long long) addr, - cur_stream, - mapped_ring); - return -EINVAL; - } - } - /* One TRB after the end of the ring segment shouldn't return a - * pointer to the current ring (although it may be a part of a - * different ring). - */ - mapped_ring = dma_to_stream_ring(stream_info, addr); - if (mapped_ring != cur_ring) { - /* One TRB before should also fail */ - addr = cur_ring->first_seg->dma - trb_size; - mapped_ring = dma_to_stream_ring(stream_info, addr); - } - if (mapped_ring == cur_ring) { - xhci_warn(xhci, "WARN: Bad DMA address 0x%08llx " - "mapped to valid stream ID %u; " - "mapped ring = %p\n", - (unsigned long long) addr, - cur_stream, - mapped_ring); - return -EINVAL; - } - } - return 0; -} -#endif /* CONFIG_USB_XHCI_HCD_DEBUGGING */ - /* * Change an endpoint's internal structure so it supports stream IDs. The * number of requested streams includes stream 0, which cannot be used by device @@ -451,36 +651,6 @@ static int xhci_test_radix_tree(struct xhci_hcd *xhci, * The number of stream contexts in the stream context array may be bigger than * the number of streams the driver wants to use. This is because the number of * stream context array entries must be a power of two. - * - * We need a radix tree for mapping physical addresses of TRBs to which stream - * ID they belong to. We need to do this because the host controller won't tell - * us which stream ring the TRB came from. We could store the stream ID in an - * event data TRB, but that doesn't help us for the cancellation case, since the - * endpoint may stop before it reaches that event data TRB. - * - * The radix tree maps the upper portion of the TRB DMA address to a ring - * segment that has the same upper portion of DMA addresses. For example, say I - * have segments of size 1KB, that are always 64-byte aligned. A segment may - * start at 0x10c91000 and end at 0x10c913f0. If I use the upper 10 bits, the - * key to the stream ID is 0x43244. I can use the DMA address of the TRB to - * pass the radix tree a key to get the right stream ID: - * - * 0x10c90fff >> 10 = 0x43243 - * 0x10c912c0 >> 10 = 0x43244 - * 0x10c91400 >> 10 = 0x43245 - * - * Obviously, only those TRBs with DMA addresses that are within the segment - * will make the radix tree return the stream ID for that ring. - * - * Caveats for the radix tree: - * - * The radix tree uses an unsigned long as a key pair. On 32-bit systems, an - * unsigned long will be 32-bits; on a 64-bit system an unsigned long will be - * 64-bits. Since we only request 32-bit DMA addresses, we can use that as the - * key on 32-bit or 64-bit systems (it would also be fine if we asked for 64-bit - * PCI DMA addresses on a 64-bit system). There might be a problem on 32-bit - * extended systems (where the DMA address can be bigger than 32-bits), - * if we allow the PCI dma mask to be bigger than 32-bits. So don't do that. */ struct xhci_stream_info *xhci_alloc_stream_info(struct xhci_hcd *xhci, unsigned int num_stream_ctxs, @@ -489,7 +659,6 @@ struct xhci_stream_info *xhci_alloc_stream_info(struct xhci_hcd *xhci, struct xhci_stream_info *stream_info; u32 cur_stream; struct xhci_ring *cur_ring; - unsigned long key; u64 addr; int ret; @@ -539,23 +708,22 @@ struct xhci_stream_info *xhci_alloc_stream_info(struct xhci_hcd *xhci, */ for (cur_stream = 1; cur_stream < num_streams; cur_stream++) { stream_info->stream_rings[cur_stream] = - xhci_ring_alloc(xhci, 1, true, mem_flags); + xhci_ring_alloc(xhci, 2, 1, TYPE_STREAM, mem_flags); cur_ring = stream_info->stream_rings[cur_stream]; if (!cur_ring) goto cleanup_rings; cur_ring->stream_id = cur_stream; + cur_ring->trb_address_map = &stream_info->trb_address_map; /* Set deq ptr, cycle bit, and stream context type */ addr = cur_ring->first_seg->dma | SCT_FOR_CTX(SCT_PRI_TR) | cur_ring->cycle_state; - stream_info->stream_ctx_array[cur_stream].stream_ring = addr; + stream_info->stream_ctx_array[cur_stream].stream_ring = + cpu_to_le64(addr); xhci_dbg(xhci, "Setting stream %d ring ptr to 0x%08llx\n", cur_stream, (unsigned long long) addr); - key = (unsigned long) - (cur_ring->first_seg->dma >> SEGMENT_SHIFT); - ret = radix_tree_insert(&stream_info->trb_address_map, - key, cur_ring); + ret = xhci_update_stream_mapping(cur_ring, mem_flags); if (ret) { xhci_ring_free(xhci, cur_ring); stream_info->stream_rings[cur_stream] = NULL; @@ -568,13 +736,6 @@ struct xhci_stream_info *xhci_alloc_stream_info(struct xhci_hcd *xhci, * was any other way, the host controller would assume the ring is * "empty" and wait forever for data to be queued to that stream ID). */ -#if XHCI_DEBUG - /* Do a little test on the radix tree to make sure it returns the - * correct values. - */ - if (xhci_test_radix_tree(xhci, num_streams, stream_info)) - goto cleanup_rings; -#endif return stream_info; @@ -582,9 +743,6 @@ cleanup_rings: for (cur_stream = 1; cur_stream < num_streams; cur_stream++) { cur_ring = stream_info->stream_rings[cur_stream]; if (cur_ring) { - addr = cur_ring->first_seg->dma; - radix_tree_delete(&stream_info->trb_address_map, - addr >> SEGMENT_SHIFT); xhci_ring_free(xhci, cur_ring); stream_info->stream_rings[cur_stream] = NULL; } @@ -612,12 +770,13 @@ void xhci_setup_streams_ep_input_ctx(struct xhci_hcd *xhci, * fls(0) = 0, fls(0x1) = 1, fls(0x10) = 2, fls(0x100) = 3, etc. */ max_primary_streams = fls(stream_info->num_stream_ctxs) - 2; - xhci_dbg(xhci, "Setting number of stream ctx array entries to %u\n", + xhci_dbg_trace(xhci, trace_xhci_dbg_context_change, + "Setting number of stream ctx array entries to %u", 1 << (max_primary_streams + 1)); - ep_ctx->ep_info &= ~EP_MAXPSTREAMS_MASK; - ep_ctx->ep_info |= EP_MAXPSTREAMS(max_primary_streams); - ep_ctx->ep_info |= EP_HAS_LSA; - ep_ctx->deq = stream_info->ctx_array_dma; + ep_ctx->ep_info &= cpu_to_le32(~EP_MAXPSTREAMS_MASK); + ep_ctx->ep_info |= cpu_to_le32(EP_MAXPSTREAMS(max_primary_streams) + | EP_HAS_LSA); + ep_ctx->deq = cpu_to_le64(stream_info->ctx_array_dma); } /* @@ -630,10 +789,9 @@ void xhci_setup_no_streams_ep_input_ctx(struct xhci_hcd *xhci, struct xhci_virt_ep *ep) { dma_addr_t addr; - ep_ctx->ep_info &= ~EP_MAXPSTREAMS_MASK; - ep_ctx->ep_info &= ~EP_HAS_LSA; + ep_ctx->ep_info &= cpu_to_le32(~(EP_MAXPSTREAMS_MASK | EP_HAS_LSA)); addr = xhci_trb_virt_to_dma(ep->ring->deq_seg, ep->ring->dequeue); - ep_ctx->deq = addr | ep->ring->cycle_state; + ep_ctx->deq = cpu_to_le64(addr | ep->ring->cycle_state); } /* Frees all stream contexts associated with the endpoint, @@ -645,7 +803,6 @@ void xhci_free_stream_info(struct xhci_hcd *xhci, { int cur_stream; struct xhci_ring *cur_ring; - dma_addr_t addr; if (!stream_info) return; @@ -654,9 +811,6 @@ void xhci_free_stream_info(struct xhci_hcd *xhci, cur_stream++) { cur_ring = stream_info->stream_rings[cur_stream]; if (cur_ring) { - addr = cur_ring->first_seg->dma; - radix_tree_delete(&stream_info->trb_address_map, - addr >> SEGMENT_SHIFT); xhci_ring_free(xhci, cur_ring); stream_info->stream_rings[cur_stream] = NULL; } @@ -669,8 +823,7 @@ void xhci_free_stream_info(struct xhci_hcd *xhci, stream_info->stream_ctx_array, stream_info->ctx_array_dma); - if (stream_info) - kfree(stream_info->stream_rings); + kfree(stream_info->stream_rings); kfree(stream_info); } @@ -686,11 +839,84 @@ static void xhci_init_endpoint_timer(struct xhci_hcd *xhci, ep->xhci = xhci; } -/* All the xhci_tds in the ring's TD list should be freed at this point */ +static void xhci_free_tt_info(struct xhci_hcd *xhci, + struct xhci_virt_device *virt_dev, + int slot_id) +{ + struct list_head *tt_list_head; + struct xhci_tt_bw_info *tt_info, *next; + bool slot_found = false; + + /* If the device never made it past the Set Address stage, + * it may not have the real_port set correctly. + */ + if (virt_dev->real_port == 0 || + virt_dev->real_port > HCS_MAX_PORTS(xhci->hcs_params1)) { + xhci_dbg(xhci, "Bad real port.\n"); + return; + } + + tt_list_head = &(xhci->rh_bw[virt_dev->real_port - 1].tts); + list_for_each_entry_safe(tt_info, next, tt_list_head, tt_list) { + /* Multi-TT hubs will have more than one entry */ + if (tt_info->slot_id == slot_id) { + slot_found = true; + list_del(&tt_info->tt_list); + kfree(tt_info); + } else if (slot_found) { + break; + } + } +} + +int xhci_alloc_tt_info(struct xhci_hcd *xhci, + struct xhci_virt_device *virt_dev, + struct usb_device *hdev, + struct usb_tt *tt, gfp_t mem_flags) +{ + struct xhci_tt_bw_info *tt_info; + unsigned int num_ports; + int i, j; + + if (!tt->multi) + num_ports = 1; + else + num_ports = hdev->maxchild; + + for (i = 0; i < num_ports; i++, tt_info++) { + struct xhci_interval_bw_table *bw_table; + + tt_info = kzalloc(sizeof(*tt_info), mem_flags); + if (!tt_info) + goto free_tts; + INIT_LIST_HEAD(&tt_info->tt_list); + list_add(&tt_info->tt_list, + &xhci->rh_bw[virt_dev->real_port - 1].tts); + tt_info->slot_id = virt_dev->udev->slot_id; + if (tt->multi) + tt_info->ttport = i+1; + bw_table = &tt_info->bw_table; + for (j = 0; j < XHCI_MAX_INTERVAL; j++) + INIT_LIST_HEAD(&bw_table->interval_bw[j].endpoints); + } + return 0; + +free_tts: + xhci_free_tt_info(xhci, virt_dev, virt_dev->udev->slot_id); + return -ENOMEM; +} + + +/* All the xhci_tds in the ring's TD list should be freed at this point. + * Should be called with xhci->lock held if there is any chance the TT lists + * will be manipulated by the configure endpoint, allocate device, or update + * hub functions while this function is removing the TT entries from the list. + */ void xhci_free_virt_device(struct xhci_hcd *xhci, int slot_id) { struct xhci_virt_device *dev; int i; + int old_active_eps = 0; /* Slot ID 0 is reserved */ if (slot_id == 0 || !xhci->devs[slot_id]) @@ -701,13 +927,29 @@ void xhci_free_virt_device(struct xhci_hcd *xhci, int slot_id) if (!dev) return; + if (dev->tt_info) + old_active_eps = dev->tt_info->active_eps; + for (i = 0; i < 31; ++i) { if (dev->eps[i].ring) xhci_ring_free(xhci, dev->eps[i].ring); if (dev->eps[i].stream_info) xhci_free_stream_info(xhci, dev->eps[i].stream_info); + /* Endpoints on the TT/root port lists should have been removed + * when usb_disable_device() was called for the device. + * We can't drop them anyway, because the udev might have gone + * away by this point, and we can't tell what speed it was. + */ + if (!list_empty(&dev->eps[i].bw_endpoint_list)) + xhci_warn(xhci, "Slot %u endpoint %u " + "not removed from BW list!\n", + slot_id, i); } + /* If this is a hub, free the TT(s) from the TT list */ + xhci_free_tt_info(xhci, dev, slot_id); + /* If necessary, update the number of active TTs on this root port */ + xhci_update_tt_active_eps(xhci, dev, old_active_eps); if (dev->ring_cache) { for (i = 0; i < dev->num_rings_cached; i++) @@ -761,10 +1003,11 @@ int xhci_alloc_virt_device(struct xhci_hcd *xhci, int slot_id, for (i = 0; i < 31; i++) { xhci_init_endpoint_timer(xhci, &dev->eps[i]); INIT_LIST_HEAD(&dev->eps[i].cancelled_td_list); + INIT_LIST_HEAD(&dev->eps[i].bw_endpoint_list); } /* Allocate endpoint 0 ring */ - dev->eps[0].ring = xhci_ring_alloc(xhci, 1, true, flags); + dev->eps[0].ring = xhci_ring_alloc(xhci, 2, 1, TYPE_CTRL, flags); if (!dev->eps[0].ring) goto fail; @@ -777,15 +1020,14 @@ int xhci_alloc_virt_device(struct xhci_hcd *xhci, int slot_id, dev->num_rings_cached = 0; init_completion(&dev->cmd_completion); - INIT_LIST_HEAD(&dev->cmd_list); dev->udev = udev; /* Point to output device context in dcbaa. */ - xhci->dcbaa->dev_context_ptrs[slot_id] = dev->out_ctx->dma; + xhci->dcbaa->dev_context_ptrs[slot_id] = cpu_to_le64(dev->out_ctx->dma); xhci_dbg(xhci, "Set slot id %d dcbaa entry %p to 0x%llx\n", - slot_id, - &xhci->dcbaa->dev_context_ptrs[slot_id], - (unsigned long long) xhci->dcbaa->dev_context_ptrs[slot_id]); + slot_id, + &xhci->dcbaa->dev_context_ptrs[slot_id], + le64_to_cpu(xhci->dcbaa->dev_context_ptrs[slot_id])); return 1; fail: @@ -810,8 +1052,9 @@ void xhci_copy_ep0_dequeue_into_input_ctx(struct xhci_hcd *xhci, * configured device has reset, so all control transfers should have * been completed or cancelled before the reset. */ - ep0_ctx->deq = xhci_trb_virt_to_dma(ep_ring->enq_seg, ep_ring->enqueue); - ep0_ctx->deq |= ep_ring->cycle_state; + ep0_ctx->deq = cpu_to_le64(xhci_trb_virt_to_dma(ep_ring->enq_seg, + ep_ring->enqueue) + | ep_ring->cycle_state); } /* @@ -823,44 +1066,24 @@ void xhci_copy_ep0_dequeue_into_input_ctx(struct xhci_hcd *xhci, * is attached to (or the roothub port its ancestor hub is attached to). All we * know is the index of that port under either the USB 2.0 or the USB 3.0 * roothub, but that doesn't give us the real index into the HW port status - * registers. Scan through the xHCI roothub port array, looking for the Nth - * entry of the correct port speed. Return the port number of that entry. + * registers. Call xhci_find_raw_port_number() to get real index. */ static u32 xhci_find_real_port_number(struct xhci_hcd *xhci, struct usb_device *udev) { struct usb_device *top_dev; - unsigned int num_similar_speed_ports; - unsigned int faked_port_num; - int i; + struct usb_hcd *hcd; + + if (udev->speed == USB_SPEED_SUPER) + hcd = xhci->shared_hcd; + else + hcd = xhci->main_hcd; for (top_dev = udev; top_dev->parent && top_dev->parent->parent; top_dev = top_dev->parent) /* Found device below root hub */; - faked_port_num = top_dev->portnum; - for (i = 0, num_similar_speed_ports = 0; - i < HCS_MAX_PORTS(xhci->hcs_params1); i++) { - u8 port_speed = xhci->port_array[i]; - - /* - * Skip ports that don't have known speeds, or have duplicate - * Extended Capabilities port speed entries. - */ - if (port_speed == 0 || port_speed == DUPLICATE_ENTRY) - continue; - /* - * USB 3.0 ports are always under a USB 3.0 hub. USB 2.0 and - * 1.1 ports are under the USB 2.0 hub. If the port speed - * matches the device speed, it's a similar speed port. - */ - if ((port_speed == 0x03) == (udev->speed == USB_SPEED_SUPER)) - num_similar_speed_ports++; - if (num_similar_speed_ports == faked_port_num) - /* Roothub ports are numbered from 1 to N */ - return i+1; - } - return 0; + return xhci_find_raw_port_number(hcd, top_dev->portnum); } /* Setup an xHCI virtual device for a Set Address command */ @@ -869,8 +1092,8 @@ int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *ud struct xhci_virt_device *dev; struct xhci_ep_ctx *ep0_ctx; struct xhci_slot_ctx *slot_ctx; - struct xhci_input_control_ctx *ctrl_ctx; u32 port_num; + u32 max_packets; struct usb_device *top_dev; dev = xhci->devs[udev->slot_id]; @@ -881,28 +1104,27 @@ int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *ud return -EINVAL; } ep0_ctx = xhci_get_ep_ctx(xhci, dev->in_ctx, 0); - ctrl_ctx = xhci_get_input_control_ctx(xhci, dev->in_ctx); slot_ctx = xhci_get_slot_ctx(xhci, dev->in_ctx); - /* 2) New slot context and endpoint 0 context are valid*/ - ctrl_ctx->add_flags = SLOT_FLAG | EP0_FLAG; - /* 3) Only the control endpoint is valid - one endpoint context */ - slot_ctx->dev_info |= LAST_CTX(1); - - slot_ctx->dev_info |= (u32) udev->route; + slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1) | udev->route); switch (udev->speed) { case USB_SPEED_SUPER: - slot_ctx->dev_info |= (u32) SLOT_SPEED_SS; + slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_SS); + max_packets = MAX_PACKET(512); break; case USB_SPEED_HIGH: - slot_ctx->dev_info |= (u32) SLOT_SPEED_HS; + slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_HS); + max_packets = MAX_PACKET(64); break; + /* USB core guesses at a 64-byte max packet first for FS devices */ case USB_SPEED_FULL: - slot_ctx->dev_info |= (u32) SLOT_SPEED_FS; + slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_FS); + max_packets = MAX_PACKET(64); break; case USB_SPEED_LOW: - slot_ctx->dev_info |= (u32) SLOT_SPEED_LS; + slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_LS); + max_packets = MAX_PACKET(8); break; case USB_SPEED_WIRELESS: xhci_dbg(xhci, "FIXME xHCI doesn't support wireless speeds\n"); @@ -910,64 +1132,72 @@ int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *ud break; default: /* Speed was set earlier, this shouldn't happen. */ - BUG(); + return -EINVAL; } /* Find the root hub port this device is under */ port_num = xhci_find_real_port_number(xhci, udev); if (!port_num) return -EINVAL; - slot_ctx->dev_info2 |= (u32) ROOT_HUB_PORT(port_num); + slot_ctx->dev_info2 |= cpu_to_le32(ROOT_HUB_PORT(port_num)); /* Set the port number in the virtual_device to the faked port number */ for (top_dev = udev; top_dev->parent && top_dev->parent->parent; top_dev = top_dev->parent) /* Found device below root hub */; - dev->port = top_dev->portnum; + dev->fake_port = top_dev->portnum; + dev->real_port = port_num; xhci_dbg(xhci, "Set root hub portnum to %d\n", port_num); - xhci_dbg(xhci, "Set fake root hub portnum to %d\n", dev->port); + xhci_dbg(xhci, "Set fake root hub portnum to %d\n", dev->fake_port); + + /* Find the right bandwidth table that this device will be a part of. + * If this is a full speed device attached directly to a root port (or a + * decendent of one), it counts as a primary bandwidth domain, not a + * secondary bandwidth domain under a TT. An xhci_tt_info structure + * will never be created for the HS root hub. + */ + if (!udev->tt || !udev->tt->hub->parent) { + dev->bw_table = &xhci->rh_bw[port_num - 1].bw_table; + } else { + struct xhci_root_port_bw_info *rh_bw; + struct xhci_tt_bw_info *tt_bw; + + rh_bw = &xhci->rh_bw[port_num - 1]; + /* Find the right TT. */ + list_for_each_entry(tt_bw, &rh_bw->tts, tt_list) { + if (tt_bw->slot_id != udev->tt->hub->slot_id) + continue; + + if (!dev->udev->tt->multi || + (udev->tt->multi && + tt_bw->ttport == dev->udev->ttport)) { + dev->bw_table = &tt_bw->bw_table; + dev->tt_info = tt_bw; + break; + } + } + if (!dev->tt_info) + xhci_warn(xhci, "WARN: Didn't find a matching TT\n"); + } /* Is this a LS/FS device under an external HS hub? */ if (udev->tt && udev->tt->hub->parent) { - slot_ctx->tt_info = udev->tt->hub->slot_id; - slot_ctx->tt_info |= udev->ttport << 8; + slot_ctx->tt_info = cpu_to_le32(udev->tt->hub->slot_id | + (udev->ttport << 8)); if (udev->tt->multi) - slot_ctx->dev_info |= DEV_MTT; + slot_ctx->dev_info |= cpu_to_le32(DEV_MTT); } xhci_dbg(xhci, "udev->tt = %p\n", udev->tt); xhci_dbg(xhci, "udev->ttport = 0x%x\n", udev->ttport); /* Step 4 - ring already allocated */ /* Step 5 */ - ep0_ctx->ep_info2 = EP_TYPE(CTRL_EP); - /* - * XXX: Not sure about wireless USB devices. - */ - switch (udev->speed) { - case USB_SPEED_SUPER: - ep0_ctx->ep_info2 |= MAX_PACKET(512); - break; - case USB_SPEED_HIGH: - /* USB core guesses at a 64-byte max packet first for FS devices */ - case USB_SPEED_FULL: - ep0_ctx->ep_info2 |= MAX_PACKET(64); - break; - case USB_SPEED_LOW: - ep0_ctx->ep_info2 |= MAX_PACKET(8); - break; - case USB_SPEED_WIRELESS: - xhci_dbg(xhci, "FIXME xHCI doesn't support wireless speeds\n"); - return -EINVAL; - break; - default: - /* New speed? */ - BUG(); - } + ep0_ctx->ep_info2 = cpu_to_le32(EP_TYPE(CTRL_EP)); + /* EP 0 can handle "burst" sizes of 1, so Max Burst Size field is 0 */ - ep0_ctx->ep_info2 |= MAX_BURST(0); - ep0_ctx->ep_info2 |= ERROR_COUNT(3); + ep0_ctx->ep_info2 |= cpu_to_le32(MAX_BURST(0) | ERROR_COUNT(3) | + max_packets); - ep0_ctx->deq = - dev->eps[0].ring->first_seg->dma; - ep0_ctx->deq |= dev->eps[0].ring->cycle_state; + ep0_ctx->deq = cpu_to_le64(dev->eps[0].ring->first_seg->dma | + dev->eps[0].ring->cycle_state); /* Steps 7 and 8 were done in xhci_alloc_virt_device() */ @@ -987,34 +1217,62 @@ static unsigned int xhci_parse_exponent_interval(struct usb_device *udev, interval = clamp_val(ep->desc.bInterval, 1, 16) - 1; if (interval != ep->desc.bInterval - 1) dev_warn(&udev->dev, - "ep %#x - rounding interval to %d microframes\n", + "ep %#x - rounding interval to %d %sframes\n", ep->desc.bEndpointAddress, - 1 << interval); + 1 << interval, + udev->speed == USB_SPEED_FULL ? "" : "micro"); + + if (udev->speed == USB_SPEED_FULL) { + /* + * Full speed isoc endpoints specify interval in frames, + * not microframes. We are using microframes everywhere, + * so adjust accordingly. + */ + interval += 3; /* 1 frame = 2^3 uframes */ + } return interval; } /* - * Convert bInterval expressed in frames (in 1-255 range) to exponent of + * Convert bInterval expressed in microframes (in 1-255 range) to exponent of * microframes, rounded down to nearest power of 2. */ -static unsigned int xhci_parse_frame_interval(struct usb_device *udev, - struct usb_host_endpoint *ep) +static unsigned int xhci_microframes_to_exponent(struct usb_device *udev, + struct usb_host_endpoint *ep, unsigned int desc_interval, + unsigned int min_exponent, unsigned int max_exponent) { unsigned int interval; - interval = fls(8 * ep->desc.bInterval) - 1; - interval = clamp_val(interval, 3, 10); - if ((1 << interval) != 8 * ep->desc.bInterval) + interval = fls(desc_interval) - 1; + interval = clamp_val(interval, min_exponent, max_exponent); + if ((1 << interval) != desc_interval) dev_warn(&udev->dev, "ep %#x - rounding interval to %d microframes, ep desc says %d microframes\n", ep->desc.bEndpointAddress, 1 << interval, - 8 * ep->desc.bInterval); + desc_interval); return interval; } +static unsigned int xhci_parse_microframe_interval(struct usb_device *udev, + struct usb_host_endpoint *ep) +{ + if (ep->desc.bInterval == 0) + return 0; + return xhci_microframes_to_exponent(udev, ep, + ep->desc.bInterval, 0, 15); +} + + +static unsigned int xhci_parse_frame_interval(struct usb_device *udev, + struct usb_host_endpoint *ep) +{ + return xhci_microframes_to_exponent(udev, ep, + ep->desc.bInterval * 8, 3, 10); +} + /* Return the polling or NAK interval. * * The polling interval is expressed in "microframes". If xHCI's Interval field @@ -1033,7 +1291,7 @@ static unsigned int xhci_get_endpoint_interval(struct usb_device *udev, /* Max NAK rate */ if (usb_endpoint_xfer_control(&ep->desc) || usb_endpoint_xfer_bulk(&ep->desc)) { - interval = ep->desc.bInterval; + interval = xhci_parse_microframe_interval(udev, ep); break; } /* Fall through - SS and HS isoc/int have same decoding */ @@ -1046,12 +1304,12 @@ static unsigned int xhci_get_endpoint_interval(struct usb_device *udev, break; case USB_SPEED_FULL: - if (usb_endpoint_xfer_int(&ep->desc)) { + if (usb_endpoint_xfer_isoc(&ep->desc)) { interval = xhci_parse_exponent_interval(udev, ep); break; } /* - * Fall through for isochronous endpoint interval decoding + * Fall through for interrupt endpoint interval decoding * since it uses the same rules as low speed interrupt * endpoints. */ @@ -1109,7 +1367,7 @@ static u32 xhci_get_endpoint_type(struct usb_device *udev, else type = EP_TYPE(INT_OUT_EP); } else { - BUG(); + type = 0; } return type; } @@ -1131,10 +1389,10 @@ static u32 xhci_get_max_esit_payload(struct xhci_hcd *xhci, return 0; if (udev->speed == USB_SPEED_SUPER) - return ep->ss_ep_comp.wBytesPerInterval; + return le16_to_cpu(ep->ss_ep_comp.wBytesPerInterval); - max_packet = GET_MAX_PACKET(ep->desc.wMaxPacketSize); - max_burst = (ep->desc.wMaxPacketSize & 0x1800) >> 11; + max_packet = GET_MAX_PACKET(usb_endpoint_maxp(&ep->desc)); + max_burst = (usb_endpoint_maxp(&ep->desc) & 0x1800) >> 11; /* A 0 in max burst means 1 transfer per ESIT */ return max_packet * (max_burst + 1); } @@ -1153,24 +1411,22 @@ int xhci_endpoint_init(struct xhci_hcd *xhci, struct xhci_ring *ep_ring; unsigned int max_packet; unsigned int max_burst; + enum xhci_ring_type type; u32 max_esit_payload; + u32 endpoint_type; ep_index = xhci_get_endpoint_index(&ep->desc); ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index); + endpoint_type = xhci_get_endpoint_type(udev, ep); + if (!endpoint_type) + return -EINVAL; + ep_ctx->ep_info2 = cpu_to_le32(endpoint_type); + + type = usb_endpoint_type(&ep->desc); /* Set up the endpoint ring */ - /* - * Isochronous endpoint ring needs bigger size because one isoc URB - * carries multiple packets and it will insert multiple tds to the - * ring. - * This should be replaced with dynamic ring resizing in the future. - */ - if (usb_endpoint_xfer_isoc(&ep->desc)) - virt_dev->eps[ep_index].new_ring = - xhci_ring_alloc(xhci, 8, true, mem_flags); - else - virt_dev->eps[ep_index].new_ring = - xhci_ring_alloc(xhci, 1, true, mem_flags); + virt_dev->eps[ep_index].new_ring = + xhci_ring_alloc(xhci, 2, 1, type, mem_flags); if (!virt_dev->eps[ep_index].new_ring) { /* Attempt to use the ring cache */ if (virt_dev->num_rings_cached == 0) @@ -1179,58 +1435,57 @@ int xhci_endpoint_init(struct xhci_hcd *xhci, virt_dev->ring_cache[virt_dev->num_rings_cached]; virt_dev->ring_cache[virt_dev->num_rings_cached] = NULL; virt_dev->num_rings_cached--; - xhci_reinit_cached_ring(xhci, virt_dev->eps[ep_index].new_ring); + xhci_reinit_cached_ring(xhci, virt_dev->eps[ep_index].new_ring, + 1, type); } virt_dev->eps[ep_index].skip = false; ep_ring = virt_dev->eps[ep_index].new_ring; - ep_ctx->deq = ep_ring->first_seg->dma | ep_ring->cycle_state; + ep_ctx->deq = cpu_to_le64(ep_ring->first_seg->dma | ep_ring->cycle_state); - ep_ctx->ep_info = xhci_get_endpoint_interval(udev, ep); - ep_ctx->ep_info |= EP_MULT(xhci_get_endpoint_mult(udev, ep)); + ep_ctx->ep_info = cpu_to_le32(xhci_get_endpoint_interval(udev, ep) + | EP_MULT(xhci_get_endpoint_mult(udev, ep))); /* FIXME dig Mult and streams info out of ep companion desc */ /* Allow 3 retries for everything but isoc; - * error count = 0 means infinite retries. + * CErr shall be set to 0 for Isoch endpoints. */ if (!usb_endpoint_xfer_isoc(&ep->desc)) - ep_ctx->ep_info2 = ERROR_COUNT(3); + ep_ctx->ep_info2 |= cpu_to_le32(ERROR_COUNT(3)); else - ep_ctx->ep_info2 = ERROR_COUNT(1); - - ep_ctx->ep_info2 |= xhci_get_endpoint_type(udev, ep); + ep_ctx->ep_info2 |= cpu_to_le32(ERROR_COUNT(0)); /* Set the max packet size and max burst */ + max_packet = GET_MAX_PACKET(usb_endpoint_maxp(&ep->desc)); + max_burst = 0; switch (udev->speed) { case USB_SPEED_SUPER: - max_packet = ep->desc.wMaxPacketSize; - ep_ctx->ep_info2 |= MAX_PACKET(max_packet); /* dig out max burst from ep companion desc */ - max_packet = ep->ss_ep_comp.bMaxBurst; - if (!max_packet) - xhci_warn(xhci, "WARN no SS endpoint bMaxBurst\n"); - ep_ctx->ep_info2 |= MAX_BURST(max_packet); + max_burst = ep->ss_ep_comp.bMaxBurst; break; case USB_SPEED_HIGH: + /* Some devices get this wrong */ + if (usb_endpoint_xfer_bulk(&ep->desc)) + max_packet = 512; /* bits 11:12 specify the number of additional transaction * opportunities per microframe (USB 2.0, section 9.6.6) */ if (usb_endpoint_xfer_isoc(&ep->desc) || usb_endpoint_xfer_int(&ep->desc)) { - max_burst = (ep->desc.wMaxPacketSize & 0x1800) >> 11; - ep_ctx->ep_info2 |= MAX_BURST(max_burst); + max_burst = (usb_endpoint_maxp(&ep->desc) + & 0x1800) >> 11; } - /* Fall through */ + break; case USB_SPEED_FULL: case USB_SPEED_LOW: - max_packet = GET_MAX_PACKET(ep->desc.wMaxPacketSize); - ep_ctx->ep_info2 |= MAX_PACKET(max_packet); break; default: BUG(); } + ep_ctx->ep_info2 |= cpu_to_le32(MAX_PACKET(max_packet) | + MAX_BURST(max_burst)); max_esit_payload = xhci_get_max_esit_payload(xhci, udev, ep); - ep_ctx->tx_info = MAX_ESIT_PAYLOAD_FOR_EP(max_esit_payload); + ep_ctx->tx_info = cpu_to_le32(MAX_ESIT_PAYLOAD_FOR_EP(max_esit_payload)); /* * XXX no idea how to calculate the average TRB buffer length for bulk @@ -1246,8 +1501,15 @@ int xhci_endpoint_init(struct xhci_hcd *xhci, * including link TRBs, No-op TRBs, and Event data TRBs. Since we don't * use Event Data TRBs, and we don't chain in a link TRB on short * transfers, we're basically dividing by 1. + * + * xHCI 1.0 specification indicates that the Average TRB Length should + * be set to 8 for control endpoints. */ - ep_ctx->tx_info |= AVG_TRB_LENGTH_FOR_EP(max_esit_payload); + if (usb_endpoint_xfer_control(&ep->desc) && xhci->hci_version == 0x100) + ep_ctx->tx_info |= cpu_to_le32(AVG_TRB_LENGTH_FOR_EP(8)); + else + ep_ctx->tx_info |= + cpu_to_le32(AVG_TRB_LENGTH_FOR_EP(max_esit_payload)); /* FIXME Debug endpoint context */ return 0; @@ -1272,6 +1534,70 @@ void xhci_endpoint_zero(struct xhci_hcd *xhci, */ } +void xhci_clear_endpoint_bw_info(struct xhci_bw_info *bw_info) +{ + bw_info->ep_interval = 0; + bw_info->mult = 0; + bw_info->num_packets = 0; + bw_info->max_packet_size = 0; + bw_info->type = 0; + bw_info->max_esit_payload = 0; +} + +void xhci_update_bw_info(struct xhci_hcd *xhci, + struct xhci_container_ctx *in_ctx, + struct xhci_input_control_ctx *ctrl_ctx, + struct xhci_virt_device *virt_dev) +{ + struct xhci_bw_info *bw_info; + struct xhci_ep_ctx *ep_ctx; + unsigned int ep_type; + int i; + + for (i = 1; i < 31; ++i) { + bw_info = &virt_dev->eps[i].bw_info; + + /* We can't tell what endpoint type is being dropped, but + * unconditionally clearing the bandwidth info for non-periodic + * endpoints should be harmless because the info will never be + * set in the first place. + */ + if (!EP_IS_ADDED(ctrl_ctx, i) && EP_IS_DROPPED(ctrl_ctx, i)) { + /* Dropped endpoint */ + xhci_clear_endpoint_bw_info(bw_info); + continue; + } + + if (EP_IS_ADDED(ctrl_ctx, i)) { + ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, i); + ep_type = CTX_TO_EP_TYPE(le32_to_cpu(ep_ctx->ep_info2)); + + /* Ignore non-periodic endpoints */ + if (ep_type != ISOC_OUT_EP && ep_type != INT_OUT_EP && + ep_type != ISOC_IN_EP && + ep_type != INT_IN_EP) + continue; + + /* Added or changed endpoint */ + bw_info->ep_interval = CTX_TO_EP_INTERVAL( + le32_to_cpu(ep_ctx->ep_info)); + /* Number of packets and mult are zero-based in the + * input context, but we want one-based for the + * interval table. + */ + bw_info->mult = CTX_TO_EP_MULT( + le32_to_cpu(ep_ctx->ep_info)) + 1; + bw_info->num_packets = CTX_TO_MAX_BURST( + le32_to_cpu(ep_ctx->ep_info2)) + 1; + bw_info->max_packet_size = MAX_PACKET_DECODED( + le32_to_cpu(ep_ctx->ep_info2)); + bw_info->type = ep_type; + bw_info->max_esit_payload = CTX_TO_MAX_ESIT_PAYLOAD( + le32_to_cpu(ep_ctx->tx_info)); + } + } +} + /* Copy output xhci_ep_ctx to the input xhci_ep_ctx copy. * Useful when you want to change one particular aspect of the endpoint and then * issue a configure endpoint command. @@ -1321,7 +1647,8 @@ static int scratchpad_alloc(struct xhci_hcd *xhci, gfp_t flags) struct device *dev = xhci_to_hcd(xhci)->self.controller; int num_sp = HCS_MAX_SCRATCHPAD(xhci->hcs_params2); - xhci_dbg(xhci, "Allocating %d scratchpad buffers\n", num_sp); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "Allocating %d scratchpad buffers", num_sp); if (!num_sp) return 0; @@ -1330,10 +1657,9 @@ static int scratchpad_alloc(struct xhci_hcd *xhci, gfp_t flags) if (!xhci->scratchpad) goto fail_sp; - xhci->scratchpad->sp_array = - pci_alloc_consistent(to_pci_dev(dev), + xhci->scratchpad->sp_array = dma_alloc_coherent(dev, num_sp * sizeof(u64), - &xhci->scratchpad->sp_dma); + &xhci->scratchpad->sp_dma, flags); if (!xhci->scratchpad->sp_array) goto fail_sp2; @@ -1347,11 +1673,11 @@ static int scratchpad_alloc(struct xhci_hcd *xhci, gfp_t flags) if (!xhci->scratchpad->sp_dma_buffers) goto fail_sp4; - xhci->dcbaa->dev_context_ptrs[0] = xhci->scratchpad->sp_dma; + xhci->dcbaa->dev_context_ptrs[0] = cpu_to_le64(xhci->scratchpad->sp_dma); for (i = 0; i < num_sp; i++) { dma_addr_t dma; - void *buf = pci_alloc_consistent(to_pci_dev(dev), - xhci->page_size, &dma); + void *buf = dma_alloc_coherent(dev, xhci->page_size, &dma, + flags); if (!buf) goto fail_sp5; @@ -1364,7 +1690,7 @@ static int scratchpad_alloc(struct xhci_hcd *xhci, gfp_t flags) fail_sp5: for (i = i - 1; i >= 0; i--) { - pci_free_consistent(to_pci_dev(dev), xhci->page_size, + dma_free_coherent(dev, xhci->page_size, xhci->scratchpad->sp_buffers[i], xhci->scratchpad->sp_dma_buffers[i]); } @@ -1374,7 +1700,7 @@ static int scratchpad_alloc(struct xhci_hcd *xhci, gfp_t flags) kfree(xhci->scratchpad->sp_buffers); fail_sp3: - pci_free_consistent(to_pci_dev(dev), num_sp * sizeof(u64), + dma_free_coherent(dev, num_sp * sizeof(u64), xhci->scratchpad->sp_array, xhci->scratchpad->sp_dma); @@ -1390,7 +1716,7 @@ static void scratchpad_free(struct xhci_hcd *xhci) { int num_sp; int i; - struct pci_dev *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller); + struct device *dev = xhci_to_hcd(xhci)->self.controller; if (!xhci->scratchpad) return; @@ -1398,13 +1724,13 @@ static void scratchpad_free(struct xhci_hcd *xhci) num_sp = HCS_MAX_SCRATCHPAD(xhci->hcs_params2); for (i = 0; i < num_sp; i++) { - pci_free_consistent(pdev, xhci->page_size, + dma_free_coherent(dev, xhci->page_size, xhci->scratchpad->sp_buffers[i], xhci->scratchpad->sp_dma_buffers[i]); } kfree(xhci->scratchpad->sp_dma_buffers); kfree(xhci->scratchpad->sp_buffers); - pci_free_consistent(pdev, num_sp * sizeof(u64), + dma_free_coherent(dev, num_sp * sizeof(u64), xhci->scratchpad->sp_array, xhci->scratchpad->sp_dma); kfree(xhci->scratchpad); @@ -1449,18 +1775,10 @@ struct xhci_command *xhci_alloc_command(struct xhci_hcd *xhci, void xhci_urb_free_priv(struct xhci_hcd *xhci, struct urb_priv *urb_priv) { - int last; - - if (!urb_priv) - return; - - last = urb_priv->length - 1; - if (last >= 0) { - int i; - for (i = 0; i <= last; i++) - kfree(urb_priv->td[i]); + if (urb_priv) { + kfree(urb_priv->td[0]); + kfree(urb_priv); } - kfree(urb_priv); } void xhci_free_command(struct xhci_hcd *xhci, @@ -1474,32 +1792,41 @@ void xhci_free_command(struct xhci_hcd *xhci, void xhci_mem_cleanup(struct xhci_hcd *xhci) { - struct pci_dev *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller); + struct device *dev = xhci_to_hcd(xhci)->self.controller; int size; - int i; + int i, j, num_ports; + + del_timer_sync(&xhci->cmd_timer); /* Free the Event Ring Segment Table and the actual Event Ring */ - if (xhci->ir_set) { - xhci_writel(xhci, 0, &xhci->ir_set->erst_size); - xhci_write_64(xhci, 0, &xhci->ir_set->erst_base); - xhci_write_64(xhci, 0, &xhci->ir_set->erst_dequeue); - } size = sizeof(struct xhci_erst_entry)*(xhci->erst.num_entries); if (xhci->erst.entries) - pci_free_consistent(pdev, size, + dma_free_coherent(dev, size, xhci->erst.entries, xhci->erst.erst_dma_addr); xhci->erst.entries = NULL; - xhci_dbg(xhci, "Freed ERST\n"); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed ERST"); if (xhci->event_ring) xhci_ring_free(xhci, xhci->event_ring); xhci->event_ring = NULL; - xhci_dbg(xhci, "Freed event ring\n"); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed event ring"); - xhci_write_64(xhci, 0, &xhci->op_regs->cmd_ring); + if (xhci->lpm_command) + xhci_free_command(xhci, xhci->lpm_command); if (xhci->cmd_ring) xhci_ring_free(xhci, xhci->cmd_ring); xhci->cmd_ring = NULL; - xhci_dbg(xhci, "Freed command ring\n"); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed command ring"); + xhci_cleanup_command_queue(xhci); + + num_ports = HCS_MAX_PORTS(xhci->hcs_params1); + for (i = 0; i < num_ports; i++) { + struct xhci_interval_bw_table *bwt = &xhci->rh_bw[i].bw_table; + for (j = 0; j < XHCI_MAX_INTERVAL; j++) { + struct list_head *ep = &bwt->interval_bw[j].endpoints; + while (!list_empty(ep)) + list_del_init(ep->next); + } + } for (i = 1; i < MAX_HC_SLOTS; ++i) xhci_free_virt_device(xhci, i); @@ -1507,36 +1834,53 @@ void xhci_mem_cleanup(struct xhci_hcd *xhci) if (xhci->segment_pool) dma_pool_destroy(xhci->segment_pool); xhci->segment_pool = NULL; - xhci_dbg(xhci, "Freed segment pool\n"); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed segment pool"); if (xhci->device_pool) dma_pool_destroy(xhci->device_pool); xhci->device_pool = NULL; - xhci_dbg(xhci, "Freed device context pool\n"); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed device context pool"); if (xhci->small_streams_pool) dma_pool_destroy(xhci->small_streams_pool); xhci->small_streams_pool = NULL; - xhci_dbg(xhci, "Freed small stream array pool\n"); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "Freed small stream array pool"); if (xhci->medium_streams_pool) dma_pool_destroy(xhci->medium_streams_pool); xhci->medium_streams_pool = NULL; - xhci_dbg(xhci, "Freed medium stream array pool\n"); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "Freed medium stream array pool"); - xhci_write_64(xhci, 0, &xhci->op_regs->dcbaa_ptr); if (xhci->dcbaa) - pci_free_consistent(pdev, sizeof(*xhci->dcbaa), + dma_free_coherent(dev, sizeof(*xhci->dcbaa), xhci->dcbaa, xhci->dcbaa->dma); xhci->dcbaa = NULL; scratchpad_free(xhci); + if (!xhci->rh_bw) + goto no_bw; + + for (i = 0; i < num_ports; i++) { + struct xhci_tt_bw_info *tt, *n; + list_for_each_entry_safe(tt, n, &xhci->rh_bw[i].tts, tt_list) { + list_del(&tt->tt_list); + kfree(tt); + } + } + +no_bw: + xhci->cmd_ring_reserved_trbs = 0; xhci->num_usb2_ports = 0; xhci->num_usb3_ports = 0; + xhci->num_active_eps = 0; kfree(xhci->usb2_ports); kfree(xhci->usb3_ports); kfree(xhci->port_array); + kfree(xhci->rh_bw); + kfree(xhci->ext_caps); xhci->page_size = 0; xhci->page_shift = 0; @@ -1717,14 +2061,15 @@ static void xhci_set_hc_event_deq(struct xhci_hcd *xhci) * there might be more events to service. */ temp &= ~ERST_EHB; - xhci_dbg(xhci, "// Write event ring dequeue pointer, " - "preserving EHB bit\n"); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "// Write event ring dequeue pointer, " + "preserving EHB bit"); xhci_write_64(xhci, ((u64) deq & (u64) ~ERST_PTR_MASK) | temp, &xhci->ir_set->erst_dequeue); } static void xhci_add_in_port(struct xhci_hcd *xhci, unsigned int num_ports, - u32 __iomem *addr, u8 major_revision) + __le32 __iomem *addr, u8 major_revision, int max_caps) { u32 temp, port_offset, port_count; int i; @@ -1738,16 +2083,41 @@ static void xhci_add_in_port(struct xhci_hcd *xhci, unsigned int num_ports, } /* Port offset and count in the third dword, see section 7.2 */ - temp = xhci_readl(xhci, addr + 2); + temp = readl(addr + 2); port_offset = XHCI_EXT_PORT_OFF(temp); port_count = XHCI_EXT_PORT_COUNT(temp); - xhci_dbg(xhci, "Ext Cap %p, port offset = %u, " - "count = %u, revision = 0x%x\n", + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "Ext Cap %p, port offset = %u, " + "count = %u, revision = 0x%x", addr, port_offset, port_count, major_revision); /* Port count includes the current port offset */ if (port_offset == 0 || (port_offset + port_count - 1) > num_ports) /* WTF? "Valid values are ‘1’ to MaxPorts" */ return; + + /* cache usb2 port capabilities */ + if (major_revision < 0x03 && xhci->num_ext_caps < max_caps) + xhci->ext_caps[xhci->num_ext_caps++] = temp; + + /* Check the host's USB2 LPM capability */ + if ((xhci->hci_version == 0x96) && (major_revision != 0x03) && + (temp & XHCI_L1C)) { + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "xHCI 0.96: support USB2 software lpm"); + xhci->sw_lpm_support = 1; + } + + if ((xhci->hci_version >= 0x100) && (major_revision != 0x03)) { + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "xHCI 1.0: support USB2 software lpm"); + xhci->sw_lpm_support = 1; + if (temp & XHCI_HLC) { + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "xHCI 1.0: support USB2 hardware lpm"); + xhci->hw_lpm_support = 1; + } + } + port_offset--; for (i = port_offset; i < (port_offset + port_count); i++) { /* Duplicate entry. Ignore the port if the revisions differ. */ @@ -1789,13 +2159,14 @@ static void xhci_add_in_port(struct xhci_hcd *xhci, unsigned int num_ports, */ static int xhci_setup_port_arrays(struct xhci_hcd *xhci, gfp_t flags) { - u32 __iomem *addr; - u32 offset; + __le32 __iomem *addr, *tmp_addr; + u32 offset, tmp_offset; unsigned int num_ports; - int i, port_index; + int i, j, port_index; + int cap_count = 0; addr = &xhci->cap_regs->hcc_params; - offset = XHCI_HCC_EXT_CAPS(xhci_readl(xhci, addr)); + offset = XHCI_HCC_EXT_CAPS(readl(addr)); if (offset == 0) { xhci_err(xhci, "No Extended Capability registers, " "unable to set up roothub.\n"); @@ -1807,19 +2178,50 @@ static int xhci_setup_port_arrays(struct xhci_hcd *xhci, gfp_t flags) if (!xhci->port_array) return -ENOMEM; + xhci->rh_bw = kzalloc(sizeof(*xhci->rh_bw)*num_ports, flags); + if (!xhci->rh_bw) + return -ENOMEM; + for (i = 0; i < num_ports; i++) { + struct xhci_interval_bw_table *bw_table; + + INIT_LIST_HEAD(&xhci->rh_bw[i].tts); + bw_table = &xhci->rh_bw[i].bw_table; + for (j = 0; j < XHCI_MAX_INTERVAL; j++) + INIT_LIST_HEAD(&bw_table->interval_bw[j].endpoints); + } + /* * For whatever reason, the first capability offset is from the * capability register base, not from the HCCPARAMS register. * See section 5.3.6 for offset calculation. */ addr = &xhci->cap_regs->hc_capbase + offset; + + tmp_addr = addr; + tmp_offset = offset; + + /* count extended protocol capability entries for later caching */ + do { + u32 cap_id; + cap_id = readl(tmp_addr); + if (XHCI_EXT_CAPS_ID(cap_id) == XHCI_EXT_CAPS_PROTOCOL) + cap_count++; + tmp_offset = XHCI_EXT_CAPS_NEXT(cap_id); + tmp_addr += tmp_offset; + } while (tmp_offset); + + xhci->ext_caps = kzalloc(sizeof(*xhci->ext_caps) * cap_count, flags); + if (!xhci->ext_caps) + return -ENOMEM; + while (1) { u32 cap_id; - cap_id = xhci_readl(xhci, addr); + cap_id = readl(addr); if (XHCI_EXT_CAPS_ID(cap_id) == XHCI_EXT_CAPS_PROTOCOL) xhci_add_in_port(xhci, num_ports, addr, - (u8) XHCI_EXT_PORT_MAJOR(cap_id)); + (u8) XHCI_EXT_PORT_MAJOR(cap_id), + cap_count); offset = XHCI_EXT_CAPS_NEXT(cap_id); if (!offset || (xhci->num_usb2_ports + xhci->num_usb3_ports) == num_ports) @@ -1835,18 +2237,21 @@ static int xhci_setup_port_arrays(struct xhci_hcd *xhci, gfp_t flags) xhci_warn(xhci, "No ports on the roothubs?\n"); return -ENODEV; } - xhci_dbg(xhci, "Found %u USB 2.0 ports and %u USB 3.0 ports.\n", + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "Found %u USB 2.0 ports and %u USB 3.0 ports.", xhci->num_usb2_ports, xhci->num_usb3_ports); /* Place limits on the number of roothub ports so that the hub * descriptors aren't longer than the USB core will allocate. */ if (xhci->num_usb3_ports > 15) { - xhci_dbg(xhci, "Limiting USB 3.0 roothub ports to 15.\n"); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "Limiting USB 3.0 roothub ports to 15."); xhci->num_usb3_ports = 15; } if (xhci->num_usb2_ports > USB_MAXCHILDREN) { - xhci_dbg(xhci, "Limiting USB 2.0 roothub ports to %u.\n", + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "Limiting USB 2.0 roothub ports to %u.", USB_MAXCHILDREN); xhci->num_usb2_ports = USB_MAXCHILDREN; } @@ -1871,8 +2276,9 @@ static int xhci_setup_port_arrays(struct xhci_hcd *xhci, gfp_t flags) xhci->usb2_ports[port_index] = &xhci->op_regs->port_status_base + NUM_PORT_REGS*i; - xhci_dbg(xhci, "USB 2.0 port at index %u, " - "addr = %p\n", i, + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "USB 2.0 port at index %u, " + "addr = %p", i, xhci->usb2_ports[port_index]); port_index++; if (port_index == xhci->num_usb2_ports) @@ -1891,8 +2297,9 @@ static int xhci_setup_port_arrays(struct xhci_hcd *xhci, gfp_t flags) xhci->usb3_ports[port_index] = &xhci->op_regs->port_status_base + NUM_PORT_REGS*i; - xhci_dbg(xhci, "USB 3.0 port at index %u, " - "addr = %p\n", i, + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "USB 3.0 port at index %u, " + "addr = %p", i, xhci->usb3_ports[port_index]); port_index++; if (port_index == xhci->num_usb3_ports) @@ -1909,60 +2316,67 @@ int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags) unsigned int val, val2; u64 val_64; struct xhci_segment *seg; - u32 page_size; + u32 page_size, temp; int i; - page_size = xhci_readl(xhci, &xhci->op_regs->page_size); - xhci_dbg(xhci, "Supported page size register = 0x%x\n", page_size); + INIT_LIST_HEAD(&xhci->cmd_list); + + page_size = readl(&xhci->op_regs->page_size); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "Supported page size register = 0x%x", page_size); for (i = 0; i < 16; i++) { if ((0x1 & page_size) != 0) break; page_size = page_size >> 1; } if (i < 16) - xhci_dbg(xhci, "Supported page size of %iK\n", (1 << (i+12)) / 1024); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "Supported page size of %iK", (1 << (i+12)) / 1024); else xhci_warn(xhci, "WARN: no supported page size\n"); /* Use 4K pages, since that's common and the minimum the HC supports */ xhci->page_shift = 12; xhci->page_size = 1 << xhci->page_shift; - xhci_dbg(xhci, "HCD page size set to %iK\n", xhci->page_size / 1024); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "HCD page size set to %iK", xhci->page_size / 1024); /* * Program the Number of Device Slots Enabled field in the CONFIG * register with the max value of slots the HC can handle. */ - val = HCS_MAX_SLOTS(xhci_readl(xhci, &xhci->cap_regs->hcs_params1)); - xhci_dbg(xhci, "// xHC can handle at most %d device slots.\n", - (unsigned int) val); - val2 = xhci_readl(xhci, &xhci->op_regs->config_reg); + val = HCS_MAX_SLOTS(readl(&xhci->cap_regs->hcs_params1)); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "// xHC can handle at most %d device slots.", val); + val2 = readl(&xhci->op_regs->config_reg); val |= (val2 & ~HCS_SLOTS_MASK); - xhci_dbg(xhci, "// Setting Max device slots reg = 0x%x.\n", - (unsigned int) val); - xhci_writel(xhci, val, &xhci->op_regs->config_reg); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "// Setting Max device slots reg = 0x%x.", val); + writel(val, &xhci->op_regs->config_reg); /* * Section 5.4.8 - doorbell array must be * "physically contiguous and 64-byte (cache line) aligned". */ - xhci->dcbaa = pci_alloc_consistent(to_pci_dev(dev), - sizeof(*xhci->dcbaa), &dma); + xhci->dcbaa = dma_alloc_coherent(dev, sizeof(*xhci->dcbaa), &dma, + GFP_KERNEL); if (!xhci->dcbaa) goto fail; memset(xhci->dcbaa, 0, sizeof *(xhci->dcbaa)); xhci->dcbaa->dma = dma; - xhci_dbg(xhci, "// Device context base array address = 0x%llx (DMA), %p (virt)\n", + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "// Device context base array address = 0x%llx (DMA), %p (virt)", (unsigned long long)xhci->dcbaa->dma, xhci->dcbaa); xhci_write_64(xhci, dma, &xhci->op_regs->dcbaa_ptr); /* * Initialize the ring segment pool. The ring must be a contiguous * structure comprised of TRBs. The TRBs must be 16 byte aligned, - * however, the command ring segment needs 64-byte aligned segments, - * so we pick the greater alignment need. + * however, the command ring segment needs 64-byte aligned segments + * and our use of dma addresses in the trb_address_map radix tree needs + * TRB_SEGMENT_SIZE alignment, so we pick the greater alignment need. */ xhci->segment_pool = dma_pool_create("xHCI ring segments", dev, - SEGMENT_SIZE, 64, xhci->page_size); + TRB_SEGMENT_SIZE, TRB_SEGMENT_SIZE, xhci->page_size); /* See Table 46 and Note on Figure 55 */ xhci->device_pool = dma_pool_create("xHCI input/output contexts", dev, @@ -1980,18 +2394,19 @@ int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags) dma_pool_create("xHCI 1KB stream ctx arrays", dev, MEDIUM_STREAM_ARRAY_SIZE, 16, 0); /* Any stream context array bigger than MEDIUM_STREAM_ARRAY_SIZE - * will be allocated with pci_alloc_consistent() + * will be allocated with dma_alloc_coherent() */ if (!xhci->small_streams_pool || !xhci->medium_streams_pool) goto fail; /* Set up the command ring to have one segments for now. */ - xhci->cmd_ring = xhci_ring_alloc(xhci, 1, true, flags); + xhci->cmd_ring = xhci_ring_alloc(xhci, 1, 1, TYPE_COMMAND, flags); if (!xhci->cmd_ring) goto fail; - xhci_dbg(xhci, "Allocated command ring at %p\n", xhci->cmd_ring); - xhci_dbg(xhci, "First segment DMA is 0x%llx\n", + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "Allocated command ring at %p", xhci->cmd_ring); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, "First segment DMA is 0x%llx", (unsigned long long)xhci->cmd_ring->first_seg->dma); /* Set the address in the Command Ring Control register */ @@ -1999,14 +2414,26 @@ int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags) val_64 = (val_64 & (u64) CMD_RING_RSVD_BITS) | (xhci->cmd_ring->first_seg->dma & (u64) ~CMD_RING_RSVD_BITS) | xhci->cmd_ring->cycle_state; - xhci_dbg(xhci, "// Setting command ring address to 0x%x\n", val); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "// Setting command ring address to 0x%x", val); xhci_write_64(xhci, val_64, &xhci->op_regs->cmd_ring); xhci_dbg_cmd_ptrs(xhci); - val = xhci_readl(xhci, &xhci->cap_regs->db_off); + xhci->lpm_command = xhci_alloc_command(xhci, true, true, flags); + if (!xhci->lpm_command) + goto fail; + + /* Reserve one command ring TRB for disabling LPM. + * Since the USB core grabs the shared usb_bus bandwidth mutex before + * disabling LPM, we only need to reserve one TRB for all devices. + */ + xhci->cmd_ring_reserved_trbs++; + + val = readl(&xhci->cap_regs->db_off); val &= DBOFF_MASK; - xhci_dbg(xhci, "// Doorbell array is located at offset 0x%x" - " from cap regs base addr\n", val); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "// Doorbell array is located at offset 0x%x" + " from cap regs base addr", val); xhci->dba = (void __iomem *) xhci->cap_regs + val; xhci_dbg_regs(xhci); xhci_print_run_regs(xhci); @@ -2017,24 +2444,28 @@ int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags) * Event ring setup: Allocate a normal ring, but also setup * the event ring segment table (ERST). Section 4.9.3. */ - xhci_dbg(xhci, "// Allocating event ring\n"); - xhci->event_ring = xhci_ring_alloc(xhci, ERST_NUM_SEGS, false, flags); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Allocating event ring"); + xhci->event_ring = xhci_ring_alloc(xhci, ERST_NUM_SEGS, 1, TYPE_EVENT, + flags); if (!xhci->event_ring) goto fail; if (xhci_check_trb_in_td_math(xhci, flags) < 0) goto fail; - xhci->erst.entries = pci_alloc_consistent(to_pci_dev(dev), - sizeof(struct xhci_erst_entry)*ERST_NUM_SEGS, &dma); + xhci->erst.entries = dma_alloc_coherent(dev, + sizeof(struct xhci_erst_entry) * ERST_NUM_SEGS, &dma, + GFP_KERNEL); if (!xhci->erst.entries) goto fail; - xhci_dbg(xhci, "// Allocated event ring segment table at 0x%llx\n", + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "// Allocated event ring segment table at 0x%llx", (unsigned long long)dma); memset(xhci->erst.entries, 0, sizeof(struct xhci_erst_entry)*ERST_NUM_SEGS); xhci->erst.num_entries = ERST_NUM_SEGS; xhci->erst.erst_dma_addr = dma; - xhci_dbg(xhci, "Set ERST to 0; private num segs = %i, virt addr = %p, dma addr = 0x%llx\n", + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "Set ERST to 0; private num segs = %i, virt addr = %p, dma addr = 0x%llx", xhci->erst.num_entries, xhci->erst.entries, (unsigned long long)xhci->erst.erst_dma_addr); @@ -2042,23 +2473,26 @@ int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags) /* set ring base address and size for each segment table entry */ for (val = 0, seg = xhci->event_ring->first_seg; val < ERST_NUM_SEGS; val++) { struct xhci_erst_entry *entry = &xhci->erst.entries[val]; - entry->seg_addr = seg->dma; - entry->seg_size = TRBS_PER_SEGMENT; + entry->seg_addr = cpu_to_le64(seg->dma); + entry->seg_size = cpu_to_le32(TRBS_PER_SEGMENT); entry->rsvd = 0; seg = seg->next; } /* set ERST count with the number of entries in the segment table */ - val = xhci_readl(xhci, &xhci->ir_set->erst_size); + val = readl(&xhci->ir_set->erst_size); val &= ERST_SIZE_MASK; val |= ERST_NUM_SEGS; - xhci_dbg(xhci, "// Write ERST size = %i to ir_set 0 (some bits preserved)\n", + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "// Write ERST size = %i to ir_set 0 (some bits preserved)", val); - xhci_writel(xhci, val, &xhci->ir_set->erst_size); + writel(val, &xhci->ir_set->erst_size); - xhci_dbg(xhci, "// Set ERST entries to point to event ring.\n"); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "// Set ERST entries to point to event ring."); /* set the segment table base address */ - xhci_dbg(xhci, "// Set ERST base address for ir_set 0 = 0x%llx\n", + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "// Set ERST base address for ir_set 0 = 0x%llx", (unsigned long long)xhci->erst.erst_dma_addr); val_64 = xhci_read_64(xhci, &xhci->ir_set->erst_base); val_64 &= ERST_PTR_MASK; @@ -2067,9 +2501,15 @@ int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags) /* Set the event ring dequeue address */ xhci_set_hc_event_deq(xhci); - xhci_dbg(xhci, "Wrote ERST address to ir_set 0.\n"); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "Wrote ERST address to ir_set 0."); xhci_print_ir_set(xhci, 0); + /* init command timeout timer */ + init_timer(&xhci->cmd_timer); + xhci->cmd_timer.data = (unsigned long) xhci; + xhci->cmd_timer.function = xhci_handle_command_timeout; + /* * XXX: Might need to set the Interrupter Moderation Register to * something other than the default (~1ms minimum between interrupts). @@ -2081,6 +2521,8 @@ int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags) for (i = 0; i < USB_MAXCHILDREN; ++i) { xhci->bus_state[0].resume_done[i] = 0; xhci->bus_state[1].resume_done[i] = 0; + /* Only the USB 2.0 completions will ever be used. */ + init_completion(&xhci->bus_state[1].rexit_done[i]); } if (scratchpad_alloc(xhci, flags)) @@ -2088,10 +2530,21 @@ int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags) if (xhci_setup_port_arrays(xhci, flags)) goto fail; + /* Enable USB 3.0 device notifications for function remote wake, which + * is necessary for allowing USB 3.0 devices to do remote wakeup from + * U3 (device suspend). + */ + temp = readl(&xhci->op_regs->dev_notification); + temp &= ~DEV_NOTE_MASK; + temp |= DEV_NOTE_FWAKE; + writel(temp, &xhci->op_regs->dev_notification); + return 0; fail: xhci_warn(xhci, "Couldn't initialize memory\n"); + xhci_halt(xhci); + xhci_reset(xhci); xhci_mem_cleanup(xhci); return -ENOMEM; } |