123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176 |
- /*
- * Copyright 2012 Tilera Corporation. All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation, version 2.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
- * NON INFRINGEMENT. See the GNU General Public License for
- * more details.
- */
- #include <linux/io.h>
- #include <linux/atomic.h>
- #include <linux/module.h>
- #include <gxio/dma_queue.h>
- /* Wait for a memory read to complete. */
- #define wait_for_value(val) \
- __asm__ __volatile__("move %0, %0" :: "r"(val))
- /* The index is in the low 16. */
- #define DMA_QUEUE_INDEX_MASK ((1 << 16) - 1)
- /*
- * The hardware descriptor-ring type.
- * This matches the types used by mpipe (MPIPE_EDMA_POST_REGION_VAL_t)
- * and trio (TRIO_PUSH_DMA_REGION_VAL_t or TRIO_PULL_DMA_REGION_VAL_t).
- * See those types for more documentation on the individual fields.
- */
- typedef union {
- struct {
- #ifndef __BIG_ENDIAN__
- uint64_t ring_idx:16;
- uint64_t count:16;
- uint64_t gen:1;
- uint64_t __reserved:31;
- #else
- uint64_t __reserved:31;
- uint64_t gen:1;
- uint64_t count:16;
- uint64_t ring_idx:16;
- #endif
- };
- uint64_t word;
- } __gxio_ring_t;
- void __gxio_dma_queue_init(__gxio_dma_queue_t *dma_queue,
- void *post_region_addr, unsigned int num_entries)
- {
- /*
- * Limit 65536 entry rings to 65535 credits because we only have a
- * 16 bit completion counter.
- */
- int64_t credits = (num_entries < 65536) ? num_entries : 65535;
- memset(dma_queue, 0, sizeof(*dma_queue));
- dma_queue->post_region_addr = post_region_addr;
- dma_queue->hw_complete_count = 0;
- dma_queue->credits_and_next_index = credits << DMA_QUEUE_CREDIT_SHIFT;
- }
- EXPORT_SYMBOL_GPL(__gxio_dma_queue_init);
- void __gxio_dma_queue_update_credits(__gxio_dma_queue_t *dma_queue)
- {
- __gxio_ring_t val;
- uint64_t count;
- uint64_t delta;
- uint64_t new_count;
- /*
- * Read the 64-bit completion count without touching the cache, so
- * we later avoid having to evict any sharers of this cache line
- * when we update it below.
- */
- uint64_t orig_hw_complete_count =
- cmpxchg(&dma_queue->hw_complete_count,
- -1, -1);
- /* Make sure the load completes before we access the hardware. */
- wait_for_value(orig_hw_complete_count);
- /* Read the 16-bit count of how many packets it has completed. */
- val.word = __gxio_mmio_read(dma_queue->post_region_addr);
- count = val.count;
- /*
- * Calculate the number of completions since we last updated the
- * 64-bit counter. It's safe to ignore the high bits because the
- * maximum credit value is 65535.
- */
- delta = (count - orig_hw_complete_count) & 0xffff;
- if (delta == 0)
- return;
- /*
- * Try to write back the count, advanced by delta. If we race with
- * another thread, this might fail, in which case we return
- * immediately on the assumption that some credits are (or at least
- * were) available.
- */
- new_count = orig_hw_complete_count + delta;
- if (cmpxchg(&dma_queue->hw_complete_count,
- orig_hw_complete_count,
- new_count) != orig_hw_complete_count)
- return;
- /*
- * We succeeded in advancing the completion count; add back the
- * corresponding number of egress credits.
- */
- __insn_fetchadd(&dma_queue->credits_and_next_index,
- (delta << DMA_QUEUE_CREDIT_SHIFT));
- }
- EXPORT_SYMBOL_GPL(__gxio_dma_queue_update_credits);
- /*
- * A separate 'blocked' method for put() so that backtraces and
- * profiles will clearly indicate that we're wasting time spinning on
- * egress availability rather than actually posting commands.
- */
- int64_t __gxio_dma_queue_wait_for_credits(__gxio_dma_queue_t *dma_queue,
- int64_t modifier)
- {
- int backoff = 16;
- int64_t old;
- do {
- int i;
- /* Back off to avoid spamming memory networks. */
- for (i = backoff; i > 0; i--)
- __insn_mfspr(SPR_PASS);
- /* Check credits again. */
- __gxio_dma_queue_update_credits(dma_queue);
- old = __insn_fetchaddgez(&dma_queue->credits_and_next_index,
- modifier);
- /* Calculate bounded exponential backoff for next iteration. */
- if (backoff < 256)
- backoff *= 2;
- } while (old + modifier < 0);
- return old;
- }
- EXPORT_SYMBOL_GPL(__gxio_dma_queue_wait_for_credits);
- int64_t __gxio_dma_queue_reserve_aux(__gxio_dma_queue_t *dma_queue,
- unsigned int num, int wait)
- {
- return __gxio_dma_queue_reserve(dma_queue, num, wait != 0, true);
- }
- EXPORT_SYMBOL_GPL(__gxio_dma_queue_reserve_aux);
- int __gxio_dma_queue_is_complete(__gxio_dma_queue_t *dma_queue,
- int64_t completion_slot, int update)
- {
- if (update) {
- if (ACCESS_ONCE(dma_queue->hw_complete_count) >
- completion_slot)
- return 1;
- __gxio_dma_queue_update_credits(dma_queue);
- }
- return ACCESS_ONCE(dma_queue->hw_complete_count) > completion_slot;
- }
- EXPORT_SYMBOL_GPL(__gxio_dma_queue_is_complete);
|