dma.h 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443
  1. /*
  2. * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
  3. *
  4. * This program is free software; you can redistribute it and/or modify it
  5. * under the terms of the GNU General Public License as published by the Free
  6. * Software Foundation; either version 2 of the License, or (at your option)
  7. * any later version.
  8. *
  9. * This program is distributed in the hope that it will be useful, but WITHOUT
  10. * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11. * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  12. * more details.
  13. *
  14. * The full GNU General Public License is included in this distribution in the
  15. * file called COPYING.
  16. */
  17. #ifndef IOATDMA_H
  18. #define IOATDMA_H
  19. #include <linux/dmaengine.h>
  20. #include <linux/init.h>
  21. #include <linux/dmapool.h>
  22. #include <linux/cache.h>
  23. #include <linux/pci_ids.h>
  24. #include <linux/circ_buf.h>
  25. #include <linux/interrupt.h>
  26. #include "registers.h"
  27. #include "hw.h"
  28. #define IOAT_DMA_VERSION "4.00"
  29. #define IOAT_DMA_DCA_ANY_CPU ~0
  30. #define to_ioatdma_device(dev) container_of(dev, struct ioatdma_device, dma_dev)
  31. #define to_dev(ioat_chan) (&(ioat_chan)->ioat_dma->pdev->dev)
  32. #define to_pdev(ioat_chan) ((ioat_chan)->ioat_dma->pdev)
  33. #define chan_num(ch) ((int)((ch)->reg_base - (ch)->ioat_dma->reg_base) / 0x80)
  34. /* ioat hardware assumes at least two sources for raid operations */
  35. #define src_cnt_to_sw(x) ((x) + 2)
  36. #define src_cnt_to_hw(x) ((x) - 2)
  37. #define ndest_to_sw(x) ((x) + 1)
  38. #define ndest_to_hw(x) ((x) - 1)
  39. #define src16_cnt_to_sw(x) ((x) + 9)
  40. #define src16_cnt_to_hw(x) ((x) - 9)
  41. /*
  42. * workaround for IOAT ver.3.0 null descriptor issue
  43. * (channel returns error when size is 0)
  44. */
  45. #define NULL_DESC_BUFFER_SIZE 1
  46. enum ioat_irq_mode {
  47. IOAT_NOIRQ = 0,
  48. IOAT_MSIX,
  49. IOAT_MSI,
  50. IOAT_INTX
  51. };
  52. /**
  53. * struct ioatdma_device - internal representation of a IOAT device
  54. * @pdev: PCI-Express device
  55. * @reg_base: MMIO register space base address
  56. * @dma_pool: for allocating DMA descriptors
  57. * @completion_pool: DMA buffers for completion ops
  58. * @sed_hw_pool: DMA super descriptor pools
  59. * @dma_dev: embedded struct dma_device
  60. * @version: version of ioatdma device
  61. * @msix_entries: irq handlers
  62. * @idx: per channel data
  63. * @dca: direct cache access context
  64. * @irq_mode: interrupt mode (INTX, MSI, MSIX)
  65. * @cap: read DMA capabilities register
  66. */
  67. struct ioatdma_device {
  68. struct pci_dev *pdev;
  69. void __iomem *reg_base;
  70. struct pci_pool *dma_pool;
  71. struct pci_pool *completion_pool;
  72. #define MAX_SED_POOLS 5
  73. struct dma_pool *sed_hw_pool[MAX_SED_POOLS];
  74. struct dma_device dma_dev;
  75. u8 version;
  76. #define IOAT_MAX_CHANS 4
  77. struct msix_entry msix_entries[IOAT_MAX_CHANS];
  78. struct ioatdma_chan *idx[IOAT_MAX_CHANS];
  79. struct dca_provider *dca;
  80. enum ioat_irq_mode irq_mode;
  81. u32 cap;
  82. };
  83. struct ioatdma_chan {
  84. struct dma_chan dma_chan;
  85. void __iomem *reg_base;
  86. dma_addr_t last_completion;
  87. spinlock_t cleanup_lock;
  88. unsigned long state;
  89. #define IOAT_CHAN_DOWN 0
  90. #define IOAT_COMPLETION_ACK 1
  91. #define IOAT_RESET_PENDING 2
  92. #define IOAT_KOBJ_INIT_FAIL 3
  93. #define IOAT_RESHAPE_PENDING 4
  94. #define IOAT_RUN 5
  95. #define IOAT_CHAN_ACTIVE 6
  96. struct timer_list timer;
  97. #define COMPLETION_TIMEOUT msecs_to_jiffies(100)
  98. #define IDLE_TIMEOUT msecs_to_jiffies(2000)
  99. #define RESET_DELAY msecs_to_jiffies(100)
  100. struct ioatdma_device *ioat_dma;
  101. dma_addr_t completion_dma;
  102. u64 *completion;
  103. struct tasklet_struct cleanup_task;
  104. struct kobject kobj;
  105. /* ioat v2 / v3 channel attributes
  106. * @xfercap_log; log2 of channel max transfer length (for fast division)
  107. * @head: allocated index
  108. * @issued: hardware notification point
  109. * @tail: cleanup index
  110. * @dmacount: identical to 'head' except for occasionally resetting to zero
  111. * @alloc_order: log2 of the number of allocated descriptors
  112. * @produce: number of descriptors to produce at submit time
  113. * @ring: software ring buffer implementation of hardware ring
  114. * @prep_lock: serializes descriptor preparation (producers)
  115. */
  116. size_t xfercap_log;
  117. u16 head;
  118. u16 issued;
  119. u16 tail;
  120. u16 dmacount;
  121. u16 alloc_order;
  122. u16 produce;
  123. struct ioat_ring_ent **ring;
  124. spinlock_t prep_lock;
  125. };
  126. struct ioat_sysfs_entry {
  127. struct attribute attr;
  128. ssize_t (*show)(struct dma_chan *, char *);
  129. };
  130. /**
  131. * struct ioat_sed_ent - wrapper around super extended hardware descriptor
  132. * @hw: hardware SED
  133. * @dma: dma address for the SED
  134. * @parent: point to the dma descriptor that's the parent
  135. * @hw_pool: descriptor pool index
  136. */
  137. struct ioat_sed_ent {
  138. struct ioat_sed_raw_descriptor *hw;
  139. dma_addr_t dma;
  140. struct ioat_ring_ent *parent;
  141. unsigned int hw_pool;
  142. };
  143. /**
  144. * struct ioat_ring_ent - wrapper around hardware descriptor
  145. * @hw: hardware DMA descriptor (for memcpy)
  146. * @xor: hardware xor descriptor
  147. * @xor_ex: hardware xor extension descriptor
  148. * @pq: hardware pq descriptor
  149. * @pq_ex: hardware pq extension descriptor
  150. * @pqu: hardware pq update descriptor
  151. * @raw: hardware raw (un-typed) descriptor
  152. * @txd: the generic software descriptor for all engines
  153. * @len: total transaction length for unmap
  154. * @result: asynchronous result of validate operations
  155. * @id: identifier for debug
  156. * @sed: pointer to super extended descriptor sw desc
  157. */
  158. struct ioat_ring_ent {
  159. union {
  160. struct ioat_dma_descriptor *hw;
  161. struct ioat_xor_descriptor *xor;
  162. struct ioat_xor_ext_descriptor *xor_ex;
  163. struct ioat_pq_descriptor *pq;
  164. struct ioat_pq_ext_descriptor *pq_ex;
  165. struct ioat_pq_update_descriptor *pqu;
  166. struct ioat_raw_descriptor *raw;
  167. };
  168. size_t len;
  169. struct dma_async_tx_descriptor txd;
  170. enum sum_check_flags *result;
  171. #ifdef DEBUG
  172. int id;
  173. #endif
  174. struct ioat_sed_ent *sed;
  175. };
  176. extern const struct sysfs_ops ioat_sysfs_ops;
  177. extern struct ioat_sysfs_entry ioat_version_attr;
  178. extern struct ioat_sysfs_entry ioat_cap_attr;
  179. extern int ioat_pending_level;
  180. extern int ioat_ring_alloc_order;
  181. extern struct kobj_type ioat_ktype;
  182. extern struct kmem_cache *ioat_cache;
  183. extern int ioat_ring_max_alloc_order;
  184. extern struct kmem_cache *ioat_sed_cache;
  185. static inline struct ioatdma_chan *to_ioat_chan(struct dma_chan *c)
  186. {
  187. return container_of(c, struct ioatdma_chan, dma_chan);
  188. }
  189. /* wrapper around hardware descriptor format + additional software fields */
  190. #ifdef DEBUG
  191. #define set_desc_id(desc, i) ((desc)->id = (i))
  192. #define desc_id(desc) ((desc)->id)
  193. #else
  194. #define set_desc_id(desc, i)
  195. #define desc_id(desc) (0)
  196. #endif
  197. static inline void
  198. __dump_desc_dbg(struct ioatdma_chan *ioat_chan, struct ioat_dma_descriptor *hw,
  199. struct dma_async_tx_descriptor *tx, int id)
  200. {
  201. struct device *dev = to_dev(ioat_chan);
  202. dev_dbg(dev, "desc[%d]: (%#llx->%#llx) cookie: %d flags: %#x"
  203. " ctl: %#10.8x (op: %#x int_en: %d compl: %d)\n", id,
  204. (unsigned long long) tx->phys,
  205. (unsigned long long) hw->next, tx->cookie, tx->flags,
  206. hw->ctl, hw->ctl_f.op, hw->ctl_f.int_en, hw->ctl_f.compl_write);
  207. }
  208. #define dump_desc_dbg(c, d) \
  209. ({ if (d) __dump_desc_dbg(c, d->hw, &d->txd, desc_id(d)); 0; })
  210. static inline struct ioatdma_chan *
  211. ioat_chan_by_index(struct ioatdma_device *ioat_dma, int index)
  212. {
  213. return ioat_dma->idx[index];
  214. }
  215. static inline u64 ioat_chansts_32(struct ioatdma_chan *ioat_chan)
  216. {
  217. u8 ver = ioat_chan->ioat_dma->version;
  218. u64 status;
  219. u32 status_lo;
  220. /* We need to read the low address first as this causes the
  221. * chipset to latch the upper bits for the subsequent read
  222. */
  223. status_lo = readl(ioat_chan->reg_base + IOAT_CHANSTS_OFFSET_LOW(ver));
  224. status = readl(ioat_chan->reg_base + IOAT_CHANSTS_OFFSET_HIGH(ver));
  225. status <<= 32;
  226. status |= status_lo;
  227. return status;
  228. }
  229. #if BITS_PER_LONG == 64
  230. static inline u64 ioat_chansts(struct ioatdma_chan *ioat_chan)
  231. {
  232. u8 ver = ioat_chan->ioat_dma->version;
  233. u64 status;
  234. /* With IOAT v3.3 the status register is 64bit. */
  235. if (ver >= IOAT_VER_3_3)
  236. status = readq(ioat_chan->reg_base + IOAT_CHANSTS_OFFSET(ver));
  237. else
  238. status = ioat_chansts_32(ioat_chan);
  239. return status;
  240. }
  241. #else
  242. #define ioat_chansts ioat_chansts_32
  243. #endif
  244. static inline u64 ioat_chansts_to_addr(u64 status)
  245. {
  246. return status & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
  247. }
  248. static inline u32 ioat_chanerr(struct ioatdma_chan *ioat_chan)
  249. {
  250. return readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
  251. }
  252. static inline void ioat_suspend(struct ioatdma_chan *ioat_chan)
  253. {
  254. u8 ver = ioat_chan->ioat_dma->version;
  255. writeb(IOAT_CHANCMD_SUSPEND,
  256. ioat_chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
  257. }
  258. static inline void ioat_reset(struct ioatdma_chan *ioat_chan)
  259. {
  260. u8 ver = ioat_chan->ioat_dma->version;
  261. writeb(IOAT_CHANCMD_RESET,
  262. ioat_chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
  263. }
  264. static inline bool ioat_reset_pending(struct ioatdma_chan *ioat_chan)
  265. {
  266. u8 ver = ioat_chan->ioat_dma->version;
  267. u8 cmd;
  268. cmd = readb(ioat_chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
  269. return (cmd & IOAT_CHANCMD_RESET) == IOAT_CHANCMD_RESET;
  270. }
  271. static inline bool is_ioat_active(unsigned long status)
  272. {
  273. return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_ACTIVE);
  274. }
  275. static inline bool is_ioat_idle(unsigned long status)
  276. {
  277. return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_DONE);
  278. }
  279. static inline bool is_ioat_halted(unsigned long status)
  280. {
  281. return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_HALTED);
  282. }
  283. static inline bool is_ioat_suspended(unsigned long status)
  284. {
  285. return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_SUSPENDED);
  286. }
  287. /* channel was fatally programmed */
  288. static inline bool is_ioat_bug(unsigned long err)
  289. {
  290. return !!err;
  291. }
  292. #define IOAT_MAX_ORDER 16
  293. #define ioat_get_alloc_order() \
  294. (min(ioat_ring_alloc_order, IOAT_MAX_ORDER))
  295. #define ioat_get_max_alloc_order() \
  296. (min(ioat_ring_max_alloc_order, IOAT_MAX_ORDER))
  297. static inline u32 ioat_ring_size(struct ioatdma_chan *ioat_chan)
  298. {
  299. return 1 << ioat_chan->alloc_order;
  300. }
  301. /* count of descriptors in flight with the engine */
  302. static inline u16 ioat_ring_active(struct ioatdma_chan *ioat_chan)
  303. {
  304. return CIRC_CNT(ioat_chan->head, ioat_chan->tail,
  305. ioat_ring_size(ioat_chan));
  306. }
  307. /* count of descriptors pending submission to hardware */
  308. static inline u16 ioat_ring_pending(struct ioatdma_chan *ioat_chan)
  309. {
  310. return CIRC_CNT(ioat_chan->head, ioat_chan->issued,
  311. ioat_ring_size(ioat_chan));
  312. }
  313. static inline u32 ioat_ring_space(struct ioatdma_chan *ioat_chan)
  314. {
  315. return ioat_ring_size(ioat_chan) - ioat_ring_active(ioat_chan);
  316. }
  317. static inline u16
  318. ioat_xferlen_to_descs(struct ioatdma_chan *ioat_chan, size_t len)
  319. {
  320. u16 num_descs = len >> ioat_chan->xfercap_log;
  321. num_descs += !!(len & ((1 << ioat_chan->xfercap_log) - 1));
  322. return num_descs;
  323. }
  324. static inline struct ioat_ring_ent *
  325. ioat_get_ring_ent(struct ioatdma_chan *ioat_chan, u16 idx)
  326. {
  327. return ioat_chan->ring[idx & (ioat_ring_size(ioat_chan) - 1)];
  328. }
  329. static inline void
  330. ioat_set_chainaddr(struct ioatdma_chan *ioat_chan, u64 addr)
  331. {
  332. writel(addr & 0x00000000FFFFFFFF,
  333. ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
  334. writel(addr >> 32,
  335. ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
  336. }
  337. /* IOAT Prep functions */
  338. struct dma_async_tx_descriptor *
  339. ioat_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
  340. dma_addr_t dma_src, size_t len, unsigned long flags);
  341. struct dma_async_tx_descriptor *
  342. ioat_prep_interrupt_lock(struct dma_chan *c, unsigned long flags);
  343. struct dma_async_tx_descriptor *
  344. ioat_prep_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
  345. unsigned int src_cnt, size_t len, unsigned long flags);
  346. struct dma_async_tx_descriptor *
  347. ioat_prep_xor_val(struct dma_chan *chan, dma_addr_t *src,
  348. unsigned int src_cnt, size_t len,
  349. enum sum_check_flags *result, unsigned long flags);
  350. struct dma_async_tx_descriptor *
  351. ioat_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
  352. unsigned int src_cnt, const unsigned char *scf, size_t len,
  353. unsigned long flags);
  354. struct dma_async_tx_descriptor *
  355. ioat_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
  356. unsigned int src_cnt, const unsigned char *scf, size_t len,
  357. enum sum_check_flags *pqres, unsigned long flags);
  358. struct dma_async_tx_descriptor *
  359. ioat_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src,
  360. unsigned int src_cnt, size_t len, unsigned long flags);
  361. struct dma_async_tx_descriptor *
  362. ioat_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src,
  363. unsigned int src_cnt, size_t len,
  364. enum sum_check_flags *result, unsigned long flags);
  365. /* IOAT Operation functions */
  366. irqreturn_t ioat_dma_do_interrupt(int irq, void *data);
  367. irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data);
  368. struct ioat_ring_ent **
  369. ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags);
  370. void ioat_start_null_desc(struct ioatdma_chan *ioat_chan);
  371. void ioat_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan);
  372. int ioat_reset_hw(struct ioatdma_chan *ioat_chan);
  373. enum dma_status
  374. ioat_tx_status(struct dma_chan *c, dma_cookie_t cookie,
  375. struct dma_tx_state *txstate);
  376. void ioat_cleanup_event(unsigned long data);
  377. void ioat_timer_event(unsigned long data);
  378. int ioat_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs);
  379. void ioat_issue_pending(struct dma_chan *chan);
  380. void ioat_timer_event(unsigned long data);
  381. /* IOAT Init functions */
  382. bool is_bwd_ioat(struct pci_dev *pdev);
  383. struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase);
  384. void ioat_kobject_add(struct ioatdma_device *ioat_dma, struct kobj_type *type);
  385. void ioat_kobject_del(struct ioatdma_device *ioat_dma);
  386. int ioat_dma_setup_interrupts(struct ioatdma_device *ioat_dma);
  387. void ioat_stop(struct ioatdma_chan *ioat_chan);
  388. #endif /* IOATDMA_H */