dma.c 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990
  1. /*
  2. * Intel I/OAT DMA Linux driver
  3. * Copyright(c) 2004 - 2015 Intel Corporation.
  4. *
  5. * This program is free software; you can redistribute it and/or modify it
  6. * under the terms and conditions of the GNU General Public License,
  7. * version 2, as published by the Free Software Foundation.
  8. *
  9. * This program is distributed in the hope that it will be useful, but WITHOUT
  10. * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11. * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  12. * more details.
  13. *
  14. * The full GNU General Public License is included in this distribution in
  15. * the file called "COPYING".
  16. *
  17. */
  18. /*
  19. * This driver supports an Intel I/OAT DMA engine, which does asynchronous
  20. * copy operations.
  21. */
  22. #include <linux/init.h>
  23. #include <linux/module.h>
  24. #include <linux/slab.h>
  25. #include <linux/pci.h>
  26. #include <linux/interrupt.h>
  27. #include <linux/dmaengine.h>
  28. #include <linux/delay.h>
  29. #include <linux/dma-mapping.h>
  30. #include <linux/workqueue.h>
  31. #include <linux/prefetch.h>
  32. #include "dma.h"
  33. #include "registers.h"
  34. #include "hw.h"
  35. #include "../dmaengine.h"
  36. static void ioat_eh(struct ioatdma_chan *ioat_chan);
  37. /**
  38. * ioat_dma_do_interrupt - handler used for single vector interrupt mode
  39. * @irq: interrupt id
  40. * @data: interrupt data
  41. */
  42. irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
  43. {
  44. struct ioatdma_device *instance = data;
  45. struct ioatdma_chan *ioat_chan;
  46. unsigned long attnstatus;
  47. int bit;
  48. u8 intrctrl;
  49. intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);
  50. if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
  51. return IRQ_NONE;
  52. if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
  53. writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
  54. return IRQ_NONE;
  55. }
  56. attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
  57. for_each_set_bit(bit, &attnstatus, BITS_PER_LONG) {
  58. ioat_chan = ioat_chan_by_index(instance, bit);
  59. if (test_bit(IOAT_RUN, &ioat_chan->state))
  60. tasklet_schedule(&ioat_chan->cleanup_task);
  61. }
  62. writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
  63. return IRQ_HANDLED;
  64. }
  65. /**
  66. * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode
  67. * @irq: interrupt id
  68. * @data: interrupt data
  69. */
  70. irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
  71. {
  72. struct ioatdma_chan *ioat_chan = data;
  73. if (test_bit(IOAT_RUN, &ioat_chan->state))
  74. tasklet_schedule(&ioat_chan->cleanup_task);
  75. return IRQ_HANDLED;
  76. }
  77. void ioat_stop(struct ioatdma_chan *ioat_chan)
  78. {
  79. struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
  80. struct pci_dev *pdev = ioat_dma->pdev;
  81. int chan_id = chan_num(ioat_chan);
  82. struct msix_entry *msix;
  83. /* 1/ stop irq from firing tasklets
  84. * 2/ stop the tasklet from re-arming irqs
  85. */
  86. clear_bit(IOAT_RUN, &ioat_chan->state);
  87. /* flush inflight interrupts */
  88. switch (ioat_dma->irq_mode) {
  89. case IOAT_MSIX:
  90. msix = &ioat_dma->msix_entries[chan_id];
  91. synchronize_irq(msix->vector);
  92. break;
  93. case IOAT_MSI:
  94. case IOAT_INTX:
  95. synchronize_irq(pdev->irq);
  96. break;
  97. default:
  98. break;
  99. }
  100. /* flush inflight timers */
  101. del_timer_sync(&ioat_chan->timer);
  102. /* flush inflight tasklet runs */
  103. tasklet_kill(&ioat_chan->cleanup_task);
  104. /* final cleanup now that everything is quiesced and can't re-arm */
  105. ioat_cleanup_event((unsigned long)&ioat_chan->dma_chan);
  106. }
  107. static void __ioat_issue_pending(struct ioatdma_chan *ioat_chan)
  108. {
  109. ioat_chan->dmacount += ioat_ring_pending(ioat_chan);
  110. ioat_chan->issued = ioat_chan->head;
  111. writew(ioat_chan->dmacount,
  112. ioat_chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
  113. dev_dbg(to_dev(ioat_chan),
  114. "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
  115. __func__, ioat_chan->head, ioat_chan->tail,
  116. ioat_chan->issued, ioat_chan->dmacount);
  117. }
  118. void ioat_issue_pending(struct dma_chan *c)
  119. {
  120. struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
  121. if (ioat_ring_pending(ioat_chan)) {
  122. spin_lock_bh(&ioat_chan->prep_lock);
  123. __ioat_issue_pending(ioat_chan);
  124. spin_unlock_bh(&ioat_chan->prep_lock);
  125. }
  126. }
  127. /**
  128. * ioat_update_pending - log pending descriptors
  129. * @ioat: ioat+ channel
  130. *
  131. * Check if the number of unsubmitted descriptors has exceeded the
  132. * watermark. Called with prep_lock held
  133. */
  134. static void ioat_update_pending(struct ioatdma_chan *ioat_chan)
  135. {
  136. if (ioat_ring_pending(ioat_chan) > ioat_pending_level)
  137. __ioat_issue_pending(ioat_chan);
  138. }
  139. static void __ioat_start_null_desc(struct ioatdma_chan *ioat_chan)
  140. {
  141. struct ioat_ring_ent *desc;
  142. struct ioat_dma_descriptor *hw;
  143. if (ioat_ring_space(ioat_chan) < 1) {
  144. dev_err(to_dev(ioat_chan),
  145. "Unable to start null desc - ring full\n");
  146. return;
  147. }
  148. dev_dbg(to_dev(ioat_chan),
  149. "%s: head: %#x tail: %#x issued: %#x\n",
  150. __func__, ioat_chan->head, ioat_chan->tail, ioat_chan->issued);
  151. desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head);
  152. hw = desc->hw;
  153. hw->ctl = 0;
  154. hw->ctl_f.null = 1;
  155. hw->ctl_f.int_en = 1;
  156. hw->ctl_f.compl_write = 1;
  157. /* set size to non-zero value (channel returns error when size is 0) */
  158. hw->size = NULL_DESC_BUFFER_SIZE;
  159. hw->src_addr = 0;
  160. hw->dst_addr = 0;
  161. async_tx_ack(&desc->txd);
  162. ioat_set_chainaddr(ioat_chan, desc->txd.phys);
  163. dump_desc_dbg(ioat_chan, desc);
  164. /* make sure descriptors are written before we submit */
  165. wmb();
  166. ioat_chan->head += 1;
  167. __ioat_issue_pending(ioat_chan);
  168. }
  169. void ioat_start_null_desc(struct ioatdma_chan *ioat_chan)
  170. {
  171. spin_lock_bh(&ioat_chan->prep_lock);
  172. if (!test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))
  173. __ioat_start_null_desc(ioat_chan);
  174. spin_unlock_bh(&ioat_chan->prep_lock);
  175. }
  176. static void __ioat_restart_chan(struct ioatdma_chan *ioat_chan)
  177. {
  178. /* set the tail to be re-issued */
  179. ioat_chan->issued = ioat_chan->tail;
  180. ioat_chan->dmacount = 0;
  181. mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
  182. dev_dbg(to_dev(ioat_chan),
  183. "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
  184. __func__, ioat_chan->head, ioat_chan->tail,
  185. ioat_chan->issued, ioat_chan->dmacount);
  186. if (ioat_ring_pending(ioat_chan)) {
  187. struct ioat_ring_ent *desc;
  188. desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail);
  189. ioat_set_chainaddr(ioat_chan, desc->txd.phys);
  190. __ioat_issue_pending(ioat_chan);
  191. } else
  192. __ioat_start_null_desc(ioat_chan);
  193. }
  194. static int ioat_quiesce(struct ioatdma_chan *ioat_chan, unsigned long tmo)
  195. {
  196. unsigned long end = jiffies + tmo;
  197. int err = 0;
  198. u32 status;
  199. status = ioat_chansts(ioat_chan);
  200. if (is_ioat_active(status) || is_ioat_idle(status))
  201. ioat_suspend(ioat_chan);
  202. while (is_ioat_active(status) || is_ioat_idle(status)) {
  203. if (tmo && time_after(jiffies, end)) {
  204. err = -ETIMEDOUT;
  205. break;
  206. }
  207. status = ioat_chansts(ioat_chan);
  208. cpu_relax();
  209. }
  210. return err;
  211. }
  212. static int ioat_reset_sync(struct ioatdma_chan *ioat_chan, unsigned long tmo)
  213. {
  214. unsigned long end = jiffies + tmo;
  215. int err = 0;
  216. ioat_reset(ioat_chan);
  217. while (ioat_reset_pending(ioat_chan)) {
  218. if (end && time_after(jiffies, end)) {
  219. err = -ETIMEDOUT;
  220. break;
  221. }
  222. cpu_relax();
  223. }
  224. return err;
  225. }
  226. static dma_cookie_t ioat_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
  227. __releases(&ioat_chan->prep_lock)
  228. {
  229. struct dma_chan *c = tx->chan;
  230. struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
  231. dma_cookie_t cookie;
  232. cookie = dma_cookie_assign(tx);
  233. dev_dbg(to_dev(ioat_chan), "%s: cookie: %d\n", __func__, cookie);
  234. if (!test_and_set_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state))
  235. mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
  236. /* make descriptor updates visible before advancing ioat->head,
  237. * this is purposefully not smp_wmb() since we are also
  238. * publishing the descriptor updates to a dma device
  239. */
  240. wmb();
  241. ioat_chan->head += ioat_chan->produce;
  242. ioat_update_pending(ioat_chan);
  243. spin_unlock_bh(&ioat_chan->prep_lock);
  244. return cookie;
  245. }
  246. static struct ioat_ring_ent *
  247. ioat_alloc_ring_ent(struct dma_chan *chan, gfp_t flags)
  248. {
  249. struct ioat_dma_descriptor *hw;
  250. struct ioat_ring_ent *desc;
  251. struct ioatdma_device *ioat_dma;
  252. dma_addr_t phys;
  253. ioat_dma = to_ioatdma_device(chan->device);
  254. hw = pci_pool_alloc(ioat_dma->dma_pool, flags, &phys);
  255. if (!hw)
  256. return NULL;
  257. memset(hw, 0, sizeof(*hw));
  258. desc = kmem_cache_zalloc(ioat_cache, flags);
  259. if (!desc) {
  260. pci_pool_free(ioat_dma->dma_pool, hw, phys);
  261. return NULL;
  262. }
  263. dma_async_tx_descriptor_init(&desc->txd, chan);
  264. desc->txd.tx_submit = ioat_tx_submit_unlock;
  265. desc->hw = hw;
  266. desc->txd.phys = phys;
  267. return desc;
  268. }
  269. void ioat_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan)
  270. {
  271. struct ioatdma_device *ioat_dma;
  272. ioat_dma = to_ioatdma_device(chan->device);
  273. pci_pool_free(ioat_dma->dma_pool, desc->hw, desc->txd.phys);
  274. kmem_cache_free(ioat_cache, desc);
  275. }
  276. struct ioat_ring_ent **
  277. ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags)
  278. {
  279. struct ioat_ring_ent **ring;
  280. int descs = 1 << order;
  281. int i;
  282. if (order > ioat_get_max_alloc_order())
  283. return NULL;
  284. /* allocate the array to hold the software ring */
  285. ring = kcalloc(descs, sizeof(*ring), flags);
  286. if (!ring)
  287. return NULL;
  288. for (i = 0; i < descs; i++) {
  289. ring[i] = ioat_alloc_ring_ent(c, flags);
  290. if (!ring[i]) {
  291. while (i--)
  292. ioat_free_ring_ent(ring[i], c);
  293. kfree(ring);
  294. return NULL;
  295. }
  296. set_desc_id(ring[i], i);
  297. }
  298. /* link descs */
  299. for (i = 0; i < descs-1; i++) {
  300. struct ioat_ring_ent *next = ring[i+1];
  301. struct ioat_dma_descriptor *hw = ring[i]->hw;
  302. hw->next = next->txd.phys;
  303. }
  304. ring[i]->hw->next = ring[0]->txd.phys;
  305. return ring;
  306. }
  307. static bool reshape_ring(struct ioatdma_chan *ioat_chan, int order)
  308. {
  309. /* reshape differs from normal ring allocation in that we want
  310. * to allocate a new software ring while only
  311. * extending/truncating the hardware ring
  312. */
  313. struct dma_chan *c = &ioat_chan->dma_chan;
  314. const u32 curr_size = ioat_ring_size(ioat_chan);
  315. const u16 active = ioat_ring_active(ioat_chan);
  316. const u32 new_size = 1 << order;
  317. struct ioat_ring_ent **ring;
  318. u32 i;
  319. if (order > ioat_get_max_alloc_order())
  320. return false;
  321. /* double check that we have at least 1 free descriptor */
  322. if (active == curr_size)
  323. return false;
  324. /* when shrinking, verify that we can hold the current active
  325. * set in the new ring
  326. */
  327. if (active >= new_size)
  328. return false;
  329. /* allocate the array to hold the software ring */
  330. ring = kcalloc(new_size, sizeof(*ring), GFP_NOWAIT);
  331. if (!ring)
  332. return false;
  333. /* allocate/trim descriptors as needed */
  334. if (new_size > curr_size) {
  335. /* copy current descriptors to the new ring */
  336. for (i = 0; i < curr_size; i++) {
  337. u16 curr_idx = (ioat_chan->tail+i) & (curr_size-1);
  338. u16 new_idx = (ioat_chan->tail+i) & (new_size-1);
  339. ring[new_idx] = ioat_chan->ring[curr_idx];
  340. set_desc_id(ring[new_idx], new_idx);
  341. }
  342. /* add new descriptors to the ring */
  343. for (i = curr_size; i < new_size; i++) {
  344. u16 new_idx = (ioat_chan->tail+i) & (new_size-1);
  345. ring[new_idx] = ioat_alloc_ring_ent(c, GFP_NOWAIT);
  346. if (!ring[new_idx]) {
  347. while (i--) {
  348. u16 new_idx = (ioat_chan->tail+i) &
  349. (new_size-1);
  350. ioat_free_ring_ent(ring[new_idx], c);
  351. }
  352. kfree(ring);
  353. return false;
  354. }
  355. set_desc_id(ring[new_idx], new_idx);
  356. }
  357. /* hw link new descriptors */
  358. for (i = curr_size-1; i < new_size; i++) {
  359. u16 new_idx = (ioat_chan->tail+i) & (new_size-1);
  360. struct ioat_ring_ent *next =
  361. ring[(new_idx+1) & (new_size-1)];
  362. struct ioat_dma_descriptor *hw = ring[new_idx]->hw;
  363. hw->next = next->txd.phys;
  364. }
  365. } else {
  366. struct ioat_dma_descriptor *hw;
  367. struct ioat_ring_ent *next;
  368. /* copy current descriptors to the new ring, dropping the
  369. * removed descriptors
  370. */
  371. for (i = 0; i < new_size; i++) {
  372. u16 curr_idx = (ioat_chan->tail+i) & (curr_size-1);
  373. u16 new_idx = (ioat_chan->tail+i) & (new_size-1);
  374. ring[new_idx] = ioat_chan->ring[curr_idx];
  375. set_desc_id(ring[new_idx], new_idx);
  376. }
  377. /* free deleted descriptors */
  378. for (i = new_size; i < curr_size; i++) {
  379. struct ioat_ring_ent *ent;
  380. ent = ioat_get_ring_ent(ioat_chan, ioat_chan->tail+i);
  381. ioat_free_ring_ent(ent, c);
  382. }
  383. /* fix up hardware ring */
  384. hw = ring[(ioat_chan->tail+new_size-1) & (new_size-1)]->hw;
  385. next = ring[(ioat_chan->tail+new_size) & (new_size-1)];
  386. hw->next = next->txd.phys;
  387. }
  388. dev_dbg(to_dev(ioat_chan), "%s: allocated %d descriptors\n",
  389. __func__, new_size);
  390. kfree(ioat_chan->ring);
  391. ioat_chan->ring = ring;
  392. ioat_chan->alloc_order = order;
  393. return true;
  394. }
  395. /**
  396. * ioat_check_space_lock - verify space and grab ring producer lock
  397. * @ioat: ioat,3 channel (ring) to operate on
  398. * @num_descs: allocation length
  399. */
  400. int ioat_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs)
  401. __acquires(&ioat_chan->prep_lock)
  402. {
  403. bool retry;
  404. retry:
  405. spin_lock_bh(&ioat_chan->prep_lock);
  406. /* never allow the last descriptor to be consumed, we need at
  407. * least one free at all times to allow for on-the-fly ring
  408. * resizing.
  409. */
  410. if (likely(ioat_ring_space(ioat_chan) > num_descs)) {
  411. dev_dbg(to_dev(ioat_chan), "%s: num_descs: %d (%x:%x:%x)\n",
  412. __func__, num_descs, ioat_chan->head,
  413. ioat_chan->tail, ioat_chan->issued);
  414. ioat_chan->produce = num_descs;
  415. return 0; /* with ioat->prep_lock held */
  416. }
  417. retry = test_and_set_bit(IOAT_RESHAPE_PENDING, &ioat_chan->state);
  418. spin_unlock_bh(&ioat_chan->prep_lock);
  419. /* is another cpu already trying to expand the ring? */
  420. if (retry)
  421. goto retry;
  422. spin_lock_bh(&ioat_chan->cleanup_lock);
  423. spin_lock_bh(&ioat_chan->prep_lock);
  424. retry = reshape_ring(ioat_chan, ioat_chan->alloc_order + 1);
  425. clear_bit(IOAT_RESHAPE_PENDING, &ioat_chan->state);
  426. spin_unlock_bh(&ioat_chan->prep_lock);
  427. spin_unlock_bh(&ioat_chan->cleanup_lock);
  428. /* if we were able to expand the ring retry the allocation */
  429. if (retry)
  430. goto retry;
  431. dev_dbg_ratelimited(to_dev(ioat_chan),
  432. "%s: ring full! num_descs: %d (%x:%x:%x)\n",
  433. __func__, num_descs, ioat_chan->head,
  434. ioat_chan->tail, ioat_chan->issued);
  435. /* progress reclaim in the allocation failure case we may be
  436. * called under bh_disabled so we need to trigger the timer
  437. * event directly
  438. */
  439. if (time_is_before_jiffies(ioat_chan->timer.expires)
  440. && timer_pending(&ioat_chan->timer)) {
  441. mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
  442. ioat_timer_event((unsigned long)ioat_chan);
  443. }
  444. return -ENOMEM;
  445. }
  446. static bool desc_has_ext(struct ioat_ring_ent *desc)
  447. {
  448. struct ioat_dma_descriptor *hw = desc->hw;
  449. if (hw->ctl_f.op == IOAT_OP_XOR ||
  450. hw->ctl_f.op == IOAT_OP_XOR_VAL) {
  451. struct ioat_xor_descriptor *xor = desc->xor;
  452. if (src_cnt_to_sw(xor->ctl_f.src_cnt) > 5)
  453. return true;
  454. } else if (hw->ctl_f.op == IOAT_OP_PQ ||
  455. hw->ctl_f.op == IOAT_OP_PQ_VAL) {
  456. struct ioat_pq_descriptor *pq = desc->pq;
  457. if (src_cnt_to_sw(pq->ctl_f.src_cnt) > 3)
  458. return true;
  459. }
  460. return false;
  461. }
  462. static void
  463. ioat_free_sed(struct ioatdma_device *ioat_dma, struct ioat_sed_ent *sed)
  464. {
  465. if (!sed)
  466. return;
  467. dma_pool_free(ioat_dma->sed_hw_pool[sed->hw_pool], sed->hw, sed->dma);
  468. kmem_cache_free(ioat_sed_cache, sed);
  469. }
  470. static u64 ioat_get_current_completion(struct ioatdma_chan *ioat_chan)
  471. {
  472. u64 phys_complete;
  473. u64 completion;
  474. completion = *ioat_chan->completion;
  475. phys_complete = ioat_chansts_to_addr(completion);
  476. dev_dbg(to_dev(ioat_chan), "%s: phys_complete: %#llx\n", __func__,
  477. (unsigned long long) phys_complete);
  478. return phys_complete;
  479. }
  480. static bool ioat_cleanup_preamble(struct ioatdma_chan *ioat_chan,
  481. u64 *phys_complete)
  482. {
  483. *phys_complete = ioat_get_current_completion(ioat_chan);
  484. if (*phys_complete == ioat_chan->last_completion)
  485. return false;
  486. clear_bit(IOAT_COMPLETION_ACK, &ioat_chan->state);
  487. mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
  488. return true;
  489. }
  490. static void
  491. desc_get_errstat(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc)
  492. {
  493. struct ioat_dma_descriptor *hw = desc->hw;
  494. switch (hw->ctl_f.op) {
  495. case IOAT_OP_PQ_VAL:
  496. case IOAT_OP_PQ_VAL_16S:
  497. {
  498. struct ioat_pq_descriptor *pq = desc->pq;
  499. /* check if there's error written */
  500. if (!pq->dwbes_f.wbes)
  501. return;
  502. /* need to set a chanerr var for checking to clear later */
  503. if (pq->dwbes_f.p_val_err)
  504. *desc->result |= SUM_CHECK_P_RESULT;
  505. if (pq->dwbes_f.q_val_err)
  506. *desc->result |= SUM_CHECK_Q_RESULT;
  507. return;
  508. }
  509. default:
  510. return;
  511. }
  512. }
  513. /**
  514. * __cleanup - reclaim used descriptors
  515. * @ioat: channel (ring) to clean
  516. */
  517. static void __cleanup(struct ioatdma_chan *ioat_chan, dma_addr_t phys_complete)
  518. {
  519. struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
  520. struct ioat_ring_ent *desc;
  521. bool seen_current = false;
  522. int idx = ioat_chan->tail, i;
  523. u16 active;
  524. dev_dbg(to_dev(ioat_chan), "%s: head: %#x tail: %#x issued: %#x\n",
  525. __func__, ioat_chan->head, ioat_chan->tail, ioat_chan->issued);
  526. /*
  527. * At restart of the channel, the completion address and the
  528. * channel status will be 0 due to starting a new chain. Since
  529. * it's new chain and the first descriptor "fails", there is
  530. * nothing to clean up. We do not want to reap the entire submitted
  531. * chain due to this 0 address value and then BUG.
  532. */
  533. if (!phys_complete)
  534. return;
  535. active = ioat_ring_active(ioat_chan);
  536. for (i = 0; i < active && !seen_current; i++) {
  537. struct dma_async_tx_descriptor *tx;
  538. smp_read_barrier_depends();
  539. prefetch(ioat_get_ring_ent(ioat_chan, idx + i + 1));
  540. desc = ioat_get_ring_ent(ioat_chan, idx + i);
  541. dump_desc_dbg(ioat_chan, desc);
  542. /* set err stat if we are using dwbes */
  543. if (ioat_dma->cap & IOAT_CAP_DWBES)
  544. desc_get_errstat(ioat_chan, desc);
  545. tx = &desc->txd;
  546. if (tx->cookie) {
  547. dma_cookie_complete(tx);
  548. dma_descriptor_unmap(tx);
  549. if (tx->callback) {
  550. tx->callback(tx->callback_param);
  551. tx->callback = NULL;
  552. }
  553. }
  554. if (tx->phys == phys_complete)
  555. seen_current = true;
  556. /* skip extended descriptors */
  557. if (desc_has_ext(desc)) {
  558. BUG_ON(i + 1 >= active);
  559. i++;
  560. }
  561. /* cleanup super extended descriptors */
  562. if (desc->sed) {
  563. ioat_free_sed(ioat_dma, desc->sed);
  564. desc->sed = NULL;
  565. }
  566. }
  567. /* finish all descriptor reads before incrementing tail */
  568. smp_mb();
  569. ioat_chan->tail = idx + i;
  570. /* no active descs have written a completion? */
  571. BUG_ON(active && !seen_current);
  572. ioat_chan->last_completion = phys_complete;
  573. if (active - i == 0) {
  574. dev_dbg(to_dev(ioat_chan), "%s: cancel completion timeout\n",
  575. __func__);
  576. mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
  577. }
  578. /* 5 microsecond delay per pending descriptor */
  579. writew(min((5 * (active - i)), IOAT_INTRDELAY_MASK),
  580. ioat_chan->ioat_dma->reg_base + IOAT_INTRDELAY_OFFSET);
  581. }
  582. static void ioat_cleanup(struct ioatdma_chan *ioat_chan)
  583. {
  584. u64 phys_complete;
  585. spin_lock_bh(&ioat_chan->cleanup_lock);
  586. if (ioat_cleanup_preamble(ioat_chan, &phys_complete))
  587. __cleanup(ioat_chan, phys_complete);
  588. if (is_ioat_halted(*ioat_chan->completion)) {
  589. u32 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
  590. if (chanerr & IOAT_CHANERR_HANDLE_MASK) {
  591. mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
  592. ioat_eh(ioat_chan);
  593. }
  594. }
  595. spin_unlock_bh(&ioat_chan->cleanup_lock);
  596. }
  597. void ioat_cleanup_event(unsigned long data)
  598. {
  599. struct ioatdma_chan *ioat_chan = to_ioat_chan((void *)data);
  600. ioat_cleanup(ioat_chan);
  601. if (!test_bit(IOAT_RUN, &ioat_chan->state))
  602. return;
  603. writew(IOAT_CHANCTRL_RUN, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
  604. }
  605. static void ioat_restart_channel(struct ioatdma_chan *ioat_chan)
  606. {
  607. u64 phys_complete;
  608. ioat_quiesce(ioat_chan, 0);
  609. if (ioat_cleanup_preamble(ioat_chan, &phys_complete))
  610. __cleanup(ioat_chan, phys_complete);
  611. __ioat_restart_chan(ioat_chan);
  612. }
  613. static void ioat_eh(struct ioatdma_chan *ioat_chan)
  614. {
  615. struct pci_dev *pdev = to_pdev(ioat_chan);
  616. struct ioat_dma_descriptor *hw;
  617. struct dma_async_tx_descriptor *tx;
  618. u64 phys_complete;
  619. struct ioat_ring_ent *desc;
  620. u32 err_handled = 0;
  621. u32 chanerr_int;
  622. u32 chanerr;
  623. /* cleanup so tail points to descriptor that caused the error */
  624. if (ioat_cleanup_preamble(ioat_chan, &phys_complete))
  625. __cleanup(ioat_chan, phys_complete);
  626. chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
  627. pci_read_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, &chanerr_int);
  628. dev_dbg(to_dev(ioat_chan), "%s: error = %x:%x\n",
  629. __func__, chanerr, chanerr_int);
  630. desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail);
  631. hw = desc->hw;
  632. dump_desc_dbg(ioat_chan, desc);
  633. switch (hw->ctl_f.op) {
  634. case IOAT_OP_XOR_VAL:
  635. if (chanerr & IOAT_CHANERR_XOR_P_OR_CRC_ERR) {
  636. *desc->result |= SUM_CHECK_P_RESULT;
  637. err_handled |= IOAT_CHANERR_XOR_P_OR_CRC_ERR;
  638. }
  639. break;
  640. case IOAT_OP_PQ_VAL:
  641. case IOAT_OP_PQ_VAL_16S:
  642. if (chanerr & IOAT_CHANERR_XOR_P_OR_CRC_ERR) {
  643. *desc->result |= SUM_CHECK_P_RESULT;
  644. err_handled |= IOAT_CHANERR_XOR_P_OR_CRC_ERR;
  645. }
  646. if (chanerr & IOAT_CHANERR_XOR_Q_ERR) {
  647. *desc->result |= SUM_CHECK_Q_RESULT;
  648. err_handled |= IOAT_CHANERR_XOR_Q_ERR;
  649. }
  650. break;
  651. }
  652. /* fault on unhandled error or spurious halt */
  653. if (chanerr ^ err_handled || chanerr == 0) {
  654. dev_err(to_dev(ioat_chan), "%s: fatal error (%x:%x)\n",
  655. __func__, chanerr, err_handled);
  656. BUG();
  657. } else { /* cleanup the faulty descriptor */
  658. tx = &desc->txd;
  659. if (tx->cookie) {
  660. dma_cookie_complete(tx);
  661. dma_descriptor_unmap(tx);
  662. if (tx->callback) {
  663. tx->callback(tx->callback_param);
  664. tx->callback = NULL;
  665. }
  666. }
  667. }
  668. writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
  669. pci_write_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, chanerr_int);
  670. /* mark faulting descriptor as complete */
  671. *ioat_chan->completion = desc->txd.phys;
  672. spin_lock_bh(&ioat_chan->prep_lock);
  673. ioat_restart_channel(ioat_chan);
  674. spin_unlock_bh(&ioat_chan->prep_lock);
  675. }
  676. static void check_active(struct ioatdma_chan *ioat_chan)
  677. {
  678. if (ioat_ring_active(ioat_chan)) {
  679. mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
  680. return;
  681. }
  682. if (test_and_clear_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state))
  683. mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
  684. else if (ioat_chan->alloc_order > ioat_get_alloc_order()) {
  685. /* if the ring is idle, empty, and oversized try to step
  686. * down the size
  687. */
  688. reshape_ring(ioat_chan, ioat_chan->alloc_order - 1);
  689. /* keep shrinking until we get back to our minimum
  690. * default size
  691. */
  692. if (ioat_chan->alloc_order > ioat_get_alloc_order())
  693. mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
  694. }
  695. }
  696. void ioat_timer_event(unsigned long data)
  697. {
  698. struct ioatdma_chan *ioat_chan = to_ioat_chan((void *)data);
  699. dma_addr_t phys_complete;
  700. u64 status;
  701. status = ioat_chansts(ioat_chan);
  702. /* when halted due to errors check for channel
  703. * programming errors before advancing the completion state
  704. */
  705. if (is_ioat_halted(status)) {
  706. u32 chanerr;
  707. chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
  708. dev_err(to_dev(ioat_chan), "%s: Channel halted (%x)\n",
  709. __func__, chanerr);
  710. if (test_bit(IOAT_RUN, &ioat_chan->state))
  711. BUG_ON(is_ioat_bug(chanerr));
  712. else /* we never got off the ground */
  713. return;
  714. }
  715. /* if we haven't made progress and we have already
  716. * acknowledged a pending completion once, then be more
  717. * forceful with a restart
  718. */
  719. spin_lock_bh(&ioat_chan->cleanup_lock);
  720. if (ioat_cleanup_preamble(ioat_chan, &phys_complete))
  721. __cleanup(ioat_chan, phys_complete);
  722. else if (test_bit(IOAT_COMPLETION_ACK, &ioat_chan->state)) {
  723. spin_lock_bh(&ioat_chan->prep_lock);
  724. ioat_restart_channel(ioat_chan);
  725. spin_unlock_bh(&ioat_chan->prep_lock);
  726. spin_unlock_bh(&ioat_chan->cleanup_lock);
  727. return;
  728. } else {
  729. set_bit(IOAT_COMPLETION_ACK, &ioat_chan->state);
  730. mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
  731. }
  732. if (ioat_ring_active(ioat_chan))
  733. mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
  734. else {
  735. spin_lock_bh(&ioat_chan->prep_lock);
  736. check_active(ioat_chan);
  737. spin_unlock_bh(&ioat_chan->prep_lock);
  738. }
  739. spin_unlock_bh(&ioat_chan->cleanup_lock);
  740. }
  741. enum dma_status
  742. ioat_tx_status(struct dma_chan *c, dma_cookie_t cookie,
  743. struct dma_tx_state *txstate)
  744. {
  745. struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
  746. enum dma_status ret;
  747. ret = dma_cookie_status(c, cookie, txstate);
  748. if (ret == DMA_COMPLETE)
  749. return ret;
  750. ioat_cleanup(ioat_chan);
  751. return dma_cookie_status(c, cookie, txstate);
  752. }
  753. static int ioat_irq_reinit(struct ioatdma_device *ioat_dma)
  754. {
  755. struct pci_dev *pdev = ioat_dma->pdev;
  756. int irq = pdev->irq, i;
  757. if (!is_bwd_ioat(pdev))
  758. return 0;
  759. switch (ioat_dma->irq_mode) {
  760. case IOAT_MSIX:
  761. for (i = 0; i < ioat_dma->dma_dev.chancnt; i++) {
  762. struct msix_entry *msix = &ioat_dma->msix_entries[i];
  763. struct ioatdma_chan *ioat_chan;
  764. ioat_chan = ioat_chan_by_index(ioat_dma, i);
  765. devm_free_irq(&pdev->dev, msix->vector, ioat_chan);
  766. }
  767. pci_disable_msix(pdev);
  768. break;
  769. case IOAT_MSI:
  770. pci_disable_msi(pdev);
  771. /* fall through */
  772. case IOAT_INTX:
  773. devm_free_irq(&pdev->dev, irq, ioat_dma);
  774. break;
  775. default:
  776. return 0;
  777. }
  778. ioat_dma->irq_mode = IOAT_NOIRQ;
  779. return ioat_dma_setup_interrupts(ioat_dma);
  780. }
  781. int ioat_reset_hw(struct ioatdma_chan *ioat_chan)
  782. {
  783. /* throw away whatever the channel was doing and get it
  784. * initialized, with ioat3 specific workarounds
  785. */
  786. struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
  787. struct pci_dev *pdev = ioat_dma->pdev;
  788. u32 chanerr;
  789. u16 dev_id;
  790. int err;
  791. ioat_quiesce(ioat_chan, msecs_to_jiffies(100));
  792. chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
  793. writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
  794. if (ioat_dma->version < IOAT_VER_3_3) {
  795. /* clear any pending errors */
  796. err = pci_read_config_dword(pdev,
  797. IOAT_PCI_CHANERR_INT_OFFSET, &chanerr);
  798. if (err) {
  799. dev_err(&pdev->dev,
  800. "channel error register unreachable\n");
  801. return err;
  802. }
  803. pci_write_config_dword(pdev,
  804. IOAT_PCI_CHANERR_INT_OFFSET, chanerr);
  805. /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
  806. * (workaround for spurious config parity error after restart)
  807. */
  808. pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id);
  809. if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0) {
  810. pci_write_config_dword(pdev,
  811. IOAT_PCI_DMAUNCERRSTS_OFFSET,
  812. 0x10);
  813. }
  814. }
  815. err = ioat_reset_sync(ioat_chan, msecs_to_jiffies(200));
  816. if (!err)
  817. err = ioat_irq_reinit(ioat_dma);
  818. if (err)
  819. dev_err(&pdev->dev, "Failed to reset: %d\n", err);
  820. return err;
  821. }