ore_raid.c 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721
  1. /*
  2. * Copyright (C) 2011
  3. * Boaz Harrosh <ooo@electrozaur.com>
  4. *
  5. * This file is part of the objects raid engine (ore).
  6. *
  7. * It is free software; you can redistribute it and/or modify
  8. * it under the terms of the GNU General Public License version 2 as published
  9. * by the Free Software Foundation.
  10. *
  11. * You should have received a copy of the GNU General Public License
  12. * along with "ore". If not, write to the Free Software Foundation, Inc:
  13. * "Free Software Foundation <info@fsf.org>"
  14. */
  15. #include <linux/gfp.h>
  16. #include <linux/async_tx.h>
  17. #include "ore_raid.h"
  18. #undef ORE_DBGMSG2
  19. #define ORE_DBGMSG2 ORE_DBGMSG
  20. static struct page *_raid_page_alloc(void)
  21. {
  22. return alloc_page(GFP_KERNEL);
  23. }
  24. static void _raid_page_free(struct page *p)
  25. {
  26. __free_page(p);
  27. }
  28. /* This struct is forward declare in ore_io_state, but is private to here.
  29. * It is put on ios->sp2d for RAID5/6 writes only. See _gen_xor_unit.
  30. *
  31. * __stripe_pages_2d is a 2d array of pages, and it is also a corner turn.
  32. * Ascending page index access is sp2d(p-minor, c-major). But storage is
  33. * sp2d[p-minor][c-major], so it can be properlly presented to the async-xor
  34. * API.
  35. */
  36. struct __stripe_pages_2d {
  37. /* Cache some hot path repeated calculations */
  38. unsigned parity;
  39. unsigned data_devs;
  40. unsigned pages_in_unit;
  41. bool needed ;
  42. /* Array size is pages_in_unit (layout->stripe_unit / PAGE_SIZE) */
  43. struct __1_page_stripe {
  44. bool alloc;
  45. unsigned write_count;
  46. struct async_submit_ctl submit;
  47. struct dma_async_tx_descriptor *tx;
  48. /* The size of this array is data_devs + parity */
  49. struct page **pages;
  50. struct page **scribble;
  51. /* bool array, size of this array is data_devs */
  52. char *page_is_read;
  53. } _1p_stripes[];
  54. };
  55. /* This can get bigger then a page. So support multiple page allocations
  56. * _sp2d_free should be called even if _sp2d_alloc fails (by returning
  57. * none-zero).
  58. */
  59. static int _sp2d_alloc(unsigned pages_in_unit, unsigned group_width,
  60. unsigned parity, struct __stripe_pages_2d **psp2d)
  61. {
  62. struct __stripe_pages_2d *sp2d;
  63. unsigned data_devs = group_width - parity;
  64. struct _alloc_all_bytes {
  65. struct __alloc_stripe_pages_2d {
  66. struct __stripe_pages_2d sp2d;
  67. struct __1_page_stripe _1p_stripes[pages_in_unit];
  68. } __asp2d;
  69. struct __alloc_1p_arrays {
  70. struct page *pages[group_width];
  71. struct page *scribble[group_width];
  72. char page_is_read[data_devs];
  73. } __a1pa[pages_in_unit];
  74. } *_aab;
  75. struct __alloc_1p_arrays *__a1pa;
  76. struct __alloc_1p_arrays *__a1pa_end;
  77. const unsigned sizeof__a1pa = sizeof(_aab->__a1pa[0]);
  78. unsigned num_a1pa, alloc_size, i;
  79. /* FIXME: check these numbers in ore_verify_layout */
  80. BUG_ON(sizeof(_aab->__asp2d) > PAGE_SIZE);
  81. BUG_ON(sizeof__a1pa > PAGE_SIZE);
  82. if (sizeof(*_aab) > PAGE_SIZE) {
  83. num_a1pa = (PAGE_SIZE - sizeof(_aab->__asp2d)) / sizeof__a1pa;
  84. alloc_size = sizeof(_aab->__asp2d) + sizeof__a1pa * num_a1pa;
  85. } else {
  86. num_a1pa = pages_in_unit;
  87. alloc_size = sizeof(*_aab);
  88. }
  89. _aab = kzalloc(alloc_size, GFP_KERNEL);
  90. if (unlikely(!_aab)) {
  91. ORE_DBGMSG("!! Failed to alloc sp2d size=%d\n", alloc_size);
  92. return -ENOMEM;
  93. }
  94. sp2d = &_aab->__asp2d.sp2d;
  95. *psp2d = sp2d; /* From here Just call _sp2d_free */
  96. __a1pa = _aab->__a1pa;
  97. __a1pa_end = __a1pa + num_a1pa;
  98. for (i = 0; i < pages_in_unit; ++i) {
  99. if (unlikely(__a1pa >= __a1pa_end)) {
  100. num_a1pa = min_t(unsigned, PAGE_SIZE / sizeof__a1pa,
  101. pages_in_unit - i);
  102. __a1pa = kcalloc(num_a1pa, sizeof__a1pa, GFP_KERNEL);
  103. if (unlikely(!__a1pa)) {
  104. ORE_DBGMSG("!! Failed to _alloc_1p_arrays=%d\n",
  105. num_a1pa);
  106. return -ENOMEM;
  107. }
  108. __a1pa_end = __a1pa + num_a1pa;
  109. /* First *pages is marked for kfree of the buffer */
  110. sp2d->_1p_stripes[i].alloc = true;
  111. }
  112. sp2d->_1p_stripes[i].pages = __a1pa->pages;
  113. sp2d->_1p_stripes[i].scribble = __a1pa->scribble ;
  114. sp2d->_1p_stripes[i].page_is_read = __a1pa->page_is_read;
  115. ++__a1pa;
  116. }
  117. sp2d->parity = parity;
  118. sp2d->data_devs = data_devs;
  119. sp2d->pages_in_unit = pages_in_unit;
  120. return 0;
  121. }
  122. static void _sp2d_reset(struct __stripe_pages_2d *sp2d,
  123. const struct _ore_r4w_op *r4w, void *priv)
  124. {
  125. unsigned data_devs = sp2d->data_devs;
  126. unsigned group_width = data_devs + sp2d->parity;
  127. int p, c;
  128. if (!sp2d->needed)
  129. return;
  130. for (c = data_devs - 1; c >= 0; --c)
  131. for (p = sp2d->pages_in_unit - 1; p >= 0; --p) {
  132. struct __1_page_stripe *_1ps = &sp2d->_1p_stripes[p];
  133. if (_1ps->page_is_read[c]) {
  134. struct page *page = _1ps->pages[c];
  135. r4w->put_page(priv, page);
  136. _1ps->page_is_read[c] = false;
  137. }
  138. }
  139. for (p = 0; p < sp2d->pages_in_unit; p++) {
  140. struct __1_page_stripe *_1ps = &sp2d->_1p_stripes[p];
  141. memset(_1ps->pages, 0, group_width * sizeof(*_1ps->pages));
  142. _1ps->write_count = 0;
  143. _1ps->tx = NULL;
  144. }
  145. sp2d->needed = false;
  146. }
  147. static void _sp2d_free(struct __stripe_pages_2d *sp2d)
  148. {
  149. unsigned i;
  150. if (!sp2d)
  151. return;
  152. for (i = 0; i < sp2d->pages_in_unit; ++i) {
  153. if (sp2d->_1p_stripes[i].alloc)
  154. kfree(sp2d->_1p_stripes[i].pages);
  155. }
  156. kfree(sp2d);
  157. }
  158. static unsigned _sp2d_min_pg(struct __stripe_pages_2d *sp2d)
  159. {
  160. unsigned p;
  161. for (p = 0; p < sp2d->pages_in_unit; p++) {
  162. struct __1_page_stripe *_1ps = &sp2d->_1p_stripes[p];
  163. if (_1ps->write_count)
  164. return p;
  165. }
  166. return ~0;
  167. }
  168. static unsigned _sp2d_max_pg(struct __stripe_pages_2d *sp2d)
  169. {
  170. int p;
  171. for (p = sp2d->pages_in_unit - 1; p >= 0; --p) {
  172. struct __1_page_stripe *_1ps = &sp2d->_1p_stripes[p];
  173. if (_1ps->write_count)
  174. return p;
  175. }
  176. return ~0;
  177. }
  178. static void _gen_xor_unit(struct __stripe_pages_2d *sp2d)
  179. {
  180. unsigned p;
  181. unsigned tx_flags = ASYNC_TX_ACK;
  182. if (sp2d->parity == 1)
  183. tx_flags |= ASYNC_TX_XOR_ZERO_DST;
  184. for (p = 0; p < sp2d->pages_in_unit; p++) {
  185. struct __1_page_stripe *_1ps = &sp2d->_1p_stripes[p];
  186. if (!_1ps->write_count)
  187. continue;
  188. init_async_submit(&_1ps->submit, tx_flags,
  189. NULL, NULL, NULL, (addr_conv_t *)_1ps->scribble);
  190. if (sp2d->parity == 1)
  191. _1ps->tx = async_xor(_1ps->pages[sp2d->data_devs],
  192. _1ps->pages, 0, sp2d->data_devs,
  193. PAGE_SIZE, &_1ps->submit);
  194. else /* parity == 2 */
  195. _1ps->tx = async_gen_syndrome(_1ps->pages, 0,
  196. sp2d->data_devs + sp2d->parity,
  197. PAGE_SIZE, &_1ps->submit);
  198. }
  199. for (p = 0; p < sp2d->pages_in_unit; p++) {
  200. struct __1_page_stripe *_1ps = &sp2d->_1p_stripes[p];
  201. /* NOTE: We wait for HW synchronously (I don't have such HW
  202. * to test with.) Is parallelism needed with today's multi
  203. * cores?
  204. */
  205. async_tx_issue_pending(_1ps->tx);
  206. }
  207. }
  208. void _ore_add_stripe_page(struct __stripe_pages_2d *sp2d,
  209. struct ore_striping_info *si, struct page *page)
  210. {
  211. struct __1_page_stripe *_1ps;
  212. sp2d->needed = true;
  213. _1ps = &sp2d->_1p_stripes[si->cur_pg];
  214. _1ps->pages[si->cur_comp] = page;
  215. ++_1ps->write_count;
  216. si->cur_pg = (si->cur_pg + 1) % sp2d->pages_in_unit;
  217. /* si->cur_comp is advanced outside at main loop */
  218. }
  219. void _ore_add_sg_seg(struct ore_per_dev_state *per_dev, unsigned cur_len,
  220. bool not_last)
  221. {
  222. struct osd_sg_entry *sge;
  223. ORE_DBGMSG("dev=%d cur_len=0x%x not_last=%d cur_sg=%d "
  224. "offset=0x%llx length=0x%x last_sgs_total=0x%x\n",
  225. per_dev->dev, cur_len, not_last, per_dev->cur_sg,
  226. _LLU(per_dev->offset), per_dev->length,
  227. per_dev->last_sgs_total);
  228. if (!per_dev->cur_sg) {
  229. sge = per_dev->sglist;
  230. /* First time we prepare two entries */
  231. if (per_dev->length) {
  232. ++per_dev->cur_sg;
  233. sge->offset = per_dev->offset;
  234. sge->len = per_dev->length;
  235. } else {
  236. /* Here the parity is the first unit of this object.
  237. * This happens every time we reach a parity device on
  238. * the same stripe as the per_dev->offset. We need to
  239. * just skip this unit.
  240. */
  241. per_dev->offset += cur_len;
  242. return;
  243. }
  244. } else {
  245. /* finalize the last one */
  246. sge = &per_dev->sglist[per_dev->cur_sg - 1];
  247. sge->len = per_dev->length - per_dev->last_sgs_total;
  248. }
  249. if (not_last) {
  250. /* Partly prepare the next one */
  251. struct osd_sg_entry *next_sge = sge + 1;
  252. ++per_dev->cur_sg;
  253. next_sge->offset = sge->offset + sge->len + cur_len;
  254. /* Save cur len so we know how mutch was added next time */
  255. per_dev->last_sgs_total = per_dev->length;
  256. next_sge->len = 0;
  257. } else if (!sge->len) {
  258. /* Optimize for when the last unit is a parity */
  259. --per_dev->cur_sg;
  260. }
  261. }
  262. static int _alloc_read_4_write(struct ore_io_state *ios)
  263. {
  264. struct ore_layout *layout = ios->layout;
  265. int ret;
  266. /* We want to only read those pages not in cache so worst case
  267. * is a stripe populated with every other page
  268. */
  269. unsigned sgs_per_dev = ios->sp2d->pages_in_unit + 2;
  270. ret = _ore_get_io_state(layout, ios->oc,
  271. layout->group_width * layout->mirrors_p1,
  272. sgs_per_dev, 0, &ios->ios_read_4_write);
  273. return ret;
  274. }
  275. /* @si contains info of the to-be-inserted page. Update of @si should be
  276. * maintained by caller. Specificaly si->dev, si->obj_offset, ...
  277. */
  278. static int _add_to_r4w(struct ore_io_state *ios, struct ore_striping_info *si,
  279. struct page *page, unsigned pg_len)
  280. {
  281. struct request_queue *q;
  282. struct ore_per_dev_state *per_dev;
  283. struct ore_io_state *read_ios;
  284. unsigned first_dev = si->dev - (si->dev %
  285. (ios->layout->group_width * ios->layout->mirrors_p1));
  286. unsigned comp = si->dev - first_dev;
  287. unsigned added_len;
  288. if (!ios->ios_read_4_write) {
  289. int ret = _alloc_read_4_write(ios);
  290. if (unlikely(ret))
  291. return ret;
  292. }
  293. read_ios = ios->ios_read_4_write;
  294. read_ios->numdevs = ios->layout->group_width * ios->layout->mirrors_p1;
  295. per_dev = &read_ios->per_dev[comp];
  296. if (!per_dev->length) {
  297. per_dev->bio = bio_kmalloc(GFP_KERNEL,
  298. ios->sp2d->pages_in_unit);
  299. if (unlikely(!per_dev->bio)) {
  300. ORE_DBGMSG("Failed to allocate BIO size=%u\n",
  301. ios->sp2d->pages_in_unit);
  302. return -ENOMEM;
  303. }
  304. per_dev->offset = si->obj_offset;
  305. per_dev->dev = si->dev;
  306. } else if (si->obj_offset != (per_dev->offset + per_dev->length)) {
  307. u64 gap = si->obj_offset - (per_dev->offset + per_dev->length);
  308. _ore_add_sg_seg(per_dev, gap, true);
  309. }
  310. q = osd_request_queue(ore_comp_dev(read_ios->oc, per_dev->dev));
  311. added_len = bio_add_pc_page(q, per_dev->bio, page, pg_len,
  312. si->obj_offset % PAGE_SIZE);
  313. if (unlikely(added_len != pg_len)) {
  314. ORE_DBGMSG("Failed to bio_add_pc_page bi_vcnt=%d\n",
  315. per_dev->bio->bi_vcnt);
  316. return -ENOMEM;
  317. }
  318. per_dev->length += pg_len;
  319. return 0;
  320. }
  321. /* read the beginning of an unaligned first page */
  322. static int _add_to_r4w_first_page(struct ore_io_state *ios, struct page *page)
  323. {
  324. struct ore_striping_info si;
  325. unsigned pg_len;
  326. ore_calc_stripe_info(ios->layout, ios->offset, 0, &si);
  327. pg_len = si.obj_offset % PAGE_SIZE;
  328. si.obj_offset -= pg_len;
  329. ORE_DBGMSG("offset=0x%llx len=0x%x index=0x%lx dev=%x\n",
  330. _LLU(si.obj_offset), pg_len, page->index, si.dev);
  331. return _add_to_r4w(ios, &si, page, pg_len);
  332. }
  333. /* read the end of an incomplete last page */
  334. static int _add_to_r4w_last_page(struct ore_io_state *ios, u64 *offset)
  335. {
  336. struct ore_striping_info si;
  337. struct page *page;
  338. unsigned pg_len, p, c;
  339. ore_calc_stripe_info(ios->layout, *offset, 0, &si);
  340. p = si.cur_pg;
  341. c = si.cur_comp;
  342. page = ios->sp2d->_1p_stripes[p].pages[c];
  343. pg_len = PAGE_SIZE - (si.unit_off % PAGE_SIZE);
  344. *offset += pg_len;
  345. ORE_DBGMSG("p=%d, c=%d next-offset=0x%llx len=0x%x dev=%x par_dev=%d\n",
  346. p, c, _LLU(*offset), pg_len, si.dev, si.par_dev);
  347. BUG_ON(!page);
  348. return _add_to_r4w(ios, &si, page, pg_len);
  349. }
  350. static void _mark_read4write_pages_uptodate(struct ore_io_state *ios, int ret)
  351. {
  352. struct bio_vec *bv;
  353. unsigned i, d;
  354. /* loop on all devices all pages */
  355. for (d = 0; d < ios->numdevs; d++) {
  356. struct bio *bio = ios->per_dev[d].bio;
  357. if (!bio)
  358. continue;
  359. bio_for_each_segment_all(bv, bio, i) {
  360. struct page *page = bv->bv_page;
  361. SetPageUptodate(page);
  362. if (PageError(page))
  363. ClearPageError(page);
  364. }
  365. }
  366. }
  367. /* read_4_write is hacked to read the start of the first stripe and/or
  368. * the end of the last stripe. If needed, with an sg-gap at each device/page.
  369. * It is assumed to be called after the to_be_written pages of the first stripe
  370. * are populating ios->sp2d[][]
  371. *
  372. * NOTE: We call ios->r4w->lock_fn for all pages needed for parity calculations
  373. * These pages are held at sp2d[p].pages[c] but with
  374. * sp2d[p].page_is_read[c] = true. At _sp2d_reset these pages are
  375. * ios->r4w->lock_fn(). The ios->r4w->lock_fn might signal that the page is
  376. * @uptodate=true, so we don't need to read it, only unlock, after IO.
  377. *
  378. * TODO: The read_4_write should calc a need_to_read_pages_count, if bigger then
  379. * to-be-written count, we should consider the xor-in-place mode.
  380. * need_to_read_pages_count is the actual number of pages not present in cache.
  381. * maybe "devs_in_group - ios->sp2d[p].write_count" is a good enough
  382. * approximation? In this mode the read pages are put in the empty places of
  383. * ios->sp2d[p][*], xor is calculated the same way. These pages are
  384. * allocated/freed and don't go through cache
  385. */
  386. static int _read_4_write_first_stripe(struct ore_io_state *ios)
  387. {
  388. struct ore_striping_info read_si;
  389. struct __stripe_pages_2d *sp2d = ios->sp2d;
  390. u64 offset = ios->si.first_stripe_start;
  391. unsigned c, p, min_p = sp2d->pages_in_unit, max_p = -1;
  392. if (offset == ios->offset) /* Go to start collect $200 */
  393. goto read_last_stripe;
  394. min_p = _sp2d_min_pg(sp2d);
  395. max_p = _sp2d_max_pg(sp2d);
  396. ORE_DBGMSG("stripe_start=0x%llx ios->offset=0x%llx min_p=%d max_p=%d\n",
  397. offset, ios->offset, min_p, max_p);
  398. for (c = 0; ; c++) {
  399. ore_calc_stripe_info(ios->layout, offset, 0, &read_si);
  400. read_si.obj_offset += min_p * PAGE_SIZE;
  401. offset += min_p * PAGE_SIZE;
  402. for (p = min_p; p <= max_p; p++) {
  403. struct __1_page_stripe *_1ps = &sp2d->_1p_stripes[p];
  404. struct page **pp = &_1ps->pages[c];
  405. bool uptodate;
  406. if (*pp) {
  407. if (ios->offset % PAGE_SIZE)
  408. /* Read the remainder of the page */
  409. _add_to_r4w_first_page(ios, *pp);
  410. /* to-be-written pages start here */
  411. goto read_last_stripe;
  412. }
  413. *pp = ios->r4w->get_page(ios->private, offset,
  414. &uptodate);
  415. if (unlikely(!*pp))
  416. return -ENOMEM;
  417. if (!uptodate)
  418. _add_to_r4w(ios, &read_si, *pp, PAGE_SIZE);
  419. /* Mark read-pages to be cache_released */
  420. _1ps->page_is_read[c] = true;
  421. read_si.obj_offset += PAGE_SIZE;
  422. offset += PAGE_SIZE;
  423. }
  424. offset += (sp2d->pages_in_unit - p) * PAGE_SIZE;
  425. }
  426. read_last_stripe:
  427. return 0;
  428. }
  429. static int _read_4_write_last_stripe(struct ore_io_state *ios)
  430. {
  431. struct ore_striping_info read_si;
  432. struct __stripe_pages_2d *sp2d = ios->sp2d;
  433. u64 offset;
  434. u64 last_stripe_end;
  435. unsigned bytes_in_stripe = ios->si.bytes_in_stripe;
  436. unsigned c, p, min_p = sp2d->pages_in_unit, max_p = -1;
  437. offset = ios->offset + ios->length;
  438. if (offset % PAGE_SIZE)
  439. _add_to_r4w_last_page(ios, &offset);
  440. /* offset will be aligned to next page */
  441. last_stripe_end = div_u64(offset + bytes_in_stripe - 1, bytes_in_stripe)
  442. * bytes_in_stripe;
  443. if (offset == last_stripe_end) /* Optimize for the aligned case */
  444. goto read_it;
  445. ore_calc_stripe_info(ios->layout, offset, 0, &read_si);
  446. p = read_si.cur_pg;
  447. c = read_si.cur_comp;
  448. if (min_p == sp2d->pages_in_unit) {
  449. /* Didn't do it yet */
  450. min_p = _sp2d_min_pg(sp2d);
  451. max_p = _sp2d_max_pg(sp2d);
  452. }
  453. ORE_DBGMSG("offset=0x%llx stripe_end=0x%llx min_p=%d max_p=%d\n",
  454. offset, last_stripe_end, min_p, max_p);
  455. while (offset < last_stripe_end) {
  456. struct __1_page_stripe *_1ps = &sp2d->_1p_stripes[p];
  457. if ((min_p <= p) && (p <= max_p)) {
  458. struct page *page;
  459. bool uptodate;
  460. BUG_ON(_1ps->pages[c]);
  461. page = ios->r4w->get_page(ios->private, offset,
  462. &uptodate);
  463. if (unlikely(!page))
  464. return -ENOMEM;
  465. _1ps->pages[c] = page;
  466. /* Mark read-pages to be cache_released */
  467. _1ps->page_is_read[c] = true;
  468. if (!uptodate)
  469. _add_to_r4w(ios, &read_si, page, PAGE_SIZE);
  470. }
  471. offset += PAGE_SIZE;
  472. if (p == (sp2d->pages_in_unit - 1)) {
  473. ++c;
  474. p = 0;
  475. ore_calc_stripe_info(ios->layout, offset, 0, &read_si);
  476. } else {
  477. read_si.obj_offset += PAGE_SIZE;
  478. ++p;
  479. }
  480. }
  481. read_it:
  482. return 0;
  483. }
  484. static int _read_4_write_execute(struct ore_io_state *ios)
  485. {
  486. struct ore_io_state *ios_read;
  487. unsigned i;
  488. int ret;
  489. ios_read = ios->ios_read_4_write;
  490. if (!ios_read)
  491. return 0;
  492. /* FIXME: Ugly to signal _sbi_read_mirror that we have bio(s). Change
  493. * to check for per_dev->bio
  494. */
  495. ios_read->pages = ios->pages;
  496. /* Now read these devices */
  497. for (i = 0; i < ios_read->numdevs; i += ios_read->layout->mirrors_p1) {
  498. ret = _ore_read_mirror(ios_read, i);
  499. if (unlikely(ret))
  500. return ret;
  501. }
  502. ret = ore_io_execute(ios_read); /* Synchronus execution */
  503. if (unlikely(ret)) {
  504. ORE_DBGMSG("!! ore_io_execute => %d\n", ret);
  505. return ret;
  506. }
  507. _mark_read4write_pages_uptodate(ios_read, ret);
  508. ore_put_io_state(ios_read);
  509. ios->ios_read_4_write = NULL; /* Might need a reuse at last stripe */
  510. return 0;
  511. }
  512. /* In writes @cur_len means length left. .i.e cur_len==0 is the last parity U */
  513. int _ore_add_parity_unit(struct ore_io_state *ios,
  514. struct ore_striping_info *si,
  515. struct ore_per_dev_state *per_dev,
  516. unsigned cur_len, bool do_xor)
  517. {
  518. if (ios->reading) {
  519. if (per_dev->cur_sg >= ios->sgs_per_dev) {
  520. ORE_DBGMSG("cur_sg(%d) >= sgs_per_dev(%d)\n" ,
  521. per_dev->cur_sg, ios->sgs_per_dev);
  522. return -ENOMEM;
  523. }
  524. _ore_add_sg_seg(per_dev, cur_len, true);
  525. } else {
  526. struct __stripe_pages_2d *sp2d = ios->sp2d;
  527. struct page **pages = ios->parity_pages + ios->cur_par_page;
  528. unsigned num_pages;
  529. unsigned array_start = 0;
  530. unsigned i;
  531. int ret;
  532. si->cur_pg = _sp2d_min_pg(sp2d);
  533. num_pages = _sp2d_max_pg(sp2d) + 1 - si->cur_pg;
  534. if (!per_dev->length) {
  535. per_dev->offset += si->cur_pg * PAGE_SIZE;
  536. /* If first stripe, Read in all read4write pages
  537. * (if needed) before we calculate the first parity.
  538. */
  539. if (do_xor)
  540. _read_4_write_first_stripe(ios);
  541. }
  542. if (!cur_len && do_xor)
  543. /* If last stripe r4w pages of last stripe */
  544. _read_4_write_last_stripe(ios);
  545. _read_4_write_execute(ios);
  546. for (i = 0; i < num_pages; i++) {
  547. pages[i] = _raid_page_alloc();
  548. if (unlikely(!pages[i]))
  549. return -ENOMEM;
  550. ++(ios->cur_par_page);
  551. }
  552. BUG_ON(si->cur_comp < sp2d->data_devs);
  553. BUG_ON(si->cur_pg + num_pages > sp2d->pages_in_unit);
  554. ret = _ore_add_stripe_unit(ios, &array_start, 0, pages,
  555. per_dev, num_pages * PAGE_SIZE);
  556. if (unlikely(ret))
  557. return ret;
  558. if (do_xor) {
  559. _gen_xor_unit(sp2d);
  560. _sp2d_reset(sp2d, ios->r4w, ios->private);
  561. }
  562. }
  563. return 0;
  564. }
  565. int _ore_post_alloc_raid_stuff(struct ore_io_state *ios)
  566. {
  567. if (ios->parity_pages) {
  568. struct ore_layout *layout = ios->layout;
  569. unsigned pages_in_unit = layout->stripe_unit / PAGE_SIZE;
  570. if (_sp2d_alloc(pages_in_unit, layout->group_width,
  571. layout->parity, &ios->sp2d)) {
  572. return -ENOMEM;
  573. }
  574. }
  575. return 0;
  576. }
  577. void _ore_free_raid_stuff(struct ore_io_state *ios)
  578. {
  579. if (ios->sp2d) { /* writing and raid */
  580. unsigned i;
  581. for (i = 0; i < ios->cur_par_page; i++) {
  582. struct page *page = ios->parity_pages[i];
  583. if (page)
  584. _raid_page_free(page);
  585. }
  586. if (ios->extra_part_alloc)
  587. kfree(ios->parity_pages);
  588. /* If IO returned an error pages might need unlocking */
  589. _sp2d_reset(ios->sp2d, ios->r4w, ios->private);
  590. _sp2d_free(ios->sp2d);
  591. } else {
  592. /* Will only be set if raid reading && sglist is big */
  593. if (ios->extra_part_alloc)
  594. kfree(ios->per_dev[0].sglist);
  595. }
  596. if (ios->ios_read_4_write)
  597. ore_put_io_state(ios->ios_read_4_write);
  598. }