writeback.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561
  1. /*
  2. * background writeback - scan btree for dirty data and write it to the backing
  3. * device
  4. *
  5. * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
  6. * Copyright 2012 Google, Inc.
  7. */
  8. #include "bcache.h"
  9. #include "btree.h"
  10. #include "debug.h"
  11. #include "writeback.h"
  12. #include <linux/delay.h>
  13. #include <linux/freezer.h>
  14. #include <linux/kthread.h>
  15. #include <trace/events/bcache.h>
  16. /* Rate limiting */
  17. static void __update_writeback_rate(struct cached_dev *dc)
  18. {
  19. struct cache_set *c = dc->disk.c;
  20. uint64_t cache_sectors = c->nbuckets * c->sb.bucket_size -
  21. bcache_flash_devs_sectors_dirty(c);
  22. uint64_t cache_dirty_target =
  23. div_u64(cache_sectors * dc->writeback_percent, 100);
  24. int64_t target = div64_u64(cache_dirty_target * bdev_sectors(dc->bdev),
  25. c->cached_dev_sectors);
  26. /* PD controller */
  27. int64_t dirty = bcache_dev_sectors_dirty(&dc->disk);
  28. int64_t derivative = dirty - dc->disk.sectors_dirty_last;
  29. int64_t proportional = dirty - target;
  30. int64_t change;
  31. dc->disk.sectors_dirty_last = dirty;
  32. /* Scale to sectors per second */
  33. proportional *= dc->writeback_rate_update_seconds;
  34. proportional = div_s64(proportional, dc->writeback_rate_p_term_inverse);
  35. derivative = div_s64(derivative, dc->writeback_rate_update_seconds);
  36. derivative = ewma_add(dc->disk.sectors_dirty_derivative, derivative,
  37. (dc->writeback_rate_d_term /
  38. dc->writeback_rate_update_seconds) ?: 1, 0);
  39. derivative *= dc->writeback_rate_d_term;
  40. derivative = div_s64(derivative, dc->writeback_rate_p_term_inverse);
  41. change = proportional + derivative;
  42. /* Don't increase writeback rate if the device isn't keeping up */
  43. if (change > 0 &&
  44. time_after64(local_clock(),
  45. dc->writeback_rate.next + NSEC_PER_MSEC))
  46. change = 0;
  47. dc->writeback_rate.rate =
  48. clamp_t(int64_t, (int64_t) dc->writeback_rate.rate + change,
  49. 1, NSEC_PER_MSEC);
  50. dc->writeback_rate_proportional = proportional;
  51. dc->writeback_rate_derivative = derivative;
  52. dc->writeback_rate_change = change;
  53. dc->writeback_rate_target = target;
  54. }
  55. static void update_writeback_rate(struct work_struct *work)
  56. {
  57. struct cached_dev *dc = container_of(to_delayed_work(work),
  58. struct cached_dev,
  59. writeback_rate_update);
  60. down_read(&dc->writeback_lock);
  61. if (atomic_read(&dc->has_dirty) &&
  62. dc->writeback_percent)
  63. __update_writeback_rate(dc);
  64. up_read(&dc->writeback_lock);
  65. schedule_delayed_work(&dc->writeback_rate_update,
  66. dc->writeback_rate_update_seconds * HZ);
  67. }
  68. static unsigned writeback_delay(struct cached_dev *dc, unsigned sectors)
  69. {
  70. if (test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) ||
  71. !dc->writeback_percent)
  72. return 0;
  73. return bch_next_delay(&dc->writeback_rate, sectors);
  74. }
  75. struct dirty_io {
  76. struct closure cl;
  77. struct cached_dev *dc;
  78. struct bio bio;
  79. };
  80. static void dirty_init(struct keybuf_key *w)
  81. {
  82. struct dirty_io *io = w->private;
  83. struct bio *bio = &io->bio;
  84. bio_init(bio);
  85. if (!io->dc->writeback_percent)
  86. bio_set_prio(bio, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0));
  87. bio->bi_iter.bi_size = KEY_SIZE(&w->key) << 9;
  88. bio->bi_max_vecs = DIV_ROUND_UP(KEY_SIZE(&w->key), PAGE_SECTORS);
  89. bio->bi_private = w;
  90. bio->bi_io_vec = bio->bi_inline_vecs;
  91. bch_bio_map(bio, NULL);
  92. }
  93. static void dirty_io_destructor(struct closure *cl)
  94. {
  95. struct dirty_io *io = container_of(cl, struct dirty_io, cl);
  96. kfree(io);
  97. }
  98. static void write_dirty_finish(struct closure *cl)
  99. {
  100. struct dirty_io *io = container_of(cl, struct dirty_io, cl);
  101. struct keybuf_key *w = io->bio.bi_private;
  102. struct cached_dev *dc = io->dc;
  103. struct bio_vec *bv;
  104. int i;
  105. bio_for_each_segment_all(bv, &io->bio, i)
  106. __free_page(bv->bv_page);
  107. /* This is kind of a dumb way of signalling errors. */
  108. if (KEY_DIRTY(&w->key)) {
  109. int ret;
  110. unsigned i;
  111. struct keylist keys;
  112. bch_keylist_init(&keys);
  113. bkey_copy(keys.top, &w->key);
  114. SET_KEY_DIRTY(keys.top, false);
  115. bch_keylist_push(&keys);
  116. for (i = 0; i < KEY_PTRS(&w->key); i++)
  117. atomic_inc(&PTR_BUCKET(dc->disk.c, &w->key, i)->pin);
  118. ret = bch_btree_insert(dc->disk.c, &keys, NULL, &w->key);
  119. if (ret)
  120. trace_bcache_writeback_collision(&w->key);
  121. atomic_long_inc(ret
  122. ? &dc->disk.c->writeback_keys_failed
  123. : &dc->disk.c->writeback_keys_done);
  124. }
  125. bch_keybuf_del(&dc->writeback_keys, w);
  126. up(&dc->in_flight);
  127. closure_return_with_destructor(cl, dirty_io_destructor);
  128. }
  129. static void dirty_endio(struct bio *bio)
  130. {
  131. struct keybuf_key *w = bio->bi_private;
  132. struct dirty_io *io = w->private;
  133. if (bio->bi_error)
  134. SET_KEY_DIRTY(&w->key, false);
  135. closure_put(&io->cl);
  136. }
  137. static void write_dirty(struct closure *cl)
  138. {
  139. struct dirty_io *io = container_of(cl, struct dirty_io, cl);
  140. struct keybuf_key *w = io->bio.bi_private;
  141. dirty_init(w);
  142. io->bio.bi_rw = WRITE;
  143. io->bio.bi_iter.bi_sector = KEY_START(&w->key);
  144. io->bio.bi_bdev = io->dc->bdev;
  145. io->bio.bi_end_io = dirty_endio;
  146. closure_bio_submit(&io->bio, cl);
  147. continue_at(cl, write_dirty_finish, io->dc->writeback_write_wq);
  148. }
  149. static void read_dirty_endio(struct bio *bio)
  150. {
  151. struct keybuf_key *w = bio->bi_private;
  152. struct dirty_io *io = w->private;
  153. bch_count_io_errors(PTR_CACHE(io->dc->disk.c, &w->key, 0),
  154. bio->bi_error, "reading dirty data from cache");
  155. dirty_endio(bio);
  156. }
  157. static void read_dirty_submit(struct closure *cl)
  158. {
  159. struct dirty_io *io = container_of(cl, struct dirty_io, cl);
  160. closure_bio_submit(&io->bio, cl);
  161. continue_at(cl, write_dirty, io->dc->writeback_write_wq);
  162. }
  163. static void read_dirty(struct cached_dev *dc)
  164. {
  165. unsigned delay = 0;
  166. struct keybuf_key *w;
  167. struct dirty_io *io;
  168. struct closure cl;
  169. closure_init_stack(&cl);
  170. /*
  171. * XXX: if we error, background writeback just spins. Should use some
  172. * mempools.
  173. */
  174. while (!kthread_should_stop()) {
  175. try_to_freeze();
  176. w = bch_keybuf_next(&dc->writeback_keys);
  177. if (!w)
  178. break;
  179. BUG_ON(ptr_stale(dc->disk.c, &w->key, 0));
  180. if (KEY_START(&w->key) != dc->last_read ||
  181. jiffies_to_msecs(delay) > 50)
  182. while (!kthread_should_stop() && delay)
  183. delay = schedule_timeout_interruptible(delay);
  184. dc->last_read = KEY_OFFSET(&w->key);
  185. io = kzalloc(sizeof(struct dirty_io) + sizeof(struct bio_vec)
  186. * DIV_ROUND_UP(KEY_SIZE(&w->key), PAGE_SECTORS),
  187. GFP_KERNEL);
  188. if (!io)
  189. goto err;
  190. w->private = io;
  191. io->dc = dc;
  192. dirty_init(w);
  193. io->bio.bi_iter.bi_sector = PTR_OFFSET(&w->key, 0);
  194. io->bio.bi_bdev = PTR_CACHE(dc->disk.c,
  195. &w->key, 0)->bdev;
  196. io->bio.bi_rw = READ;
  197. io->bio.bi_end_io = read_dirty_endio;
  198. if (bio_alloc_pages(&io->bio, GFP_KERNEL))
  199. goto err_free;
  200. trace_bcache_writeback(&w->key);
  201. down(&dc->in_flight);
  202. closure_call(&io->cl, read_dirty_submit, NULL, &cl);
  203. delay = writeback_delay(dc, KEY_SIZE(&w->key));
  204. }
  205. if (0) {
  206. err_free:
  207. kfree(w->private);
  208. err:
  209. bch_keybuf_del(&dc->writeback_keys, w);
  210. }
  211. /*
  212. * Wait for outstanding writeback IOs to finish (and keybuf slots to be
  213. * freed) before refilling again
  214. */
  215. closure_sync(&cl);
  216. }
  217. /* Scan for dirty data */
  218. void bcache_dev_sectors_dirty_add(struct cache_set *c, unsigned inode,
  219. uint64_t offset, int nr_sectors)
  220. {
  221. struct bcache_device *d = c->devices[inode];
  222. unsigned stripe_offset, stripe, sectors_dirty;
  223. if (!d)
  224. return;
  225. stripe = offset_to_stripe(d, offset);
  226. stripe_offset = offset & (d->stripe_size - 1);
  227. while (nr_sectors) {
  228. int s = min_t(unsigned, abs(nr_sectors),
  229. d->stripe_size - stripe_offset);
  230. if (nr_sectors < 0)
  231. s = -s;
  232. if (stripe >= d->nr_stripes)
  233. return;
  234. sectors_dirty = atomic_add_return(s,
  235. d->stripe_sectors_dirty + stripe);
  236. if (sectors_dirty == d->stripe_size)
  237. set_bit(stripe, d->full_dirty_stripes);
  238. else
  239. clear_bit(stripe, d->full_dirty_stripes);
  240. nr_sectors -= s;
  241. stripe_offset = 0;
  242. stripe++;
  243. }
  244. }
  245. static bool dirty_pred(struct keybuf *buf, struct bkey *k)
  246. {
  247. struct cached_dev *dc = container_of(buf, struct cached_dev, writeback_keys);
  248. BUG_ON(KEY_INODE(k) != dc->disk.id);
  249. return KEY_DIRTY(k);
  250. }
  251. static void refill_full_stripes(struct cached_dev *dc)
  252. {
  253. struct keybuf *buf = &dc->writeback_keys;
  254. unsigned start_stripe, stripe, next_stripe;
  255. bool wrapped = false;
  256. stripe = offset_to_stripe(&dc->disk, KEY_OFFSET(&buf->last_scanned));
  257. if (stripe >= dc->disk.nr_stripes)
  258. stripe = 0;
  259. start_stripe = stripe;
  260. while (1) {
  261. stripe = find_next_bit(dc->disk.full_dirty_stripes,
  262. dc->disk.nr_stripes, stripe);
  263. if (stripe == dc->disk.nr_stripes)
  264. goto next;
  265. next_stripe = find_next_zero_bit(dc->disk.full_dirty_stripes,
  266. dc->disk.nr_stripes, stripe);
  267. buf->last_scanned = KEY(dc->disk.id,
  268. stripe * dc->disk.stripe_size, 0);
  269. bch_refill_keybuf(dc->disk.c, buf,
  270. &KEY(dc->disk.id,
  271. next_stripe * dc->disk.stripe_size, 0),
  272. dirty_pred);
  273. if (array_freelist_empty(&buf->freelist))
  274. return;
  275. stripe = next_stripe;
  276. next:
  277. if (wrapped && stripe > start_stripe)
  278. return;
  279. if (stripe == dc->disk.nr_stripes) {
  280. stripe = 0;
  281. wrapped = true;
  282. }
  283. }
  284. }
  285. /*
  286. * Returns true if we scanned the entire disk
  287. */
  288. static bool refill_dirty(struct cached_dev *dc)
  289. {
  290. struct keybuf *buf = &dc->writeback_keys;
  291. struct bkey start = KEY(dc->disk.id, 0, 0);
  292. struct bkey end = KEY(dc->disk.id, MAX_KEY_OFFSET, 0);
  293. struct bkey start_pos;
  294. /*
  295. * make sure keybuf pos is inside the range for this disk - at bringup
  296. * we might not be attached yet so this disk's inode nr isn't
  297. * initialized then
  298. */
  299. if (bkey_cmp(&buf->last_scanned, &start) < 0 ||
  300. bkey_cmp(&buf->last_scanned, &end) > 0)
  301. buf->last_scanned = start;
  302. if (dc->partial_stripes_expensive) {
  303. refill_full_stripes(dc);
  304. if (array_freelist_empty(&buf->freelist))
  305. return false;
  306. }
  307. start_pos = buf->last_scanned;
  308. bch_refill_keybuf(dc->disk.c, buf, &end, dirty_pred);
  309. if (bkey_cmp(&buf->last_scanned, &end) < 0)
  310. return false;
  311. /*
  312. * If we get to the end start scanning again from the beginning, and
  313. * only scan up to where we initially started scanning from:
  314. */
  315. buf->last_scanned = start;
  316. bch_refill_keybuf(dc->disk.c, buf, &start_pos, dirty_pred);
  317. return bkey_cmp(&buf->last_scanned, &start_pos) >= 0;
  318. }
  319. static int bch_writeback_thread(void *arg)
  320. {
  321. struct cached_dev *dc = arg;
  322. bool searched_full_index;
  323. while (!kthread_should_stop()) {
  324. down_write(&dc->writeback_lock);
  325. set_current_state(TASK_INTERRUPTIBLE);
  326. /*
  327. * If the bache device is detaching, skip here and continue
  328. * to perform writeback. Otherwise, if no dirty data on cache,
  329. * or there is dirty data on cache but writeback is disabled,
  330. * the writeback thread should sleep here and wait for others
  331. * to wake up it.
  332. */
  333. if (!test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) &&
  334. (!atomic_read(&dc->has_dirty) || !dc->writeback_running)) {
  335. up_write(&dc->writeback_lock);
  336. if (kthread_should_stop()) {
  337. set_current_state(TASK_RUNNING);
  338. return 0;
  339. }
  340. try_to_freeze();
  341. schedule();
  342. continue;
  343. }
  344. set_current_state(TASK_RUNNING);
  345. searched_full_index = refill_dirty(dc);
  346. if (searched_full_index &&
  347. RB_EMPTY_ROOT(&dc->writeback_keys.keys)) {
  348. atomic_set(&dc->has_dirty, 0);
  349. cached_dev_put(dc);
  350. SET_BDEV_STATE(&dc->sb, BDEV_STATE_CLEAN);
  351. bch_write_bdev_super(dc, NULL);
  352. /*
  353. * If bcache device is detaching via sysfs interface,
  354. * writeback thread should stop after there is no dirty
  355. * data on cache. BCACHE_DEV_DETACHING flag is set in
  356. * bch_cached_dev_detach().
  357. */
  358. if (test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags)) {
  359. up_write(&dc->writeback_lock);
  360. break;
  361. }
  362. }
  363. up_write(&dc->writeback_lock);
  364. bch_ratelimit_reset(&dc->writeback_rate);
  365. read_dirty(dc);
  366. if (searched_full_index) {
  367. unsigned delay = dc->writeback_delay * HZ;
  368. while (delay &&
  369. !kthread_should_stop() &&
  370. !test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags))
  371. delay = schedule_timeout_interruptible(delay);
  372. }
  373. }
  374. return 0;
  375. }
  376. /* Init */
  377. struct sectors_dirty_init {
  378. struct btree_op op;
  379. unsigned inode;
  380. };
  381. static int sectors_dirty_init_fn(struct btree_op *_op, struct btree *b,
  382. struct bkey *k)
  383. {
  384. struct sectors_dirty_init *op = container_of(_op,
  385. struct sectors_dirty_init, op);
  386. if (KEY_INODE(k) > op->inode)
  387. return MAP_DONE;
  388. if (KEY_DIRTY(k))
  389. bcache_dev_sectors_dirty_add(b->c, KEY_INODE(k),
  390. KEY_START(k), KEY_SIZE(k));
  391. return MAP_CONTINUE;
  392. }
  393. void bch_sectors_dirty_init(struct bcache_device *d)
  394. {
  395. struct sectors_dirty_init op;
  396. bch_btree_op_init(&op.op, -1);
  397. op.inode = d->id;
  398. bch_btree_map_keys(&op.op, d->c, &KEY(op.inode, 0, 0),
  399. sectors_dirty_init_fn, 0);
  400. d->sectors_dirty_last = bcache_dev_sectors_dirty(d);
  401. }
  402. void bch_cached_dev_writeback_init(struct cached_dev *dc)
  403. {
  404. sema_init(&dc->in_flight, 64);
  405. init_rwsem(&dc->writeback_lock);
  406. bch_keybuf_init(&dc->writeback_keys);
  407. dc->writeback_metadata = true;
  408. dc->writeback_running = true;
  409. dc->writeback_percent = 10;
  410. dc->writeback_delay = 30;
  411. dc->writeback_rate.rate = 1024;
  412. dc->writeback_rate_update_seconds = 5;
  413. dc->writeback_rate_d_term = 30;
  414. dc->writeback_rate_p_term_inverse = 6000;
  415. INIT_DELAYED_WORK(&dc->writeback_rate_update, update_writeback_rate);
  416. }
  417. int bch_cached_dev_writeback_start(struct cached_dev *dc)
  418. {
  419. dc->writeback_write_wq = alloc_workqueue("bcache_writeback_wq",
  420. WQ_MEM_RECLAIM, 0);
  421. if (!dc->writeback_write_wq)
  422. return -ENOMEM;
  423. dc->writeback_thread = kthread_create(bch_writeback_thread, dc,
  424. "bcache_writeback");
  425. if (IS_ERR(dc->writeback_thread))
  426. return PTR_ERR(dc->writeback_thread);
  427. schedule_delayed_work(&dc->writeback_rate_update,
  428. dc->writeback_rate_update_seconds * HZ);
  429. bch_writeback_queue(dc);
  430. return 0;
  431. }