super.c 51 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153
  1. /*
  2. * bcache setup/teardown code, and some metadata io - read a superblock and
  3. * figure out what to do with it.
  4. *
  5. * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
  6. * Copyright 2012 Google, Inc.
  7. */
  8. #include "bcache.h"
  9. #include "btree.h"
  10. #include "debug.h"
  11. #include "extents.h"
  12. #include "request.h"
  13. #include "writeback.h"
  14. #include <linux/blkdev.h>
  15. #include <linux/buffer_head.h>
  16. #include <linux/debugfs.h>
  17. #include <linux/genhd.h>
  18. #include <linux/idr.h>
  19. #include <linux/kthread.h>
  20. #include <linux/module.h>
  21. #include <linux/random.h>
  22. #include <linux/reboot.h>
  23. #include <linux/sysfs.h>
  24. MODULE_LICENSE("GPL");
  25. MODULE_AUTHOR("Kent Overstreet <kent.overstreet@gmail.com>");
  26. static const char bcache_magic[] = {
  27. 0xc6, 0x85, 0x73, 0xf6, 0x4e, 0x1a, 0x45, 0xca,
  28. 0x82, 0x65, 0xf5, 0x7f, 0x48, 0xba, 0x6d, 0x81
  29. };
  30. static const char invalid_uuid[] = {
  31. 0xa0, 0x3e, 0xf8, 0xed, 0x3e, 0xe1, 0xb8, 0x78,
  32. 0xc8, 0x50, 0xfc, 0x5e, 0xcb, 0x16, 0xcd, 0x99
  33. };
  34. /* Default is -1; we skip past it for struct cached_dev's cache mode */
  35. const char * const bch_cache_modes[] = {
  36. "default",
  37. "writethrough",
  38. "writeback",
  39. "writearound",
  40. "none",
  41. NULL
  42. };
  43. static struct kobject *bcache_kobj;
  44. struct mutex bch_register_lock;
  45. LIST_HEAD(bch_cache_sets);
  46. static LIST_HEAD(uncached_devices);
  47. static int bcache_major;
  48. static DEFINE_IDA(bcache_minor);
  49. static wait_queue_head_t unregister_wait;
  50. struct workqueue_struct *bcache_wq;
  51. #define BTREE_MAX_PAGES (256 * 1024 / PAGE_SIZE)
  52. /* Superblock */
  53. static const char *read_super(struct cache_sb *sb, struct block_device *bdev,
  54. struct page **res)
  55. {
  56. const char *err;
  57. struct cache_sb *s;
  58. struct buffer_head *bh = __bread(bdev, 1, SB_SIZE);
  59. unsigned i;
  60. if (!bh)
  61. return "IO error";
  62. s = (struct cache_sb *) bh->b_data;
  63. sb->offset = le64_to_cpu(s->offset);
  64. sb->version = le64_to_cpu(s->version);
  65. memcpy(sb->magic, s->magic, 16);
  66. memcpy(sb->uuid, s->uuid, 16);
  67. memcpy(sb->set_uuid, s->set_uuid, 16);
  68. memcpy(sb->label, s->label, SB_LABEL_SIZE);
  69. sb->flags = le64_to_cpu(s->flags);
  70. sb->seq = le64_to_cpu(s->seq);
  71. sb->last_mount = le32_to_cpu(s->last_mount);
  72. sb->first_bucket = le16_to_cpu(s->first_bucket);
  73. sb->keys = le16_to_cpu(s->keys);
  74. for (i = 0; i < SB_JOURNAL_BUCKETS; i++)
  75. sb->d[i] = le64_to_cpu(s->d[i]);
  76. pr_debug("read sb version %llu, flags %llu, seq %llu, journal size %u",
  77. sb->version, sb->flags, sb->seq, sb->keys);
  78. err = "Not a bcache superblock";
  79. if (sb->offset != SB_SECTOR)
  80. goto err;
  81. if (memcmp(sb->magic, bcache_magic, 16))
  82. goto err;
  83. err = "Too many journal buckets";
  84. if (sb->keys > SB_JOURNAL_BUCKETS)
  85. goto err;
  86. err = "Bad checksum";
  87. if (s->csum != csum_set(s))
  88. goto err;
  89. err = "Bad UUID";
  90. if (bch_is_zero(sb->uuid, 16))
  91. goto err;
  92. sb->block_size = le16_to_cpu(s->block_size);
  93. err = "Superblock block size smaller than device block size";
  94. if (sb->block_size << 9 < bdev_logical_block_size(bdev))
  95. goto err;
  96. switch (sb->version) {
  97. case BCACHE_SB_VERSION_BDEV:
  98. sb->data_offset = BDEV_DATA_START_DEFAULT;
  99. break;
  100. case BCACHE_SB_VERSION_BDEV_WITH_OFFSET:
  101. sb->data_offset = le64_to_cpu(s->data_offset);
  102. err = "Bad data offset";
  103. if (sb->data_offset < BDEV_DATA_START_DEFAULT)
  104. goto err;
  105. break;
  106. case BCACHE_SB_VERSION_CDEV:
  107. case BCACHE_SB_VERSION_CDEV_WITH_UUID:
  108. sb->nbuckets = le64_to_cpu(s->nbuckets);
  109. sb->block_size = le16_to_cpu(s->block_size);
  110. sb->bucket_size = le16_to_cpu(s->bucket_size);
  111. sb->nr_in_set = le16_to_cpu(s->nr_in_set);
  112. sb->nr_this_dev = le16_to_cpu(s->nr_this_dev);
  113. err = "Too many buckets";
  114. if (sb->nbuckets > LONG_MAX)
  115. goto err;
  116. err = "Not enough buckets";
  117. if (sb->nbuckets < 1 << 7)
  118. goto err;
  119. err = "Bad block/bucket size";
  120. if (!is_power_of_2(sb->block_size) ||
  121. sb->block_size > PAGE_SECTORS ||
  122. !is_power_of_2(sb->bucket_size) ||
  123. sb->bucket_size < PAGE_SECTORS)
  124. goto err;
  125. err = "Invalid superblock: device too small";
  126. if (get_capacity(bdev->bd_disk) < sb->bucket_size * sb->nbuckets)
  127. goto err;
  128. err = "Bad UUID";
  129. if (bch_is_zero(sb->set_uuid, 16))
  130. goto err;
  131. err = "Bad cache device number in set";
  132. if (!sb->nr_in_set ||
  133. sb->nr_in_set <= sb->nr_this_dev ||
  134. sb->nr_in_set > MAX_CACHES_PER_SET)
  135. goto err;
  136. err = "Journal buckets not sequential";
  137. for (i = 0; i < sb->keys; i++)
  138. if (sb->d[i] != sb->first_bucket + i)
  139. goto err;
  140. err = "Too many journal buckets";
  141. if (sb->first_bucket + sb->keys > sb->nbuckets)
  142. goto err;
  143. err = "Invalid superblock: first bucket comes before end of super";
  144. if (sb->first_bucket * sb->bucket_size < 16)
  145. goto err;
  146. break;
  147. default:
  148. err = "Unsupported superblock version";
  149. goto err;
  150. }
  151. sb->last_mount = get_seconds();
  152. err = NULL;
  153. get_page(bh->b_page);
  154. *res = bh->b_page;
  155. err:
  156. put_bh(bh);
  157. return err;
  158. }
  159. static void write_bdev_super_endio(struct bio *bio)
  160. {
  161. struct cached_dev *dc = bio->bi_private;
  162. /* XXX: error checking */
  163. closure_put(&dc->sb_write);
  164. }
  165. static void __write_super(struct cache_sb *sb, struct bio *bio)
  166. {
  167. struct cache_sb *out = page_address(bio->bi_io_vec[0].bv_page);
  168. unsigned i;
  169. bio->bi_iter.bi_sector = SB_SECTOR;
  170. bio->bi_rw = REQ_SYNC|REQ_META;
  171. bio->bi_iter.bi_size = SB_SIZE;
  172. bch_bio_map(bio, NULL);
  173. out->offset = cpu_to_le64(sb->offset);
  174. out->version = cpu_to_le64(sb->version);
  175. memcpy(out->uuid, sb->uuid, 16);
  176. memcpy(out->set_uuid, sb->set_uuid, 16);
  177. memcpy(out->label, sb->label, SB_LABEL_SIZE);
  178. out->flags = cpu_to_le64(sb->flags);
  179. out->seq = cpu_to_le64(sb->seq);
  180. out->last_mount = cpu_to_le32(sb->last_mount);
  181. out->first_bucket = cpu_to_le16(sb->first_bucket);
  182. out->keys = cpu_to_le16(sb->keys);
  183. for (i = 0; i < sb->keys; i++)
  184. out->d[i] = cpu_to_le64(sb->d[i]);
  185. out->csum = csum_set(out);
  186. pr_debug("ver %llu, flags %llu, seq %llu",
  187. sb->version, sb->flags, sb->seq);
  188. submit_bio(REQ_WRITE, bio);
  189. }
  190. static void bch_write_bdev_super_unlock(struct closure *cl)
  191. {
  192. struct cached_dev *dc = container_of(cl, struct cached_dev, sb_write);
  193. up(&dc->sb_write_mutex);
  194. }
  195. void bch_write_bdev_super(struct cached_dev *dc, struct closure *parent)
  196. {
  197. struct closure *cl = &dc->sb_write;
  198. struct bio *bio = &dc->sb_bio;
  199. down(&dc->sb_write_mutex);
  200. closure_init(cl, parent);
  201. bio_reset(bio);
  202. bio->bi_bdev = dc->bdev;
  203. bio->bi_end_io = write_bdev_super_endio;
  204. bio->bi_private = dc;
  205. closure_get(cl);
  206. __write_super(&dc->sb, bio);
  207. closure_return_with_destructor(cl, bch_write_bdev_super_unlock);
  208. }
  209. static void write_super_endio(struct bio *bio)
  210. {
  211. struct cache *ca = bio->bi_private;
  212. bch_count_io_errors(ca, bio->bi_error, "writing superblock");
  213. closure_put(&ca->set->sb_write);
  214. }
  215. static void bcache_write_super_unlock(struct closure *cl)
  216. {
  217. struct cache_set *c = container_of(cl, struct cache_set, sb_write);
  218. up(&c->sb_write_mutex);
  219. }
  220. void bcache_write_super(struct cache_set *c)
  221. {
  222. struct closure *cl = &c->sb_write;
  223. struct cache *ca;
  224. unsigned i;
  225. down(&c->sb_write_mutex);
  226. closure_init(cl, &c->cl);
  227. c->sb.seq++;
  228. for_each_cache(ca, c, i) {
  229. struct bio *bio = &ca->sb_bio;
  230. ca->sb.version = BCACHE_SB_VERSION_CDEV_WITH_UUID;
  231. ca->sb.seq = c->sb.seq;
  232. ca->sb.last_mount = c->sb.last_mount;
  233. SET_CACHE_SYNC(&ca->sb, CACHE_SYNC(&c->sb));
  234. bio_reset(bio);
  235. bio->bi_bdev = ca->bdev;
  236. bio->bi_end_io = write_super_endio;
  237. bio->bi_private = ca;
  238. closure_get(cl);
  239. __write_super(&ca->sb, bio);
  240. }
  241. closure_return_with_destructor(cl, bcache_write_super_unlock);
  242. }
  243. /* UUID io */
  244. static void uuid_endio(struct bio *bio)
  245. {
  246. struct closure *cl = bio->bi_private;
  247. struct cache_set *c = container_of(cl, struct cache_set, uuid_write);
  248. cache_set_err_on(bio->bi_error, c, "accessing uuids");
  249. bch_bbio_free(bio, c);
  250. closure_put(cl);
  251. }
  252. static void uuid_io_unlock(struct closure *cl)
  253. {
  254. struct cache_set *c = container_of(cl, struct cache_set, uuid_write);
  255. up(&c->uuid_write_mutex);
  256. }
  257. static void uuid_io(struct cache_set *c, unsigned long rw,
  258. struct bkey *k, struct closure *parent)
  259. {
  260. struct closure *cl = &c->uuid_write;
  261. struct uuid_entry *u;
  262. unsigned i;
  263. char buf[80];
  264. BUG_ON(!parent);
  265. down(&c->uuid_write_mutex);
  266. closure_init(cl, parent);
  267. for (i = 0; i < KEY_PTRS(k); i++) {
  268. struct bio *bio = bch_bbio_alloc(c);
  269. bio->bi_rw = REQ_SYNC|REQ_META|rw;
  270. bio->bi_iter.bi_size = KEY_SIZE(k) << 9;
  271. bio->bi_end_io = uuid_endio;
  272. bio->bi_private = cl;
  273. bch_bio_map(bio, c->uuids);
  274. bch_submit_bbio(bio, c, k, i);
  275. if (!(rw & WRITE))
  276. break;
  277. }
  278. bch_extent_to_text(buf, sizeof(buf), k);
  279. pr_debug("%s UUIDs at %s", rw & REQ_WRITE ? "wrote" : "read", buf);
  280. for (u = c->uuids; u < c->uuids + c->nr_uuids; u++)
  281. if (!bch_is_zero(u->uuid, 16))
  282. pr_debug("Slot %zi: %pU: %s: 1st: %u last: %u inv: %u",
  283. u - c->uuids, u->uuid, u->label,
  284. u->first_reg, u->last_reg, u->invalidated);
  285. closure_return_with_destructor(cl, uuid_io_unlock);
  286. }
  287. static char *uuid_read(struct cache_set *c, struct jset *j, struct closure *cl)
  288. {
  289. struct bkey *k = &j->uuid_bucket;
  290. if (__bch_btree_ptr_invalid(c, k))
  291. return "bad uuid pointer";
  292. bkey_copy(&c->uuid_bucket, k);
  293. uuid_io(c, READ_SYNC, k, cl);
  294. if (j->version < BCACHE_JSET_VERSION_UUIDv1) {
  295. struct uuid_entry_v0 *u0 = (void *) c->uuids;
  296. struct uuid_entry *u1 = (void *) c->uuids;
  297. int i;
  298. closure_sync(cl);
  299. /*
  300. * Since the new uuid entry is bigger than the old, we have to
  301. * convert starting at the highest memory address and work down
  302. * in order to do it in place
  303. */
  304. for (i = c->nr_uuids - 1;
  305. i >= 0;
  306. --i) {
  307. memcpy(u1[i].uuid, u0[i].uuid, 16);
  308. memcpy(u1[i].label, u0[i].label, 32);
  309. u1[i].first_reg = u0[i].first_reg;
  310. u1[i].last_reg = u0[i].last_reg;
  311. u1[i].invalidated = u0[i].invalidated;
  312. u1[i].flags = 0;
  313. u1[i].sectors = 0;
  314. }
  315. }
  316. return NULL;
  317. }
  318. static int __uuid_write(struct cache_set *c)
  319. {
  320. BKEY_PADDED(key) k;
  321. struct closure cl;
  322. closure_init_stack(&cl);
  323. lockdep_assert_held(&bch_register_lock);
  324. if (bch_bucket_alloc_set(c, RESERVE_BTREE, &k.key, 1, true))
  325. return 1;
  326. SET_KEY_SIZE(&k.key, c->sb.bucket_size);
  327. uuid_io(c, REQ_WRITE, &k.key, &cl);
  328. closure_sync(&cl);
  329. bkey_copy(&c->uuid_bucket, &k.key);
  330. bkey_put(c, &k.key);
  331. return 0;
  332. }
  333. int bch_uuid_write(struct cache_set *c)
  334. {
  335. int ret = __uuid_write(c);
  336. if (!ret)
  337. bch_journal_meta(c, NULL);
  338. return ret;
  339. }
  340. static struct uuid_entry *uuid_find(struct cache_set *c, const char *uuid)
  341. {
  342. struct uuid_entry *u;
  343. for (u = c->uuids;
  344. u < c->uuids + c->nr_uuids; u++)
  345. if (!memcmp(u->uuid, uuid, 16))
  346. return u;
  347. return NULL;
  348. }
  349. static struct uuid_entry *uuid_find_empty(struct cache_set *c)
  350. {
  351. static const char zero_uuid[16] = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";
  352. return uuid_find(c, zero_uuid);
  353. }
  354. /*
  355. * Bucket priorities/gens:
  356. *
  357. * For each bucket, we store on disk its
  358. * 8 bit gen
  359. * 16 bit priority
  360. *
  361. * See alloc.c for an explanation of the gen. The priority is used to implement
  362. * lru (and in the future other) cache replacement policies; for most purposes
  363. * it's just an opaque integer.
  364. *
  365. * The gens and the priorities don't have a whole lot to do with each other, and
  366. * it's actually the gens that must be written out at specific times - it's no
  367. * big deal if the priorities don't get written, if we lose them we just reuse
  368. * buckets in suboptimal order.
  369. *
  370. * On disk they're stored in a packed array, and in as many buckets are required
  371. * to fit them all. The buckets we use to store them form a list; the journal
  372. * header points to the first bucket, the first bucket points to the second
  373. * bucket, et cetera.
  374. *
  375. * This code is used by the allocation code; periodically (whenever it runs out
  376. * of buckets to allocate from) the allocation code will invalidate some
  377. * buckets, but it can't use those buckets until their new gens are safely on
  378. * disk.
  379. */
  380. static void prio_endio(struct bio *bio)
  381. {
  382. struct cache *ca = bio->bi_private;
  383. cache_set_err_on(bio->bi_error, ca->set, "accessing priorities");
  384. bch_bbio_free(bio, ca->set);
  385. closure_put(&ca->prio);
  386. }
  387. static void prio_io(struct cache *ca, uint64_t bucket, unsigned long rw)
  388. {
  389. struct closure *cl = &ca->prio;
  390. struct bio *bio = bch_bbio_alloc(ca->set);
  391. closure_init_stack(cl);
  392. bio->bi_iter.bi_sector = bucket * ca->sb.bucket_size;
  393. bio->bi_bdev = ca->bdev;
  394. bio->bi_rw = REQ_SYNC|REQ_META|rw;
  395. bio->bi_iter.bi_size = bucket_bytes(ca);
  396. bio->bi_end_io = prio_endio;
  397. bio->bi_private = ca;
  398. bch_bio_map(bio, ca->disk_buckets);
  399. closure_bio_submit(bio, &ca->prio);
  400. closure_sync(cl);
  401. }
  402. void bch_prio_write(struct cache *ca)
  403. {
  404. int i;
  405. struct bucket *b;
  406. struct closure cl;
  407. closure_init_stack(&cl);
  408. lockdep_assert_held(&ca->set->bucket_lock);
  409. ca->disk_buckets->seq++;
  410. atomic_long_add(ca->sb.bucket_size * prio_buckets(ca),
  411. &ca->meta_sectors_written);
  412. //pr_debug("free %zu, free_inc %zu, unused %zu", fifo_used(&ca->free),
  413. // fifo_used(&ca->free_inc), fifo_used(&ca->unused));
  414. for (i = prio_buckets(ca) - 1; i >= 0; --i) {
  415. long bucket;
  416. struct prio_set *p = ca->disk_buckets;
  417. struct bucket_disk *d = p->data;
  418. struct bucket_disk *end = d + prios_per_bucket(ca);
  419. for (b = ca->buckets + i * prios_per_bucket(ca);
  420. b < ca->buckets + ca->sb.nbuckets && d < end;
  421. b++, d++) {
  422. d->prio = cpu_to_le16(b->prio);
  423. d->gen = b->gen;
  424. }
  425. p->next_bucket = ca->prio_buckets[i + 1];
  426. p->magic = pset_magic(&ca->sb);
  427. p->csum = bch_crc64(&p->magic, bucket_bytes(ca) - 8);
  428. bucket = bch_bucket_alloc(ca, RESERVE_PRIO, true);
  429. BUG_ON(bucket == -1);
  430. mutex_unlock(&ca->set->bucket_lock);
  431. prio_io(ca, bucket, REQ_WRITE);
  432. mutex_lock(&ca->set->bucket_lock);
  433. ca->prio_buckets[i] = bucket;
  434. atomic_dec_bug(&ca->buckets[bucket].pin);
  435. }
  436. mutex_unlock(&ca->set->bucket_lock);
  437. bch_journal_meta(ca->set, &cl);
  438. closure_sync(&cl);
  439. mutex_lock(&ca->set->bucket_lock);
  440. /*
  441. * Don't want the old priorities to get garbage collected until after we
  442. * finish writing the new ones, and they're journalled
  443. */
  444. for (i = 0; i < prio_buckets(ca); i++) {
  445. if (ca->prio_last_buckets[i])
  446. __bch_bucket_free(ca,
  447. &ca->buckets[ca->prio_last_buckets[i]]);
  448. ca->prio_last_buckets[i] = ca->prio_buckets[i];
  449. }
  450. }
  451. static void prio_read(struct cache *ca, uint64_t bucket)
  452. {
  453. struct prio_set *p = ca->disk_buckets;
  454. struct bucket_disk *d = p->data + prios_per_bucket(ca), *end = d;
  455. struct bucket *b;
  456. unsigned bucket_nr = 0;
  457. for (b = ca->buckets;
  458. b < ca->buckets + ca->sb.nbuckets;
  459. b++, d++) {
  460. if (d == end) {
  461. ca->prio_buckets[bucket_nr] = bucket;
  462. ca->prio_last_buckets[bucket_nr] = bucket;
  463. bucket_nr++;
  464. prio_io(ca, bucket, READ_SYNC);
  465. if (p->csum != bch_crc64(&p->magic, bucket_bytes(ca) - 8))
  466. pr_warn("bad csum reading priorities");
  467. if (p->magic != pset_magic(&ca->sb))
  468. pr_warn("bad magic reading priorities");
  469. bucket = p->next_bucket;
  470. d = p->data;
  471. }
  472. b->prio = le16_to_cpu(d->prio);
  473. b->gen = b->last_gc = d->gen;
  474. }
  475. }
  476. /* Bcache device */
  477. static int open_dev(struct block_device *b, fmode_t mode)
  478. {
  479. struct bcache_device *d = b->bd_disk->private_data;
  480. if (test_bit(BCACHE_DEV_CLOSING, &d->flags))
  481. return -ENXIO;
  482. closure_get(&d->cl);
  483. return 0;
  484. }
  485. static void release_dev(struct gendisk *b, fmode_t mode)
  486. {
  487. struct bcache_device *d = b->private_data;
  488. closure_put(&d->cl);
  489. }
  490. static int ioctl_dev(struct block_device *b, fmode_t mode,
  491. unsigned int cmd, unsigned long arg)
  492. {
  493. struct bcache_device *d = b->bd_disk->private_data;
  494. return d->ioctl(d, mode, cmd, arg);
  495. }
  496. static const struct block_device_operations bcache_ops = {
  497. .open = open_dev,
  498. .release = release_dev,
  499. .ioctl = ioctl_dev,
  500. .owner = THIS_MODULE,
  501. };
  502. void bcache_device_stop(struct bcache_device *d)
  503. {
  504. if (!test_and_set_bit(BCACHE_DEV_CLOSING, &d->flags))
  505. closure_queue(&d->cl);
  506. }
  507. static void bcache_device_unlink(struct bcache_device *d)
  508. {
  509. lockdep_assert_held(&bch_register_lock);
  510. if (d->c && !test_and_set_bit(BCACHE_DEV_UNLINK_DONE, &d->flags)) {
  511. unsigned i;
  512. struct cache *ca;
  513. sysfs_remove_link(&d->c->kobj, d->name);
  514. sysfs_remove_link(&d->kobj, "cache");
  515. for_each_cache(ca, d->c, i)
  516. bd_unlink_disk_holder(ca->bdev, d->disk);
  517. }
  518. }
  519. static void bcache_device_link(struct bcache_device *d, struct cache_set *c,
  520. const char *name)
  521. {
  522. unsigned i;
  523. struct cache *ca;
  524. for_each_cache(ca, d->c, i)
  525. bd_link_disk_holder(ca->bdev, d->disk);
  526. snprintf(d->name, BCACHEDEVNAME_SIZE,
  527. "%s%u", name, d->id);
  528. WARN(sysfs_create_link(&d->kobj, &c->kobj, "cache") ||
  529. sysfs_create_link(&c->kobj, &d->kobj, d->name),
  530. "Couldn't create device <-> cache set symlinks");
  531. clear_bit(BCACHE_DEV_UNLINK_DONE, &d->flags);
  532. }
  533. static void bcache_device_detach(struct bcache_device *d)
  534. {
  535. lockdep_assert_held(&bch_register_lock);
  536. if (test_bit(BCACHE_DEV_DETACHING, &d->flags)) {
  537. struct uuid_entry *u = d->c->uuids + d->id;
  538. SET_UUID_FLASH_ONLY(u, 0);
  539. memcpy(u->uuid, invalid_uuid, 16);
  540. u->invalidated = cpu_to_le32(get_seconds());
  541. bch_uuid_write(d->c);
  542. }
  543. bcache_device_unlink(d);
  544. d->c->devices[d->id] = NULL;
  545. closure_put(&d->c->caching);
  546. d->c = NULL;
  547. }
  548. static void bcache_device_attach(struct bcache_device *d, struct cache_set *c,
  549. unsigned id)
  550. {
  551. d->id = id;
  552. d->c = c;
  553. c->devices[id] = d;
  554. closure_get(&c->caching);
  555. }
  556. static void bcache_device_free(struct bcache_device *d)
  557. {
  558. lockdep_assert_held(&bch_register_lock);
  559. pr_info("%s stopped", d->disk->disk_name);
  560. if (d->c)
  561. bcache_device_detach(d);
  562. if (d->disk && d->disk->flags & GENHD_FL_UP)
  563. del_gendisk(d->disk);
  564. if (d->disk && d->disk->queue)
  565. blk_cleanup_queue(d->disk->queue);
  566. if (d->disk) {
  567. ida_simple_remove(&bcache_minor, d->disk->first_minor);
  568. put_disk(d->disk);
  569. }
  570. if (d->bio_split)
  571. bioset_free(d->bio_split);
  572. kvfree(d->full_dirty_stripes);
  573. kvfree(d->stripe_sectors_dirty);
  574. closure_debug_destroy(&d->cl);
  575. }
  576. static int bcache_device_init(struct bcache_device *d, unsigned block_size,
  577. sector_t sectors)
  578. {
  579. struct request_queue *q;
  580. size_t n;
  581. int minor;
  582. if (!d->stripe_size)
  583. d->stripe_size = 1 << 31;
  584. d->nr_stripes = DIV_ROUND_UP_ULL(sectors, d->stripe_size);
  585. if (!d->nr_stripes ||
  586. d->nr_stripes > INT_MAX ||
  587. d->nr_stripes > SIZE_MAX / sizeof(atomic_t)) {
  588. pr_err("nr_stripes too large");
  589. return -ENOMEM;
  590. }
  591. n = d->nr_stripes * sizeof(atomic_t);
  592. d->stripe_sectors_dirty = n < PAGE_SIZE << 6
  593. ? kzalloc(n, GFP_KERNEL)
  594. : vzalloc(n);
  595. if (!d->stripe_sectors_dirty)
  596. return -ENOMEM;
  597. n = BITS_TO_LONGS(d->nr_stripes) * sizeof(unsigned long);
  598. d->full_dirty_stripes = n < PAGE_SIZE << 6
  599. ? kzalloc(n, GFP_KERNEL)
  600. : vzalloc(n);
  601. if (!d->full_dirty_stripes)
  602. return -ENOMEM;
  603. minor = ida_simple_get(&bcache_minor, 0, MINORMASK + 1, GFP_KERNEL);
  604. if (minor < 0)
  605. return minor;
  606. if (!(d->bio_split = bioset_create(4, offsetof(struct bbio, bio))) ||
  607. !(d->disk = alloc_disk(1))) {
  608. ida_simple_remove(&bcache_minor, minor);
  609. return -ENOMEM;
  610. }
  611. set_capacity(d->disk, sectors);
  612. snprintf(d->disk->disk_name, DISK_NAME_LEN, "bcache%i", minor);
  613. d->disk->major = bcache_major;
  614. d->disk->first_minor = minor;
  615. d->disk->fops = &bcache_ops;
  616. d->disk->private_data = d;
  617. q = blk_alloc_queue(GFP_KERNEL);
  618. if (!q)
  619. return -ENOMEM;
  620. blk_queue_make_request(q, NULL);
  621. d->disk->queue = q;
  622. q->queuedata = d;
  623. q->backing_dev_info.congested_data = d;
  624. q->limits.max_hw_sectors = UINT_MAX;
  625. q->limits.max_sectors = UINT_MAX;
  626. q->limits.max_segment_size = UINT_MAX;
  627. q->limits.max_segments = BIO_MAX_PAGES;
  628. blk_queue_max_discard_sectors(q, UINT_MAX);
  629. q->limits.discard_granularity = 512;
  630. q->limits.io_min = block_size;
  631. q->limits.logical_block_size = block_size;
  632. q->limits.physical_block_size = block_size;
  633. set_bit(QUEUE_FLAG_NONROT, &d->disk->queue->queue_flags);
  634. clear_bit(QUEUE_FLAG_ADD_RANDOM, &d->disk->queue->queue_flags);
  635. set_bit(QUEUE_FLAG_DISCARD, &d->disk->queue->queue_flags);
  636. blk_queue_flush(q, REQ_FLUSH|REQ_FUA);
  637. return 0;
  638. }
  639. /* Cached device */
  640. static void calc_cached_dev_sectors(struct cache_set *c)
  641. {
  642. uint64_t sectors = 0;
  643. struct cached_dev *dc;
  644. list_for_each_entry(dc, &c->cached_devs, list)
  645. sectors += bdev_sectors(dc->bdev);
  646. c->cached_dev_sectors = sectors;
  647. }
  648. void bch_cached_dev_run(struct cached_dev *dc)
  649. {
  650. struct bcache_device *d = &dc->disk;
  651. char buf[SB_LABEL_SIZE + 1];
  652. char *env[] = {
  653. "DRIVER=bcache",
  654. kasprintf(GFP_KERNEL, "CACHED_UUID=%pU", dc->sb.uuid),
  655. NULL,
  656. NULL,
  657. };
  658. memcpy(buf, dc->sb.label, SB_LABEL_SIZE);
  659. buf[SB_LABEL_SIZE] = '\0';
  660. env[2] = kasprintf(GFP_KERNEL, "CACHED_LABEL=%s", buf);
  661. if (atomic_xchg(&dc->running, 1)) {
  662. kfree(env[1]);
  663. kfree(env[2]);
  664. return;
  665. }
  666. if (!d->c &&
  667. BDEV_STATE(&dc->sb) != BDEV_STATE_NONE) {
  668. struct closure cl;
  669. closure_init_stack(&cl);
  670. SET_BDEV_STATE(&dc->sb, BDEV_STATE_STALE);
  671. bch_write_bdev_super(dc, &cl);
  672. closure_sync(&cl);
  673. }
  674. add_disk(d->disk);
  675. bd_link_disk_holder(dc->bdev, dc->disk.disk);
  676. /* won't show up in the uevent file, use udevadm monitor -e instead
  677. * only class / kset properties are persistent */
  678. kobject_uevent_env(&disk_to_dev(d->disk)->kobj, KOBJ_CHANGE, env);
  679. kfree(env[1]);
  680. kfree(env[2]);
  681. if (sysfs_create_link(&d->kobj, &disk_to_dev(d->disk)->kobj, "dev") ||
  682. sysfs_create_link(&disk_to_dev(d->disk)->kobj, &d->kobj, "bcache"))
  683. pr_debug("error creating sysfs link");
  684. }
  685. static void cached_dev_detach_finish(struct work_struct *w)
  686. {
  687. struct cached_dev *dc = container_of(w, struct cached_dev, detach);
  688. char buf[BDEVNAME_SIZE];
  689. struct closure cl;
  690. closure_init_stack(&cl);
  691. BUG_ON(!test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags));
  692. BUG_ON(atomic_read(&dc->count));
  693. mutex_lock(&bch_register_lock);
  694. cancel_delayed_work_sync(&dc->writeback_rate_update);
  695. if (!IS_ERR_OR_NULL(dc->writeback_thread)) {
  696. kthread_stop(dc->writeback_thread);
  697. dc->writeback_thread = NULL;
  698. }
  699. memset(&dc->sb.set_uuid, 0, 16);
  700. SET_BDEV_STATE(&dc->sb, BDEV_STATE_NONE);
  701. bch_write_bdev_super(dc, &cl);
  702. closure_sync(&cl);
  703. bcache_device_detach(&dc->disk);
  704. list_move(&dc->list, &uncached_devices);
  705. clear_bit(BCACHE_DEV_DETACHING, &dc->disk.flags);
  706. clear_bit(BCACHE_DEV_UNLINK_DONE, &dc->disk.flags);
  707. mutex_unlock(&bch_register_lock);
  708. pr_info("Caching disabled for %s", bdevname(dc->bdev, buf));
  709. /* Drop ref we took in cached_dev_detach() */
  710. closure_put(&dc->disk.cl);
  711. }
  712. void bch_cached_dev_detach(struct cached_dev *dc)
  713. {
  714. lockdep_assert_held(&bch_register_lock);
  715. if (test_bit(BCACHE_DEV_CLOSING, &dc->disk.flags))
  716. return;
  717. if (test_and_set_bit(BCACHE_DEV_DETACHING, &dc->disk.flags))
  718. return;
  719. /*
  720. * Block the device from being closed and freed until we're finished
  721. * detaching
  722. */
  723. closure_get(&dc->disk.cl);
  724. bch_writeback_queue(dc);
  725. cached_dev_put(dc);
  726. }
  727. int bch_cached_dev_attach(struct cached_dev *dc, struct cache_set *c,
  728. uint8_t *set_uuid)
  729. {
  730. uint32_t rtime = cpu_to_le32(get_seconds());
  731. struct uuid_entry *u;
  732. char buf[BDEVNAME_SIZE];
  733. struct cached_dev *exist_dc, *t;
  734. bdevname(dc->bdev, buf);
  735. if ((set_uuid && memcmp(set_uuid, c->sb.set_uuid, 16)) ||
  736. (!set_uuid && memcmp(dc->sb.set_uuid, c->sb.set_uuid, 16)))
  737. return -ENOENT;
  738. if (dc->disk.c) {
  739. pr_err("Can't attach %s: already attached", buf);
  740. return -EINVAL;
  741. }
  742. if (test_bit(CACHE_SET_STOPPING, &c->flags)) {
  743. pr_err("Can't attach %s: shutting down", buf);
  744. return -EINVAL;
  745. }
  746. if (dc->sb.block_size < c->sb.block_size) {
  747. /* Will die */
  748. pr_err("Couldn't attach %s: block size less than set's block size",
  749. buf);
  750. return -EINVAL;
  751. }
  752. /* Check whether already attached */
  753. list_for_each_entry_safe(exist_dc, t, &c->cached_devs, list) {
  754. if (!memcmp(dc->sb.uuid, exist_dc->sb.uuid, 16)) {
  755. pr_err("Tried to attach %s but duplicate UUID already attached",
  756. buf);
  757. return -EINVAL;
  758. }
  759. }
  760. u = uuid_find(c, dc->sb.uuid);
  761. if (u &&
  762. (BDEV_STATE(&dc->sb) == BDEV_STATE_STALE ||
  763. BDEV_STATE(&dc->sb) == BDEV_STATE_NONE)) {
  764. memcpy(u->uuid, invalid_uuid, 16);
  765. u->invalidated = cpu_to_le32(get_seconds());
  766. u = NULL;
  767. }
  768. if (!u) {
  769. if (BDEV_STATE(&dc->sb) == BDEV_STATE_DIRTY) {
  770. pr_err("Couldn't find uuid for %s in set", buf);
  771. return -ENOENT;
  772. }
  773. u = uuid_find_empty(c);
  774. if (!u) {
  775. pr_err("Not caching %s, no room for UUID", buf);
  776. return -EINVAL;
  777. }
  778. }
  779. /* Deadlocks since we're called via sysfs...
  780. sysfs_remove_file(&dc->kobj, &sysfs_attach);
  781. */
  782. if (bch_is_zero(u->uuid, 16)) {
  783. struct closure cl;
  784. closure_init_stack(&cl);
  785. memcpy(u->uuid, dc->sb.uuid, 16);
  786. memcpy(u->label, dc->sb.label, SB_LABEL_SIZE);
  787. u->first_reg = u->last_reg = rtime;
  788. bch_uuid_write(c);
  789. memcpy(dc->sb.set_uuid, c->sb.set_uuid, 16);
  790. SET_BDEV_STATE(&dc->sb, BDEV_STATE_CLEAN);
  791. bch_write_bdev_super(dc, &cl);
  792. closure_sync(&cl);
  793. } else {
  794. u->last_reg = rtime;
  795. bch_uuid_write(c);
  796. }
  797. bcache_device_attach(&dc->disk, c, u - c->uuids);
  798. list_move(&dc->list, &c->cached_devs);
  799. calc_cached_dev_sectors(c);
  800. smp_wmb();
  801. /*
  802. * dc->c must be set before dc->count != 0 - paired with the mb in
  803. * cached_dev_get()
  804. */
  805. atomic_set(&dc->count, 1);
  806. /* Block writeback thread, but spawn it */
  807. down_write(&dc->writeback_lock);
  808. if (bch_cached_dev_writeback_start(dc)) {
  809. up_write(&dc->writeback_lock);
  810. return -ENOMEM;
  811. }
  812. if (BDEV_STATE(&dc->sb) == BDEV_STATE_DIRTY) {
  813. bch_sectors_dirty_init(&dc->disk);
  814. atomic_set(&dc->has_dirty, 1);
  815. atomic_inc(&dc->count);
  816. bch_writeback_queue(dc);
  817. }
  818. bch_cached_dev_run(dc);
  819. bcache_device_link(&dc->disk, c, "bdev");
  820. /* Allow the writeback thread to proceed */
  821. up_write(&dc->writeback_lock);
  822. pr_info("Caching %s as %s on set %pU",
  823. bdevname(dc->bdev, buf), dc->disk.disk->disk_name,
  824. dc->disk.c->sb.set_uuid);
  825. return 0;
  826. }
  827. void bch_cached_dev_release(struct kobject *kobj)
  828. {
  829. struct cached_dev *dc = container_of(kobj, struct cached_dev,
  830. disk.kobj);
  831. kfree(dc);
  832. module_put(THIS_MODULE);
  833. }
  834. static void cached_dev_free(struct closure *cl)
  835. {
  836. struct cached_dev *dc = container_of(cl, struct cached_dev, disk.cl);
  837. cancel_delayed_work_sync(&dc->writeback_rate_update);
  838. if (!IS_ERR_OR_NULL(dc->writeback_thread))
  839. kthread_stop(dc->writeback_thread);
  840. if (dc->writeback_write_wq)
  841. destroy_workqueue(dc->writeback_write_wq);
  842. mutex_lock(&bch_register_lock);
  843. if (atomic_read(&dc->running))
  844. bd_unlink_disk_holder(dc->bdev, dc->disk.disk);
  845. bcache_device_free(&dc->disk);
  846. list_del(&dc->list);
  847. mutex_unlock(&bch_register_lock);
  848. if (!IS_ERR_OR_NULL(dc->bdev))
  849. blkdev_put(dc->bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL);
  850. wake_up(&unregister_wait);
  851. kobject_put(&dc->disk.kobj);
  852. }
  853. static void cached_dev_flush(struct closure *cl)
  854. {
  855. struct cached_dev *dc = container_of(cl, struct cached_dev, disk.cl);
  856. struct bcache_device *d = &dc->disk;
  857. mutex_lock(&bch_register_lock);
  858. bcache_device_unlink(d);
  859. mutex_unlock(&bch_register_lock);
  860. bch_cache_accounting_destroy(&dc->accounting);
  861. kobject_del(&d->kobj);
  862. continue_at(cl, cached_dev_free, system_wq);
  863. }
  864. static int cached_dev_init(struct cached_dev *dc, unsigned block_size)
  865. {
  866. int ret;
  867. struct io *io;
  868. struct request_queue *q = bdev_get_queue(dc->bdev);
  869. __module_get(THIS_MODULE);
  870. INIT_LIST_HEAD(&dc->list);
  871. closure_init(&dc->disk.cl, NULL);
  872. set_closure_fn(&dc->disk.cl, cached_dev_flush, system_wq);
  873. kobject_init(&dc->disk.kobj, &bch_cached_dev_ktype);
  874. INIT_WORK(&dc->detach, cached_dev_detach_finish);
  875. sema_init(&dc->sb_write_mutex, 1);
  876. INIT_LIST_HEAD(&dc->io_lru);
  877. spin_lock_init(&dc->io_lock);
  878. bch_cache_accounting_init(&dc->accounting, &dc->disk.cl);
  879. dc->sequential_cutoff = 4 << 20;
  880. for (io = dc->io; io < dc->io + RECENT_IO; io++) {
  881. list_add(&io->lru, &dc->io_lru);
  882. hlist_add_head(&io->hash, dc->io_hash + RECENT_IO);
  883. }
  884. dc->disk.stripe_size = q->limits.io_opt >> 9;
  885. if (dc->disk.stripe_size)
  886. dc->partial_stripes_expensive =
  887. q->limits.raid_partial_stripes_expensive;
  888. ret = bcache_device_init(&dc->disk, block_size,
  889. dc->bdev->bd_part->nr_sects - dc->sb.data_offset);
  890. if (ret)
  891. return ret;
  892. set_capacity(dc->disk.disk,
  893. dc->bdev->bd_part->nr_sects - dc->sb.data_offset);
  894. dc->disk.disk->queue->backing_dev_info.ra_pages =
  895. max(dc->disk.disk->queue->backing_dev_info.ra_pages,
  896. q->backing_dev_info.ra_pages);
  897. bch_cached_dev_request_init(dc);
  898. bch_cached_dev_writeback_init(dc);
  899. return 0;
  900. }
  901. /* Cached device - bcache superblock */
  902. static void register_bdev(struct cache_sb *sb, struct page *sb_page,
  903. struct block_device *bdev,
  904. struct cached_dev *dc)
  905. {
  906. char name[BDEVNAME_SIZE];
  907. const char *err = "cannot allocate memory";
  908. struct cache_set *c;
  909. memcpy(&dc->sb, sb, sizeof(struct cache_sb));
  910. dc->bdev = bdev;
  911. dc->bdev->bd_holder = dc;
  912. bio_init(&dc->sb_bio);
  913. dc->sb_bio.bi_max_vecs = 1;
  914. dc->sb_bio.bi_io_vec = dc->sb_bio.bi_inline_vecs;
  915. dc->sb_bio.bi_io_vec[0].bv_page = sb_page;
  916. get_page(sb_page);
  917. if (cached_dev_init(dc, sb->block_size << 9))
  918. goto err;
  919. err = "error creating kobject";
  920. if (kobject_add(&dc->disk.kobj, &part_to_dev(bdev->bd_part)->kobj,
  921. "bcache"))
  922. goto err;
  923. if (bch_cache_accounting_add_kobjs(&dc->accounting, &dc->disk.kobj))
  924. goto err;
  925. pr_info("registered backing device %s", bdevname(bdev, name));
  926. list_add(&dc->list, &uncached_devices);
  927. list_for_each_entry(c, &bch_cache_sets, list)
  928. bch_cached_dev_attach(dc, c, NULL);
  929. if (BDEV_STATE(&dc->sb) == BDEV_STATE_NONE ||
  930. BDEV_STATE(&dc->sb) == BDEV_STATE_STALE)
  931. bch_cached_dev_run(dc);
  932. return;
  933. err:
  934. pr_notice("error opening %s: %s", bdevname(bdev, name), err);
  935. bcache_device_stop(&dc->disk);
  936. }
  937. /* Flash only volumes */
  938. void bch_flash_dev_release(struct kobject *kobj)
  939. {
  940. struct bcache_device *d = container_of(kobj, struct bcache_device,
  941. kobj);
  942. kfree(d);
  943. }
  944. static void flash_dev_free(struct closure *cl)
  945. {
  946. struct bcache_device *d = container_of(cl, struct bcache_device, cl);
  947. mutex_lock(&bch_register_lock);
  948. bcache_device_free(d);
  949. mutex_unlock(&bch_register_lock);
  950. kobject_put(&d->kobj);
  951. }
  952. static void flash_dev_flush(struct closure *cl)
  953. {
  954. struct bcache_device *d = container_of(cl, struct bcache_device, cl);
  955. mutex_lock(&bch_register_lock);
  956. bcache_device_unlink(d);
  957. mutex_unlock(&bch_register_lock);
  958. kobject_del(&d->kobj);
  959. continue_at(cl, flash_dev_free, system_wq);
  960. }
  961. static int flash_dev_run(struct cache_set *c, struct uuid_entry *u)
  962. {
  963. struct bcache_device *d = kzalloc(sizeof(struct bcache_device),
  964. GFP_KERNEL);
  965. if (!d)
  966. return -ENOMEM;
  967. closure_init(&d->cl, NULL);
  968. set_closure_fn(&d->cl, flash_dev_flush, system_wq);
  969. kobject_init(&d->kobj, &bch_flash_dev_ktype);
  970. if (bcache_device_init(d, block_bytes(c), u->sectors))
  971. goto err;
  972. bcache_device_attach(d, c, u - c->uuids);
  973. bch_sectors_dirty_init(d);
  974. bch_flash_dev_request_init(d);
  975. add_disk(d->disk);
  976. if (kobject_add(&d->kobj, &disk_to_dev(d->disk)->kobj, "bcache"))
  977. goto err;
  978. bcache_device_link(d, c, "volume");
  979. return 0;
  980. err:
  981. kobject_put(&d->kobj);
  982. return -ENOMEM;
  983. }
  984. static int flash_devs_run(struct cache_set *c)
  985. {
  986. int ret = 0;
  987. struct uuid_entry *u;
  988. for (u = c->uuids;
  989. u < c->uuids + c->nr_uuids && !ret;
  990. u++)
  991. if (UUID_FLASH_ONLY(u))
  992. ret = flash_dev_run(c, u);
  993. return ret;
  994. }
  995. int bch_flash_dev_create(struct cache_set *c, uint64_t size)
  996. {
  997. struct uuid_entry *u;
  998. if (test_bit(CACHE_SET_STOPPING, &c->flags))
  999. return -EINTR;
  1000. if (!test_bit(CACHE_SET_RUNNING, &c->flags))
  1001. return -EPERM;
  1002. u = uuid_find_empty(c);
  1003. if (!u) {
  1004. pr_err("Can't create volume, no room for UUID");
  1005. return -EINVAL;
  1006. }
  1007. get_random_bytes(u->uuid, 16);
  1008. memset(u->label, 0, 32);
  1009. u->first_reg = u->last_reg = cpu_to_le32(get_seconds());
  1010. SET_UUID_FLASH_ONLY(u, 1);
  1011. u->sectors = size >> 9;
  1012. bch_uuid_write(c);
  1013. return flash_dev_run(c, u);
  1014. }
  1015. /* Cache set */
  1016. __printf(2, 3)
  1017. bool bch_cache_set_error(struct cache_set *c, const char *fmt, ...)
  1018. {
  1019. va_list args;
  1020. if (c->on_error != ON_ERROR_PANIC &&
  1021. test_bit(CACHE_SET_STOPPING, &c->flags))
  1022. return false;
  1023. /* XXX: we can be called from atomic context
  1024. acquire_console_sem();
  1025. */
  1026. printk(KERN_ERR "bcache: error on %pU: ", c->sb.set_uuid);
  1027. va_start(args, fmt);
  1028. vprintk(fmt, args);
  1029. va_end(args);
  1030. printk(", disabling caching\n");
  1031. if (c->on_error == ON_ERROR_PANIC)
  1032. panic("panic forced after error\n");
  1033. bch_cache_set_unregister(c);
  1034. return true;
  1035. }
  1036. void bch_cache_set_release(struct kobject *kobj)
  1037. {
  1038. struct cache_set *c = container_of(kobj, struct cache_set, kobj);
  1039. kfree(c);
  1040. module_put(THIS_MODULE);
  1041. }
  1042. static void cache_set_free(struct closure *cl)
  1043. {
  1044. struct cache_set *c = container_of(cl, struct cache_set, cl);
  1045. struct cache *ca;
  1046. unsigned i;
  1047. if (!IS_ERR_OR_NULL(c->debug))
  1048. debugfs_remove(c->debug);
  1049. bch_open_buckets_free(c);
  1050. bch_btree_cache_free(c);
  1051. bch_journal_free(c);
  1052. for_each_cache(ca, c, i)
  1053. if (ca) {
  1054. ca->set = NULL;
  1055. c->cache[ca->sb.nr_this_dev] = NULL;
  1056. kobject_put(&ca->kobj);
  1057. }
  1058. bch_bset_sort_state_free(&c->sort);
  1059. free_pages((unsigned long) c->uuids, ilog2(bucket_pages(c)));
  1060. if (c->moving_gc_wq)
  1061. destroy_workqueue(c->moving_gc_wq);
  1062. if (c->bio_split)
  1063. bioset_free(c->bio_split);
  1064. if (c->fill_iter)
  1065. mempool_destroy(c->fill_iter);
  1066. if (c->bio_meta)
  1067. mempool_destroy(c->bio_meta);
  1068. if (c->search)
  1069. mempool_destroy(c->search);
  1070. kfree(c->devices);
  1071. mutex_lock(&bch_register_lock);
  1072. list_del(&c->list);
  1073. mutex_unlock(&bch_register_lock);
  1074. pr_info("Cache set %pU unregistered", c->sb.set_uuid);
  1075. wake_up(&unregister_wait);
  1076. closure_debug_destroy(&c->cl);
  1077. kobject_put(&c->kobj);
  1078. }
  1079. static void cache_set_flush(struct closure *cl)
  1080. {
  1081. struct cache_set *c = container_of(cl, struct cache_set, caching);
  1082. struct cache *ca;
  1083. struct btree *b;
  1084. unsigned i;
  1085. if (!c)
  1086. closure_return(cl);
  1087. bch_cache_accounting_destroy(&c->accounting);
  1088. kobject_put(&c->internal);
  1089. kobject_del(&c->kobj);
  1090. if (c->gc_thread)
  1091. kthread_stop(c->gc_thread);
  1092. if (!IS_ERR_OR_NULL(c->root))
  1093. list_add(&c->root->list, &c->btree_cache);
  1094. /* Should skip this if we're unregistering because of an error */
  1095. list_for_each_entry(b, &c->btree_cache, list) {
  1096. mutex_lock(&b->write_lock);
  1097. if (btree_node_dirty(b))
  1098. __bch_btree_node_write(b, NULL);
  1099. mutex_unlock(&b->write_lock);
  1100. }
  1101. for_each_cache(ca, c, i)
  1102. if (ca->alloc_thread)
  1103. kthread_stop(ca->alloc_thread);
  1104. if (c->journal.cur) {
  1105. cancel_delayed_work_sync(&c->journal.work);
  1106. /* flush last journal entry if needed */
  1107. c->journal.work.work.func(&c->journal.work.work);
  1108. }
  1109. closure_return(cl);
  1110. }
  1111. static void __cache_set_unregister(struct closure *cl)
  1112. {
  1113. struct cache_set *c = container_of(cl, struct cache_set, caching);
  1114. struct cached_dev *dc;
  1115. size_t i;
  1116. mutex_lock(&bch_register_lock);
  1117. for (i = 0; i < c->nr_uuids; i++)
  1118. if (c->devices[i]) {
  1119. if (!UUID_FLASH_ONLY(&c->uuids[i]) &&
  1120. test_bit(CACHE_SET_UNREGISTERING, &c->flags)) {
  1121. dc = container_of(c->devices[i],
  1122. struct cached_dev, disk);
  1123. bch_cached_dev_detach(dc);
  1124. } else {
  1125. bcache_device_stop(c->devices[i]);
  1126. }
  1127. }
  1128. mutex_unlock(&bch_register_lock);
  1129. continue_at(cl, cache_set_flush, system_wq);
  1130. }
  1131. void bch_cache_set_stop(struct cache_set *c)
  1132. {
  1133. if (!test_and_set_bit(CACHE_SET_STOPPING, &c->flags))
  1134. closure_queue(&c->caching);
  1135. }
  1136. void bch_cache_set_unregister(struct cache_set *c)
  1137. {
  1138. set_bit(CACHE_SET_UNREGISTERING, &c->flags);
  1139. bch_cache_set_stop(c);
  1140. }
  1141. #define alloc_bucket_pages(gfp, c) \
  1142. ((void *) __get_free_pages(__GFP_ZERO|gfp, ilog2(bucket_pages(c))))
  1143. struct cache_set *bch_cache_set_alloc(struct cache_sb *sb)
  1144. {
  1145. int iter_size;
  1146. struct cache_set *c = kzalloc(sizeof(struct cache_set), GFP_KERNEL);
  1147. if (!c)
  1148. return NULL;
  1149. __module_get(THIS_MODULE);
  1150. closure_init(&c->cl, NULL);
  1151. set_closure_fn(&c->cl, cache_set_free, system_wq);
  1152. closure_init(&c->caching, &c->cl);
  1153. set_closure_fn(&c->caching, __cache_set_unregister, system_wq);
  1154. /* Maybe create continue_at_noreturn() and use it here? */
  1155. closure_set_stopped(&c->cl);
  1156. closure_put(&c->cl);
  1157. kobject_init(&c->kobj, &bch_cache_set_ktype);
  1158. kobject_init(&c->internal, &bch_cache_set_internal_ktype);
  1159. bch_cache_accounting_init(&c->accounting, &c->cl);
  1160. memcpy(c->sb.set_uuid, sb->set_uuid, 16);
  1161. c->sb.block_size = sb->block_size;
  1162. c->sb.bucket_size = sb->bucket_size;
  1163. c->sb.nr_in_set = sb->nr_in_set;
  1164. c->sb.last_mount = sb->last_mount;
  1165. c->bucket_bits = ilog2(sb->bucket_size);
  1166. c->block_bits = ilog2(sb->block_size);
  1167. c->nr_uuids = bucket_bytes(c) / sizeof(struct uuid_entry);
  1168. c->btree_pages = bucket_pages(c);
  1169. if (c->btree_pages > BTREE_MAX_PAGES)
  1170. c->btree_pages = max_t(int, c->btree_pages / 4,
  1171. BTREE_MAX_PAGES);
  1172. sema_init(&c->sb_write_mutex, 1);
  1173. mutex_init(&c->bucket_lock);
  1174. init_waitqueue_head(&c->btree_cache_wait);
  1175. init_waitqueue_head(&c->bucket_wait);
  1176. init_waitqueue_head(&c->gc_wait);
  1177. sema_init(&c->uuid_write_mutex, 1);
  1178. spin_lock_init(&c->btree_gc_time.lock);
  1179. spin_lock_init(&c->btree_split_time.lock);
  1180. spin_lock_init(&c->btree_read_time.lock);
  1181. bch_moving_init_cache_set(c);
  1182. INIT_LIST_HEAD(&c->list);
  1183. INIT_LIST_HEAD(&c->cached_devs);
  1184. INIT_LIST_HEAD(&c->btree_cache);
  1185. INIT_LIST_HEAD(&c->btree_cache_freeable);
  1186. INIT_LIST_HEAD(&c->btree_cache_freed);
  1187. INIT_LIST_HEAD(&c->data_buckets);
  1188. c->search = mempool_create_slab_pool(32, bch_search_cache);
  1189. if (!c->search)
  1190. goto err;
  1191. iter_size = (sb->bucket_size / sb->block_size + 1) *
  1192. sizeof(struct btree_iter_set);
  1193. if (!(c->devices = kzalloc(c->nr_uuids * sizeof(void *), GFP_KERNEL)) ||
  1194. !(c->bio_meta = mempool_create_kmalloc_pool(2,
  1195. sizeof(struct bbio) + sizeof(struct bio_vec) *
  1196. bucket_pages(c))) ||
  1197. !(c->fill_iter = mempool_create_kmalloc_pool(1, iter_size)) ||
  1198. !(c->bio_split = bioset_create(4, offsetof(struct bbio, bio))) ||
  1199. !(c->uuids = alloc_bucket_pages(GFP_KERNEL, c)) ||
  1200. !(c->moving_gc_wq = create_workqueue("bcache_gc")) ||
  1201. bch_journal_alloc(c) ||
  1202. bch_btree_cache_alloc(c) ||
  1203. bch_open_buckets_alloc(c) ||
  1204. bch_bset_sort_state_init(&c->sort, ilog2(c->btree_pages)))
  1205. goto err;
  1206. c->congested_read_threshold_us = 2000;
  1207. c->congested_write_threshold_us = 20000;
  1208. c->error_limit = 8 << IO_ERROR_SHIFT;
  1209. return c;
  1210. err:
  1211. bch_cache_set_unregister(c);
  1212. return NULL;
  1213. }
  1214. static void run_cache_set(struct cache_set *c)
  1215. {
  1216. const char *err = "cannot allocate memory";
  1217. struct cached_dev *dc, *t;
  1218. struct cache *ca;
  1219. struct closure cl;
  1220. unsigned i;
  1221. closure_init_stack(&cl);
  1222. for_each_cache(ca, c, i)
  1223. c->nbuckets += ca->sb.nbuckets;
  1224. set_gc_sectors(c);
  1225. if (CACHE_SYNC(&c->sb)) {
  1226. LIST_HEAD(journal);
  1227. struct bkey *k;
  1228. struct jset *j;
  1229. err = "cannot allocate memory for journal";
  1230. if (bch_journal_read(c, &journal))
  1231. goto err;
  1232. pr_debug("btree_journal_read() done");
  1233. err = "no journal entries found";
  1234. if (list_empty(&journal))
  1235. goto err;
  1236. j = &list_entry(journal.prev, struct journal_replay, list)->j;
  1237. err = "IO error reading priorities";
  1238. for_each_cache(ca, c, i)
  1239. prio_read(ca, j->prio_bucket[ca->sb.nr_this_dev]);
  1240. /*
  1241. * If prio_read() fails it'll call cache_set_error and we'll
  1242. * tear everything down right away, but if we perhaps checked
  1243. * sooner we could avoid journal replay.
  1244. */
  1245. k = &j->btree_root;
  1246. err = "bad btree root";
  1247. if (__bch_btree_ptr_invalid(c, k))
  1248. goto err;
  1249. err = "error reading btree root";
  1250. c->root = bch_btree_node_get(c, NULL, k, j->btree_level, true, NULL);
  1251. if (IS_ERR_OR_NULL(c->root))
  1252. goto err;
  1253. list_del_init(&c->root->list);
  1254. rw_unlock(true, c->root);
  1255. err = uuid_read(c, j, &cl);
  1256. if (err)
  1257. goto err;
  1258. err = "error in recovery";
  1259. if (bch_btree_check(c))
  1260. goto err;
  1261. bch_journal_mark(c, &journal);
  1262. bch_initial_gc_finish(c);
  1263. pr_debug("btree_check() done");
  1264. /*
  1265. * bcache_journal_next() can't happen sooner, or
  1266. * btree_gc_finish() will give spurious errors about last_gc >
  1267. * gc_gen - this is a hack but oh well.
  1268. */
  1269. bch_journal_next(&c->journal);
  1270. err = "error starting allocator thread";
  1271. for_each_cache(ca, c, i)
  1272. if (bch_cache_allocator_start(ca))
  1273. goto err;
  1274. /*
  1275. * First place it's safe to allocate: btree_check() and
  1276. * btree_gc_finish() have to run before we have buckets to
  1277. * allocate, and bch_bucket_alloc_set() might cause a journal
  1278. * entry to be written so bcache_journal_next() has to be called
  1279. * first.
  1280. *
  1281. * If the uuids were in the old format we have to rewrite them
  1282. * before the next journal entry is written:
  1283. */
  1284. if (j->version < BCACHE_JSET_VERSION_UUID)
  1285. __uuid_write(c);
  1286. bch_journal_replay(c, &journal);
  1287. } else {
  1288. pr_notice("invalidating existing data");
  1289. for_each_cache(ca, c, i) {
  1290. unsigned j;
  1291. ca->sb.keys = clamp_t(int, ca->sb.nbuckets >> 7,
  1292. 2, SB_JOURNAL_BUCKETS);
  1293. for (j = 0; j < ca->sb.keys; j++)
  1294. ca->sb.d[j] = ca->sb.first_bucket + j;
  1295. }
  1296. bch_initial_gc_finish(c);
  1297. err = "error starting allocator thread";
  1298. for_each_cache(ca, c, i)
  1299. if (bch_cache_allocator_start(ca))
  1300. goto err;
  1301. mutex_lock(&c->bucket_lock);
  1302. for_each_cache(ca, c, i)
  1303. bch_prio_write(ca);
  1304. mutex_unlock(&c->bucket_lock);
  1305. err = "cannot allocate new UUID bucket";
  1306. if (__uuid_write(c))
  1307. goto err;
  1308. err = "cannot allocate new btree root";
  1309. c->root = __bch_btree_node_alloc(c, NULL, 0, true, NULL);
  1310. if (IS_ERR_OR_NULL(c->root))
  1311. goto err;
  1312. mutex_lock(&c->root->write_lock);
  1313. bkey_copy_key(&c->root->key, &MAX_KEY);
  1314. bch_btree_node_write(c->root, &cl);
  1315. mutex_unlock(&c->root->write_lock);
  1316. bch_btree_set_root(c->root);
  1317. rw_unlock(true, c->root);
  1318. /*
  1319. * We don't want to write the first journal entry until
  1320. * everything is set up - fortunately journal entries won't be
  1321. * written until the SET_CACHE_SYNC() here:
  1322. */
  1323. SET_CACHE_SYNC(&c->sb, true);
  1324. bch_journal_next(&c->journal);
  1325. bch_journal_meta(c, &cl);
  1326. }
  1327. err = "error starting gc thread";
  1328. if (bch_gc_thread_start(c))
  1329. goto err;
  1330. closure_sync(&cl);
  1331. c->sb.last_mount = get_seconds();
  1332. bcache_write_super(c);
  1333. list_for_each_entry_safe(dc, t, &uncached_devices, list)
  1334. bch_cached_dev_attach(dc, c, NULL);
  1335. flash_devs_run(c);
  1336. set_bit(CACHE_SET_RUNNING, &c->flags);
  1337. return;
  1338. err:
  1339. closure_sync(&cl);
  1340. /* XXX: test this, it's broken */
  1341. bch_cache_set_error(c, "%s", err);
  1342. }
  1343. static bool can_attach_cache(struct cache *ca, struct cache_set *c)
  1344. {
  1345. return ca->sb.block_size == c->sb.block_size &&
  1346. ca->sb.bucket_size == c->sb.bucket_size &&
  1347. ca->sb.nr_in_set == c->sb.nr_in_set;
  1348. }
  1349. static const char *register_cache_set(struct cache *ca)
  1350. {
  1351. char buf[12];
  1352. const char *err = "cannot allocate memory";
  1353. struct cache_set *c;
  1354. list_for_each_entry(c, &bch_cache_sets, list)
  1355. if (!memcmp(c->sb.set_uuid, ca->sb.set_uuid, 16)) {
  1356. if (c->cache[ca->sb.nr_this_dev])
  1357. return "duplicate cache set member";
  1358. if (!can_attach_cache(ca, c))
  1359. return "cache sb does not match set";
  1360. if (!CACHE_SYNC(&ca->sb))
  1361. SET_CACHE_SYNC(&c->sb, false);
  1362. goto found;
  1363. }
  1364. c = bch_cache_set_alloc(&ca->sb);
  1365. if (!c)
  1366. return err;
  1367. err = "error creating kobject";
  1368. if (kobject_add(&c->kobj, bcache_kobj, "%pU", c->sb.set_uuid) ||
  1369. kobject_add(&c->internal, &c->kobj, "internal"))
  1370. goto err;
  1371. if (bch_cache_accounting_add_kobjs(&c->accounting, &c->kobj))
  1372. goto err;
  1373. bch_debug_init_cache_set(c);
  1374. list_add(&c->list, &bch_cache_sets);
  1375. found:
  1376. sprintf(buf, "cache%i", ca->sb.nr_this_dev);
  1377. if (sysfs_create_link(&ca->kobj, &c->kobj, "set") ||
  1378. sysfs_create_link(&c->kobj, &ca->kobj, buf))
  1379. goto err;
  1380. if (ca->sb.seq > c->sb.seq) {
  1381. c->sb.version = ca->sb.version;
  1382. memcpy(c->sb.set_uuid, ca->sb.set_uuid, 16);
  1383. c->sb.flags = ca->sb.flags;
  1384. c->sb.seq = ca->sb.seq;
  1385. pr_debug("set version = %llu", c->sb.version);
  1386. }
  1387. kobject_get(&ca->kobj);
  1388. ca->set = c;
  1389. ca->set->cache[ca->sb.nr_this_dev] = ca;
  1390. c->cache_by_alloc[c->caches_loaded++] = ca;
  1391. if (c->caches_loaded == c->sb.nr_in_set)
  1392. run_cache_set(c);
  1393. return NULL;
  1394. err:
  1395. bch_cache_set_unregister(c);
  1396. return err;
  1397. }
  1398. /* Cache device */
  1399. void bch_cache_release(struct kobject *kobj)
  1400. {
  1401. struct cache *ca = container_of(kobj, struct cache, kobj);
  1402. unsigned i;
  1403. if (ca->set) {
  1404. BUG_ON(ca->set->cache[ca->sb.nr_this_dev] != ca);
  1405. ca->set->cache[ca->sb.nr_this_dev] = NULL;
  1406. }
  1407. free_pages((unsigned long) ca->disk_buckets, ilog2(bucket_pages(ca)));
  1408. kfree(ca->prio_buckets);
  1409. vfree(ca->buckets);
  1410. free_heap(&ca->heap);
  1411. free_fifo(&ca->free_inc);
  1412. for (i = 0; i < RESERVE_NR; i++)
  1413. free_fifo(&ca->free[i]);
  1414. if (ca->sb_bio.bi_inline_vecs[0].bv_page)
  1415. put_page(ca->sb_bio.bi_io_vec[0].bv_page);
  1416. if (!IS_ERR_OR_NULL(ca->bdev))
  1417. blkdev_put(ca->bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL);
  1418. kfree(ca);
  1419. module_put(THIS_MODULE);
  1420. }
  1421. static int cache_alloc(struct cache_sb *sb, struct cache *ca)
  1422. {
  1423. size_t free;
  1424. size_t btree_buckets;
  1425. struct bucket *b;
  1426. __module_get(THIS_MODULE);
  1427. kobject_init(&ca->kobj, &bch_cache_ktype);
  1428. bio_init(&ca->journal.bio);
  1429. ca->journal.bio.bi_max_vecs = 8;
  1430. ca->journal.bio.bi_io_vec = ca->journal.bio.bi_inline_vecs;
  1431. /*
  1432. * when ca->sb.njournal_buckets is not zero, journal exists,
  1433. * and in bch_journal_replay(), tree node may split,
  1434. * so bucket of RESERVE_BTREE type is needed,
  1435. * the worst situation is all journal buckets are valid journal,
  1436. * and all the keys need to replay,
  1437. * so the number of RESERVE_BTREE type buckets should be as much
  1438. * as journal buckets
  1439. */
  1440. btree_buckets = ca->sb.njournal_buckets ?: 8;
  1441. free = roundup_pow_of_two(ca->sb.nbuckets) >> 10;
  1442. if (!init_fifo(&ca->free[RESERVE_BTREE], btree_buckets, GFP_KERNEL) ||
  1443. !init_fifo_exact(&ca->free[RESERVE_PRIO], prio_buckets(ca), GFP_KERNEL) ||
  1444. !init_fifo(&ca->free[RESERVE_MOVINGGC], free, GFP_KERNEL) ||
  1445. !init_fifo(&ca->free[RESERVE_NONE], free, GFP_KERNEL) ||
  1446. !init_fifo(&ca->free_inc, free << 2, GFP_KERNEL) ||
  1447. !init_heap(&ca->heap, free << 3, GFP_KERNEL) ||
  1448. !(ca->buckets = vzalloc(sizeof(struct bucket) *
  1449. ca->sb.nbuckets)) ||
  1450. !(ca->prio_buckets = kzalloc(sizeof(uint64_t) * prio_buckets(ca) *
  1451. 2, GFP_KERNEL)) ||
  1452. !(ca->disk_buckets = alloc_bucket_pages(GFP_KERNEL, ca)))
  1453. return -ENOMEM;
  1454. ca->prio_last_buckets = ca->prio_buckets + prio_buckets(ca);
  1455. for_each_bucket(b, ca)
  1456. atomic_set(&b->pin, 0);
  1457. return 0;
  1458. }
  1459. static int register_cache(struct cache_sb *sb, struct page *sb_page,
  1460. struct block_device *bdev, struct cache *ca)
  1461. {
  1462. char name[BDEVNAME_SIZE];
  1463. const char *err = NULL;
  1464. int ret = 0;
  1465. memcpy(&ca->sb, sb, sizeof(struct cache_sb));
  1466. ca->bdev = bdev;
  1467. ca->bdev->bd_holder = ca;
  1468. bio_init(&ca->sb_bio);
  1469. ca->sb_bio.bi_max_vecs = 1;
  1470. ca->sb_bio.bi_io_vec = ca->sb_bio.bi_inline_vecs;
  1471. ca->sb_bio.bi_io_vec[0].bv_page = sb_page;
  1472. get_page(sb_page);
  1473. if (blk_queue_discard(bdev_get_queue(ca->bdev)))
  1474. ca->discard = CACHE_DISCARD(&ca->sb);
  1475. ret = cache_alloc(sb, ca);
  1476. if (ret != 0)
  1477. goto err;
  1478. if (kobject_add(&ca->kobj, &part_to_dev(bdev->bd_part)->kobj, "bcache")) {
  1479. err = "error calling kobject_add";
  1480. ret = -ENOMEM;
  1481. goto out;
  1482. }
  1483. mutex_lock(&bch_register_lock);
  1484. err = register_cache_set(ca);
  1485. mutex_unlock(&bch_register_lock);
  1486. if (err) {
  1487. ret = -ENODEV;
  1488. goto out;
  1489. }
  1490. pr_info("registered cache device %s", bdevname(bdev, name));
  1491. out:
  1492. kobject_put(&ca->kobj);
  1493. err:
  1494. if (err)
  1495. pr_notice("error opening %s: %s", bdevname(bdev, name), err);
  1496. return ret;
  1497. }
  1498. /* Global interfaces/init */
  1499. static ssize_t register_bcache(struct kobject *, struct kobj_attribute *,
  1500. const char *, size_t);
  1501. kobj_attribute_write(register, register_bcache);
  1502. kobj_attribute_write(register_quiet, register_bcache);
  1503. static bool bch_is_open_backing(struct block_device *bdev) {
  1504. struct cache_set *c, *tc;
  1505. struct cached_dev *dc, *t;
  1506. list_for_each_entry_safe(c, tc, &bch_cache_sets, list)
  1507. list_for_each_entry_safe(dc, t, &c->cached_devs, list)
  1508. if (dc->bdev == bdev)
  1509. return true;
  1510. list_for_each_entry_safe(dc, t, &uncached_devices, list)
  1511. if (dc->bdev == bdev)
  1512. return true;
  1513. return false;
  1514. }
  1515. static bool bch_is_open_cache(struct block_device *bdev) {
  1516. struct cache_set *c, *tc;
  1517. struct cache *ca;
  1518. unsigned i;
  1519. list_for_each_entry_safe(c, tc, &bch_cache_sets, list)
  1520. for_each_cache(ca, c, i)
  1521. if (ca->bdev == bdev)
  1522. return true;
  1523. return false;
  1524. }
  1525. static bool bch_is_open(struct block_device *bdev) {
  1526. return bch_is_open_cache(bdev) || bch_is_open_backing(bdev);
  1527. }
  1528. static ssize_t register_bcache(struct kobject *k, struct kobj_attribute *attr,
  1529. const char *buffer, size_t size)
  1530. {
  1531. ssize_t ret = size;
  1532. const char *err = "cannot allocate memory";
  1533. char *path = NULL;
  1534. struct cache_sb *sb = NULL;
  1535. struct block_device *bdev = NULL;
  1536. struct page *sb_page = NULL;
  1537. if (!try_module_get(THIS_MODULE))
  1538. return -EBUSY;
  1539. if (!(path = kstrndup(buffer, size, GFP_KERNEL)) ||
  1540. !(sb = kmalloc(sizeof(struct cache_sb), GFP_KERNEL)))
  1541. goto err;
  1542. err = "failed to open device";
  1543. bdev = blkdev_get_by_path(strim(path),
  1544. FMODE_READ|FMODE_WRITE|FMODE_EXCL,
  1545. sb);
  1546. if (IS_ERR(bdev)) {
  1547. if (bdev == ERR_PTR(-EBUSY)) {
  1548. bdev = lookup_bdev(strim(path));
  1549. mutex_lock(&bch_register_lock);
  1550. if (!IS_ERR(bdev) && bch_is_open(bdev))
  1551. err = "device already registered";
  1552. else
  1553. err = "device busy";
  1554. mutex_unlock(&bch_register_lock);
  1555. if (!IS_ERR(bdev))
  1556. bdput(bdev);
  1557. if (attr == &ksysfs_register_quiet)
  1558. goto out;
  1559. }
  1560. goto err;
  1561. }
  1562. err = "failed to set blocksize";
  1563. if (set_blocksize(bdev, 4096))
  1564. goto err_close;
  1565. err = read_super(sb, bdev, &sb_page);
  1566. if (err)
  1567. goto err_close;
  1568. if (SB_IS_BDEV(sb)) {
  1569. struct cached_dev *dc = kzalloc(sizeof(*dc), GFP_KERNEL);
  1570. if (!dc)
  1571. goto err_close;
  1572. mutex_lock(&bch_register_lock);
  1573. register_bdev(sb, sb_page, bdev, dc);
  1574. mutex_unlock(&bch_register_lock);
  1575. } else {
  1576. struct cache *ca = kzalloc(sizeof(*ca), GFP_KERNEL);
  1577. if (!ca)
  1578. goto err_close;
  1579. if (register_cache(sb, sb_page, bdev, ca) != 0)
  1580. goto err_close;
  1581. }
  1582. out:
  1583. if (sb_page)
  1584. put_page(sb_page);
  1585. kfree(sb);
  1586. kfree(path);
  1587. module_put(THIS_MODULE);
  1588. return ret;
  1589. err_close:
  1590. blkdev_put(bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL);
  1591. err:
  1592. pr_info("error opening %s: %s", path, err);
  1593. ret = -EINVAL;
  1594. goto out;
  1595. }
  1596. static int bcache_reboot(struct notifier_block *n, unsigned long code, void *x)
  1597. {
  1598. if (code == SYS_DOWN ||
  1599. code == SYS_HALT ||
  1600. code == SYS_POWER_OFF) {
  1601. DEFINE_WAIT(wait);
  1602. unsigned long start = jiffies;
  1603. bool stopped = false;
  1604. struct cache_set *c, *tc;
  1605. struct cached_dev *dc, *tdc;
  1606. mutex_lock(&bch_register_lock);
  1607. if (list_empty(&bch_cache_sets) &&
  1608. list_empty(&uncached_devices))
  1609. goto out;
  1610. pr_info("Stopping all devices:");
  1611. list_for_each_entry_safe(c, tc, &bch_cache_sets, list)
  1612. bch_cache_set_stop(c);
  1613. list_for_each_entry_safe(dc, tdc, &uncached_devices, list)
  1614. bcache_device_stop(&dc->disk);
  1615. /* What's a condition variable? */
  1616. while (1) {
  1617. long timeout = start + 2 * HZ - jiffies;
  1618. stopped = list_empty(&bch_cache_sets) &&
  1619. list_empty(&uncached_devices);
  1620. if (timeout < 0 || stopped)
  1621. break;
  1622. prepare_to_wait(&unregister_wait, &wait,
  1623. TASK_UNINTERRUPTIBLE);
  1624. mutex_unlock(&bch_register_lock);
  1625. schedule_timeout(timeout);
  1626. mutex_lock(&bch_register_lock);
  1627. }
  1628. finish_wait(&unregister_wait, &wait);
  1629. if (stopped)
  1630. pr_info("All devices stopped");
  1631. else
  1632. pr_notice("Timeout waiting for devices to be closed");
  1633. out:
  1634. mutex_unlock(&bch_register_lock);
  1635. }
  1636. return NOTIFY_DONE;
  1637. }
  1638. static struct notifier_block reboot = {
  1639. .notifier_call = bcache_reboot,
  1640. .priority = INT_MAX, /* before any real devices */
  1641. };
  1642. static void bcache_exit(void)
  1643. {
  1644. bch_debug_exit();
  1645. bch_request_exit();
  1646. if (bcache_kobj)
  1647. kobject_put(bcache_kobj);
  1648. if (bcache_wq)
  1649. destroy_workqueue(bcache_wq);
  1650. if (bcache_major)
  1651. unregister_blkdev(bcache_major, "bcache");
  1652. unregister_reboot_notifier(&reboot);
  1653. mutex_destroy(&bch_register_lock);
  1654. }
  1655. static int __init bcache_init(void)
  1656. {
  1657. static const struct attribute *files[] = {
  1658. &ksysfs_register.attr,
  1659. &ksysfs_register_quiet.attr,
  1660. NULL
  1661. };
  1662. mutex_init(&bch_register_lock);
  1663. init_waitqueue_head(&unregister_wait);
  1664. register_reboot_notifier(&reboot);
  1665. closure_debug_init();
  1666. bcache_major = register_blkdev(0, "bcache");
  1667. if (bcache_major < 0) {
  1668. unregister_reboot_notifier(&reboot);
  1669. mutex_destroy(&bch_register_lock);
  1670. return bcache_major;
  1671. }
  1672. if (!(bcache_wq = create_workqueue("bcache")) ||
  1673. !(bcache_kobj = kobject_create_and_add("bcache", fs_kobj)) ||
  1674. bch_request_init() ||
  1675. bch_debug_init(bcache_kobj) ||
  1676. sysfs_create_files(bcache_kobj, files))
  1677. goto err;
  1678. return 0;
  1679. err:
  1680. bcache_exit();
  1681. return -ENOMEM;
  1682. }
  1683. module_exit(bcache_exit);
  1684. module_init(bcache_init);