highmem.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488
  1. /*
  2. * High memory handling common code and variables.
  3. *
  4. * (C) 1999 Andrea Arcangeli, SuSE GmbH, andrea@suse.de
  5. * Gerhard Wichert, Siemens AG, Gerhard.Wichert@pdb.siemens.de
  6. *
  7. *
  8. * Redesigned the x86 32-bit VM architecture to deal with
  9. * 64-bit physical space. With current x86 CPUs this
  10. * means up to 64 Gigabytes physical RAM.
  11. *
  12. * Rewrote high memory support to move the page cache into
  13. * high memory. Implemented permanent (schedulable) kmaps
  14. * based on Linus' idea.
  15. *
  16. * Copyright (C) 1999 Ingo Molnar <mingo@redhat.com>
  17. */
  18. #include <linux/mm.h>
  19. #include <linux/export.h>
  20. #include <linux/swap.h>
  21. #include <linux/bio.h>
  22. #include <linux/pagemap.h>
  23. #include <linux/mempool.h>
  24. #include <linux/blkdev.h>
  25. #include <linux/init.h>
  26. #include <linux/hash.h>
  27. #include <linux/highmem.h>
  28. #include <linux/kgdb.h>
  29. #include <asm/tlbflush.h>
  30. #if defined(CONFIG_HIGHMEM) || defined(CONFIG_X86_32)
  31. DEFINE_PER_CPU(int, __kmap_atomic_idx);
  32. #endif
  33. /*
  34. * Virtual_count is not a pure "count".
  35. * 0 means that it is not mapped, and has not been mapped
  36. * since a TLB flush - it is usable.
  37. * 1 means that there are no users, but it has been mapped
  38. * since the last TLB flush - so we can't use it.
  39. * n means that there are (n-1) current users of it.
  40. */
  41. #ifdef CONFIG_HIGHMEM
  42. /*
  43. * Architecture with aliasing data cache may define the following family of
  44. * helper functions in its asm/highmem.h to control cache color of virtual
  45. * addresses where physical memory pages are mapped by kmap.
  46. */
  47. #ifndef get_pkmap_color
  48. /*
  49. * Determine color of virtual address where the page should be mapped.
  50. */
  51. static inline unsigned int get_pkmap_color(struct page *page)
  52. {
  53. return 0;
  54. }
  55. #define get_pkmap_color get_pkmap_color
  56. /*
  57. * Get next index for mapping inside PKMAP region for page with given color.
  58. */
  59. static inline unsigned int get_next_pkmap_nr(unsigned int color)
  60. {
  61. static unsigned int last_pkmap_nr;
  62. last_pkmap_nr = (last_pkmap_nr + 1) & LAST_PKMAP_MASK;
  63. return last_pkmap_nr;
  64. }
  65. /*
  66. * Determine if page index inside PKMAP region (pkmap_nr) of given color
  67. * has wrapped around PKMAP region end. When this happens an attempt to
  68. * flush all unused PKMAP slots is made.
  69. */
  70. static inline int no_more_pkmaps(unsigned int pkmap_nr, unsigned int color)
  71. {
  72. return pkmap_nr == 0;
  73. }
  74. /*
  75. * Get the number of PKMAP entries of the given color. If no free slot is
  76. * found after checking that many entries, kmap will sleep waiting for
  77. * someone to call kunmap and free PKMAP slot.
  78. */
  79. static inline int get_pkmap_entries_count(unsigned int color)
  80. {
  81. return LAST_PKMAP;
  82. }
  83. /*
  84. * Get head of a wait queue for PKMAP entries of the given color.
  85. * Wait queues for different mapping colors should be independent to avoid
  86. * unnecessary wakeups caused by freeing of slots of other colors.
  87. */
  88. static inline wait_queue_head_t *get_pkmap_wait_queue_head(unsigned int color)
  89. {
  90. static DECLARE_WAIT_QUEUE_HEAD(pkmap_map_wait);
  91. return &pkmap_map_wait;
  92. }
  93. #endif
  94. unsigned long totalhigh_pages __read_mostly;
  95. EXPORT_SYMBOL(totalhigh_pages);
  96. EXPORT_PER_CPU_SYMBOL(__kmap_atomic_idx);
  97. unsigned int nr_free_highpages (void)
  98. {
  99. pg_data_t *pgdat;
  100. unsigned int pages = 0;
  101. for_each_online_pgdat(pgdat) {
  102. pages += zone_page_state(&pgdat->node_zones[ZONE_HIGHMEM],
  103. NR_FREE_PAGES);
  104. if (zone_movable_is_highmem())
  105. pages += zone_page_state(
  106. &pgdat->node_zones[ZONE_MOVABLE],
  107. NR_FREE_PAGES);
  108. }
  109. return pages;
  110. }
  111. static int pkmap_count[LAST_PKMAP];
  112. static __cacheline_aligned_in_smp DEFINE_SPINLOCK(kmap_lock);
  113. pte_t * pkmap_page_table;
  114. /*
  115. * Most architectures have no use for kmap_high_get(), so let's abstract
  116. * the disabling of IRQ out of the locking in that case to save on a
  117. * potential useless overhead.
  118. */
  119. #ifdef ARCH_NEEDS_KMAP_HIGH_GET
  120. #define lock_kmap() spin_lock_irq(&kmap_lock)
  121. #define unlock_kmap() spin_unlock_irq(&kmap_lock)
  122. #define lock_kmap_any(flags) spin_lock_irqsave(&kmap_lock, flags)
  123. #define unlock_kmap_any(flags) spin_unlock_irqrestore(&kmap_lock, flags)
  124. #else
  125. #define lock_kmap() spin_lock(&kmap_lock)
  126. #define unlock_kmap() spin_unlock(&kmap_lock)
  127. #define lock_kmap_any(flags) \
  128. do { spin_lock(&kmap_lock); (void)(flags); } while (0)
  129. #define unlock_kmap_any(flags) \
  130. do { spin_unlock(&kmap_lock); (void)(flags); } while (0)
  131. #endif
  132. struct page *kmap_to_page(void *vaddr)
  133. {
  134. unsigned long addr = (unsigned long)vaddr;
  135. if (addr >= PKMAP_ADDR(0) && addr < PKMAP_ADDR(LAST_PKMAP)) {
  136. int i = PKMAP_NR(addr);
  137. return pte_page(pkmap_page_table[i]);
  138. }
  139. return virt_to_page(addr);
  140. }
  141. EXPORT_SYMBOL(kmap_to_page);
  142. static void flush_all_zero_pkmaps(void)
  143. {
  144. int i;
  145. int need_flush = 0;
  146. flush_cache_kmaps();
  147. for (i = 0; i < LAST_PKMAP; i++) {
  148. struct page *page;
  149. /*
  150. * zero means we don't have anything to do,
  151. * >1 means that it is still in use. Only
  152. * a count of 1 means that it is free but
  153. * needs to be unmapped
  154. */
  155. if (pkmap_count[i] != 1)
  156. continue;
  157. pkmap_count[i] = 0;
  158. /* sanity check */
  159. BUG_ON(pte_none(pkmap_page_table[i]));
  160. /*
  161. * Don't need an atomic fetch-and-clear op here;
  162. * no-one has the page mapped, and cannot get at
  163. * its virtual address (and hence PTE) without first
  164. * getting the kmap_lock (which is held here).
  165. * So no dangers, even with speculative execution.
  166. */
  167. page = pte_page(pkmap_page_table[i]);
  168. pte_clear(&init_mm, PKMAP_ADDR(i), &pkmap_page_table[i]);
  169. set_page_address(page, NULL);
  170. need_flush = 1;
  171. }
  172. if (need_flush)
  173. flush_tlb_kernel_range(PKMAP_ADDR(0), PKMAP_ADDR(LAST_PKMAP));
  174. }
  175. /**
  176. * kmap_flush_unused - flush all unused kmap mappings in order to remove stray mappings
  177. */
  178. void kmap_flush_unused(void)
  179. {
  180. lock_kmap();
  181. flush_all_zero_pkmaps();
  182. unlock_kmap();
  183. }
  184. static inline unsigned long map_new_virtual(struct page *page)
  185. {
  186. unsigned long vaddr;
  187. int count;
  188. unsigned int last_pkmap_nr;
  189. unsigned int color = get_pkmap_color(page);
  190. start:
  191. count = get_pkmap_entries_count(color);
  192. /* Find an empty entry */
  193. for (;;) {
  194. last_pkmap_nr = get_next_pkmap_nr(color);
  195. if (no_more_pkmaps(last_pkmap_nr, color)) {
  196. flush_all_zero_pkmaps();
  197. count = get_pkmap_entries_count(color);
  198. }
  199. if (!pkmap_count[last_pkmap_nr])
  200. break; /* Found a usable entry */
  201. if (--count)
  202. continue;
  203. /*
  204. * Sleep for somebody else to unmap their entries
  205. */
  206. {
  207. DECLARE_WAITQUEUE(wait, current);
  208. wait_queue_head_t *pkmap_map_wait =
  209. get_pkmap_wait_queue_head(color);
  210. __set_current_state(TASK_UNINTERRUPTIBLE);
  211. add_wait_queue(pkmap_map_wait, &wait);
  212. unlock_kmap();
  213. schedule();
  214. remove_wait_queue(pkmap_map_wait, &wait);
  215. lock_kmap();
  216. /* Somebody else might have mapped it while we slept */
  217. if (page_address(page))
  218. return (unsigned long)page_address(page);
  219. /* Re-start */
  220. goto start;
  221. }
  222. }
  223. vaddr = PKMAP_ADDR(last_pkmap_nr);
  224. set_pte_at(&init_mm, vaddr,
  225. &(pkmap_page_table[last_pkmap_nr]), mk_pte(page, kmap_prot));
  226. pkmap_count[last_pkmap_nr] = 1;
  227. set_page_address(page, (void *)vaddr);
  228. return vaddr;
  229. }
  230. /**
  231. * kmap_high - map a highmem page into memory
  232. * @page: &struct page to map
  233. *
  234. * Returns the page's virtual memory address.
  235. *
  236. * We cannot call this from interrupts, as it may block.
  237. */
  238. void *kmap_high(struct page *page)
  239. {
  240. unsigned long vaddr;
  241. /*
  242. * For highmem pages, we can't trust "virtual" until
  243. * after we have the lock.
  244. */
  245. lock_kmap();
  246. vaddr = (unsigned long)page_address(page);
  247. if (!vaddr)
  248. vaddr = map_new_virtual(page);
  249. pkmap_count[PKMAP_NR(vaddr)]++;
  250. BUG_ON(pkmap_count[PKMAP_NR(vaddr)] < 2);
  251. unlock_kmap();
  252. return (void*) vaddr;
  253. }
  254. EXPORT_SYMBOL(kmap_high);
  255. #ifdef ARCH_NEEDS_KMAP_HIGH_GET
  256. /**
  257. * kmap_high_get - pin a highmem page into memory
  258. * @page: &struct page to pin
  259. *
  260. * Returns the page's current virtual memory address, or NULL if no mapping
  261. * exists. If and only if a non null address is returned then a
  262. * matching call to kunmap_high() is necessary.
  263. *
  264. * This can be called from any context.
  265. */
  266. void *kmap_high_get(struct page *page)
  267. {
  268. unsigned long vaddr, flags;
  269. lock_kmap_any(flags);
  270. vaddr = (unsigned long)page_address(page);
  271. if (vaddr) {
  272. BUG_ON(pkmap_count[PKMAP_NR(vaddr)] < 1);
  273. pkmap_count[PKMAP_NR(vaddr)]++;
  274. }
  275. unlock_kmap_any(flags);
  276. return (void*) vaddr;
  277. }
  278. #endif
  279. /**
  280. * kunmap_high - unmap a highmem page into memory
  281. * @page: &struct page to unmap
  282. *
  283. * If ARCH_NEEDS_KMAP_HIGH_GET is not defined then this may be called
  284. * only from user context.
  285. */
  286. void kunmap_high(struct page *page)
  287. {
  288. unsigned long vaddr;
  289. unsigned long nr;
  290. unsigned long flags;
  291. int need_wakeup;
  292. unsigned int color = get_pkmap_color(page);
  293. wait_queue_head_t *pkmap_map_wait;
  294. lock_kmap_any(flags);
  295. vaddr = (unsigned long)page_address(page);
  296. BUG_ON(!vaddr);
  297. nr = PKMAP_NR(vaddr);
  298. /*
  299. * A count must never go down to zero
  300. * without a TLB flush!
  301. */
  302. need_wakeup = 0;
  303. switch (--pkmap_count[nr]) {
  304. case 0:
  305. BUG();
  306. case 1:
  307. /*
  308. * Avoid an unnecessary wake_up() function call.
  309. * The common case is pkmap_count[] == 1, but
  310. * no waiters.
  311. * The tasks queued in the wait-queue are guarded
  312. * by both the lock in the wait-queue-head and by
  313. * the kmap_lock. As the kmap_lock is held here,
  314. * no need for the wait-queue-head's lock. Simply
  315. * test if the queue is empty.
  316. */
  317. pkmap_map_wait = get_pkmap_wait_queue_head(color);
  318. need_wakeup = waitqueue_active(pkmap_map_wait);
  319. }
  320. unlock_kmap_any(flags);
  321. /* do wake-up, if needed, race-free outside of the spin lock */
  322. if (need_wakeup)
  323. wake_up(pkmap_map_wait);
  324. }
  325. EXPORT_SYMBOL(kunmap_high);
  326. #endif
  327. #if defined(HASHED_PAGE_VIRTUAL)
  328. #define PA_HASH_ORDER 7
  329. /*
  330. * Describes one page->virtual association
  331. */
  332. struct page_address_map {
  333. struct page *page;
  334. void *virtual;
  335. struct list_head list;
  336. };
  337. static struct page_address_map page_address_maps[LAST_PKMAP];
  338. /*
  339. * Hash table bucket
  340. */
  341. static struct page_address_slot {
  342. struct list_head lh; /* List of page_address_maps */
  343. spinlock_t lock; /* Protect this bucket's list */
  344. } ____cacheline_aligned_in_smp page_address_htable[1<<PA_HASH_ORDER];
  345. static struct page_address_slot *page_slot(const struct page *page)
  346. {
  347. return &page_address_htable[hash_ptr(page, PA_HASH_ORDER)];
  348. }
  349. /**
  350. * page_address - get the mapped virtual address of a page
  351. * @page: &struct page to get the virtual address of
  352. *
  353. * Returns the page's virtual address.
  354. */
  355. void *page_address(const struct page *page)
  356. {
  357. unsigned long flags;
  358. void *ret;
  359. struct page_address_slot *pas;
  360. if (!PageHighMem(page))
  361. return lowmem_page_address(page);
  362. pas = page_slot(page);
  363. ret = NULL;
  364. spin_lock_irqsave(&pas->lock, flags);
  365. if (!list_empty(&pas->lh)) {
  366. struct page_address_map *pam;
  367. list_for_each_entry(pam, &pas->lh, list) {
  368. if (pam->page == page) {
  369. ret = pam->virtual;
  370. goto done;
  371. }
  372. }
  373. }
  374. done:
  375. spin_unlock_irqrestore(&pas->lock, flags);
  376. return ret;
  377. }
  378. EXPORT_SYMBOL(page_address);
  379. /**
  380. * set_page_address - set a page's virtual address
  381. * @page: &struct page to set
  382. * @virtual: virtual address to use
  383. */
  384. void set_page_address(struct page *page, void *virtual)
  385. {
  386. unsigned long flags;
  387. struct page_address_slot *pas;
  388. struct page_address_map *pam;
  389. BUG_ON(!PageHighMem(page));
  390. pas = page_slot(page);
  391. if (virtual) { /* Add */
  392. pam = &page_address_maps[PKMAP_NR((unsigned long)virtual)];
  393. pam->page = page;
  394. pam->virtual = virtual;
  395. spin_lock_irqsave(&pas->lock, flags);
  396. list_add_tail(&pam->list, &pas->lh);
  397. spin_unlock_irqrestore(&pas->lock, flags);
  398. } else { /* Remove */
  399. spin_lock_irqsave(&pas->lock, flags);
  400. list_for_each_entry(pam, &pas->lh, list) {
  401. if (pam->page == page) {
  402. list_del(&pam->list);
  403. spin_unlock_irqrestore(&pas->lock, flags);
  404. goto done;
  405. }
  406. }
  407. spin_unlock_irqrestore(&pas->lock, flags);
  408. }
  409. done:
  410. return;
  411. }
  412. void __init page_address_init(void)
  413. {
  414. int i;
  415. for (i = 0; i < ARRAY_SIZE(page_address_htable); i++) {
  416. INIT_LIST_HEAD(&page_address_htable[i].lh);
  417. spin_lock_init(&page_address_htable[i].lock);
  418. }
  419. }
  420. #endif /* defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL) */