topology.c 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506
  1. /*
  2. * Copyright IBM Corp. 2007, 2011
  3. * Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>
  4. */
  5. #define KMSG_COMPONENT "cpu"
  6. #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  7. #include <linux/workqueue.h>
  8. #include <linux/cpuset.h>
  9. #include <linux/device.h>
  10. #include <linux/export.h>
  11. #include <linux/kernel.h>
  12. #include <linux/sched.h>
  13. #include <linux/delay.h>
  14. #include <linux/init.h>
  15. #include <linux/slab.h>
  16. #include <linux/cpu.h>
  17. #include <linux/smp.h>
  18. #include <linux/mm.h>
  19. #include <linux/nodemask.h>
  20. #include <linux/node.h>
  21. #include <asm/sysinfo.h>
  22. #include <asm/numa.h>
  23. #define PTF_HORIZONTAL (0UL)
  24. #define PTF_VERTICAL (1UL)
  25. #define PTF_CHECK (2UL)
  26. struct mask_info {
  27. struct mask_info *next;
  28. unsigned char id;
  29. cpumask_t mask;
  30. };
  31. static void set_topology_timer(void);
  32. static void topology_work_fn(struct work_struct *work);
  33. static struct sysinfo_15_1_x *tl_info;
  34. static int topology_enabled = 1;
  35. static DECLARE_WORK(topology_work, topology_work_fn);
  36. /*
  37. * Socket/Book linked lists and per_cpu(cpu_topology) updates are
  38. * protected by "sched_domains_mutex".
  39. */
  40. static struct mask_info socket_info;
  41. static struct mask_info book_info;
  42. DEFINE_PER_CPU(struct cpu_topology_s390, cpu_topology);
  43. EXPORT_PER_CPU_SYMBOL_GPL(cpu_topology);
  44. static cpumask_t cpu_group_map(struct mask_info *info, unsigned int cpu)
  45. {
  46. cpumask_t mask;
  47. cpumask_copy(&mask, cpumask_of(cpu));
  48. if (!topology_enabled || !MACHINE_HAS_TOPOLOGY)
  49. return mask;
  50. for (; info; info = info->next) {
  51. if (cpumask_test_cpu(cpu, &info->mask))
  52. return info->mask;
  53. }
  54. return mask;
  55. }
  56. static cpumask_t cpu_thread_map(unsigned int cpu)
  57. {
  58. cpumask_t mask;
  59. int i;
  60. cpumask_copy(&mask, cpumask_of(cpu));
  61. if (!topology_enabled || !MACHINE_HAS_TOPOLOGY)
  62. return mask;
  63. cpu -= cpu % (smp_cpu_mtid + 1);
  64. for (i = 0; i <= smp_cpu_mtid; i++)
  65. if (cpu_present(cpu + i))
  66. cpumask_set_cpu(cpu + i, &mask);
  67. return mask;
  68. }
  69. static struct mask_info *add_cpus_to_mask(struct topology_core *tl_core,
  70. struct mask_info *book,
  71. struct mask_info *socket,
  72. int one_socket_per_cpu)
  73. {
  74. struct cpu_topology_s390 *topo;
  75. unsigned int core;
  76. for_each_set_bit(core, &tl_core->mask[0], TOPOLOGY_CORE_BITS) {
  77. unsigned int rcore;
  78. int lcpu, i;
  79. rcore = TOPOLOGY_CORE_BITS - 1 - core + tl_core->origin;
  80. lcpu = smp_find_processor_id(rcore << smp_cpu_mt_shift);
  81. if (lcpu < 0)
  82. continue;
  83. for (i = 0; i <= smp_cpu_mtid; i++) {
  84. topo = &per_cpu(cpu_topology, lcpu + i);
  85. topo->book_id = book->id;
  86. topo->core_id = rcore;
  87. topo->thread_id = lcpu + i;
  88. cpumask_set_cpu(lcpu + i, &book->mask);
  89. cpumask_set_cpu(lcpu + i, &socket->mask);
  90. if (one_socket_per_cpu)
  91. topo->socket_id = rcore;
  92. else
  93. topo->socket_id = socket->id;
  94. smp_cpu_set_polarization(lcpu + i, tl_core->pp);
  95. }
  96. if (one_socket_per_cpu)
  97. socket = socket->next;
  98. }
  99. return socket;
  100. }
  101. static void clear_masks(void)
  102. {
  103. struct mask_info *info;
  104. info = &socket_info;
  105. while (info) {
  106. cpumask_clear(&info->mask);
  107. info = info->next;
  108. }
  109. info = &book_info;
  110. while (info) {
  111. cpumask_clear(&info->mask);
  112. info = info->next;
  113. }
  114. }
  115. static union topology_entry *next_tle(union topology_entry *tle)
  116. {
  117. if (!tle->nl)
  118. return (union topology_entry *)((struct topology_core *)tle + 1);
  119. return (union topology_entry *)((struct topology_container *)tle + 1);
  120. }
  121. static void __tl_to_masks_generic(struct sysinfo_15_1_x *info)
  122. {
  123. struct mask_info *socket = &socket_info;
  124. struct mask_info *book = &book_info;
  125. union topology_entry *tle, *end;
  126. tle = info->tle;
  127. end = (union topology_entry *)((unsigned long)info + info->length);
  128. while (tle < end) {
  129. switch (tle->nl) {
  130. case 2:
  131. book = book->next;
  132. book->id = tle->container.id;
  133. break;
  134. case 1:
  135. socket = socket->next;
  136. socket->id = tle->container.id;
  137. break;
  138. case 0:
  139. add_cpus_to_mask(&tle->cpu, book, socket, 0);
  140. break;
  141. default:
  142. clear_masks();
  143. return;
  144. }
  145. tle = next_tle(tle);
  146. }
  147. }
  148. static void __tl_to_masks_z10(struct sysinfo_15_1_x *info)
  149. {
  150. struct mask_info *socket = &socket_info;
  151. struct mask_info *book = &book_info;
  152. union topology_entry *tle, *end;
  153. tle = info->tle;
  154. end = (union topology_entry *)((unsigned long)info + info->length);
  155. while (tle < end) {
  156. switch (tle->nl) {
  157. case 1:
  158. book = book->next;
  159. book->id = tle->container.id;
  160. break;
  161. case 0:
  162. socket = add_cpus_to_mask(&tle->cpu, book, socket, 1);
  163. break;
  164. default:
  165. clear_masks();
  166. return;
  167. }
  168. tle = next_tle(tle);
  169. }
  170. }
  171. static void tl_to_masks(struct sysinfo_15_1_x *info)
  172. {
  173. struct cpuid cpu_id;
  174. get_cpu_id(&cpu_id);
  175. clear_masks();
  176. switch (cpu_id.machine) {
  177. case 0x2097:
  178. case 0x2098:
  179. __tl_to_masks_z10(info);
  180. break;
  181. default:
  182. __tl_to_masks_generic(info);
  183. }
  184. }
  185. static void topology_update_polarization_simple(void)
  186. {
  187. int cpu;
  188. mutex_lock(&smp_cpu_state_mutex);
  189. for_each_possible_cpu(cpu)
  190. smp_cpu_set_polarization(cpu, POLARIZATION_HRZ);
  191. mutex_unlock(&smp_cpu_state_mutex);
  192. }
  193. static int ptf(unsigned long fc)
  194. {
  195. int rc;
  196. asm volatile(
  197. " .insn rre,0xb9a20000,%1,%1\n"
  198. " ipm %0\n"
  199. " srl %0,28\n"
  200. : "=d" (rc)
  201. : "d" (fc) : "cc");
  202. return rc;
  203. }
  204. int topology_set_cpu_management(int fc)
  205. {
  206. int cpu, rc;
  207. if (!MACHINE_HAS_TOPOLOGY)
  208. return -EOPNOTSUPP;
  209. if (fc)
  210. rc = ptf(PTF_VERTICAL);
  211. else
  212. rc = ptf(PTF_HORIZONTAL);
  213. if (rc)
  214. return -EBUSY;
  215. for_each_possible_cpu(cpu)
  216. smp_cpu_set_polarization(cpu, POLARIZATION_UNKNOWN);
  217. return rc;
  218. }
  219. static void update_cpu_masks(void)
  220. {
  221. struct cpu_topology_s390 *topo;
  222. int cpu;
  223. for_each_possible_cpu(cpu) {
  224. topo = &per_cpu(cpu_topology, cpu);
  225. topo->thread_mask = cpu_thread_map(cpu);
  226. topo->core_mask = cpu_group_map(&socket_info, cpu);
  227. topo->book_mask = cpu_group_map(&book_info, cpu);
  228. if (!MACHINE_HAS_TOPOLOGY) {
  229. topo->thread_id = cpu;
  230. topo->core_id = cpu;
  231. topo->socket_id = cpu;
  232. topo->book_id = cpu;
  233. }
  234. }
  235. numa_update_cpu_topology();
  236. }
  237. void store_topology(struct sysinfo_15_1_x *info)
  238. {
  239. if (topology_max_mnest >= 3)
  240. stsi(info, 15, 1, 3);
  241. else
  242. stsi(info, 15, 1, 2);
  243. }
  244. int arch_update_cpu_topology(void)
  245. {
  246. struct sysinfo_15_1_x *info = tl_info;
  247. struct device *dev;
  248. int cpu, rc = 0;
  249. if (MACHINE_HAS_TOPOLOGY) {
  250. rc = 1;
  251. store_topology(info);
  252. tl_to_masks(info);
  253. }
  254. update_cpu_masks();
  255. if (!MACHINE_HAS_TOPOLOGY)
  256. topology_update_polarization_simple();
  257. for_each_online_cpu(cpu) {
  258. dev = get_cpu_device(cpu);
  259. kobject_uevent(&dev->kobj, KOBJ_CHANGE);
  260. }
  261. return rc;
  262. }
  263. static void topology_work_fn(struct work_struct *work)
  264. {
  265. rebuild_sched_domains();
  266. }
  267. void topology_schedule_update(void)
  268. {
  269. schedule_work(&topology_work);
  270. }
  271. static void topology_timer_fn(unsigned long ignored)
  272. {
  273. if (ptf(PTF_CHECK))
  274. topology_schedule_update();
  275. set_topology_timer();
  276. }
  277. static struct timer_list topology_timer =
  278. TIMER_DEFERRED_INITIALIZER(topology_timer_fn, 0, 0);
  279. static atomic_t topology_poll = ATOMIC_INIT(0);
  280. static void set_topology_timer(void)
  281. {
  282. if (atomic_add_unless(&topology_poll, -1, 0))
  283. mod_timer(&topology_timer, jiffies + HZ / 10);
  284. else
  285. mod_timer(&topology_timer, jiffies + HZ * 60);
  286. }
  287. void topology_expect_change(void)
  288. {
  289. if (!MACHINE_HAS_TOPOLOGY)
  290. return;
  291. /* This is racy, but it doesn't matter since it is just a heuristic.
  292. * Worst case is that we poll in a higher frequency for a bit longer.
  293. */
  294. if (atomic_read(&topology_poll) > 60)
  295. return;
  296. atomic_add(60, &topology_poll);
  297. set_topology_timer();
  298. }
  299. static int cpu_management;
  300. static ssize_t dispatching_show(struct device *dev,
  301. struct device_attribute *attr,
  302. char *buf)
  303. {
  304. ssize_t count;
  305. mutex_lock(&smp_cpu_state_mutex);
  306. count = sprintf(buf, "%d\n", cpu_management);
  307. mutex_unlock(&smp_cpu_state_mutex);
  308. return count;
  309. }
  310. static ssize_t dispatching_store(struct device *dev,
  311. struct device_attribute *attr,
  312. const char *buf,
  313. size_t count)
  314. {
  315. int val, rc;
  316. char delim;
  317. if (sscanf(buf, "%d %c", &val, &delim) != 1)
  318. return -EINVAL;
  319. if (val != 0 && val != 1)
  320. return -EINVAL;
  321. rc = 0;
  322. get_online_cpus();
  323. mutex_lock(&smp_cpu_state_mutex);
  324. if (cpu_management == val)
  325. goto out;
  326. rc = topology_set_cpu_management(val);
  327. if (rc)
  328. goto out;
  329. cpu_management = val;
  330. topology_expect_change();
  331. out:
  332. mutex_unlock(&smp_cpu_state_mutex);
  333. put_online_cpus();
  334. return rc ? rc : count;
  335. }
  336. static DEVICE_ATTR(dispatching, 0644, dispatching_show,
  337. dispatching_store);
  338. static ssize_t cpu_polarization_show(struct device *dev,
  339. struct device_attribute *attr, char *buf)
  340. {
  341. int cpu = dev->id;
  342. ssize_t count;
  343. mutex_lock(&smp_cpu_state_mutex);
  344. switch (smp_cpu_get_polarization(cpu)) {
  345. case POLARIZATION_HRZ:
  346. count = sprintf(buf, "horizontal\n");
  347. break;
  348. case POLARIZATION_VL:
  349. count = sprintf(buf, "vertical:low\n");
  350. break;
  351. case POLARIZATION_VM:
  352. count = sprintf(buf, "vertical:medium\n");
  353. break;
  354. case POLARIZATION_VH:
  355. count = sprintf(buf, "vertical:high\n");
  356. break;
  357. default:
  358. count = sprintf(buf, "unknown\n");
  359. break;
  360. }
  361. mutex_unlock(&smp_cpu_state_mutex);
  362. return count;
  363. }
  364. static DEVICE_ATTR(polarization, 0444, cpu_polarization_show, NULL);
  365. static struct attribute *topology_cpu_attrs[] = {
  366. &dev_attr_polarization.attr,
  367. NULL,
  368. };
  369. static struct attribute_group topology_cpu_attr_group = {
  370. .attrs = topology_cpu_attrs,
  371. };
  372. int topology_cpu_init(struct cpu *cpu)
  373. {
  374. return sysfs_create_group(&cpu->dev.kobj, &topology_cpu_attr_group);
  375. }
  376. static const struct cpumask *cpu_thread_mask(int cpu)
  377. {
  378. return &per_cpu(cpu_topology, cpu).thread_mask;
  379. }
  380. const struct cpumask *cpu_coregroup_mask(int cpu)
  381. {
  382. return &per_cpu(cpu_topology, cpu).core_mask;
  383. }
  384. static const struct cpumask *cpu_book_mask(int cpu)
  385. {
  386. return &per_cpu(cpu_topology, cpu).book_mask;
  387. }
  388. static int __init early_parse_topology(char *p)
  389. {
  390. if (strncmp(p, "off", 3))
  391. return 0;
  392. topology_enabled = 0;
  393. return 0;
  394. }
  395. early_param("topology", early_parse_topology);
  396. static struct sched_domain_topology_level s390_topology[] = {
  397. { cpu_thread_mask, cpu_smt_flags, SD_INIT_NAME(SMT) },
  398. { cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) },
  399. { cpu_book_mask, SD_INIT_NAME(BOOK) },
  400. { cpu_cpu_mask, SD_INIT_NAME(DIE) },
  401. { NULL, },
  402. };
  403. static void __init alloc_masks(struct sysinfo_15_1_x *info,
  404. struct mask_info *mask, int offset)
  405. {
  406. int i, nr_masks;
  407. nr_masks = info->mag[TOPOLOGY_NR_MAG - offset];
  408. for (i = 0; i < info->mnest - offset; i++)
  409. nr_masks *= info->mag[TOPOLOGY_NR_MAG - offset - 1 - i];
  410. nr_masks = max(nr_masks, 1);
  411. for (i = 0; i < nr_masks; i++) {
  412. mask->next = kzalloc(sizeof(*mask->next), GFP_KERNEL);
  413. mask = mask->next;
  414. }
  415. }
  416. static int __init s390_topology_init(void)
  417. {
  418. struct sysinfo_15_1_x *info;
  419. int i;
  420. if (!MACHINE_HAS_TOPOLOGY)
  421. return 0;
  422. tl_info = (struct sysinfo_15_1_x *)__get_free_page(GFP_KERNEL);
  423. info = tl_info;
  424. store_topology(info);
  425. pr_info("The CPU configuration topology of the machine is:");
  426. for (i = 0; i < TOPOLOGY_NR_MAG; i++)
  427. printk(KERN_CONT " %d", info->mag[i]);
  428. printk(KERN_CONT " / %d\n", info->mnest);
  429. alloc_masks(info, &socket_info, 1);
  430. alloc_masks(info, &book_info, 2);
  431. set_sched_topology(s390_topology);
  432. return 0;
  433. }
  434. early_initcall(s390_topology_init);
  435. static int __init topology_init(void)
  436. {
  437. if (MACHINE_HAS_TOPOLOGY)
  438. set_topology_timer();
  439. else
  440. topology_update_polarization_simple();
  441. return device_create_file(cpu_subsys.dev_root, &dev_attr_dispatching);
  442. }
  443. device_initcall(topology_init);