amdgpu_uvd.c 25 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049
  1. /*
  2. * Copyright 2011 Advanced Micro Devices, Inc.
  3. * All Rights Reserved.
  4. *
  5. * Permission is hereby granted, free of charge, to any person obtaining a
  6. * copy of this software and associated documentation files (the
  7. * "Software"), to deal in the Software without restriction, including
  8. * without limitation the rights to use, copy, modify, merge, publish,
  9. * distribute, sub license, and/or sell copies of the Software, and to
  10. * permit persons to whom the Software is furnished to do so, subject to
  11. * the following conditions:
  12. *
  13. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15. * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  16. * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  17. * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  18. * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  19. * USE OR OTHER DEALINGS IN THE SOFTWARE.
  20. *
  21. * The above copyright notice and this permission notice (including the
  22. * next paragraph) shall be included in all copies or substantial portions
  23. * of the Software.
  24. *
  25. */
  26. /*
  27. * Authors:
  28. * Christian König <deathsimple@vodafone.de>
  29. */
  30. #include <linux/firmware.h>
  31. #include <linux/module.h>
  32. #include <drm/drmP.h>
  33. #include <drm/drm.h>
  34. #include "amdgpu.h"
  35. #include "amdgpu_pm.h"
  36. #include "amdgpu_uvd.h"
  37. #include "cikd.h"
  38. #include "uvd/uvd_4_2_d.h"
  39. /* 1 second timeout */
  40. #define UVD_IDLE_TIMEOUT_MS 1000
  41. /* Firmware Names */
  42. #ifdef CONFIG_DRM_AMDGPU_CIK
  43. #define FIRMWARE_BONAIRE "radeon/bonaire_uvd.bin"
  44. #define FIRMWARE_KABINI "radeon/kabini_uvd.bin"
  45. #define FIRMWARE_KAVERI "radeon/kaveri_uvd.bin"
  46. #define FIRMWARE_HAWAII "radeon/hawaii_uvd.bin"
  47. #define FIRMWARE_MULLINS "radeon/mullins_uvd.bin"
  48. #endif
  49. #define FIRMWARE_TONGA "amdgpu/tonga_uvd.bin"
  50. #define FIRMWARE_CARRIZO "amdgpu/carrizo_uvd.bin"
  51. #define FIRMWARE_FIJI "amdgpu/fiji_uvd.bin"
  52. #define FIRMWARE_STONEY "amdgpu/stoney_uvd.bin"
  53. /**
  54. * amdgpu_uvd_cs_ctx - Command submission parser context
  55. *
  56. * Used for emulating virtual memory support on UVD 4.2.
  57. */
  58. struct amdgpu_uvd_cs_ctx {
  59. struct amdgpu_cs_parser *parser;
  60. unsigned reg, count;
  61. unsigned data0, data1;
  62. unsigned idx;
  63. unsigned ib_idx;
  64. /* does the IB has a msg command */
  65. bool has_msg_cmd;
  66. /* minimum buffer sizes */
  67. unsigned *buf_sizes;
  68. };
  69. #ifdef CONFIG_DRM_AMDGPU_CIK
  70. MODULE_FIRMWARE(FIRMWARE_BONAIRE);
  71. MODULE_FIRMWARE(FIRMWARE_KABINI);
  72. MODULE_FIRMWARE(FIRMWARE_KAVERI);
  73. MODULE_FIRMWARE(FIRMWARE_HAWAII);
  74. MODULE_FIRMWARE(FIRMWARE_MULLINS);
  75. #endif
  76. MODULE_FIRMWARE(FIRMWARE_TONGA);
  77. MODULE_FIRMWARE(FIRMWARE_CARRIZO);
  78. MODULE_FIRMWARE(FIRMWARE_FIJI);
  79. MODULE_FIRMWARE(FIRMWARE_STONEY);
  80. static void amdgpu_uvd_note_usage(struct amdgpu_device *adev);
  81. static void amdgpu_uvd_idle_work_handler(struct work_struct *work);
  82. int amdgpu_uvd_sw_init(struct amdgpu_device *adev)
  83. {
  84. unsigned long bo_size;
  85. const char *fw_name;
  86. const struct common_firmware_header *hdr;
  87. unsigned version_major, version_minor, family_id;
  88. int i, r;
  89. INIT_DELAYED_WORK(&adev->uvd.idle_work, amdgpu_uvd_idle_work_handler);
  90. switch (adev->asic_type) {
  91. #ifdef CONFIG_DRM_AMDGPU_CIK
  92. case CHIP_BONAIRE:
  93. fw_name = FIRMWARE_BONAIRE;
  94. break;
  95. case CHIP_KABINI:
  96. fw_name = FIRMWARE_KABINI;
  97. break;
  98. case CHIP_KAVERI:
  99. fw_name = FIRMWARE_KAVERI;
  100. break;
  101. case CHIP_HAWAII:
  102. fw_name = FIRMWARE_HAWAII;
  103. break;
  104. case CHIP_MULLINS:
  105. fw_name = FIRMWARE_MULLINS;
  106. break;
  107. #endif
  108. case CHIP_TONGA:
  109. fw_name = FIRMWARE_TONGA;
  110. break;
  111. case CHIP_FIJI:
  112. fw_name = FIRMWARE_FIJI;
  113. break;
  114. case CHIP_CARRIZO:
  115. fw_name = FIRMWARE_CARRIZO;
  116. break;
  117. case CHIP_STONEY:
  118. fw_name = FIRMWARE_STONEY;
  119. break;
  120. default:
  121. return -EINVAL;
  122. }
  123. r = request_firmware(&adev->uvd.fw, fw_name, adev->dev);
  124. if (r) {
  125. dev_err(adev->dev, "amdgpu_uvd: Can't load firmware \"%s\"\n",
  126. fw_name);
  127. return r;
  128. }
  129. r = amdgpu_ucode_validate(adev->uvd.fw);
  130. if (r) {
  131. dev_err(adev->dev, "amdgpu_uvd: Can't validate firmware \"%s\"\n",
  132. fw_name);
  133. release_firmware(adev->uvd.fw);
  134. adev->uvd.fw = NULL;
  135. return r;
  136. }
  137. hdr = (const struct common_firmware_header *)adev->uvd.fw->data;
  138. family_id = le32_to_cpu(hdr->ucode_version) & 0xff;
  139. version_major = (le32_to_cpu(hdr->ucode_version) >> 24) & 0xff;
  140. version_minor = (le32_to_cpu(hdr->ucode_version) >> 8) & 0xff;
  141. DRM_INFO("Found UVD firmware Version: %hu.%hu Family ID: %hu\n",
  142. version_major, version_minor, family_id);
  143. adev->uvd.fw_version = ((version_major << 24) | (version_minor << 16) |
  144. (family_id << 8));
  145. bo_size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8)
  146. + AMDGPU_UVD_STACK_SIZE + AMDGPU_UVD_HEAP_SIZE;
  147. r = amdgpu_bo_create(adev, bo_size, PAGE_SIZE, true,
  148. AMDGPU_GEM_DOMAIN_VRAM,
  149. AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
  150. NULL, NULL, &adev->uvd.vcpu_bo);
  151. if (r) {
  152. dev_err(adev->dev, "(%d) failed to allocate UVD bo\n", r);
  153. return r;
  154. }
  155. r = amdgpu_bo_reserve(adev->uvd.vcpu_bo, false);
  156. if (r) {
  157. amdgpu_bo_unref(&adev->uvd.vcpu_bo);
  158. dev_err(adev->dev, "(%d) failed to reserve UVD bo\n", r);
  159. return r;
  160. }
  161. r = amdgpu_bo_pin(adev->uvd.vcpu_bo, AMDGPU_GEM_DOMAIN_VRAM,
  162. &adev->uvd.gpu_addr);
  163. if (r) {
  164. amdgpu_bo_unreserve(adev->uvd.vcpu_bo);
  165. amdgpu_bo_unref(&adev->uvd.vcpu_bo);
  166. dev_err(adev->dev, "(%d) UVD bo pin failed\n", r);
  167. return r;
  168. }
  169. r = amdgpu_bo_kmap(adev->uvd.vcpu_bo, &adev->uvd.cpu_addr);
  170. if (r) {
  171. dev_err(adev->dev, "(%d) UVD map failed\n", r);
  172. return r;
  173. }
  174. amdgpu_bo_unreserve(adev->uvd.vcpu_bo);
  175. for (i = 0; i < AMDGPU_MAX_UVD_HANDLES; ++i) {
  176. atomic_set(&adev->uvd.handles[i], 0);
  177. adev->uvd.filp[i] = NULL;
  178. }
  179. /* from uvd v5.0 HW addressing capacity increased to 64 bits */
  180. if (!amdgpu_ip_block_version_cmp(adev, AMD_IP_BLOCK_TYPE_UVD, 5, 0))
  181. adev->uvd.address_64_bit = true;
  182. return 0;
  183. }
  184. int amdgpu_uvd_sw_fini(struct amdgpu_device *adev)
  185. {
  186. int r;
  187. if (adev->uvd.vcpu_bo == NULL)
  188. return 0;
  189. r = amdgpu_bo_reserve(adev->uvd.vcpu_bo, false);
  190. if (!r) {
  191. amdgpu_bo_kunmap(adev->uvd.vcpu_bo);
  192. amdgpu_bo_unpin(adev->uvd.vcpu_bo);
  193. amdgpu_bo_unreserve(adev->uvd.vcpu_bo);
  194. }
  195. amdgpu_bo_unref(&adev->uvd.vcpu_bo);
  196. amdgpu_ring_fini(&adev->uvd.ring);
  197. release_firmware(adev->uvd.fw);
  198. return 0;
  199. }
  200. int amdgpu_uvd_suspend(struct amdgpu_device *adev)
  201. {
  202. struct amdgpu_ring *ring = &adev->uvd.ring;
  203. int i, r;
  204. if (adev->uvd.vcpu_bo == NULL)
  205. return 0;
  206. for (i = 0; i < AMDGPU_MAX_UVD_HANDLES; ++i) {
  207. uint32_t handle = atomic_read(&adev->uvd.handles[i]);
  208. if (handle != 0) {
  209. struct fence *fence;
  210. amdgpu_uvd_note_usage(adev);
  211. r = amdgpu_uvd_get_destroy_msg(ring, handle, &fence);
  212. if (r) {
  213. DRM_ERROR("Error destroying UVD (%d)!\n", r);
  214. continue;
  215. }
  216. fence_wait(fence, false);
  217. fence_put(fence);
  218. adev->uvd.filp[i] = NULL;
  219. atomic_set(&adev->uvd.handles[i], 0);
  220. }
  221. }
  222. return 0;
  223. }
  224. int amdgpu_uvd_resume(struct amdgpu_device *adev)
  225. {
  226. unsigned size;
  227. void *ptr;
  228. const struct common_firmware_header *hdr;
  229. unsigned offset;
  230. if (adev->uvd.vcpu_bo == NULL)
  231. return -EINVAL;
  232. hdr = (const struct common_firmware_header *)adev->uvd.fw->data;
  233. offset = le32_to_cpu(hdr->ucode_array_offset_bytes);
  234. memcpy(adev->uvd.cpu_addr, (adev->uvd.fw->data) + offset,
  235. (adev->uvd.fw->size) - offset);
  236. cancel_delayed_work_sync(&adev->uvd.idle_work);
  237. size = amdgpu_bo_size(adev->uvd.vcpu_bo);
  238. size -= le32_to_cpu(hdr->ucode_size_bytes);
  239. ptr = adev->uvd.cpu_addr;
  240. ptr += le32_to_cpu(hdr->ucode_size_bytes);
  241. memset(ptr, 0, size);
  242. return 0;
  243. }
  244. void amdgpu_uvd_free_handles(struct amdgpu_device *adev, struct drm_file *filp)
  245. {
  246. struct amdgpu_ring *ring = &adev->uvd.ring;
  247. int i, r;
  248. for (i = 0; i < AMDGPU_MAX_UVD_HANDLES; ++i) {
  249. uint32_t handle = atomic_read(&adev->uvd.handles[i]);
  250. if (handle != 0 && adev->uvd.filp[i] == filp) {
  251. struct fence *fence;
  252. amdgpu_uvd_note_usage(adev);
  253. r = amdgpu_uvd_get_destroy_msg(ring, handle, &fence);
  254. if (r) {
  255. DRM_ERROR("Error destroying UVD (%d)!\n", r);
  256. continue;
  257. }
  258. fence_wait(fence, false);
  259. fence_put(fence);
  260. adev->uvd.filp[i] = NULL;
  261. atomic_set(&adev->uvd.handles[i], 0);
  262. }
  263. }
  264. }
  265. static void amdgpu_uvd_force_into_uvd_segment(struct amdgpu_bo *rbo)
  266. {
  267. int i;
  268. for (i = 0; i < rbo->placement.num_placement; ++i) {
  269. rbo->placements[i].fpfn = 0 >> PAGE_SHIFT;
  270. rbo->placements[i].lpfn = (256 * 1024 * 1024) >> PAGE_SHIFT;
  271. }
  272. }
  273. /**
  274. * amdgpu_uvd_cs_pass1 - first parsing round
  275. *
  276. * @ctx: UVD parser context
  277. *
  278. * Make sure UVD message and feedback buffers are in VRAM and
  279. * nobody is violating an 256MB boundary.
  280. */
  281. static int amdgpu_uvd_cs_pass1(struct amdgpu_uvd_cs_ctx *ctx)
  282. {
  283. struct amdgpu_bo_va_mapping *mapping;
  284. struct amdgpu_bo *bo;
  285. uint32_t cmd, lo, hi;
  286. uint64_t addr;
  287. int r = 0;
  288. lo = amdgpu_get_ib_value(ctx->parser, ctx->ib_idx, ctx->data0);
  289. hi = amdgpu_get_ib_value(ctx->parser, ctx->ib_idx, ctx->data1);
  290. addr = ((uint64_t)lo) | (((uint64_t)hi) << 32);
  291. mapping = amdgpu_cs_find_mapping(ctx->parser, addr, &bo);
  292. if (mapping == NULL) {
  293. DRM_ERROR("Can't find BO for addr 0x%08Lx\n", addr);
  294. return -EINVAL;
  295. }
  296. if (!ctx->parser->adev->uvd.address_64_bit) {
  297. /* check if it's a message or feedback command */
  298. cmd = amdgpu_get_ib_value(ctx->parser, ctx->ib_idx, ctx->idx) >> 1;
  299. if (cmd == 0x0 || cmd == 0x3) {
  300. /* yes, force it into VRAM */
  301. uint32_t domain = AMDGPU_GEM_DOMAIN_VRAM;
  302. amdgpu_ttm_placement_from_domain(bo, domain);
  303. }
  304. amdgpu_uvd_force_into_uvd_segment(bo);
  305. r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
  306. }
  307. return r;
  308. }
  309. /**
  310. * amdgpu_uvd_cs_msg_decode - handle UVD decode message
  311. *
  312. * @msg: pointer to message structure
  313. * @buf_sizes: returned buffer sizes
  314. *
  315. * Peek into the decode message and calculate the necessary buffer sizes.
  316. */
  317. static int amdgpu_uvd_cs_msg_decode(uint32_t *msg, unsigned buf_sizes[])
  318. {
  319. unsigned stream_type = msg[4];
  320. unsigned width = msg[6];
  321. unsigned height = msg[7];
  322. unsigned dpb_size = msg[9];
  323. unsigned pitch = msg[28];
  324. unsigned level = msg[57];
  325. unsigned width_in_mb = width / 16;
  326. unsigned height_in_mb = ALIGN(height / 16, 2);
  327. unsigned fs_in_mb = width_in_mb * height_in_mb;
  328. unsigned image_size, tmp, min_dpb_size, num_dpb_buffer;
  329. unsigned min_ctx_size = 0;
  330. image_size = width * height;
  331. image_size += image_size / 2;
  332. image_size = ALIGN(image_size, 1024);
  333. switch (stream_type) {
  334. case 0: /* H264 */
  335. case 7: /* H264 Perf */
  336. switch(level) {
  337. case 30:
  338. num_dpb_buffer = 8100 / fs_in_mb;
  339. break;
  340. case 31:
  341. num_dpb_buffer = 18000 / fs_in_mb;
  342. break;
  343. case 32:
  344. num_dpb_buffer = 20480 / fs_in_mb;
  345. break;
  346. case 41:
  347. num_dpb_buffer = 32768 / fs_in_mb;
  348. break;
  349. case 42:
  350. num_dpb_buffer = 34816 / fs_in_mb;
  351. break;
  352. case 50:
  353. num_dpb_buffer = 110400 / fs_in_mb;
  354. break;
  355. case 51:
  356. num_dpb_buffer = 184320 / fs_in_mb;
  357. break;
  358. default:
  359. num_dpb_buffer = 184320 / fs_in_mb;
  360. break;
  361. }
  362. num_dpb_buffer++;
  363. if (num_dpb_buffer > 17)
  364. num_dpb_buffer = 17;
  365. /* reference picture buffer */
  366. min_dpb_size = image_size * num_dpb_buffer;
  367. /* macroblock context buffer */
  368. min_dpb_size += width_in_mb * height_in_mb * num_dpb_buffer * 192;
  369. /* IT surface buffer */
  370. min_dpb_size += width_in_mb * height_in_mb * 32;
  371. break;
  372. case 1: /* VC1 */
  373. /* reference picture buffer */
  374. min_dpb_size = image_size * 3;
  375. /* CONTEXT_BUFFER */
  376. min_dpb_size += width_in_mb * height_in_mb * 128;
  377. /* IT surface buffer */
  378. min_dpb_size += width_in_mb * 64;
  379. /* DB surface buffer */
  380. min_dpb_size += width_in_mb * 128;
  381. /* BP */
  382. tmp = max(width_in_mb, height_in_mb);
  383. min_dpb_size += ALIGN(tmp * 7 * 16, 64);
  384. break;
  385. case 3: /* MPEG2 */
  386. /* reference picture buffer */
  387. min_dpb_size = image_size * 3;
  388. break;
  389. case 4: /* MPEG4 */
  390. /* reference picture buffer */
  391. min_dpb_size = image_size * 3;
  392. /* CM */
  393. min_dpb_size += width_in_mb * height_in_mb * 64;
  394. /* IT surface buffer */
  395. min_dpb_size += ALIGN(width_in_mb * height_in_mb * 32, 64);
  396. break;
  397. case 16: /* H265 */
  398. image_size = (ALIGN(width, 16) * ALIGN(height, 16) * 3) / 2;
  399. image_size = ALIGN(image_size, 256);
  400. num_dpb_buffer = (le32_to_cpu(msg[59]) & 0xff) + 2;
  401. min_dpb_size = image_size * num_dpb_buffer;
  402. min_ctx_size = ((width + 255) / 16) * ((height + 255) / 16)
  403. * 16 * num_dpb_buffer + 52 * 1024;
  404. break;
  405. default:
  406. DRM_ERROR("UVD codec not handled %d!\n", stream_type);
  407. return -EINVAL;
  408. }
  409. if (width > pitch) {
  410. DRM_ERROR("Invalid UVD decoding target pitch!\n");
  411. return -EINVAL;
  412. }
  413. if (dpb_size < min_dpb_size) {
  414. DRM_ERROR("Invalid dpb_size in UVD message (%d / %d)!\n",
  415. dpb_size, min_dpb_size);
  416. return -EINVAL;
  417. }
  418. buf_sizes[0x1] = dpb_size;
  419. buf_sizes[0x2] = image_size;
  420. buf_sizes[0x4] = min_ctx_size;
  421. return 0;
  422. }
  423. /**
  424. * amdgpu_uvd_cs_msg - handle UVD message
  425. *
  426. * @ctx: UVD parser context
  427. * @bo: buffer object containing the message
  428. * @offset: offset into the buffer object
  429. *
  430. * Peek into the UVD message and extract the session id.
  431. * Make sure that we don't open up to many sessions.
  432. */
  433. static int amdgpu_uvd_cs_msg(struct amdgpu_uvd_cs_ctx *ctx,
  434. struct amdgpu_bo *bo, unsigned offset)
  435. {
  436. struct amdgpu_device *adev = ctx->parser->adev;
  437. int32_t *msg, msg_type, handle;
  438. void *ptr;
  439. long r;
  440. int i;
  441. if (offset & 0x3F) {
  442. DRM_ERROR("UVD messages must be 64 byte aligned!\n");
  443. return -EINVAL;
  444. }
  445. r = reservation_object_wait_timeout_rcu(bo->tbo.resv, true, false,
  446. MAX_SCHEDULE_TIMEOUT);
  447. if (r < 0) {
  448. DRM_ERROR("Failed waiting for UVD message (%ld)!\n", r);
  449. return r;
  450. }
  451. r = amdgpu_bo_kmap(bo, &ptr);
  452. if (r) {
  453. DRM_ERROR("Failed mapping the UVD message (%ld)!\n", r);
  454. return r;
  455. }
  456. msg = ptr + offset;
  457. msg_type = msg[1];
  458. handle = msg[2];
  459. if (handle == 0) {
  460. DRM_ERROR("Invalid UVD handle!\n");
  461. return -EINVAL;
  462. }
  463. switch (msg_type) {
  464. case 0:
  465. /* it's a create msg, calc image size (width * height) */
  466. amdgpu_bo_kunmap(bo);
  467. /* try to alloc a new handle */
  468. for (i = 0; i < AMDGPU_MAX_UVD_HANDLES; ++i) {
  469. if (atomic_read(&adev->uvd.handles[i]) == handle) {
  470. DRM_ERROR("Handle 0x%x already in use!\n", handle);
  471. return -EINVAL;
  472. }
  473. if (!atomic_cmpxchg(&adev->uvd.handles[i], 0, handle)) {
  474. adev->uvd.filp[i] = ctx->parser->filp;
  475. return 0;
  476. }
  477. }
  478. DRM_ERROR("No more free UVD handles!\n");
  479. return -EINVAL;
  480. case 1:
  481. /* it's a decode msg, calc buffer sizes */
  482. r = amdgpu_uvd_cs_msg_decode(msg, ctx->buf_sizes);
  483. amdgpu_bo_kunmap(bo);
  484. if (r)
  485. return r;
  486. /* validate the handle */
  487. for (i = 0; i < AMDGPU_MAX_UVD_HANDLES; ++i) {
  488. if (atomic_read(&adev->uvd.handles[i]) == handle) {
  489. if (adev->uvd.filp[i] != ctx->parser->filp) {
  490. DRM_ERROR("UVD handle collision detected!\n");
  491. return -EINVAL;
  492. }
  493. return 0;
  494. }
  495. }
  496. DRM_ERROR("Invalid UVD handle 0x%x!\n", handle);
  497. return -ENOENT;
  498. case 2:
  499. /* it's a destroy msg, free the handle */
  500. for (i = 0; i < AMDGPU_MAX_UVD_HANDLES; ++i)
  501. atomic_cmpxchg(&adev->uvd.handles[i], handle, 0);
  502. amdgpu_bo_kunmap(bo);
  503. return 0;
  504. default:
  505. DRM_ERROR("Illegal UVD message type (%d)!\n", msg_type);
  506. return -EINVAL;
  507. }
  508. BUG();
  509. return -EINVAL;
  510. }
  511. /**
  512. * amdgpu_uvd_cs_pass2 - second parsing round
  513. *
  514. * @ctx: UVD parser context
  515. *
  516. * Patch buffer addresses, make sure buffer sizes are correct.
  517. */
  518. static int amdgpu_uvd_cs_pass2(struct amdgpu_uvd_cs_ctx *ctx)
  519. {
  520. struct amdgpu_bo_va_mapping *mapping;
  521. struct amdgpu_bo *bo;
  522. struct amdgpu_ib *ib;
  523. uint32_t cmd, lo, hi;
  524. uint64_t start, end;
  525. uint64_t addr;
  526. int r;
  527. lo = amdgpu_get_ib_value(ctx->parser, ctx->ib_idx, ctx->data0);
  528. hi = amdgpu_get_ib_value(ctx->parser, ctx->ib_idx, ctx->data1);
  529. addr = ((uint64_t)lo) | (((uint64_t)hi) << 32);
  530. mapping = amdgpu_cs_find_mapping(ctx->parser, addr, &bo);
  531. if (mapping == NULL)
  532. return -EINVAL;
  533. start = amdgpu_bo_gpu_offset(bo);
  534. end = (mapping->it.last + 1 - mapping->it.start);
  535. end = end * AMDGPU_GPU_PAGE_SIZE + start;
  536. addr -= ((uint64_t)mapping->it.start) * AMDGPU_GPU_PAGE_SIZE;
  537. start += addr;
  538. ib = &ctx->parser->ibs[ctx->ib_idx];
  539. ib->ptr[ctx->data0] = start & 0xFFFFFFFF;
  540. ib->ptr[ctx->data1] = start >> 32;
  541. cmd = amdgpu_get_ib_value(ctx->parser, ctx->ib_idx, ctx->idx) >> 1;
  542. if (cmd < 0x4) {
  543. if ((end - start) < ctx->buf_sizes[cmd]) {
  544. DRM_ERROR("buffer (%d) to small (%d / %d)!\n", cmd,
  545. (unsigned)(end - start),
  546. ctx->buf_sizes[cmd]);
  547. return -EINVAL;
  548. }
  549. } else if (cmd == 0x206) {
  550. if ((end - start) < ctx->buf_sizes[4]) {
  551. DRM_ERROR("buffer (%d) to small (%d / %d)!\n", cmd,
  552. (unsigned)(end - start),
  553. ctx->buf_sizes[4]);
  554. return -EINVAL;
  555. }
  556. } else if ((cmd != 0x100) && (cmd != 0x204)) {
  557. DRM_ERROR("invalid UVD command %X!\n", cmd);
  558. return -EINVAL;
  559. }
  560. if (!ctx->parser->adev->uvd.address_64_bit) {
  561. if ((start >> 28) != ((end - 1) >> 28)) {
  562. DRM_ERROR("reloc %LX-%LX crossing 256MB boundary!\n",
  563. start, end);
  564. return -EINVAL;
  565. }
  566. if ((cmd == 0 || cmd == 0x3) &&
  567. (start >> 28) != (ctx->parser->adev->uvd.gpu_addr >> 28)) {
  568. DRM_ERROR("msg/fb buffer %LX-%LX out of 256MB segment!\n",
  569. start, end);
  570. return -EINVAL;
  571. }
  572. }
  573. if (cmd == 0) {
  574. ctx->has_msg_cmd = true;
  575. r = amdgpu_uvd_cs_msg(ctx, bo, addr);
  576. if (r)
  577. return r;
  578. } else if (!ctx->has_msg_cmd) {
  579. DRM_ERROR("Message needed before other commands are send!\n");
  580. return -EINVAL;
  581. }
  582. return 0;
  583. }
  584. /**
  585. * amdgpu_uvd_cs_reg - parse register writes
  586. *
  587. * @ctx: UVD parser context
  588. * @cb: callback function
  589. *
  590. * Parse the register writes, call cb on each complete command.
  591. */
  592. static int amdgpu_uvd_cs_reg(struct amdgpu_uvd_cs_ctx *ctx,
  593. int (*cb)(struct amdgpu_uvd_cs_ctx *ctx))
  594. {
  595. struct amdgpu_ib *ib = &ctx->parser->ibs[ctx->ib_idx];
  596. int i, r;
  597. ctx->idx++;
  598. for (i = 0; i <= ctx->count; ++i) {
  599. unsigned reg = ctx->reg + i;
  600. if (ctx->idx >= ib->length_dw) {
  601. DRM_ERROR("Register command after end of CS!\n");
  602. return -EINVAL;
  603. }
  604. switch (reg) {
  605. case mmUVD_GPCOM_VCPU_DATA0:
  606. ctx->data0 = ctx->idx;
  607. break;
  608. case mmUVD_GPCOM_VCPU_DATA1:
  609. ctx->data1 = ctx->idx;
  610. break;
  611. case mmUVD_GPCOM_VCPU_CMD:
  612. r = cb(ctx);
  613. if (r)
  614. return r;
  615. break;
  616. case mmUVD_ENGINE_CNTL:
  617. break;
  618. default:
  619. DRM_ERROR("Invalid reg 0x%X!\n", reg);
  620. return -EINVAL;
  621. }
  622. ctx->idx++;
  623. }
  624. return 0;
  625. }
  626. /**
  627. * amdgpu_uvd_cs_packets - parse UVD packets
  628. *
  629. * @ctx: UVD parser context
  630. * @cb: callback function
  631. *
  632. * Parse the command stream packets.
  633. */
  634. static int amdgpu_uvd_cs_packets(struct amdgpu_uvd_cs_ctx *ctx,
  635. int (*cb)(struct amdgpu_uvd_cs_ctx *ctx))
  636. {
  637. struct amdgpu_ib *ib = &ctx->parser->ibs[ctx->ib_idx];
  638. int r;
  639. for (ctx->idx = 0 ; ctx->idx < ib->length_dw; ) {
  640. uint32_t cmd = amdgpu_get_ib_value(ctx->parser, ctx->ib_idx, ctx->idx);
  641. unsigned type = CP_PACKET_GET_TYPE(cmd);
  642. switch (type) {
  643. case PACKET_TYPE0:
  644. ctx->reg = CP_PACKET0_GET_REG(cmd);
  645. ctx->count = CP_PACKET_GET_COUNT(cmd);
  646. r = amdgpu_uvd_cs_reg(ctx, cb);
  647. if (r)
  648. return r;
  649. break;
  650. case PACKET_TYPE2:
  651. ++ctx->idx;
  652. break;
  653. default:
  654. DRM_ERROR("Unknown packet type %d !\n", type);
  655. return -EINVAL;
  656. }
  657. }
  658. return 0;
  659. }
  660. /**
  661. * amdgpu_uvd_ring_parse_cs - UVD command submission parser
  662. *
  663. * @parser: Command submission parser context
  664. *
  665. * Parse the command stream, patch in addresses as necessary.
  666. */
  667. int amdgpu_uvd_ring_parse_cs(struct amdgpu_cs_parser *parser, uint32_t ib_idx)
  668. {
  669. struct amdgpu_uvd_cs_ctx ctx = {};
  670. unsigned buf_sizes[] = {
  671. [0x00000000] = 2048,
  672. [0x00000001] = 0xFFFFFFFF,
  673. [0x00000002] = 0xFFFFFFFF,
  674. [0x00000003] = 2048,
  675. [0x00000004] = 0xFFFFFFFF,
  676. };
  677. struct amdgpu_ib *ib = &parser->ibs[ib_idx];
  678. int r;
  679. if (ib->length_dw % 16) {
  680. DRM_ERROR("UVD IB length (%d) not 16 dwords aligned!\n",
  681. ib->length_dw);
  682. return -EINVAL;
  683. }
  684. ctx.parser = parser;
  685. ctx.buf_sizes = buf_sizes;
  686. ctx.ib_idx = ib_idx;
  687. /* first round, make sure the buffers are actually in the UVD segment */
  688. r = amdgpu_uvd_cs_packets(&ctx, amdgpu_uvd_cs_pass1);
  689. if (r)
  690. return r;
  691. /* second round, patch buffer addresses into the command stream */
  692. r = amdgpu_uvd_cs_packets(&ctx, amdgpu_uvd_cs_pass2);
  693. if (r)
  694. return r;
  695. if (!ctx.has_msg_cmd) {
  696. DRM_ERROR("UVD-IBs need a msg command!\n");
  697. return -EINVAL;
  698. }
  699. amdgpu_uvd_note_usage(ctx.parser->adev);
  700. return 0;
  701. }
  702. static int amdgpu_uvd_free_job(
  703. struct amdgpu_job *job)
  704. {
  705. amdgpu_ib_free(job->adev, job->ibs);
  706. kfree(job->ibs);
  707. return 0;
  708. }
  709. static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring,
  710. struct amdgpu_bo *bo,
  711. struct fence **fence)
  712. {
  713. struct ttm_validate_buffer tv;
  714. struct ww_acquire_ctx ticket;
  715. struct list_head head;
  716. struct amdgpu_ib *ib = NULL;
  717. struct fence *f = NULL;
  718. struct amdgpu_device *adev = ring->adev;
  719. uint64_t addr;
  720. int i, r;
  721. memset(&tv, 0, sizeof(tv));
  722. tv.bo = &bo->tbo;
  723. INIT_LIST_HEAD(&head);
  724. list_add(&tv.head, &head);
  725. r = ttm_eu_reserve_buffers(&ticket, &head, true, NULL);
  726. if (r)
  727. return r;
  728. if (!bo->adev->uvd.address_64_bit) {
  729. amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_VRAM);
  730. amdgpu_uvd_force_into_uvd_segment(bo);
  731. }
  732. r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
  733. if (r)
  734. goto err;
  735. ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL);
  736. if (!ib) {
  737. r = -ENOMEM;
  738. goto err;
  739. }
  740. r = amdgpu_ib_get(ring, NULL, 64, ib);
  741. if (r)
  742. goto err1;
  743. addr = amdgpu_bo_gpu_offset(bo);
  744. ib->ptr[0] = PACKET0(mmUVD_GPCOM_VCPU_DATA0, 0);
  745. ib->ptr[1] = addr;
  746. ib->ptr[2] = PACKET0(mmUVD_GPCOM_VCPU_DATA1, 0);
  747. ib->ptr[3] = addr >> 32;
  748. ib->ptr[4] = PACKET0(mmUVD_GPCOM_VCPU_CMD, 0);
  749. ib->ptr[5] = 0;
  750. for (i = 6; i < 16; ++i)
  751. ib->ptr[i] = PACKET2(0);
  752. ib->length_dw = 16;
  753. r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1,
  754. &amdgpu_uvd_free_job,
  755. AMDGPU_FENCE_OWNER_UNDEFINED,
  756. &f);
  757. if (r)
  758. goto err2;
  759. ttm_eu_fence_buffer_objects(&ticket, &head, f);
  760. if (fence)
  761. *fence = fence_get(f);
  762. amdgpu_bo_unref(&bo);
  763. fence_put(f);
  764. if (amdgpu_enable_scheduler)
  765. return 0;
  766. amdgpu_ib_free(ring->adev, ib);
  767. kfree(ib);
  768. return 0;
  769. err2:
  770. amdgpu_ib_free(ring->adev, ib);
  771. err1:
  772. kfree(ib);
  773. err:
  774. ttm_eu_backoff_reservation(&ticket, &head);
  775. return r;
  776. }
  777. /* multiple fence commands without any stream commands in between can
  778. crash the vcpu so just try to emmit a dummy create/destroy msg to
  779. avoid this */
  780. int amdgpu_uvd_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
  781. struct fence **fence)
  782. {
  783. struct amdgpu_device *adev = ring->adev;
  784. struct amdgpu_bo *bo;
  785. uint32_t *msg;
  786. int r, i;
  787. r = amdgpu_bo_create(adev, 1024, PAGE_SIZE, true,
  788. AMDGPU_GEM_DOMAIN_VRAM,
  789. AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
  790. NULL, NULL, &bo);
  791. if (r)
  792. return r;
  793. r = amdgpu_bo_reserve(bo, false);
  794. if (r) {
  795. amdgpu_bo_unref(&bo);
  796. return r;
  797. }
  798. r = amdgpu_bo_kmap(bo, (void **)&msg);
  799. if (r) {
  800. amdgpu_bo_unreserve(bo);
  801. amdgpu_bo_unref(&bo);
  802. return r;
  803. }
  804. /* stitch together an UVD create msg */
  805. msg[0] = cpu_to_le32(0x00000de4);
  806. msg[1] = cpu_to_le32(0x00000000);
  807. msg[2] = cpu_to_le32(handle);
  808. msg[3] = cpu_to_le32(0x00000000);
  809. msg[4] = cpu_to_le32(0x00000000);
  810. msg[5] = cpu_to_le32(0x00000000);
  811. msg[6] = cpu_to_le32(0x00000000);
  812. msg[7] = cpu_to_le32(0x00000780);
  813. msg[8] = cpu_to_le32(0x00000440);
  814. msg[9] = cpu_to_le32(0x00000000);
  815. msg[10] = cpu_to_le32(0x01b37000);
  816. for (i = 11; i < 1024; ++i)
  817. msg[i] = cpu_to_le32(0x0);
  818. amdgpu_bo_kunmap(bo);
  819. amdgpu_bo_unreserve(bo);
  820. return amdgpu_uvd_send_msg(ring, bo, fence);
  821. }
  822. int amdgpu_uvd_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
  823. struct fence **fence)
  824. {
  825. struct amdgpu_device *adev = ring->adev;
  826. struct amdgpu_bo *bo;
  827. uint32_t *msg;
  828. int r, i;
  829. r = amdgpu_bo_create(adev, 1024, PAGE_SIZE, true,
  830. AMDGPU_GEM_DOMAIN_VRAM,
  831. AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
  832. NULL, NULL, &bo);
  833. if (r)
  834. return r;
  835. r = amdgpu_bo_reserve(bo, false);
  836. if (r) {
  837. amdgpu_bo_unref(&bo);
  838. return r;
  839. }
  840. r = amdgpu_bo_kmap(bo, (void **)&msg);
  841. if (r) {
  842. amdgpu_bo_unreserve(bo);
  843. amdgpu_bo_unref(&bo);
  844. return r;
  845. }
  846. /* stitch together an UVD destroy msg */
  847. msg[0] = cpu_to_le32(0x00000de4);
  848. msg[1] = cpu_to_le32(0x00000002);
  849. msg[2] = cpu_to_le32(handle);
  850. msg[3] = cpu_to_le32(0x00000000);
  851. for (i = 4; i < 1024; ++i)
  852. msg[i] = cpu_to_le32(0x0);
  853. amdgpu_bo_kunmap(bo);
  854. amdgpu_bo_unreserve(bo);
  855. return amdgpu_uvd_send_msg(ring, bo, fence);
  856. }
  857. static void amdgpu_uvd_idle_work_handler(struct work_struct *work)
  858. {
  859. struct amdgpu_device *adev =
  860. container_of(work, struct amdgpu_device, uvd.idle_work.work);
  861. unsigned i, fences, handles = 0;
  862. fences = amdgpu_fence_count_emitted(&adev->uvd.ring);
  863. for (i = 0; i < AMDGPU_MAX_UVD_HANDLES; ++i)
  864. if (atomic_read(&adev->uvd.handles[i]))
  865. ++handles;
  866. if (fences == 0 && handles == 0) {
  867. if (adev->pm.dpm_enabled) {
  868. amdgpu_dpm_enable_uvd(adev, false);
  869. } else {
  870. amdgpu_asic_set_uvd_clocks(adev, 0, 0);
  871. }
  872. } else {
  873. schedule_delayed_work(&adev->uvd.idle_work,
  874. msecs_to_jiffies(UVD_IDLE_TIMEOUT_MS));
  875. }
  876. }
  877. static void amdgpu_uvd_note_usage(struct amdgpu_device *adev)
  878. {
  879. bool set_clocks = !cancel_delayed_work_sync(&adev->uvd.idle_work);
  880. set_clocks &= schedule_delayed_work(&adev->uvd.idle_work,
  881. msecs_to_jiffies(UVD_IDLE_TIMEOUT_MS));
  882. if (set_clocks) {
  883. if (adev->pm.dpm_enabled) {
  884. amdgpu_dpm_enable_uvd(adev, true);
  885. } else {
  886. amdgpu_asic_set_uvd_clocks(adev, 53300, 40000);
  887. }
  888. }
  889. }