Source: lib/media/segment_utils.js

  1. /*! @license
  2. * Shaka Player
  3. * Copyright 2016 Google LLC
  4. * SPDX-License-Identifier: Apache-2.0
  5. */
  6. goog.provide('shaka.media.SegmentUtils');
  7. goog.require('goog.asserts');
  8. goog.require('shaka.log');
  9. goog.require('shaka.media.Capabilities');
  10. goog.require('shaka.media.ClosedCaptionParser');
  11. goog.require('shaka.util.BufferUtils');
  12. goog.require('shaka.util.ManifestParserUtils');
  13. goog.require('shaka.util.MimeUtils');
  14. goog.require('shaka.util.Mp4BoxParsers');
  15. goog.require('shaka.util.Mp4Parser');
  16. goog.require('shaka.util.TsParser');
  17. /**
  18. * @summary Utility functions for segment parsing.
  19. */
  20. shaka.media.SegmentUtils = class {
  21. /**
  22. * @param {string} mimeType
  23. * @return {shaka.media.SegmentUtils.BasicInfo}
  24. */
  25. static getBasicInfoFromMimeType(mimeType) {
  26. const baseMimeType = shaka.util.MimeUtils.getBasicType(mimeType);
  27. const type = baseMimeType.split('/')[0];
  28. const codecs = shaka.util.MimeUtils.getCodecs(mimeType);
  29. return {
  30. type: type,
  31. mimeType: baseMimeType,
  32. codecs: codecs,
  33. language: null,
  34. height: null,
  35. width: null,
  36. channelCount: null,
  37. sampleRate: null,
  38. closedCaptions: new Map(),
  39. videoRange: null,
  40. colorGamut: null,
  41. frameRate: null,
  42. };
  43. }
  44. /**
  45. * @param {!BufferSource} data
  46. * @param {boolean} disableAudio
  47. * @param {boolean} disableVideo
  48. * @param {boolean} disableText
  49. * @return {?shaka.media.SegmentUtils.BasicInfo}
  50. */
  51. static getBasicInfoFromTs(data, disableAudio, disableVideo, disableText) {
  52. const uint8ArrayData = shaka.util.BufferUtils.toUint8(data);
  53. const tsParser = new shaka.util.TsParser().parse(uint8ArrayData);
  54. const tsCodecs = tsParser.getCodecs();
  55. const videoInfo = tsParser.getVideoInfo();
  56. const codecs = [];
  57. let hasAudio = false;
  58. let hasVideo = false;
  59. if (!disableAudio) {
  60. switch (tsCodecs.audio) {
  61. case 'aac':
  62. case 'aac-loas':
  63. if (tsParser.getAudioData().length) {
  64. codecs.push('mp4a.40.2');
  65. hasAudio = true;
  66. }
  67. break;
  68. case 'mp3':
  69. if (tsParser.getAudioData().length) {
  70. codecs.push('mp4a.40.34');
  71. hasAudio = true;
  72. }
  73. break;
  74. case 'ac3':
  75. if (tsParser.getAudioData().length) {
  76. codecs.push('ac-3');
  77. hasAudio = true;
  78. }
  79. break;
  80. case 'ec3':
  81. if (tsParser.getAudioData().length) {
  82. codecs.push('ec-3');
  83. hasAudio = true;
  84. }
  85. break;
  86. case 'opus':
  87. if (tsParser.getAudioData().length) {
  88. codecs.push('opus');
  89. hasAudio = true;
  90. }
  91. break;
  92. }
  93. }
  94. if (!disableVideo) {
  95. switch (tsCodecs.video) {
  96. case 'avc':
  97. if (videoInfo.codec) {
  98. codecs.push(videoInfo.codec);
  99. } else {
  100. codecs.push('avc1.42E01E');
  101. }
  102. hasVideo = true;
  103. break;
  104. case 'hvc':
  105. if (videoInfo.codec) {
  106. codecs.push(videoInfo.codec);
  107. } else {
  108. codecs.push('hvc1.1.6.L93.90');
  109. }
  110. hasVideo = true;
  111. break;
  112. case 'av1':
  113. codecs.push('av01.0.01M.08');
  114. hasVideo = true;
  115. break;
  116. }
  117. }
  118. if (!codecs.length) {
  119. return null;
  120. }
  121. const onlyAudio = hasAudio && !hasVideo;
  122. const closedCaptions = new Map();
  123. if (hasVideo && !disableText) {
  124. const captionParser = new shaka.media.ClosedCaptionParser('video/mp2t');
  125. captionParser.parseFrom(data);
  126. for (const stream of captionParser.getStreams()) {
  127. closedCaptions.set(stream, stream);
  128. }
  129. captionParser.reset();
  130. }
  131. return {
  132. type: onlyAudio ? 'audio' : 'video',
  133. mimeType: 'video/mp2t',
  134. codecs: codecs.join(', '),
  135. language: null,
  136. height: videoInfo.height,
  137. width: videoInfo.width,
  138. channelCount: null,
  139. sampleRate: null,
  140. closedCaptions: closedCaptions,
  141. videoRange: null,
  142. colorGamut: null,
  143. frameRate: videoInfo.frameRate,
  144. };
  145. }
  146. /**
  147. * @param {?BufferSource} initData
  148. * @param {!BufferSource} data
  149. * @param {boolean} disableText
  150. * @return {?shaka.media.SegmentUtils.BasicInfo}
  151. */
  152. static getBasicInfoFromMp4(initData, data, disableText) {
  153. const Mp4Parser = shaka.util.Mp4Parser;
  154. const SegmentUtils = shaka.media.SegmentUtils;
  155. const audioCodecs = [];
  156. let videoCodecs = [];
  157. let hasAudio = false;
  158. let hasVideo = false;
  159. const addCodec = (codec) => {
  160. const codecLC = codec.toLowerCase();
  161. switch (codecLC) {
  162. case 'avc1':
  163. case 'avc3':
  164. videoCodecs.push(codecLC + '.42E01E');
  165. hasVideo = true;
  166. break;
  167. case 'hev1':
  168. case 'hvc1':
  169. videoCodecs.push(codecLC + '.1.6.L93.90');
  170. hasVideo = true;
  171. break;
  172. case 'dvh1':
  173. case 'dvhe':
  174. videoCodecs.push(codecLC + '.05.04');
  175. hasVideo = true;
  176. break;
  177. case 'vp09':
  178. videoCodecs.push(codecLC + '.00.10.08');
  179. hasVideo = true;
  180. break;
  181. case 'av01':
  182. videoCodecs.push(codecLC + '.0.01M.08');
  183. hasVideo = true;
  184. break;
  185. case 'mp4a':
  186. // We assume AAC, but this can be wrong since mp4a supports
  187. // others codecs
  188. audioCodecs.push('mp4a.40.2');
  189. hasAudio = true;
  190. break;
  191. case 'ac-3':
  192. case 'ec-3':
  193. case 'ac-4':
  194. case 'opus':
  195. case 'flac':
  196. audioCodecs.push(codecLC);
  197. hasAudio = true;
  198. break;
  199. case 'apac':
  200. audioCodecs.push('apac.31.00');
  201. hasAudio = true;
  202. break;
  203. }
  204. };
  205. const codecBoxParser = (box) => addCodec(box.name);
  206. /** @type {?string} */
  207. let language = null;
  208. /** @type {?string} */
  209. let height = null;
  210. /** @type {?string} */
  211. let width = null;
  212. /** @type {?number} */
  213. let channelCount = null;
  214. /** @type {?number} */
  215. let sampleRate = null;
  216. /** @type {?string} */
  217. let realVideoRange = null;
  218. /** @type {?string} */
  219. let realColorGamut = null;
  220. /** @type {?string} */
  221. const realFrameRate = null;
  222. /** @type {?string} */
  223. let baseBox;
  224. const genericAudioBox = (box) => {
  225. const parsedAudioSampleEntryBox =
  226. shaka.util.Mp4BoxParsers.audioSampleEntry(box.reader);
  227. channelCount = parsedAudioSampleEntryBox.channelCount;
  228. sampleRate = parsedAudioSampleEntryBox.sampleRate;
  229. codecBoxParser(box);
  230. };
  231. const genericVideoBox = (box) => {
  232. baseBox = box.name;
  233. const parsedVisualSampleEntryBox =
  234. shaka.util.Mp4BoxParsers.visualSampleEntry(box.reader);
  235. width = String(parsedVisualSampleEntryBox.width);
  236. height = String(parsedVisualSampleEntryBox.height);
  237. if (box.reader.hasMoreData()) {
  238. Mp4Parser.children(box);
  239. }
  240. };
  241. new Mp4Parser()
  242. .box('moov', Mp4Parser.children)
  243. .box('trak', Mp4Parser.children)
  244. .box('mdia', Mp4Parser.children)
  245. .fullBox('mdhd', (box) => {
  246. goog.asserts.assert(
  247. box.version != null,
  248. 'MDHD is a full box and should have a valid version.');
  249. const parsedMDHDBox = shaka.util.Mp4BoxParsers.parseMDHD(
  250. box.reader, box.version);
  251. language = parsedMDHDBox.language;
  252. })
  253. .box('minf', Mp4Parser.children)
  254. .box('stbl', Mp4Parser.children)
  255. .fullBox('stsd', Mp4Parser.sampleDescription)
  256. // AUDIO
  257. // These are the various boxes that signal a codec.
  258. .box('mp4a', (box) => {
  259. const parsedAudioSampleEntryBox =
  260. shaka.util.Mp4BoxParsers.audioSampleEntry(box.reader);
  261. channelCount = parsedAudioSampleEntryBox.channelCount;
  262. sampleRate = parsedAudioSampleEntryBox.sampleRate;
  263. if (box.reader.hasMoreData()) {
  264. Mp4Parser.children(box);
  265. } else {
  266. codecBoxParser(box);
  267. }
  268. })
  269. .box('esds', (box) => {
  270. const parsedESDSBox = shaka.util.Mp4BoxParsers.parseESDS(box.reader);
  271. audioCodecs.push(parsedESDSBox.codec);
  272. hasAudio = true;
  273. })
  274. .box('ac-3', genericAudioBox)
  275. .box('ec-3', genericAudioBox)
  276. .box('ac-4', genericAudioBox)
  277. .box('Opus', genericAudioBox)
  278. .box('fLaC', genericAudioBox)
  279. .box('apac', genericAudioBox)
  280. // VIDEO
  281. // These are the various boxes that signal a codec.
  282. .box('avc1', genericVideoBox)
  283. .box('avc3', genericVideoBox)
  284. .box('hev1', genericVideoBox)
  285. .box('hvc1', genericVideoBox)
  286. .box('dva1', genericVideoBox)
  287. .box('dvav', genericVideoBox)
  288. .box('dvh1', genericVideoBox)
  289. .box('dvhe', genericVideoBox)
  290. .box('vp09', genericVideoBox)
  291. .box('av01', genericVideoBox)
  292. .box('avcC', (box) => {
  293. let codecBase = baseBox || '';
  294. switch (baseBox) {
  295. case 'dvav':
  296. codecBase = 'avc3';
  297. break;
  298. case 'dva1':
  299. codecBase = 'avc1';
  300. break;
  301. }
  302. const parsedAVCCBox = shaka.util.Mp4BoxParsers.parseAVCC(
  303. codecBase, box.reader, box.name);
  304. videoCodecs.push(parsedAVCCBox.codec);
  305. hasVideo = true;
  306. })
  307. .box('hvcC', (box) => {
  308. let codecBase = baseBox || '';
  309. switch (baseBox) {
  310. case 'dvh1':
  311. codecBase = 'hvc1';
  312. break;
  313. case 'dvhe':
  314. codecBase = 'hev1';
  315. break;
  316. }
  317. const parsedHVCCBox = shaka.util.Mp4BoxParsers.parseHVCC(
  318. codecBase, box.reader, box.name);
  319. videoCodecs.push(parsedHVCCBox.codec);
  320. hasVideo = true;
  321. })
  322. .box('dvcC', (box) => {
  323. let codecBase = baseBox || '';
  324. switch (baseBox) {
  325. case 'hvc1':
  326. codecBase = 'dvh1';
  327. break;
  328. case 'hev1':
  329. codecBase = 'dvhe';
  330. break;
  331. case 'avc1':
  332. codecBase = 'dva1';
  333. break;
  334. case 'avc3':
  335. codecBase = 'dvav';
  336. break;
  337. case 'av01':
  338. codecBase = 'dav1';
  339. break;
  340. }
  341. const parsedDVCCBox = shaka.util.Mp4BoxParsers.parseDVCC(
  342. codecBase, box.reader, box.name);
  343. videoCodecs.push(parsedDVCCBox.codec);
  344. hasVideo = true;
  345. })
  346. .box('dvvC', (box) => {
  347. let codecBase = baseBox || '';
  348. switch (baseBox) {
  349. case 'hvc1':
  350. codecBase = 'dvh1';
  351. break;
  352. case 'hev1':
  353. codecBase = 'dvhe';
  354. break;
  355. case 'avc1':
  356. codecBase = 'dva1';
  357. break;
  358. case 'avc3':
  359. codecBase = 'dvav';
  360. break;
  361. case 'av01':
  362. codecBase = 'dav1';
  363. break;
  364. }
  365. const parsedDVCCBox = shaka.util.Mp4BoxParsers.parseDVVC(
  366. codecBase, box.reader, box.name);
  367. videoCodecs.push(parsedDVCCBox.codec);
  368. hasVideo = true;
  369. })
  370. .fullBox('vpcC', (box) => {
  371. const codecBase = baseBox || '';
  372. const parsedVPCCBox = shaka.util.Mp4BoxParsers.parseVPCC(
  373. codecBase, box.reader, box.name);
  374. videoCodecs.push(parsedVPCCBox.codec);
  375. hasVideo = true;
  376. })
  377. .box('av1C', (box) => {
  378. let codecBase = baseBox || '';
  379. switch (baseBox) {
  380. case 'dav1':
  381. codecBase = 'av01';
  382. break;
  383. }
  384. const parsedAV1CBox = shaka.util.Mp4BoxParsers.parseAV1C(
  385. codecBase, box.reader, box.name);
  386. videoCodecs.push(parsedAV1CBox.codec);
  387. hasVideo = true;
  388. })
  389. // This signals an encrypted sample, which we can go inside of to
  390. // find the codec used.
  391. // Note: If encrypted, you can only have audio or video, not both.
  392. .box('enca', Mp4Parser.audioSampleEntry)
  393. .box('encv', Mp4Parser.visualSampleEntry)
  394. .box('sinf', Mp4Parser.children)
  395. .box('frma', (box) => {
  396. const {codec} = shaka.util.Mp4BoxParsers.parseFRMA(box.reader);
  397. addCodec(codec);
  398. })
  399. .box('colr', (box) => {
  400. videoCodecs = videoCodecs.map((codec) => {
  401. if (codec.startsWith('av01.')) {
  402. return shaka.util.Mp4BoxParsers.updateAV1CodecWithCOLRBox(
  403. codec, box.reader);
  404. }
  405. return codec;
  406. });
  407. const {videoRange, colorGamut} =
  408. shaka.util.Mp4BoxParsers.parseCOLR(box.reader);
  409. realVideoRange = videoRange;
  410. realColorGamut = colorGamut;
  411. })
  412. .parse(initData || data,
  413. /* partialOkay= */ true, /* stopOnPartial= */ true);
  414. if (!audioCodecs.length && !videoCodecs.length) {
  415. return null;
  416. }
  417. const onlyAudio = hasAudio && !hasVideo;
  418. const closedCaptions = new Map();
  419. if (hasVideo && !disableText) {
  420. const captionParser = new shaka.media.ClosedCaptionParser('video/mp4');
  421. if (initData) {
  422. captionParser.init(initData);
  423. }
  424. try {
  425. captionParser.parseFrom(data);
  426. for (const stream of captionParser.getStreams()) {
  427. closedCaptions.set(stream, stream);
  428. }
  429. } catch (e) {
  430. shaka.log.debug('Error detecting CC streams', e);
  431. }
  432. captionParser.reset();
  433. }
  434. const codecs = audioCodecs.concat(videoCodecs);
  435. return {
  436. type: onlyAudio ? 'audio' : 'video',
  437. mimeType: onlyAudio ? 'audio/mp4' : 'video/mp4',
  438. codecs: SegmentUtils.codecsFiltering(codecs).join(', '),
  439. language: language,
  440. height: height,
  441. width: width,
  442. channelCount: channelCount,
  443. sampleRate: sampleRate,
  444. closedCaptions: closedCaptions,
  445. videoRange: realVideoRange,
  446. colorGamut: realColorGamut,
  447. frameRate: realFrameRate,
  448. };
  449. }
  450. /**
  451. * @param {!Array<string>} codecs
  452. * @return {!Array<string>} codecs
  453. */
  454. static codecsFiltering(codecs) {
  455. const ContentType = shaka.util.ManifestParserUtils.ContentType;
  456. const ManifestParserUtils = shaka.util.ManifestParserUtils;
  457. const SegmentUtils = shaka.media.SegmentUtils;
  458. const allCodecs = SegmentUtils.filterDuplicateCodecs_(codecs);
  459. const audioCodecs =
  460. ManifestParserUtils.guessAllCodecsSafe(ContentType.AUDIO, allCodecs);
  461. const videoCodecs =
  462. ManifestParserUtils.guessAllCodecsSafe(ContentType.VIDEO, allCodecs);
  463. const textCodecs =
  464. ManifestParserUtils.guessAllCodecsSafe(ContentType.TEXT, allCodecs);
  465. const validVideoCodecs = SegmentUtils.chooseBetterCodecs_(videoCodecs);
  466. const finalCodecs =
  467. audioCodecs.concat(validVideoCodecs).concat(textCodecs);
  468. if (allCodecs.length && !finalCodecs.length) {
  469. return allCodecs;
  470. }
  471. return finalCodecs;
  472. }
  473. /**
  474. * @param {!Array<string>} codecs
  475. * @return {!Array<string>} codecs
  476. * @private
  477. */
  478. static filterDuplicateCodecs_(codecs) {
  479. // Filter out duplicate codecs.
  480. const seen = new Set();
  481. const ret = [];
  482. for (const codec of codecs) {
  483. const shortCodec = shaka.util.MimeUtils.getCodecBase(codec);
  484. if (!seen.has(shortCodec)) {
  485. ret.push(codec);
  486. seen.add(shortCodec);
  487. } else {
  488. shaka.log.debug('Ignoring duplicate codec');
  489. }
  490. }
  491. return ret;
  492. }
  493. /**
  494. * Prioritizes Dolby Vision if supported. This is necessary because with
  495. * Dolby Vision we could have hvcC and dvcC boxes at the same time.
  496. *
  497. * @param {!Array<string>} codecs
  498. * @return {!Array<string>} codecs
  499. * @private
  500. */
  501. static chooseBetterCodecs_(codecs) {
  502. if (codecs.length <= 1) {
  503. return codecs;
  504. }
  505. const dolbyVision = codecs.find((codec) => {
  506. return codec.startsWith('dvav.') ||
  507. codec.startsWith('dva1.') ||
  508. codec.startsWith('dvh1.') ||
  509. codec.startsWith('dvhe.') ||
  510. codec.startsWith('dav1.') ||
  511. codec.startsWith('dvc1.') ||
  512. codec.startsWith('dvi1.');
  513. });
  514. if (!dolbyVision) {
  515. return codecs;
  516. }
  517. const type = `video/mp4; codecs="${dolbyVision}"`;
  518. if (shaka.media.Capabilities.isTypeSupported(type)) {
  519. return [dolbyVision];
  520. }
  521. return codecs.filter((codec) => codec != dolbyVision);
  522. }
  523. /**
  524. * @param {!BufferSource} data
  525. * @return {?string}
  526. */
  527. static getDefaultKID(data) {
  528. const Mp4Parser = shaka.util.Mp4Parser;
  529. let defaultKID = null;
  530. new Mp4Parser()
  531. .box('moov', Mp4Parser.children)
  532. .box('trak', Mp4Parser.children)
  533. .box('mdia', Mp4Parser.children)
  534. .box('minf', Mp4Parser.children)
  535. .box('stbl', Mp4Parser.children)
  536. .fullBox('stsd', Mp4Parser.sampleDescription)
  537. .box('encv', Mp4Parser.visualSampleEntry)
  538. .box('enca', Mp4Parser.audioSampleEntry)
  539. .box('sinf', Mp4Parser.children)
  540. .box('schi', Mp4Parser.children)
  541. .fullBox('tenc', (box) => {
  542. const parsedTENCBox = shaka.util.Mp4BoxParsers.parseTENC(box.reader);
  543. defaultKID = parsedTENCBox.defaultKID;
  544. })
  545. .parse(data, /* partialOkay= */ true);
  546. return defaultKID;
  547. }
  548. /**
  549. * @param {!BufferSource} rawResult
  550. * @param {shaka.extern.aesKey} aesKey
  551. * @param {number} position
  552. * @return {!Promise<!BufferSource>}
  553. */
  554. static async aesDecrypt(rawResult, aesKey, position) {
  555. const key = aesKey;
  556. if (!key.cryptoKey) {
  557. goog.asserts.assert(key.fetchKey, 'If AES cryptoKey was not ' +
  558. 'preloaded, fetchKey function should be provided');
  559. await key.fetchKey();
  560. goog.asserts.assert(key.cryptoKey, 'AES cryptoKey should now be set');
  561. }
  562. let iv = key.iv;
  563. if (!iv) {
  564. iv = shaka.util.BufferUtils.toUint8(new ArrayBuffer(16));
  565. let sequence = key.firstMediaSequenceNumber + position;
  566. for (let i = iv.byteLength - 1; i >= 0; i--) {
  567. iv[i] = sequence & 0xff;
  568. sequence >>= 8;
  569. }
  570. }
  571. let algorithm;
  572. if (aesKey.blockCipherMode == 'CBC') {
  573. algorithm = {
  574. name: 'AES-CBC',
  575. iv,
  576. };
  577. } else {
  578. algorithm = {
  579. name: 'AES-CTR',
  580. counter: iv,
  581. // NIST SP800-38A standard suggests that the counter should occupy half
  582. // of the counter block
  583. length: 64,
  584. };
  585. }
  586. return window.crypto.subtle.decrypt(algorithm, key.cryptoKey, rawResult);
  587. }
  588. };
  589. /**
  590. * @typedef {{
  591. * type: string,
  592. * mimeType: string,
  593. * codecs: string,
  594. * language: ?string,
  595. * height: ?string,
  596. * width: ?string,
  597. * channelCount: ?number,
  598. * sampleRate: ?number,
  599. * closedCaptions: Map<string, string>,
  600. * videoRange: ?string,
  601. * colorGamut: ?string,
  602. * frameRate: ?string,
  603. * }}
  604. *
  605. * @property {string} type
  606. * @property {string} mimeType
  607. * @property {string} codecs
  608. * @property {?string} language
  609. * @property {?string} height
  610. * @property {?string} width
  611. * @property {?number} channelCount
  612. * @property {?number} sampleRate
  613. * @property {Map<string, string>} closedCaptions
  614. * @property {?string} videoRange
  615. * @property {?string} colorGamut
  616. * @property {?string} frameRate
  617. */
  618. shaka.media.SegmentUtils.BasicInfo;