/*! @license * Shaka Player * Copyright 2016 Google LLC * SPDX-License-Identifier: Apache-2.0 */ goog.provide('shaka.media.MediaSourceEngine'); goog.require('goog.asserts'); goog.require('shaka.log'); goog.require('shaka.config.CodecSwitchingStrategy'); goog.require('shaka.device.DeviceFactory'); goog.require('shaka.media.ClearKeyWebCryptoDecryptor'); goog.require('shaka.media.Capabilities'); goog.require('shaka.media.ContentWorkarounds'); goog.require('shaka.media.ClosedCaptionParser'); goog.require('shaka.media.ManifestParser'); goog.require('shaka.media.SegmentReference'); goog.require('shaka.media.SegmentUtils'); goog.require('shaka.media.TimeRangesUtils'); goog.require('shaka.metadata.Id3Utils'); goog.require('shaka.text.TextEngine'); goog.require('shaka.transmuxer.TransmuxerEngine'); goog.require('shaka.transmuxer.TransmuxerProxy'); goog.require('shaka.util.ArrayUtils'); goog.require('shaka.util.BufferUtils'); goog.require('shaka.util.Destroyer'); goog.require('shaka.util.Dom'); goog.require('shaka.util.Error'); goog.require('shaka.util.EventManager'); goog.require('shaka.util.FakeEvent'); goog.require('shaka.util.IDestroyable'); goog.require('shaka.util.ManifestParserUtils'); goog.require('shaka.util.MediaElementEvent'); goog.require('shaka.util.MimeUtils'); goog.require('shaka.util.Mp4BoxParsers'); goog.require('shaka.util.Mp4Parser'); goog.require('shaka.util.NumberUtils'); goog.require('shaka.util.TimeUtils'); goog.require('shaka.util.TsParser'); goog.require('shaka.lcevc.Dec'); /** * @summary * MediaSourceEngine wraps all operations on MediaSource and SourceBuffers. * All asynchronous operations return a Promise, and all operations are * internally synchronized and serialized as needed. Operations that can * be done in parallel will be done in parallel. * * @implements {shaka.util.IDestroyable} */ shaka.media.MediaSourceEngine = class { /** * @param {HTMLMediaElement} video The video element, whose source is tied to * MediaSource during the lifetime of the MediaSourceEngine. * @param {!shaka.extern.TextDisplayer} textDisplayer * The text displayer that will be used with the text engine. * MediaSourceEngine takes ownership of the displayer. When * MediaSourceEngine is destroyed, it will destroy the displayer. * @param {!shaka.media.MediaSourceEngine.PlayerInterface} playerInterface * Interface for common player methods. * @param {shaka.extern.MediaSourceConfiguration} config * @param {?shaka.lcevc.Dec} [lcevcDec] Optional - LCEVC Decoder Object */ constructor(video, textDisplayer, playerInterface, config, lcevcDec) { /** @private {HTMLMediaElement} */ this.video_ = video; /** @private {?shaka.media.MediaSourceEngine.PlayerInterface} */ this.playerInterface_ = playerInterface; /** @private {?shaka.extern.MediaSourceConfiguration} */ this.config_ = config; /** @private {shaka.extern.TextDisplayer} */ this.textDisplayer_ = textDisplayer; /** * @private {!Map} */ this.sourceBuffers_ = new Map(); /** * @private {!Map} */ this.sourceBufferTypes_ = new Map(); /** * The timestamp offset last requested for each SourceBuffer, before the * legacy Edge rounding workaround in setTimestampOffset_ is applied. The * value read back from the SourceBuffer includes that workaround, so it * cannot be compared against a freshly calculated offset. * * @private {!Map} */ this.requestedTimestampOffsets_ = new Map(); /** * @private {!Map} */ this.expectedEncryption_ = new Map(); /** @private {shaka.text.TextEngine} */ this.textEngine_ = null; /** @private {boolean} */ this.segmentRelativeVttTiming_ = false; /** @private {?shaka.lcevc.Dec} */ this.lcevcDec_ = lcevcDec || null; /** * @private {!Map>} */ this.queues_ = new Map(); /** @private {shaka.util.EventManager} */ this.eventManager_ = new shaka.util.EventManager(); /** * @private {!Map} */ this.transmuxers_ = new Map(); /** @private {?shaka.media.ClosedCaptionParser} */ this.captionParser_ = null; /** @private {Promise.PromiseWithResolvers} */ this.mediaSourceOpen_ = Promise.withResolvers(); /** @private {string} */ this.url_ = ''; /** @private {boolean} */ this.playbackHasBegun_ = false; /** @private {boolean} */ this.streamingAllowed_ = true; /** @private {boolean} */ this.usingRemotePlayback_ = false; /** @private {HTMLSourceElement} */ this.source_ = null; /** * Fallback source element with direct media URI, used for casting * purposes. * @private {HTMLSourceElement} */ this.secondarySource_ = null; /** @private {MediaSource} */ this.mediaSource_ = this.createMediaSource(this.mediaSourceOpen_); /** @private {boolean} */ this.reloadingMediaSource_ = false; /** @private {boolean} */ this.playAfterReset_ = false; /** @type {!shaka.util.Destroyer} */ this.destroyer_ = new shaka.util.Destroyer(() => this.doDestroy_()); /** @private {boolean} */ this.sequenceMode_ = false; /** @private {string} */ this.manifestType_ = shaka.media.ManifestParser.UNKNOWN; /** @private {boolean} */ this.ignoreManifestTimestampsInSegmentsMode_ = false; /** @private {boolean} */ this.attemptTimestampOffsetCalculation_ = false; /** @private {boolean} */ this.needSplitMuxedContent_ = false; /** @private {?number} */ this.lastDuration_ = null; /** * @private {!Map} */ this.tsParsers_ = new Map(); /** @private {?number} */ this.firstVideoTimestamp_ = null; /** @private {?number} */ this.firstVideoReferenceStartTime_ = null; /** @private {?number} */ this.firstAudioTimestamp_ = null; /** @private {?number} */ this.firstAudioReferenceStartTime_ = null; /** @private {!Promise.PromiseWithResolvers} */ this.audioCompensation_ = Promise.withResolvers(); const device = shaka.device.DeviceFactory.getDevice(); const remote = device.getRemote(this.video_); if (remote) { this.usingRemotePlayback_ = remote.state != 'disconnected'; this.eventManager_.listenMulti( remote, [ 'connect', 'connecting', 'disconnect', ], () => { this.usingRemotePlayback_ = remote.state != 'disconnected'; }); } /** @private {boolean} */ this.usesLiveSeekableRange_ = false; /** * @private {!Map} */ this.clearKeyDecryptors_ = new Map(); } /** * Create a MediaSource object, attach it to the video element, and return it. * Resolves the given promise when the MediaSource is ready. * * Replaced by unit tests. * * @param {!Promise.PromiseWithResolvers} p * @return {!MediaSource} */ createMediaSource(p) { this.streamingAllowed_ = true; /** @type {!MediaSource} */ let mediaSource; if (window.ManagedMediaSource) { if (!this.secondarySource_) { this.video_.disableRemotePlayback = true; } mediaSource = new ManagedMediaSource(); this.eventManager_.listen( mediaSource, 'startstreaming', () => { shaka.log.info('MMS startstreaming'); this.streamingAllowed_ = true; }); this.eventManager_.listen( mediaSource, 'endstreaming', () => { shaka.log.info('MMS endstreaming'); this.streamingAllowed_ = false; }); } else { mediaSource = new MediaSource(); } // Set up MediaSource on the video element. this.eventManager_.listenOnce( mediaSource, 'sourceopen', () => this.onSourceOpen_(p)); // Correctly set when playback has begun. this.eventManager_.listenOnce(this.video_, 'playing', () => { this.playbackHasBegun_ = true; }); // Store the object URL for releasing it later. this.url_ = shaka.media.MediaSourceEngine.createObjectURL(mediaSource); // Remove all sources. this.removeSources_(); // Set or .src, depending on config. if (this.config_.useSourceElements) { this.source_ = shaka.util.Dom.createSourceElement(this.url_); this.video_.appendChild(this.source_); if (this.secondarySource_) { this.video_.appendChild(this.secondarySource_); } this.video_.load(); } else { this.video_.src = this.url_; } return mediaSource; } /** * @param {string} uri * @param {string} mimeType */ addSecondarySource(uri, mimeType) { if (!this.video_ || !window.ManagedMediaSource || !this.mediaSource_) { shaka.log.warning( 'Secondary source is used only with ManagedMediaSource'); return; } if (!this.config_.useSourceElements) { return; } if (this.secondarySource_?.parentElement === this.video_) { this.video_.removeChild(this.secondarySource_); } this.secondarySource_ = shaka.util.Dom.createSourceElement(uri, mimeType); this.video_.appendChild(this.secondarySource_); this.video_.disableRemotePlayback = false; } /** * @param {!Promise.PromiseWithResolvers} p * @private */ onSourceOpen_(p) { goog.asserts.assert(this.url_, 'Must have object URL'); // Release the object URL that was previously created, to prevent memory // leak. // createObjectURL creates a strong reference to the MediaSource object // inside the browser. Setting the src of the video then creates another // reference within the video element. revokeObjectURL will remove the // strong reference to the MediaSource object, and allow it to be // garbage-collected later. URL.revokeObjectURL(this.url_); p.resolve(); } /** * Returns a map of MediaSource support for well-known types. * * @return {!Object} */ static probeSupport() { const MimeUtils = shaka.util.MimeUtils; const testMimeTypes = [ // MP4 types 'video/mp4; codecs="avc1.42E01E"', 'video/mp4; codecs="avc3.42E01E"', 'video/mp4; codecs="hev1.1.6.L93.90"', 'video/mp4; codecs="hvc1.1.6.L93.90"', 'video/mp4; codecs="hev1.2.4.L153.B0"; eotf="smpte2084"', // HDR HEVC 'video/mp4; codecs="hvc1.2.4.L153.B0"; eotf="smpte2084"', // HDR HEVC 'video/mp4; codecs="hvc1.2.20000000.L153.B0"', 'video/mp4; codecs="hvc1.2.4.L120.b0"', 'video/mp4; codecs="hvc1.2.4.L123.b0"', 'video/mp4; codecs="vp9"', 'video/mp4; codecs="vp09.00.10.08"', 'video/mp4; codecs="av01.0.01M.08"', 'video/mp4; codecs="av01.0.31M.10.0.111.09.16.09.0"', // AV1 PQ 'video/mp4; codecs="av01.0.31M.10.0.112.09.18.09.0"', // AV1 HLG 'video/mp4; codecs="dvh1.05.01"', // Dolby Vision p5 'video/mp4; codecs="dvh1.08.01"', // Dolby Vision p8 'video/mp4; codecs="dav1.10.01"', // Dolby Vision p10 'video/mp4; codecs="dvh1.20.01"', // Dolby Vision p20 'audio/mp4; codecs="mp4a.40.2"', 'audio/mp4; codecs="ac-3"', 'audio/mp4; codecs="ec-3"', 'audio/mp4; codecs="ac-4.02.01.03"', 'audio/mp4; codecs="opus"', 'audio/mp4; codecs="flac"', 'audio/mp4; codecs="dtsc"', // DTS Digital Surround 'audio/mp4; codecs="dtse"', // DTS Express 'audio/mp4; codecs="dtsx"', // DTS:X 'audio/mp4; codecs="apac.31.00"', // IAMF is only defined for CMAF/MP4 so far. 'audio/mp4; codecs="iamf.000.000.Opus"', 'audio/mp4; codecs="iamf.000.000.mp4a.40.2"', 'audio/mp4; codecs="iamf.000.000.fLaC"', 'audio/mp4; codecs="iamf.000.000.ipcm"', // WebM types 'video/webm; codecs="vp8"', 'video/webm; codecs="vp9"', 'video/webm; codecs="vp09.00.10.08"', 'audio/webm; codecs="vorbis"', 'audio/webm; codecs="opus"', // MPEG2 TS types (video/ is also used for audio: https://bit.ly/TsMse) 'video/mp2t; codecs="avc1.42E01E"', 'video/mp2t; codecs="avc3.42E01E"', 'video/mp2t; codecs="hvc1.1.6.L93.90"', 'video/mp2t; codecs="mp4a.40.2"', 'video/mp2t; codecs="ac-3"', 'video/mp2t; codecs="ec-3"', // WebVTT types 'text/vtt', 'application/mp4; codecs="wvtt"', // TTML types 'application/ttml+xml', 'application/mp4; codecs="stpp"', // Containerless types ...MimeUtils.RAW_FORMATS, ]; const support = {}; const device = shaka.device.DeviceFactory.getDevice(); for (const type of testMimeTypes) { if (shaka.text.TextEngine.isTypeSupported(type)) { support[type] = true; } else if (device.supportsMediaSource()) { const baseMimeType = MimeUtils.getBasicType(type); const codecs = MimeUtils.getCorrectAudioCodecs( MimeUtils.getCodecs(type), baseMimeType); const newType = MimeUtils.getFullType(baseMimeType, codecs); support[type] = shaka.media.Capabilities.isTypeSupported(newType) || shaka.transmuxer.TransmuxerEngine.isSupported(newType); } else { support[type] = device.supportsMediaType(type); } const basicType = type.split(';')[0]; support[basicType] = support[basicType] || support[type]; } return support; } /** @override */ destroy() { return this.destroyer_.destroy(); } /** @private */ async doDestroy_() { let cleanup = []; for (const [key, q] of this.queues_) { // Make a local copy of the queue and the first item. const inProgress = q[0]; const contentType = /** @type {string} */(key); // Drop everything else out of the original queue. this.queues_.set(contentType, q.slice(0, 1)); // We will wait for this item to complete/fail. if (inProgress) { cleanup.push(inProgress.p.promise.catch(() => {})); } // The rest will be rejected silently if possible. for (const item of q.slice(1)) { item.p.reject(shaka.util.Destroyer.destroyedError()); } } if (this.textEngine_) { cleanup.push(this.textEngine_.destroy()); } await Promise.all(cleanup); cleanup = []; for (const transmuxer of this.transmuxers_.values()) { transmuxer.destroy(); } for (const clearKeyDecryptor of this.clearKeyDecryptors_.values()) { cleanup.push(clearKeyDecryptor.destroy()); } if (this.eventManager_) { this.eventManager_.release(); this.eventManager_ = null; } if (this.video_) { // "unload" the video element. this.removeSources_(); this.video_.load(); this.video_.disableRemotePlayback = false; } this.video_ = null; this.source_ = null; this.secondarySource_ = null; this.config_ = null; this.mediaSource_ = null; this.textEngine_ = null; this.textDisplayer_ = null; this.sourceBuffers_.clear(); this.requestedTimestampOffsets_.clear(); this.expectedEncryption_.clear(); this.clearKeyDecryptors_.clear(); this.transmuxers_.clear(); this.captionParser_ = null; if (goog.DEBUG) { for (const [contentType, q] of this.queues_) { goog.asserts.assert( q.length == 0, contentType + ' queue should be empty after destroy!'); } } this.queues_.clear(); // This object is owned by Player this.lcevcDec_ = null; this.tsParsers_.clear(); this.playerInterface_ = null; await Promise.all(cleanup); } /** * Remove all sources from the video. * * @private */ removeSources_() { if (!this.video_) { return; } // Both .src and must be gone before load() for the element to // empty to HAVE_NOTHING, which DRM teardown's setMediaKeys(null) depends // on. this.video_.removeAttribute('src'); if (this.source_?.parentElement === this.video_) { this.video_.removeChild(this.source_); } if (this.secondarySource_?.parentElement === this.video_) { this.video_.removeChild(this.secondarySource_); } } /** * @return {!Promise} Resolved when MediaSource is open and attached to the * media element. This process is actually initiated by the constructor. */ open() { return this.mediaSourceOpen_.promise; } /** * Initialize MediaSourceEngine. * * Note that it is not valid to call this multiple times, except to add or * reinitialize text streams. * * @param {!Map} streamsByType * A map of content types to streams. * @param {boolean=} sequenceMode * If true, the media segments are appended to the SourceBuffer in strict * sequence. * @param {string=} manifestType * Indicates the type of the manifest. * @param {boolean=} ignoreManifestTimestampsInSegmentsMode * If true, don't adjust the timestamp offset to account for manifest * segment durations being out of sync with segment durations. In other * words, assume that there are no gaps in the segments when appending * to the SourceBuffer, even if the manifest and segment times disagree. * Indicates if the manifest has text streams. * * @return {!Promise} */ async init(streamsByType, sequenceMode=false, manifestType=shaka.media.ManifestParser.UNKNOWN, ignoreManifestTimestampsInSegmentsMode=false) { const ContentType = shaka.util.ManifestParserUtils.ContentType; await this.mediaSourceOpen_.promise; if (this.ended() || this.closed()) { shaka.log.alwaysError('Expected MediaSource to be open during init(); ' + 'reopening the media source.'); this.mediaSourceOpen_ = Promise.withResolvers(); this.mediaSource_ = this.createMediaSource(this.mediaSourceOpen_); await this.mediaSourceOpen_.promise; } this.sequenceMode_ = sequenceMode; this.manifestType_ = manifestType; this.ignoreManifestTimestampsInSegmentsMode_ = ignoreManifestTimestampsInSegmentsMode; this.attemptTimestampOffsetCalculation_ = !this.sequenceMode_ && this.manifestType_ == shaka.media.ManifestParser.HLS && !this.ignoreManifestTimestampsInSegmentsMode_; this.tsParsers_.clear(); this.firstVideoTimestamp_ = null; this.firstVideoReferenceStartTime_ = null; this.firstAudioTimestamp_ = null; this.firstAudioReferenceStartTime_ = null; this.audioCompensation_ = Promise.withResolvers(); for (const contentType of streamsByType.keys()) { const stream = streamsByType.get(contentType); this.initSourceBuffer_(contentType, stream, stream.codecs); if (this.needSplitMuxedContent_) { this.queues_.set(ContentType.AUDIO, []); this.queues_.set(ContentType.VIDEO, []); } else { this.queues_.set(contentType, []); } } const audio = streamsByType.get(ContentType.AUDIO); if (audio && audio.isAudioMuxedInVideo) { this.needSplitMuxedContent_ = true; } } /** * Initialize a specific SourceBuffer. * * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @param {shaka.extern.Stream} stream * @param {string} codecs * @private */ initSourceBuffer_(contentType, stream, codecs) { const ContentType = shaka.util.ManifestParserUtils.ContentType; if (contentType == ContentType.AUDIO && codecs) { codecs = shaka.util.MimeUtils.getCorrectAudioCodecs( codecs, stream.mimeType); } if (contentType == ContentType.VIDEO && codecs && stream.mimeType == 'video/mp4') { codecs = shaka.util.MimeUtils.getCorrectCodecs( codecs, stream.mimeType); } let mimeType = shaka.util.MimeUtils.getFullType( stream.mimeType, codecs); if (contentType == ContentType.TEXT) { this.reinitText(mimeType, stream.external); } else { let needTransmux = this.config_.forceTransmux; if (!shaka.media.Capabilities.isTypeSupported(mimeType) || (!this.sequenceMode_ && shaka.util.MimeUtils.RAW_FORMATS.includes(mimeType))) { needTransmux = true; } const mimeTypeWithAllCodecs = shaka.util.MimeUtils.getFullTypeWithAllCodecs( stream.mimeType, codecs); if (needTransmux) { const audioCodec = shaka.util.ManifestParserUtils.guessCodecsSafe( ContentType.AUDIO, (codecs || '').split(',')); const videoCodec = shaka.util.ManifestParserUtils.guessCodecsSafe( ContentType.VIDEO, (codecs || '').split(',')); if (audioCodec && videoCodec) { this.needSplitMuxedContent_ = true; this.initSourceBuffer_(ContentType.AUDIO, stream, audioCodec); this.initSourceBuffer_(ContentType.VIDEO, stream, videoCodec); return; } const transmuxerPlugin = shaka.transmuxer.TransmuxerEngine .findTransmuxer(mimeTypeWithAllCodecs); if (transmuxerPlugin) { const transmuxer = new shaka.transmuxer.TransmuxerProxy( transmuxerPlugin(), this.config_.transmuxWorkerUrl); this.transmuxers_.set(contentType, transmuxer); mimeType = transmuxer.convertCodecs(contentType, mimeTypeWithAllCodecs); } } const type = this.addExtraFeaturesToMimeType_(mimeType); this.destroyer_.ensureNotDestroyed(); let sourceBuffer; try { sourceBuffer = this.mediaSource_.addSourceBuffer(type); } catch (exception) { throw new shaka.util.Error( shaka.util.Error.Severity.CRITICAL, shaka.util.Error.Category.MEDIA, shaka.util.Error.Code.MEDIA_SOURCE_OPERATION_THREW, exception, 'The mediaSource_ status was ' + this.mediaSource_.readyState + ' expected \'open\'', null); } if (this.sequenceMode_) { sourceBuffer.mode = shaka.media.MediaSourceEngine.SourceBufferMode_.SEQUENCE; } this.eventManager_.listen( sourceBuffer, 'error', () => this.onError_(contentType)); this.eventManager_.listen( sourceBuffer, 'updateend', () => this.onUpdateEnd_(contentType)); this.sourceBuffers_.set(contentType, sourceBuffer); this.sourceBufferTypes_.set(contentType, mimeType); this.expectedEncryption_.set(contentType, !!stream.drmInfos.length); const drmInfo = this.playerInterface_.getDrmInfo(); if (shaka.media.ClearKeyWebCryptoDecryptor.shouldUse(drmInfo)) { const decryptor = new shaka.media.ClearKeyWebCryptoDecryptor(); this.clearKeyDecryptors_.set(contentType, decryptor); } else { this.clearKeyDecryptors_.get(contentType)?.destroy(); this.clearKeyDecryptors_.delete(contentType); } } } /** * Called by the Player to provide an updated configuration any time it * changes. Must be called at least once before init(). * * @param {shaka.extern.MediaSourceConfiguration} config */ configure(config) { this.config_ = config; if (this.textEngine_) { this.textEngine_.setModifyCueCallback(config.modifyCueCallback); } } /** * Indicate if the streaming is allowed by MediaSourceEngine. * If we using MediaSource we always returns true. * * @return {boolean} */ isStreamingAllowed() { return this.streamingAllowed_ && !this.usingRemotePlayback_ && !this.reloadingMediaSource_; } /** * Indicate if the MediaSource is currently being reloaded, e.g. because of * a codec switch or an error recovery. * * @return {boolean} */ isReloadingMediaSource() { return this.reloadingMediaSource_; } /** * Reinitialize the TextEngine for a new text type. * @param {string} mimeType * @param {boolean} external */ reinitText(mimeType, external) { if (!this.textEngine_) { this.textEngine_ = new shaka.text.TextEngine( this.textDisplayer_, this.manifestType_); if (this.textEngine_) { this.textEngine_.setModifyCueCallback(this.config_.modifyCueCallback); } } this.textEngine_.initParser(mimeType, external, this.segmentRelativeVttTiming_); } /** * @return {boolean} True if the MediaSource is in an "ended" state, or if the * object has been destroyed. */ ended() { if (this.reloadingMediaSource_) { return false; } return this.mediaSource_ ? this.mediaSource_.readyState == 'ended' : true; } /** * @return {boolean} True if the MediaSource is in an "closed" state, or if * the object has been destroyed. */ closed() { if (this.reloadingMediaSource_) { return false; } return this.mediaSource_ ? this.mediaSource_.readyState == 'closed' : true; } /** * Gets the first timestamp in buffer for the given content type. * * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @return {?number} The timestamp in seconds, or null if nothing is buffered. */ bufferStart(contentType) { if (!this.sourceBuffers_.size) { return null; } const ContentType = shaka.util.ManifestParserUtils.ContentType; if (contentType == ContentType.TEXT) { return this.textEngine_.bufferStart(); } return shaka.media.TimeRangesUtils.bufferStart( this.getBuffered_(contentType)); } /** * Gets the last timestamp in buffer for the given content type. * * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @return {?number} The timestamp in seconds, or null if nothing is buffered. */ bufferEnd(contentType) { if (!this.sourceBuffers_.size) { return null; } const ContentType = shaka.util.ManifestParserUtils.ContentType; if (contentType == ContentType.TEXT) { return this.textEngine_.bufferEnd(); } return shaka.media.TimeRangesUtils.bufferEnd( this.getBuffered_(contentType)); } /** * Determines if the given time is inside the buffered range of the given * content type. * * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @param {number} time Playhead time * @return {boolean} */ isBuffered(contentType, time) { const ContentType = shaka.util.ManifestParserUtils.ContentType; if (contentType == ContentType.TEXT) { return this.textEngine_.isBuffered(time); } else { const buffered = this.getBuffered_(contentType); return shaka.media.TimeRangesUtils.isBuffered(buffered, time); } } /** * Computes how far ahead of the given timestamp is buffered for the given * content type. * * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @param {number} time * @return {number} The amount of time buffered ahead in seconds. */ bufferedAheadOf(contentType, time) { const ContentType = shaka.util.ManifestParserUtils.ContentType; if (contentType == ContentType.TEXT) { return this.textEngine_.bufferedAheadOf(time); } else { const buffered = this.getBuffered_(contentType); return shaka.media.TimeRangesUtils.bufferedAheadOf(buffered, time); } } /** * Computes how far behind the given timestamp is buffered for the given * content type. This is used when playing in reverse, where the buffer is * filled behind the playhead instead of ahead of it. * * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @param {number} time * @return {number} The amount of time buffered behind in seconds. */ bufferedBehindOf(contentType, time) { const ContentType = shaka.util.ManifestParserUtils.ContentType; if (contentType == ContentType.TEXT) { return this.textEngine_.bufferedBehindOf(time); } else { const buffered = this.getBuffered_(contentType); return shaka.media.TimeRangesUtils.bufferedBehindOf(buffered, time); } } /** * Returns info about what is currently buffered. * @return {shaka.extern.BufferedInfo} */ getBufferedInfo() { const ContentType = shaka.util.ManifestParserUtils.ContentType; const TimeRangesUtils = shaka.media.TimeRangesUtils; const info = { total: this.reloadingMediaSource_ ? [] : TimeRangesUtils.getBufferedInfo(this.video_.buffered), audio: TimeRangesUtils.getBufferedInfo(this.getBuffered_(ContentType.AUDIO)), video: TimeRangesUtils.getBufferedInfo(this.getBuffered_(ContentType.VIDEO)), text: [], }; if (this.textEngine_) { const start = this.textEngine_.bufferStart(); const end = this.textEngine_.bufferEnd(); if (start != null && end != null) { info.text.push({start: start, end: end}); } } return info; } /** * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @return {boolean} */ hasSourceBufferFor(contentType) { return this.sourceBuffers_.has(contentType); } /** * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @return {TimeRanges} The buffered ranges for the given content type, or * null if the buffered ranges could not be obtained. * @private */ getBuffered_(contentType) { if (!this.canPerformOperations_()) { return null; } try { return this.sourceBuffers_.get(contentType).buffered; } catch (exception) { if (this.sourceBuffers_.has(contentType)) { // Note: previous MediaSource errors may cause access to |buffered| to // throw. shaka.log.error('failed to get buffered range for ' + contentType, exception); } return null; } } /** * Create a new closed caption parser. This will ONLY be replaced by tests as * a way to inject fake closed caption parser instances. * * @param {string} mimeType * @return {!shaka.media.ClosedCaptionParser} */ getCaptionParser(mimeType) { return new shaka.media.ClosedCaptionParser(mimeType); } /** * This method is only public for testing. * * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @param {!BufferSource} data * @param {!shaka.media.SegmentReference} reference The segment reference * we are appending * @param {shaka.extern.Stream} stream * @param {!string} mimeType * @param {boolean=} isChunkedData True if we add to the buffer from the * partial read of the segment. * @return {{timestamp: ?number, metadata: !Array}} */ getTimestampAndDispatchMetadata(contentType, data, reference, stream, mimeType, isChunkedData = false) { let timestamp = null; let metadata = []; const uint8ArrayData = shaka.util.BufferUtils.toUint8(data); // Packed audio is sometimes published with a container extension (e.g. // .ts), so its mimeType claims MPEG-2 TS. The transmuxer already detects // this and handles it as packed audio, so we must also read the timestamp // from its ID3 tag; otherwise the timestamp offset is never computed. const isPackedAudio = shaka.util.MimeUtils.RAW_FORMATS.includes(mimeType) || (contentType == shaka.util.ManifestParserUtils.ContentType.AUDIO && !mimeType.includes('/mp4') && !mimeType.includes('/webm') && !shaka.util.TsParser.probe(uint8ArrayData) && !!shaka.media.SegmentUtils.getRawAudioMimeType(uint8ArrayData)); if (isPackedAudio) { const frames = shaka.metadata.Id3Utils.getID3Frames(uint8ArrayData); if (frames.length && reference) { const metadataTimestamp = frames.find((frame) => { return frame.description === 'com.apple.streaming.transportStreamTimestamp'; }); if (metadataTimestamp && typeof metadataTimestamp.data == 'number') { timestamp = Math.round(metadataTimestamp.data) / 1000; } /** @private {shaka.extern.ID3Metadata} */ const id3Metadata = { cueTime: reference.startTime, data: uint8ArrayData, frames: frames, dts: reference.startTime, pts: reference.startTime, }; this.playerInterface_.onMetadata( [id3Metadata], /* offset= */ 0, reference.endTime); } } else if (mimeType.includes('/mp4') && reference && reference.initSegmentReference) { const Mp4Parser = shaka.util.Mp4Parser; const parser = new Mp4Parser(); const hasEmsg = ((stream.emsgSchemeIdUris != null && stream.emsgSchemeIdUris.length > 0) || this.config_.dispatchAllEmsgBoxes); const emsgEvents = []; if (hasEmsg) { parser.fullBox('emsg', (box) => { const emsg = this.parseEMSG_(reference, stream.emsgSchemeIdUris, box); if (emsg) { emsgEvents.push(emsg); } }); } const timescale = reference.initSegmentReference.timescale; const hasTimescale = timescale && !isNaN(timescale); let startTime = 0; let parsedMedia = false; if (hasTimescale) { parser .fullBox('prft', (box) => { goog.asserts.assert(typeof timescale == 'number', 'Timescale should be a number!'); this.parsePrft_(timescale, box); }) .boxes(Mp4Parser.FRAGMENT_PATH, Mp4Parser.children) .fullBox('tfdt', (box) => { if (!parsedMedia) { goog.asserts.assert( box.version == 0 || box.version == 1, 'TFDT version can only be 0 or 1'); goog.asserts.assert(typeof timescale == 'number', 'Timescale should be a number!'); const parsed = shaka.util.Mp4BoxParsers.parseTFDTInaccurate( box.reader, box.version); startTime = parsed.baseMediaDecodeTime / timescale; parsedMedia = true; if (!hasEmsg) { box.parser.stop(); } } }); } if (hasEmsg || hasTimescale) { parser.parse(data, /* partialOkay= */ false, isChunkedData); } if (parsedMedia && reference.timestampOffset == 0) { timestamp = startTime; } for (const emsg of emsgEvents) { this.dispatchEmsg_(emsg, reference, timestamp); } } else if (!mimeType.includes('/mp4') && !mimeType.includes('/webm') && shaka.util.TsParser.probe(uint8ArrayData)) { const tsParser = this.tsParsers_.getOrInsertComputed( contentType, () => new shaka.util.TsParser()); tsParser.clearData(); tsParser.setDiscontinuitySequence(reference.discontinuitySequence); tsParser.parse(uint8ArrayData); const startTime = tsParser.getStartTimeFromPesHeaders(contentType); if (startTime != null) { timestamp = startTime; } metadata = tsParser.getMetadata(); } return {timestamp, metadata}; } /** * Parse the EMSG box from a MP4 container. * * @param {!shaka.media.SegmentReference} reference * @param {?Array} emsgSchemeIdUris Array of emsg * scheme_id_uri for which emsg boxes should be parsed. * @param {!shaka.extern.ParsedBox} box * @return {?shaka.extern.EmsgInfo} * @private * https://dashif-documents.azurewebsites.net/Events/master/event.html#emsg-format * aligned(8) class DASHEventMessageBox * extends FullBox(‘emsg’, version, flags = 0){ * if (version==0) { * string scheme_id_uri; * string value; * unsigned int(32) timescale; * unsigned int(32) presentation_time_delta; * unsigned int(32) event_duration; * unsigned int(32) id; * } else if (version==1) { * unsigned int(32) timescale; * unsigned int(64) presentation_time; * unsigned int(32) event_duration; * unsigned int(32) id; * string scheme_id_uri; * string value; * } * unsigned int(8) message_data[]; */ parseEMSG_(reference, emsgSchemeIdUris, box) { let timescale; let id; let eventDuration; let schemeId; let startTime; let presentationTimeDelta; let value; if (box.version === 0) { schemeId = box.reader.readTerminatedString(); value = box.reader.readTerminatedString(); timescale = box.reader.readUint32(); presentationTimeDelta = box.reader.readUint32(); eventDuration = box.reader.readUint32(); id = box.reader.readUint32(); startTime = reference.startTime + (presentationTimeDelta / timescale); } else { timescale = box.reader.readUint32(); const pts = box.reader.readUint64(); startTime = (pts / timescale) + reference.timestampOffset; presentationTimeDelta = startTime - reference.startTime; eventDuration = box.reader.readUint32(); id = box.reader.readUint32(); schemeId = box.reader.readTerminatedString(); value = box.reader.readTerminatedString(); } const messageData = box.reader.readBytes( box.reader.getLength() - box.reader.getPosition(), // The message data gets stored. // Clone the message data so we don't hold the segment in memory. /* clone= */ true); // See DASH sec. 5.10.3.3.1 // If a DASH client detects an event message box with a scheme that is not // defined in MPD, the client is expected to ignore it. if ((emsgSchemeIdUris && emsgSchemeIdUris.includes(schemeId)) || this.config_.dispatchAllEmsgBoxes) { // See DASH sec. 5.10.4.1 // A special scheme in DASH used to signal manifest updates. if (schemeId == 'urn:mpeg:dash:event:2012') { this.playerInterface_.onManifestUpdate(); } else { // All other schemes are dispatched as a general 'emsg' event. let endTime = startTime; // See: https://aomediacodec.github.io/id3-emsg/ // ID3 EMSG events do not carry a duration if (schemeId !== 'https://aomedia.org/emsg/ID3' || eventDuration !== 0xFFFFFFFF) { endTime += eventDuration / timescale; } /** @type {shaka.extern.EmsgInfo} */ const emsg = { startTime: startTime, endTime: endTime, schemeIdUri: schemeId, value: value, timescale: timescale, presentationTimeDelta: presentationTimeDelta, eventDuration: eventDuration, id: id, messageData: messageData, }; return emsg; } } return null; } /** * @param {shaka.extern.EmsgInfo} emsg * @param {!shaka.media.SegmentReference} reference * @param {?number} timestamp * @private */ dispatchEmsg_(emsg, reference, timestamp) { const isVersion0 = shaka.util.NumberUtils.isFloatEqual(emsg.startTime, reference.startTime + (emsg.presentationTimeDelta / emsg.timescale)); if (timestamp && this.manifestType_ == shaka.media.ManifestParser.HLS && !isVersion0) { const diff = -timestamp + reference.startTime; emsg.startTime = emsg.startTime + diff; emsg.endTime = emsg.endTime + diff; } this.playerInterface_.onEmsg(emsg); // Additionally, ID3 events generate a 'metadata' event. This is a // pre-parsed version of the metadata blob already dispatched in the // 'emsg' event. if (emsg.messageData && (emsg.schemeIdUri == 'https://aomedia.org/emsg/ID3' || emsg.schemeIdUri == 'https://developer.apple.com/streaming/emsg-id3')) { // See https://aomediacodec.github.io/id3-emsg/ const frames = shaka.metadata.Id3Utils.getID3Frames(emsg.messageData); if (frames.length) { const startTime = emsg.startTime; /** @private {shaka.extern.ID3Metadata} */ const metadata = { cueTime: startTime, data: emsg.messageData, frames: frames, dts: startTime, pts: startTime, }; this.playerInterface_.onMetadata( [metadata], /* offset= */ 0, emsg.endTime); } } } /** * Parse PRFT box. * @param {number} timescale * @param {!shaka.extern.ParsedBox} box * @private */ parsePrft_(timescale, box) { goog.asserts.assert( box.version == 0 || box.version == 1, 'PRFT version can only be 0 or 1'); const parsed = shaka.util.Mp4BoxParsers.parsePRFTInaccurate( box.reader, box.version); const wallClockTime = shaka.util.TimeUtils.convertNtp(parsed.ntpTimestamp); const programStartDate = new Date(wallClockTime - (parsed.mediaTime / timescale) * 1000); /** @type {shaka.extern.ProducerReferenceTime} */ const prftInfo = { wallClockTime, programStartDate, }; const eventName = shaka.util.FakeEvent.EventName.Prft; const data = (new Map()).set('detail', prftInfo); const event = new shaka.util.FakeEvent( eventName, data); this.playerInterface_.onEvent(event); } /** * Dispatch buffer appending events including the init segment and reference. * Use a null check on the reference to determine the init segment. * @param {string} contentType The chunk content type. * @param {!BufferSource} data The chunk or init segment data. * @param {boolean} isInitData Determines if the data is an init segment. * @private */ dispatchBufferAppending_(contentType, data, isInitData = false) { // make a bufferAppending event with init segment data const eventName = shaka.util.FakeEvent.EventName.BufferAppending; const eventData = (new Map()) .set('data', data) .set('contentType', contentType) .set('isInitData', isInitData); const event = new shaka.util.FakeEvent( eventName, eventData); this.playerInterface_.onEvent(event); } /** * Enqueue an operation to append data to the SourceBuffer. * Start and end times are needed for TextEngine, but not for MediaSource. * Start and end times may be null for initialization segments; if present * they are relative to the presentation timeline. * * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @param {!BufferSource} data * @param {?shaka.media.SegmentReference} reference The segment reference * we are appending, or null for init segments * @param {shaka.extern.Stream} stream * @param {?boolean} hasClosedCaptions True if the buffer contains CEA closed * captions * @param {boolean=} seeked True if we just seeked * @param {boolean=} adaptation True if we just automatically switched active * variant(s). * @param {boolean=} isChunkedData True if we add to the buffer from the * partial read of the segment. * @param {boolean=} fromSplit * @param {number=} continuityTimeline an optional continuity timeline * @return {!Promise} */ async appendBuffer( contentType, data, reference, stream, hasClosedCaptions, seeked = false, adaptation = false, isChunkedData = false, fromSplit = false, continuityTimeline) { const ContentType = shaka.util.ManifestParserUtils.ContentType; if (contentType == ContentType.TEXT) { await this.textEngine_.appendBuffer( data, reference ? reference.startTime : null, reference ? reference.endTime : null, reference ? reference.getUris()[0] : null, reference ? reference.discontinuitySequence : -1); return { mediaTimestamp: null, }; } if (!fromSplit && this.needSplitMuxedContent_) { // Video first: its segments are cut on keyframes, so its embedded // timestamps land exactly on the segment boundary. Audio's own // timestamps do not (frames rarely align with the cut point), so in // segments mode we anchor audio's timestampOffset to video's instead of // recomputing it independently. That requires video's offset for this // segment to already be known, hence the ordering. const videoAppendBufferInfo = await this.appendBuffer( ContentType.VIDEO, data, reference, stream, hasClosedCaptions, seeked, adaptation, isChunkedData, /* fromSplit= */ true); const audioAppendBufferInfo = await this.appendBuffer( ContentType.AUDIO, data, reference, stream, hasClosedCaptions, seeked, adaptation, isChunkedData, /* fromSplit= */ true); return videoAppendBufferInfo.mediaTimestamp != null ? videoAppendBufferInfo : audioAppendBufferInfo; } if (!this.sourceBuffers_.has(contentType)) { shaka.log.warning('Attempted to restore a non-existent source buffer'); return { mediaTimestamp: null, }; } // If the stream belongs to a Preselection, give the application a chance // to rewrite the segment so the media engine decodes the selected // experience (for example, an AC-4 presentation). if (stream.preselection) { /** @type {shaka.extern.PreselectionSegmentInfo} */ const preselectionSegmentInfo = { data, preselection: stream.preselection, stream, contentType, isInitSegment: reference == null, }; const modifiedInfo = this.config_.modifyPreselectionSegmentCallback( preselectionSegmentInfo); if (modifiedInfo && modifiedInfo.data) { data = modifiedInfo.data; } } let timestampOffset = this.requestedTimestampOffsets_.has(contentType) ? this.requestedTimestampOffsets_.get(contentType) : this.sourceBuffers_.get(contentType).timestampOffset; let mediaTimestamp = null; // If we buffered to the end of the presentation the MediaSource was put // into the 'ended' state (e.g. before switching to reverse playback). In // that state abort() and the timestampOffset setter throw, so any append // that needs them (seeks, adaptations, reverse) must first transition the // MediaSource back to 'open'. Enqueue the reopen ahead of them. if (this.ended()) { this.enqueueOperation_( contentType, () => this.reopenMediaSource_(contentType), null); } let mimeType = this.sourceBufferTypes_.get(contentType); if (this.transmuxers_.has(contentType)) { mimeType = this.transmuxers_.get(contentType).getOriginalMimeType(); } if (reference) { const {timestamp, metadata} = this.getTimestampAndDispatchMetadata( contentType, data, reference, stream, mimeType, isChunkedData); if (timestamp != null) { mediaTimestamp = timestamp; if (this.firstVideoTimestamp_ == null && contentType == ContentType.VIDEO) { this.firstVideoTimestamp_ = timestamp; this.firstVideoReferenceStartTime_ = reference.startTime; if (this.firstAudioTimestamp_ != null) { let compensation = 0; // Only apply compensation if video and audio segment startTime // match, to avoid introducing sync issues. if (this.firstVideoReferenceStartTime_ == this.firstAudioReferenceStartTime_) { compensation = this.firstVideoTimestamp_ - this.firstAudioTimestamp_; } this.audioCompensation_.resolve(compensation); } } if (this.firstAudioTimestamp_ == null && contentType == ContentType.AUDIO) { this.firstAudioTimestamp_ = timestamp; this.firstAudioReferenceStartTime_ = reference.startTime; if (this.firstVideoTimestamp_ != null) { let compensation = 0; // Only apply compensation if video and audio segment startTime // match, to avoid introducing sync issues. if (this.firstVideoReferenceStartTime_ == this.firstAudioReferenceStartTime_) { compensation = this.firstVideoTimestamp_ - this.firstAudioTimestamp_; } this.audioCompensation_.resolve(compensation); } } let calculatedTimestampOffset; const SourceBufferMode = shaka.media.MediaSourceEngine.SourceBufferMode_; const anchorAudioToVideo = contentType == ContentType.AUDIO && this.needSplitMuxedContent_ && !this.sequenceMode_ && this.firstVideoTimestamp_ != null && this.sourceBuffers_.has(ContentType.VIDEO) && // Some platforms put an audio-only mp4 SourceBuffer into // 'sequence' mode even though Shaka is using 'segments' mode // overall (observed for MP3-in-MP4 on Chrome). In that mode the // browser derives each append's placement from where the // previous one ended, not from timestampOffset, so anchoring to // video here would be silently ignored; fall through to the // per-segment recomputation below instead. this.sourceBuffers_.get(ContentType.AUDIO).mode == SourceBufferMode.SEGMENTS; if (anchorAudioToVideo) { // Muxed content split into separate audio/video SourceBuffers: in // segments mode, video's per-segment offset is trustworthy because // segments are cut on keyframes, so video's embedded timestamp // lands exactly on the segment boundary. Audio's own embedded // timestamp does not (frames rarely align with the cut point, so it // jitters by a fraction of a frame from segment to segment). // Recomputing audio's offset independently every segment chases // that harmless jitter, which forces an abort() + splice on nearly // every append (see abort_ above) and slowly erodes the buffered // audio. Anchor audio to video's offset instead, adjusted by the // fixed compensation between the two tracks measured on the first // segment. See https://github.com/shaka-project/shaka-player/issues/10341 const videoTimestampOffset = this.requestedTimestampOffsets_.has(ContentType.VIDEO) ? this.requestedTimestampOffsets_.get(ContentType.VIDEO) : this.sourceBuffers_.get(ContentType.VIDEO).timestampOffset; const compensation = await this.audioCompensation_.promise; calculatedTimestampOffset = videoTimestampOffset + compensation; } else { let realTimestamp = timestamp; const RAW_FORMATS = shaka.util.MimeUtils.RAW_FORMATS; // For formats without containers and using segments mode, we need // to adjust TimestampOffset relative to 0 because segments do not // have any timestamp information. if (!this.sequenceMode_ && RAW_FORMATS.includes(this.sourceBufferTypes_.get(contentType))) { realTimestamp = 0; } calculatedTimestampOffset = reference.startTime - realTimestamp; } const timestampOffsetDifference = Math.abs(timestampOffset - calculatedTimestampOffset); if ((timestampOffsetDifference >= 0.001 || seeked || adaptation) && (!isChunkedData || calculatedTimestampOffset > 0 || !timestampOffset)) { timestampOffset = calculatedTimestampOffset; if (this.attemptTimestampOffsetCalculation_) { this.enqueueOperation_( contentType, () => this.abort_(contentType), null); this.enqueueOperation_( contentType, () => this.setTimestampOffset_(contentType, timestampOffset), null); } } // Timestamps can only be reliably extracted from video, not audio. // Packed audio formats do not have internal timestamps at all. // Prefer video for this when available. const isBestSourceBufferForTimestamps = contentType == ContentType.VIDEO || !(this.sourceBuffers_.has(ContentType.VIDEO)); if (isBestSourceBufferForTimestamps && this.textEngine_) { this.textEngine_.setTimestampOffset( timestampOffset, reference.discontinuitySequence); this.textEngine_.setContainerIsMpegTs( (mimeType || '').includes('mp2t')); } } if (metadata.length) { this.playerInterface_.onMetadata(metadata, timestampOffset, reference ? reference.endTime : null); } } /** * The initialization segment a transmuxer produced on this append, if any. * Closed captions are parsed out of the transmuxed media below, so this is * what describes it, not whatever initialization segment arrived from the * network. * @type {?BufferSource} */ let transmuxedInitSegment = null; if (this.transmuxers_.has(contentType)) { const transmuxerOutput = await this.transmuxers_.get(contentType).transmux( data, stream, reference, this.mediaSource_.duration, contentType); if (ArrayBuffer.isView(transmuxerOutput)) { data = /** @type {!Uint8Array} */(transmuxerOutput); } else { const output = /** @type {!shaka.extern.TransmuxerOutput} */(transmuxerOutput); if (output.init != null) { const initData = output.init; transmuxedInitSegment = initData; this.enqueueOperation_(contentType, () => { this.append_(contentType, initData, timestampOffset, stream, true); }, reference ? reference.getUris()[0] : null); } data = output.data; } } // Closed captions are parsed AFTER transmuxing, out of what is actually // going to be appended. Transmuxed containers are always repackaged into // MP4, so one parser covers every path that reaches a SourceBuffer, and a // packaging only has to preserve the SEI NAL units to get captions for // free. Parsing the pre-transmux bytes instead needed a parser per source // container, and gave nothing at all to containers that carry the // elementary stream raw. if (hasClosedCaptions && contentType == ContentType.VIDEO) { if (!this.textEngine_) { this.reinitText(shaka.util.MimeUtils.CEA608_CLOSED_CAPTION_MIMETYPE, /* external= */ false); } if (!this.captionParser_) { // The SourceBuffer's own type, which is what `data` is by now. const bufferType = this.sourceBufferTypes_.get(contentType) || mimeType; this.captionParser_ = this.getCaptionParser( bufferType.split(';', 1)[0]); } if (transmuxedInitSegment || !reference) { this.captionParser_.init( transmuxedInitSegment || data, adaptation, continuityTimeline); } if (reference) { const closedCaptions = this.captionParser_.parseFrom(data); if (closedCaptions.length) { this.textEngine_.storeAndAppendClosedCaptions( closedCaptions, timestampOffset); } } } data = this.workAroundBrokenPlatforms_( stream, data, reference, contentType); if (this.clearKeyDecryptors_.has(contentType)) { const isMp4 = shaka.util.MimeUtils.getContainerType( this.sourceBufferTypes_.get(contentType)) == 'mp4'; const drmInfo = this.playerInterface_.getDrmInfo(); if (isMp4 && drmInfo) { const isInitSegment = reference === null; const decryptor = this.clearKeyDecryptors_.get(contentType); data = await decryptor.decrypt(data, isInitSegment, drmInfo); } } if (reference && this.sequenceMode_ && contentType != ContentType.TEXT) { // In sequence mode, for non-text streams, if we just cleared the buffer // and are either performing an unbuffered seek or handling an automatic // adaptation, we need to set a new timestampOffset on the sourceBuffer. if (seeked || adaptation) { let timestampOffset = reference.startTime; // Audio and video may not be aligned, so we will compensate for audio // if necessary. if (this.manifestType_ == shaka.media.ManifestParser.HLS && !this.needSplitMuxedContent_ && contentType == ContentType.AUDIO && this.sourceBuffers_.has(ContentType.VIDEO)) { const compensation = await this.audioCompensation_.promise; // Only apply compensation if the difference is greater than 150ms if (Math.abs(compensation) > 0.15) { timestampOffset -= compensation; } } // The logic to call abort() before setting the timestampOffset is // extended during unbuffered seeks or automatic adaptations; it is // possible for the append state to be PARSING_MEDIA_SEGMENT from the // previous SourceBuffer#appendBuffer() call. this.enqueueOperation_( contentType, () => this.abort_(contentType), null); this.enqueueOperation_( contentType, () => this.setTimestampOffset_(contentType, timestampOffset), null); } } let bufferedBefore = null; await this.enqueueOperation_(contentType, () => { if (goog.DEBUG && reference && !reference.isPreload() && !isChunkedData) { bufferedBefore = this.getBuffered_(contentType); } this.append_(contentType, data, timestampOffset, stream, !reference); }, reference ? reference.getUris()[0] : null); if (goog.DEBUG && reference && !reference.isPreload() && !isChunkedData && this.manifestType_ !== shaka.media.ManifestParser.MSF) { const bufferedAfter = this.getBuffered_(contentType); const newBuffered = shaka.media.TimeRangesUtils.computeAddedRange( bufferedBefore, bufferedAfter); if (newBuffered) { const segmentDuration = reference.endTime - reference.startTime; const timeAdded = newBuffered.end - newBuffered.start; // Check end times instead of start times. We may be overwriting a // buffer and only the end changes, and that would be fine. // Also, exclude tiny segments. Sometimes alignment segments as small // as 33ms are seen in Google DAI content. For such tiny segments, // half a segment duration would be no issue. const offset = Math.abs(newBuffered.end - reference.endTime); if (segmentDuration > 0.100 && (offset > segmentDuration / 2 || Math.abs(segmentDuration - timeAdded) > 0.030)) { shaka.log.error('Possible encoding problem detected!', 'Unexpected buffered range for reference', reference, 'from URIs', reference.getUris(), 'should be', {start: reference.startTime, end: reference.endTime}, 'but got', newBuffered); } } } return { mediaTimestamp: mediaTimestamp, }; } /** * Set the selected closed captions Id and language. * * @param {string} id */ setSelectedClosedCaptionId(id) { const VIDEO = shaka.util.ManifestParserUtils.ContentType.VIDEO; const videoBufferEndTime = this.bufferEnd(VIDEO) || 0; this.textEngine_.setSelectedClosedCaptionId(id, videoBufferEndTime); } /** Disable embedded closed captions. */ clearSelectedClosedCaptionId() { if (this.textEngine_) { this.textEngine_.setSelectedClosedCaptionId('', 0); } } /** * Enqueue an operation to remove data from the SourceBuffer. * * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @param {number} startTime relative to the start of the presentation * @param {number} endTime relative to the start of the presentation * @param {Array=} continuityTimelines a list of continuity timelines * that are still available on the stream. * @return {!Promise} */ async remove(contentType, startTime, endTime, continuityTimelines) { const ContentType = shaka.util.ManifestParserUtils.ContentType; if (contentType == ContentType.VIDEO && this.captionParser_) { this.captionParser_.remove(continuityTimelines); // Get actual TextEngine buffer start, as it's not the same as video // buffer and TextEngine does not support multiple buffered ranges. const textStart = this.textEngine_.bufferStart() || 0; this.textEngine_.remove(textStart, endTime, /* removeCC= */ true); } if (contentType == ContentType.TEXT) { await this.textEngine_.remove(startTime, endTime); } else if (endTime > startTime) { await this.enqueueOperation_( contentType, () => this.remove_(contentType, startTime, endTime), null); if (this.needSplitMuxedContent_) { await this.enqueueOperation_( ContentType.AUDIO, () => this.remove_(ContentType.AUDIO, startTime, endTime), null); } } } /** * Enqueue an operation to clear the SourceBuffer. * * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @return {!Promise} */ async clear(contentType) { const ContentType = shaka.util.ManifestParserUtils.ContentType; if (contentType == ContentType.TEXT) { if (!this.textEngine_) { return; } await this.textEngine_.remove(0, Infinity); } else { // if we have CEA captions, we should clear those too. if (contentType === ContentType.VIDEO && this.captionParser_ && this.textEngine_) { await this.textEngine_.remove(0, Infinity, /* removeCC= */ true); } // Note that not all platforms allow clearing to Infinity. await this.enqueueOperation_( contentType, () => this.remove_(contentType, 0, this.mediaSource_.duration), null); if (this.needSplitMuxedContent_) { await this.enqueueOperation_( ContentType.AUDIO, () => this.remove_( ContentType.AUDIO, 0, this.mediaSource_.duration), null); } } } /** * Fully reset the state of the caption parser owned by MediaSourceEngine. */ resetCaptionParser() { if (this.captionParser_) { this.captionParser_.reset(); } } /** * Enqueue an operation to flush the SourceBuffer. * This is a workaround for platforms that do not replace the old content * after the buffer has been cleared. * * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @return {!Promise} */ async flush(contentType) { const ContentType = shaka.util.ManifestParserUtils.ContentType; if (contentType == ContentType.TEXT) { // Nothing to flush for text. return; } await this.enqueueOperation_( contentType, () => this.flush_(contentType), null); if (this.needSplitMuxedContent_) { await this.enqueueOperation_( ContentType.AUDIO, () => this.flush_(ContentType.AUDIO), null); } } /** * Sets the timestamp offset and append window end for the given content type. * * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @param {number} timestampOffset The timestamp offset. Segments which start * at time t will be inserted at time t + timestampOffset instead. This * value does not affect segments which have already been inserted. * @param {number} appendWindowStart The timestamp to set the append window * start to. For future appends, frames/samples with timestamps less than * this value will be dropped. * @param {number} appendWindowEnd The timestamp to set the append window end * to. For future appends, frames/samples with timestamps greater than this * value will be dropped. * @param {boolean} ignoreTimestampOffset If true, the timestampOffset will * not be applied in this step. * @param {string} mimeType * @param {string} codecs * @param {!Map} streamsByType * A map of content types to streams. * * @return {!Promise} */ async setStreamProperties( contentType, timestampOffset, appendWindowStart, appendWindowEnd, ignoreTimestampOffset, mimeType, codecs, streamsByType) { const ContentType = shaka.util.ManifestParserUtils.ContentType; if (contentType == ContentType.TEXT) { if (!ignoreTimestampOffset) { this.textEngine_.setTimestampOffset(timestampOffset); } this.textEngine_.setAppendWindow(appendWindowStart, appendWindowEnd); return; } const operations = []; const hasChangedCodecs = await this.codecSwitchIfNecessary_( contentType, mimeType, codecs, streamsByType); if (!hasChangedCodecs) { // Queue an abort() to help MSE splice together overlapping segments. // We set appendWindowEnd when we change periods in DASH content, and the // period transition may result in overlap. // // An abort() also helps with MPEG2-TS. When we append a TS segment, we // always enter a PARSING_MEDIA_SEGMENT state and we can't change the // timestamp offset. By calling abort(), we reset the state so we can // set it. operations.push(this.enqueueOperation_( contentType, () => this.abort_(contentType), null)); if (this.needSplitMuxedContent_) { operations.push(this.enqueueOperation_( ContentType.AUDIO, () => this.abort_(ContentType.AUDIO), null)); } } if (!ignoreTimestampOffset) { operations.push(this.enqueueOperation_( contentType, () => this.setTimestampOffset_(contentType, timestampOffset), null)); if (this.needSplitMuxedContent_) { operations.push(this.enqueueOperation_( ContentType.AUDIO, () => this.setTimestampOffset_( ContentType.AUDIO, timestampOffset), null)); } } if (appendWindowStart != 0 || appendWindowEnd != Infinity) { operations.push(this.enqueueOperation_( contentType, () => this.setAppendWindow_( contentType, appendWindowStart, appendWindowEnd), null)); if (this.needSplitMuxedContent_) { operations.push(this.enqueueOperation_( ContentType.AUDIO, () => this.setAppendWindow_( ContentType.AUDIO, appendWindowStart, appendWindowEnd), null)); } } if (operations.length) { await Promise.all(operations); } } /** * Adjust timestamp offset to maintain AV sync across discontinuities. * * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @param {number} timestampOffset * @return {!Promise} */ async resync(contentType, timestampOffset) { const ContentType = shaka.util.ManifestParserUtils.ContentType; if (contentType == ContentType.TEXT) { // This operation is for audio and video only. return; } if (!this.sequenceMode_) { return; } // Avoid changing timestampOffset when the difference is less than 100 ms // from the end of the current buffer. const bufferEnd = this.bufferEnd(contentType); if (bufferEnd && Math.abs(bufferEnd - timestampOffset) < 0.15) { return; } // Queue an abort() to help MSE splice together overlapping segments. // We set appendWindowEnd when we change periods in DASH content, and the // period transition may result in overlap. // // An abort() also helps with MPEG2-TS. When we append a TS segment, we // always enter a PARSING_MEDIA_SEGMENT state and we can't change the // timestamp offset. By calling abort(), we reset the state so we can // set it. this.enqueueOperation_( contentType, () => this.abort_(contentType), null); if (this.needSplitMuxedContent_) { this.enqueueOperation_( ContentType.AUDIO, () => this.abort_(ContentType.AUDIO), null); } await this.enqueueOperation_( contentType, () => this.setTimestampOffset_(contentType, timestampOffset), null); if (this.needSplitMuxedContent_) { await this.enqueueOperation_( ContentType.AUDIO, () => this.setTimestampOffset_(ContentType.AUDIO, timestampOffset), null); } } /** * @param {string=} reason Valid reasons are 'network' and 'decode'. * @return {!Promise} * @see http://w3c.github.io/media-source/#idl-def-EndOfStreamError */ async endOfStream(reason) { await this.enqueueBlockingOperation_(() => { // If endOfStream() has already been called on the media source, // don't call it again. Also do not call if readyState is // 'closed' (not attached to video element) since it is not a // valid operation. if (this.ended() || this.closed()) { return; } // Tizen won't let us pass undefined, but it will let us omit the // argument. if (reason) { this.mediaSource_.endOfStream(reason); } else { this.mediaSource_.endOfStream(); } }); } /** * @param {number} duration * @return {!Promise} */ async setDuration(duration) { await this.enqueueBlockingOperation_(() => { // https://www.w3.org/TR/media-source-2/#duration-change-algorithm // "Duration reductions that would truncate currently buffered media // are disallowed. // When truncation is necessary, use remove() to reduce the buffered // range before updating duration." // But in some platforms, truncating the duration causes the // buffer range removal algorithm to run which triggers an // 'updateend' event to fire. // To handle this scenario, we have to insert a dummy operation into // the beginning of each queue, which the 'updateend' handler will remove. // Using config to disable it by default and enable only // on relevant platforms. if (this.config_.durationReductionEmitsUpdateEnd && duration < this.mediaSource_.duration) { for (const contentType of this.sourceBuffers_.keys()) { const dummyOperation = { start: () => {}, p: Promise.withResolvers(), uri: null, }; this.queues_.get(contentType).unshift(dummyOperation); } } this.mediaSource_.duration = duration; this.lastDuration_ = duration; }); } /** * Get the current MediaSource duration. * * @return {number} */ getDuration() { return this.mediaSource_.duration; } /** * Updates the live seekable range. * * @param {number} startTime * @param {number} endTime */ async setLiveSeekableRange(startTime, endTime) { if (!this.canPerformOperations_() || this.mediaSource_.duration !== Infinity) { return; } goog.asserts.assert('setLiveSeekableRange' in this.mediaSource_, 'Using setLiveSeekableRange on not supported platform'); this.usesLiveSeekableRange_ = true; await this.enqueueBlockingOperation_(() => { if (!this.canPerformOperations_() || this.mediaSource_.duration !== Infinity) { return; } this.mediaSource_.setLiveSeekableRange(startTime, endTime); }); } /** * Clear the current live seekable range. */ async clearLiveSeekableRange() { if (!this.canPerformOperations_() || !this.usesLiveSeekableRange_ || this.mediaSource_.duration !== Infinity) { return; } goog.asserts.assert('clearLiveSeekableRange' in this.mediaSource_, 'Using clearLiveSeekableRange on not supported platform'); this.usesLiveSeekableRange_ = false; await this.enqueueBlockingOperation_(() => { if (!this.canPerformOperations_() || this.mediaSource_.duration !== Infinity) { return; } this.mediaSource_.clearLiveSeekableRange(); }); } /** * Append dependency data. * @param {BufferSource} data * @param {number} timestampOffset * @param {shaka.extern.Stream} stream */ appendDependency(data, timestampOffset, stream) { if (this.lcevcDec_) { // Append buffers to the LCEVC Dec for parsing and storing // of LCEVC data. this.lcevcDec_.appendBuffer(data, timestampOffset, stream); } } /** * Remove dependency data. * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @param {number} startTime relative to the start of the presentation * @param {number} endTime relative to the start of the presentation */ removeDependency(contentType, startTime, endTime) { if (this.lcevcDec_) { // Remove buffers from the LCEVC Dec. this.lcevcDec_.removeBuffer(contentType, startTime, endTime); } } /** * Append data to the SourceBuffer. * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @param {BufferSource} data * @param {number} timestampOffset * @param {shaka.extern.Stream} stream * @param {boolean} isInitData Determines if the data is an init segment. * @private */ append_(contentType, data, timestampOffset, stream, isInitData = false) { this.appendDependency(data, timestampOffset, stream); this.dispatchBufferAppending_(contentType, data, isInitData); // This will trigger an 'updateend' event. this.sourceBuffers_.get(contentType).appendBuffer(data); } /** * Remove data from the SourceBuffer. * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @param {number} startTime relative to the start of the presentation * @param {number} endTime relative to the start of the presentation * @private */ remove_(contentType, startTime, endTime) { if (endTime <= startTime) { // Ignore removal of inverted or empty ranges. // Fake 'updateend' event to resolve the operation. this.onUpdateEnd_(contentType); return; } this.removeDependency(contentType, startTime, endTime); // This will trigger an 'updateend' event. this.sourceBuffers_.get(contentType).remove(startTime, endTime); } /** * Call abort() on the SourceBuffer. * This resets MSE's last_decode_timestamp on all track buffers, which should * trigger the splicing logic for overlapping segments. * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @private */ abort_(contentType) { const sourceBuffer = this.sourceBuffers_.get(contentType); // Save the append window, which is reset on abort(). const appendWindowStart = sourceBuffer.appendWindowStart; const appendWindowEnd = sourceBuffer.appendWindowEnd; // This will not trigger an 'updateend' event, since nothing is happening. // This is only to reset MSE internals, not to abort an actual operation. sourceBuffer.abort(); // Restore the append window. sourceBuffer.appendWindowStart = appendWindowStart; sourceBuffer.appendWindowEnd = appendWindowEnd; // Fake an 'updateend' event to resolve the operation. this.onUpdateEnd_(contentType); } /** * Transition the MediaSource from the 'ended' state back to 'open' so that * further appends (and the abort()/timestampOffset operations they rely on) * are allowed again. appendBuffer() and remove() are the only operations * that reopen an ended MediaSource, so we remove an empty range just past the * end of the buffered content, which reopens it without discarding anything. * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @private */ reopenMediaSource_(contentType) { const sourceBuffer = this.sourceBuffers_.get(contentType); if (!this.ended()) { // Already reopened by a previous operation in the queue. this.onUpdateEnd_(contentType); return; } const buffered = sourceBuffer.buffered; const start = buffered.length ? buffered.end(buffered.length - 1) : 0; // This reopens the MediaSource and fires a real 'updateend' event, which // resolves the enqueued operation. sourceBuffer.remove(start, start + 0.001); } /** * Nudge the playhead to force the media pipeline to be flushed. * This is necessary on some platforms to get new content to replace old * content. * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @private */ flush_(contentType) { // Never use flush_ if there's data. It causes a hiccup in playback. goog.asserts.assert( this.video_.buffered.length == 0, 'MediaSourceEngine.flush_ should ' + 'only be used after clearing all data!'); // Seeking forces the pipeline to be flushed. this.video_.currentTime -= 0.001; // Fake an 'updateend' event to resolve the operation. this.onUpdateEnd_(contentType); } /** * Set the SourceBuffer's timestamp offset. * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @param {number} timestampOffset * @private */ setTimestampOffset_(contentType, timestampOffset) { // Remember what was asked for, before the workaround below may skew it. // Callers compare a newly calculated offset against the current one to // decide whether it actually changed, and the value read back from the // SourceBuffer carries the extra 0.001 forever. Comparing against it // would report a 0.001 difference on every segment of any stream whose // offset is negative, which is every stream whose media timestamps do not // start near zero. this.requestedTimestampOffsets_.set(contentType, timestampOffset); // Work around for // https://github.com/shaka-project/shaka-player/issues/1281: on affected // platforms, rounding the timestampOffset can push the first frame just // below appendWindowStart, which makes MSE drop everything up to the next // keyframe. Nudge the offset forward so the first keyframe survives. if (timestampOffset < 0 && shaka.device.DeviceFactory.getDevice().requiresTimestampOffsetFudge()) { timestampOffset += 0.001; } this.sourceBuffers_.get(contentType).timestampOffset = timestampOffset; // Fake an 'updateend' event to resolve the operation. this.onUpdateEnd_(contentType); } /** * Set the SourceBuffer's append window end. * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @param {number} appendWindowStart * @param {number} appendWindowEnd * @private */ setAppendWindow_(contentType, appendWindowStart, appendWindowEnd) { const sourceBuffer = this.sourceBuffers_.get(contentType); if (sourceBuffer.appendWindowEnd !== appendWindowEnd || sourceBuffer.appendWindowStart !== appendWindowStart) { // You can't set start > end, so first set start to 0, then set the new // end, then set the new start. That way, there are no intermediate // states which are invalid. sourceBuffer.appendWindowStart = 0; sourceBuffer.appendWindowEnd = appendWindowEnd; sourceBuffer.appendWindowStart = appendWindowStart; } // Fake an 'updateend' event to resolve the operation. this.onUpdateEnd_(contentType); } /** * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @private */ onError_(contentType) { const operation = this.queues_.get(contentType)[0]; goog.asserts.assert(operation, 'Spurious error event!'); goog.asserts.assert(!this.sourceBuffers_.get(contentType).updating, 'SourceBuffer should not be updating on error!'); const code = this.video_.error ? this.video_.error.code : 0; operation.p.reject(new shaka.util.Error( shaka.util.Error.Severity.CRITICAL, shaka.util.Error.Category.MEDIA, shaka.util.Error.Code.MEDIA_SOURCE_OPERATION_FAILED, code, operation.uri)); // Do not pop from queue. An 'updateend' event will fire next, and to // avoid synchronizing these two event handlers, we will allow that one to // pop from the queue as normal. Note that because the operation has // already been rejected, the call to resolve() in the 'updateend' handler // will have no effect. } /** * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @private */ onUpdateEnd_(contentType) { // If we're reloading or have been destroyed, clear the queue for this // content type. if (this.reloadingMediaSource_ || this.destroyer_.destroyed()) { // Resolve any pending operations in this content type's queue const queue = this.queues_.get(contentType); if (queue && queue.length) { // Resolve the first operation that triggered this updateEnd const firstOperation = queue[0]; if (firstOperation && firstOperation.p) { firstOperation.p.resolve(); } // Clear the rest of the queue this.queues_.set(contentType, []); } return; } const operation = this.queues_.get(contentType)[0]; goog.asserts.assert(operation, 'Spurious updateend event!'); if (!operation) { return; } goog.asserts.assert(!this.sourceBuffers_.get(contentType).updating, 'SourceBuffer should not be updating on updateend!'); operation.p.resolve(); this.popFromQueue_(contentType); } /** * Enqueue an operation and start it if appropriate. * * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @param {function()} start * @param {?string} uri * @return {!Promise} * @private */ enqueueOperation_(contentType, start, uri) { this.destroyer_.ensureNotDestroyed(); const operation = { start: start, p: Promise.withResolvers(), uri, }; this.queues_.get(contentType).push(operation); if (this.queues_.get(contentType).length == 1) { this.startOperation_(contentType); } return operation.p.promise; } /** * Enqueue an operation which must block all other operations on all * SourceBuffers. * * @param {function():(Promise|undefined)} run * @return {!Promise} * @private */ async enqueueBlockingOperation_(run) { this.destroyer_.ensureNotDestroyed(); /** @type {!Array} */ const allWaiters = []; /** @type {!Array} */ const contentTypes = Array.from(this.sourceBuffers_.keys()); const blockOperations = new Map(); // Enqueue a 'wait' operation onto each queue. // This operation signals its readiness when it starts. // When all wait operations are ready, the real operation takes place. for (const contentType of contentTypes) { const ready = Promise.withResolvers(); const operation = { start: () => ready.resolve(), p: ready, uri: null, }; const queue = this.queues_.get(contentType); queue.push(operation); allWaiters.push(ready.promise); blockOperations.set(contentType, operation); if (queue.length == 1) { operation.start(); } } // Return a Promise to the real operation, which waits to begin until // there are no other in-progress operations on any SourceBuffers. try { await Promise.all(allWaiters); } catch (error) { if (this.destroyer_.destroyed()) { for (const contentType of contentTypes) { const queue = this.queues_.get(contentType); if (blockOperations.has(contentType)) { shaka.util.ArrayUtils.remove( queue, blockOperations.get(contentType)); } } return; } throw error; } if (goog.DEBUG) { // If we did it correctly, nothing is updating. for (const contentType of contentTypes) { goog.asserts.assert( this.sourceBuffers_.get(contentType).updating == false, 'SourceBuffers should not be updating after a blocking op!'); } } // Run the real operation, which can be asynchronous. try { await run(); } catch (exception) { const errorMessage = this.video_.error ? shaka.util.Error.getMediaElementErrorDetails(this.video_.error) : 'No error in the media element'; throw new shaka.util.Error( shaka.util.Error.Severity.CRITICAL, shaka.util.Error.Category.MEDIA, shaka.util.Error.Code.MEDIA_SOURCE_OPERATION_THREW, exception, errorMessage, null); } finally { // Unblock the queues. for (const contentType of contentTypes) { this.popFromQueue_(contentType); } } } /** * Pop from the front of the queue and start a new operation. * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @private */ popFromQueue_(contentType) { goog.asserts.assert(this.queues_.has(contentType), 'Queue should exist'); // Remove the in-progress operation, which is now complete. this.queues_.get(contentType).shift(); this.startOperation_(contentType); } /** * Starts the next operation in the queue. * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @private */ startOperation_(contentType) { // Retrieve the next operation, if any, from the queue and start it. const next = this.queues_.get(contentType)[0]; if (next) { try { next.start(); } catch (exception) { if (exception.name == 'QuotaExceededError') { next.p.reject(new shaka.util.Error( shaka.util.Error.Severity.CRITICAL, shaka.util.Error.Category.MEDIA, shaka.util.Error.Code.QUOTA_EXCEEDED_ERROR, contentType)); } else if (!this.isStreamingAllowed()) { next.p.reject(new shaka.util.Error( shaka.util.Error.Severity.CRITICAL, shaka.util.Error.Category.MEDIA, shaka.util.Error.Code.STREAMING_NOT_ALLOWED, contentType)); } else { const errorMessage = this.video_.error ? shaka.util.Error.getMediaElementErrorDetails(this.video_.error) : 'No error in the media element'; next.p.reject(new shaka.util.Error( shaka.util.Error.Severity.CRITICAL, shaka.util.Error.Category.MEDIA, shaka.util.Error.Code.MEDIA_SOURCE_OPERATION_THREW, exception, errorMessage, next.uri)); } this.popFromQueue_(contentType); } } } /** * @return {!shaka.extern.TextDisplayer} */ getTextDisplayer() { goog.asserts.assert( this.textDisplayer_, 'TextDisplayer should only be null when this is destroyed'); return this.textDisplayer_; } /** * @param {!shaka.extern.TextDisplayer} textDisplayer */ setTextDisplayer(textDisplayer) { this.textDisplayer_ = textDisplayer; if (this.textEngine_) { this.textEngine_.setDisplayer(textDisplayer); } } /** * @param {boolean} segmentRelativeVttTiming */ setSegmentRelativeVttTiming(segmentRelativeVttTiming) { this.segmentRelativeVttTiming_ = segmentRelativeVttTiming; } /** * Apply platform-specific transformations to this segment to work around * issues in the platform. * * @param {shaka.extern.Stream} stream * @param {!BufferSource} segment * @param {?shaka.media.SegmentReference} reference * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @return {!BufferSource} * @private */ workAroundBrokenPlatforms_(stream, segment, reference, contentType) { const ContentType = shaka.util.ManifestParserUtils.ContentType; const isMp4 = shaka.util.MimeUtils.getContainerType( this.sourceBufferTypes_.get(contentType)) == 'mp4'; if (!isMp4) { return segment; } const isInitSegment = reference === null; const encryptionExpected = this.expectedEncryption_.get(contentType); const keySystem = this.playerInterface_.getKeySystem(); let isEncrypted = false; if (reference && reference.initSegmentReference) { isEncrypted = reference.initSegmentReference.encrypted; } const uri = reference ? reference.getUris()[0] : null; const device = shaka.device.DeviceFactory.getDevice(); if (this.config_.correctEc3Enca && isInitSegment && contentType === ContentType.AUDIO) { segment = shaka.media.ContentWorkarounds.correctEnca(segment); } if (isInitSegment && device.requiresDvvcWorkaround(stream)) { segment = shaka.media.ContentWorkarounds.freeDvvcBox(segment); } // If: // 1. the configuration tells to insert fake encryption, // 2. and this is an init segment or media segment, // 3. and encryption is expected, // 4. and the platform requires encryption in all init or media segments // of current content type, // then insert fake encryption metadata for init segments that lack it. // The MP4 requirement is because we can currently only do this // transformation on MP4 containers. // See: https://github.com/shaka-project/shaka-player/issues/2759 if (this.config_.insertFakeEncryptionInInit && encryptionExpected && device.requiresEncryptionInfoInAllInitSegments(keySystem, contentType)) { if (isInitSegment) { shaka.log.debug('Forcing fake encryption information in init segment.'); segment = shaka.media.ContentWorkarounds.fakeEncryption( stream, segment, uri, keySystem); } else if (!isEncrypted && device.requiresTfhdFix(contentType)) { shaka.log.debug( 'Forcing fake encryption information in media segment.'); segment = shaka.media.ContentWorkarounds.fakeMediaEncryption(segment); } } if (isInitSegment && device.requiresEC3InitSegments()) { shaka.log.debug('Forcing fake EC-3 information in init segment.'); segment = shaka.media.ContentWorkarounds.fakeEC3(segment); } // Some I-frame only playlists clip the media segment byte range without // rewriting the moof/mdat, leaving the fragment declaring more samples than // it actually contains. Repair those before appending. if (!isInitSegment && stream.isIframe && this.config_.repairIFrames) { segment = shaka.media.ContentWorkarounds.fixBrokenIframes(segment); } return segment; } /** * Prepare the SourceBuffer to parse a potentially new type or codec. * * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @param {string} mimeType * @param {?shaka.extern.Transmuxer} transmuxer * @private */ change_(contentType, mimeType, transmuxer) { const ContentType = shaka.util.ManifestParserUtils.ContentType; if (contentType === ContentType.TEXT) { shaka.log.debug(`Change not supported for ${contentType}`); return; } const sourceBuffer = this.sourceBufferTypes_.get(contentType); shaka.log.debug( `Change Type: ${sourceBuffer} -> ${mimeType}`); const device = shaka.device.DeviceFactory.getDevice(); const keySystem = this.playerInterface_.getKeySystem(); if (device.supportsSmoothCodecSwitching(keySystem)) { if (this.transmuxers_.has(contentType)) { this.transmuxers_.get(contentType).destroy(); this.transmuxers_.delete(contentType); } if (transmuxer) { this.transmuxers_.set(contentType, transmuxer); } const type = this.addExtraFeaturesToMimeType_(mimeType); this.sourceBuffers_.get(contentType).changeType(type); this.sourceBufferTypes_.set(contentType, mimeType); } else { shaka.log.debug('Change Type not supported'); } // Fake an 'updateend' event to resolve the operation. this.onUpdateEnd_(contentType); } /** * Enqueue an operation to prepare the SourceBuffer to parse a potentially new * type or codec. * * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @param {string} mimeType * @param {?shaka.extern.Transmuxer} transmuxer * @return {!Promise} */ changeType(contentType, mimeType, transmuxer) { return this.enqueueOperation_( contentType, () => this.change_(contentType, mimeType, transmuxer), null); } /** * Resets the MediaSource and re-adds source buffers due to codec mismatch * * @param {!Map} streamsByType * @private */ async reset_(streamsByType) { if (this.reloadingMediaSource_ || this.usingRemotePlayback_) { return; } const ContentType = shaka.util.ManifestParserUtils.ContentType; this.reloadingMediaSource_ = true; this.needSplitMuxedContent_ = false; const currentTime = this.video_.currentTime; // When codec switching if the user is currently paused we don't want // to trigger a play when switching codec. // Playing can also end up in a paused state after a codec switch // so we need to remember the current states. const previousAutoPlayState = this.video_.autoplay; if (!this.video_.paused) { this.playAfterReset_ = true; } if (this.playbackHasBegun_) { // Only set autoplay to false if the video playback has already begun. // When a codec switch happens before playback has begun this can cause // autoplay not to work as expected. this.video_.autoplay = false; } try { this.eventManager_.removeAll(); for (const transmuxer of this.transmuxers_.values()) { transmuxer.destroy(); } for (const clearKeyDecryptor of this.clearKeyDecryptors_.values()) { clearKeyDecryptor.destroy(); } for (const sourceBuffer of this.sourceBuffers_.values()) { try { this.mediaSource_.removeSourceBuffer(sourceBuffer); } catch (e) { shaka.log.debug('Exception on removeSourceBuffer', e); } } this.transmuxers_.clear(); this.sourceBuffers_.clear(); this.requestedTimestampOffsets_.clear(); this.clearKeyDecryptors_.clear(); const previousDuration = this.mediaSource_.duration; this.mediaSourceOpen_ = Promise.withResolvers(); this.mediaSource_ = this.createMediaSource(this.mediaSourceOpen_); await this.mediaSourceOpen_.promise; if (!isNaN(previousDuration) && previousDuration) { this.mediaSource_.duration = previousDuration; } else if (!isNaN(this.lastDuration_) && this.lastDuration_) { this.mediaSource_.duration = this.lastDuration_; } const sourceBufferAdded = Promise.withResolvers(); const sourceBuffers = /** @type {EventTarget} */(this.mediaSource_.sourceBuffers); const totalOfBuffers = streamsByType.size; let numberOfSourceBufferAdded = 0; const onSourceBufferAdded = () => { numberOfSourceBufferAdded++; if (numberOfSourceBufferAdded === totalOfBuffers) { sourceBufferAdded.resolve(); this.eventManager_.unlisten(sourceBuffers, 'addsourcebuffer', onSourceBufferAdded); } }; this.eventManager_.listen(sourceBuffers, 'addsourcebuffer', onSourceBufferAdded); for (const contentType of streamsByType.keys()) { const stream = streamsByType.get(contentType); this.initSourceBuffer_(contentType, stream, stream.codecs); } const audio = streamsByType.get(ContentType.AUDIO); if (audio && audio.isAudioMuxedInVideo) { this.needSplitMuxedContent_ = true; } if (this.needSplitMuxedContent_ && !this.queues_.has(ContentType.AUDIO)) { this.queues_.set(ContentType.AUDIO, []); } await sourceBufferAdded.promise; } finally { this.reloadingMediaSource_ = false; this.destroyer_.ensureNotDestroyed(); this.eventManager_.listenOnce(this.video_, shaka.util.MediaElementEvent.LOADED_METADATA, () => { // Don't use ensureNotDestroyed() from this event listener, because // that results in an uncaught exception. Instead, just check the // flag. if (this.destroyer_.destroyed()) { return; } // Fake a seek to catchup the playhead. this.video_.currentTime = currentTime; }); this.eventManager_.listenOnce(this.video_, shaka.util.MediaElementEvent.CAN_PLAY_THROUGH, () => { // Don't use ensureNotDestroyed() from this event listener, because // that results in an uncaught exception. Instead, just check the // flag. if (this.destroyer_.destroyed()) { return; } this.video_.autoplay = previousAutoPlayState; if (this.playAfterReset_) { this.playAfterReset_ = false; this.video_.play(); } }); } } /** * Resets the Media Source * @param {!Map} streamsByType * @return {!Promise} */ reset(streamsByType) { return this.enqueueBlockingOperation_( () => this.reset_(streamsByType)); } /** * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @param {string} mimeType * @param {string} codecs * @return {{ * transmuxer: ?shaka.extern.Transmuxer, * transmuxerMuxed: boolean, * basicType: string, * codec: string, * mimeType: string, * }} * @private */ getRealInfo_(contentType, mimeType, codecs) { const ContentType = shaka.util.ManifestParserUtils.ContentType; const MimeUtils = shaka.util.MimeUtils; if (mimeType == 'video/mp4') { codecs = MimeUtils.getCorrectCodecs(codecs, mimeType); } /** @type {?shaka.extern.Transmuxer} */ let transmuxer; let transmuxerMuxed = false; const audioCodec = shaka.util.ManifestParserUtils.guessCodecsSafe( ContentType.AUDIO, (codecs || '').split(',')); const videoCodec = shaka.util.ManifestParserUtils.guessCodecsSafe( ContentType.VIDEO, (codecs || '').split(',')); let codec = videoCodec; if (contentType == ContentType.AUDIO) { codec = audioCodec; } if (!codec) { codec = codecs; } let newMimeType = shaka.util.MimeUtils.getFullType(mimeType, codec); const currentBasicType = MimeUtils.getBasicType( this.sourceBufferTypes_.get(contentType)); let needTransmux = this.config_.forceTransmux; if (!shaka.media.Capabilities.isTypeSupported(newMimeType) || (!this.sequenceMode_ && shaka.util.MimeUtils.RAW_FORMATS.includes(newMimeType))) { needTransmux = true; } else if (!needTransmux && mimeType != currentBasicType) { const device = shaka.device.DeviceFactory.getDevice(); needTransmux = !device.supportsContainerChangeType() && shaka.util.MimeUtils.RAW_FORMATS.includes(mimeType); } const TransmuxerEngine = shaka.transmuxer.TransmuxerEngine; if (needTransmux) { const newMimeTypeWithAllCodecs = shaka.util.MimeUtils.getFullTypeWithAllCodecs(mimeType, codec); const transmuxerPlugin = TransmuxerEngine.findTransmuxer(newMimeTypeWithAllCodecs); if (transmuxerPlugin) { transmuxer = new shaka.transmuxer.TransmuxerProxy( transmuxerPlugin(), this.config_.transmuxWorkerUrl); if (audioCodec && videoCodec) { transmuxerMuxed = true; } newMimeType = transmuxer.convertCodecs(contentType, newMimeTypeWithAllCodecs); } } if (!transmuxer && contentType === ContentType.VIDEO && audioCodec) { newMimeType = shaka.util.MimeUtils.getFullType(mimeType, codecs); } const newCodec = MimeUtils.getNormalizedCodec( MimeUtils.getCodecs(newMimeType)); const newBasicType = MimeUtils.getBasicType(newMimeType); return { transmuxer, transmuxerMuxed, basicType: newBasicType, codec: newCodec, mimeType: newMimeType, }; } /** * Codec switch if necessary, this will not resolve until the codec * switch is over. * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @param {string} mimeType * @param {string} codecs * @param {!Map} streamsByType * @return {{ * type: string, * newMimeType: string, * transmuxer: ?shaka.extern.Transmuxer, * }} * @private */ getInfoAboutResetOrChangeType_(contentType, mimeType, codecs, streamsByType) { const ContentType = shaka.util.ManifestParserUtils.ContentType; if (contentType == ContentType.TEXT) { return { type: shaka.media.MediaSourceEngine.ResetMode_.NONE, newMimeType: mimeType, transmuxer: null, }; } const MimeUtils = shaka.util.MimeUtils; const currentCodec = MimeUtils.getNormalizedCodec( MimeUtils.getCodecs(this.sourceBufferTypes_.get(contentType))); const currentBasicType = MimeUtils.getBasicType( this.sourceBufferTypes_.get(contentType)); const realInfo = this.getRealInfo_(contentType, mimeType, codecs); const transmuxer = realInfo.transmuxer; const transmuxerMuxed = realInfo.transmuxerMuxed; const newBasicType = realInfo.basicType; const newCodec = realInfo.codec; const newMimeType = realInfo.mimeType; let muxedContentCheck = true; if (transmuxerMuxed && this.sourceBufferTypes_.has(ContentType.AUDIO)) { const muxedRealInfo = this.getRealInfo_(ContentType.AUDIO, mimeType, codecs); const muxedCurrentCodec = MimeUtils.getNormalizedCodec( MimeUtils.getCodecs(this.sourceBufferTypes_.get(ContentType.AUDIO))); const muxedCurrentBasicType = MimeUtils.getBasicType( this.sourceBufferTypes_.get(ContentType.AUDIO)); muxedContentCheck = muxedCurrentCodec == muxedRealInfo.codec && muxedCurrentBasicType == muxedRealInfo.basicType; if (muxedRealInfo.transmuxer) { muxedRealInfo.transmuxer.destroy(); } } // Current/new codecs base and basic type match then no need to switch if (currentCodec === newCodec && currentBasicType === newBasicType && muxedContentCheck) { return { type: shaka.media.MediaSourceEngine.ResetMode_.NONE, newMimeType, transmuxer, }; } let allowChangeType = true; if ((this.needSplitMuxedContent_ && !streamsByType.has(ContentType.AUDIO)) || (transmuxerMuxed && transmuxer && !this.transmuxers_.has(contentType))) { allowChangeType = false; } const device = shaka.device.DeviceFactory.getDevice(); const keySystem = this.playerInterface_.getKeySystem(); if (allowChangeType && this.config_.codecSwitchingStrategy === shaka.config.CodecSwitchingStrategy.SMOOTH && device.supportsSmoothCodecSwitching(keySystem)) { return { type: shaka.media.MediaSourceEngine.ResetMode_.CHANGE_TYPE, newMimeType, transmuxer, }; } else { if (transmuxer) { transmuxer.destroy(); } return { type: shaka.media.MediaSourceEngine.ResetMode_.RESET, newMimeType, transmuxer: null, }; } } /** * Codec switch if necessary, this will not resolve until the codec * switch is over. * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @param {string} mimeType * @param {string} codecs * @param {!Map} streamsByType * @return {!Promise} true if there was a codec switch, * false otherwise. * @private */ async codecSwitchIfNecessary_(contentType, mimeType, codecs, streamsByType) { const ContentType = shaka.util.ManifestParserUtils.ContentType; const {type, transmuxer, newMimeType} = this.getInfoAboutResetOrChangeType_( contentType, mimeType, codecs, streamsByType); const newAudioStream = streamsByType.get(ContentType.AUDIO); if (newAudioStream) { this.needSplitMuxedContent_ = newAudioStream.isAudioMuxedInVideo; } if (type == shaka.media.MediaSourceEngine.ResetMode_.NONE) { if (this.transmuxers_.has(contentType) && !transmuxer) { this.transmuxers_.get(contentType).destroy(); this.transmuxers_.delete(contentType); } else if (!this.transmuxers_.has(contentType) && transmuxer) { this.transmuxers_.set(contentType, transmuxer); } else if (transmuxer) { // Compare if the transmuxer is different if (this.transmuxers_.has(contentType) && this.transmuxers_.get(contentType).transmux !== transmuxer.transmux) { this.transmuxers_.get(contentType).destroy(); this.transmuxers_.set(contentType, transmuxer); } else { transmuxer.destroy(); } } return false; } if (type == shaka.media.MediaSourceEngine.ResetMode_.CHANGE_TYPE) { await this.changeType(contentType, newMimeType, transmuxer); } else if (type == shaka.media.MediaSourceEngine.ResetMode_.RESET) { if (transmuxer) { transmuxer.destroy(); } await this.reset(streamsByType); } return true; } /** * Returns true if it's necessary reset the media source to load the * new stream. * * @param {shaka.util.ManifestParserUtils.ContentType} contentType * @param {string} mimeType * @param {string} codecs * @return {boolean} */ isResetMediaSourceNecessary(contentType, mimeType, codecs, streamsByType) { const info = this.getInfoAboutResetOrChangeType_( contentType, mimeType, codecs, streamsByType); if (info.transmuxer) { info.transmuxer.destroy(); } return info.type == shaka.media.MediaSourceEngine.ResetMode_.RESET; } /** * Update LCEVC Decoder object when ready for LCEVC Decode. * @param {?shaka.lcevc.Dec} lcevcDec */ updateLcevcDec(lcevcDec) { this.lcevcDec_ = lcevcDec; } /** * @param {string} mimeType * @return {string} * @private */ addExtraFeaturesToMimeType_(mimeType) { const extraFeatures = this.config_.addExtraFeaturesToSourceBuffer(mimeType); const extendedType = mimeType + extraFeatures; shaka.log.debug('Using full mime type', extendedType); return extendedType; } /** * @return {boolean} * @private */ canPerformOperations_() { if (!this.mediaSource_ || this.destroyer_.destroyed() || this.video_.error || this.usingRemotePlayback_ || this.reloadingMediaSource_ || this.closed()) { return false; } return true; } }; /** * Internal reference to window.URL.createObjectURL function to avoid * compatibility issues with other libraries and frameworks such as React * Native. For use in unit tests only, not meant for external use. * * @type {function(?):string} */ shaka.media.MediaSourceEngine.createObjectURL = window.URL.createObjectURL; /** * @typedef {{ * mediaTimestamp: ?number, * }} * * @property {?number} mediaTimestamp * The segment start timestamp parsed from the media segment, in seconds, or * null if Shaka could not parse one. */ shaka.media.MediaSourceEngine.AppendBufferInfo; /** * @typedef {{ * start: function(), * p: !Promise.PromiseWithResolvers, * uri: ?string, * }} * * @summary An operation in queue. * @property {function()} start * The function which starts the operation. * @property {!Promise.PromiseWithResolvers} p * The Promise with resolvers which is associated with this operation. * @property {?string} uri * A segment URI (if any) associated with this operation. */ shaka.media.MediaSourceEngine.Operation; /** * @enum {string} * @private */ shaka.media.MediaSourceEngine.SourceBufferMode_ = { SEQUENCE: 'sequence', SEGMENTS: 'segments', }; /** * @enum {string} * @private */ shaka.media.MediaSourceEngine.ResetMode_ = { NONE: 'none', RESET: 'reset', CHANGE_TYPE: 'changeType', }; /** * @typedef {{ * getKeySystem: function():string, * onMetadata: function(!Array, number, ?number), * onEmsg: function(!shaka.extern.EmsgInfo), * onEvent: function(!Event), * onManifestUpdate: function(), * getDrmInfo: function():?shaka.extern.DrmInfo, * }} * * @summary Player interface * @property {function():string} getKeySystem * Gets currently used key system or empty if not used. * @property {function( * !Array, number, ?number)} onMetadata * Callback to use when metadata arrives. * @property {function(!shaka.extern.EmsgInfo)} onEmsg * Callback to use when EMSG arrives. * @property {function(!Event)} onEvent * Called when an event occurs that should be sent to the app. * @property {function()} onManifestUpdate * Called when an embedded 'emsg' box should trigger a manifest update. * @property {function():?shaka.extern.DrmInfo} getDrmInfo * Gets currently used drmInfo or null if not used. */ shaka.media.MediaSourceEngine.PlayerInterface;