I am trying to stream low latency video to the browser using the MediaSource API. For context, I am receiving live H.264 video over a WebRTC data channel (with custom reliable delivery protocol), muxing into a fragmented MP4 container in the browser and feeding this data to the MediaSource API.
In this scenario I will often not have enough data to feed to the API, either because: 1) no frames are sent from the server because nothing has visually changed, or 2) network hiccups cause data to be delayed.
In this "buffer underflow" case, I notice behaviour differences between Safari and Chrome:
Chrome will happily wait for more data to arrive and continue playing the video where it left off, maintaining a constant short buffer time between the HTMLVideoElement.currentTime
and the end of the SourceBuffer
.
Safari will pause the video element as soon as no more data is available and I must forcefully continue playing the video with HTMLVideoElement.play
, and also update the HTMLVideoElement.currentTime
back to near the end of the SourceBuffer
end time. This causes noticeable hitches in the video playback.
Is there any way to achieve the behaviour seen in Chrome, in Safari? I think this might be related to "low delay" mode which is present in the MediaSource implementation for Chrome (which I believe is turned on after inferring information from the fed video stream) - is there a way to trigger something similar in Safari?
See repro code below. In this case I have already muxed the video into fragmented MP4, and I am feeding it into the MediaSource API using setInterval
rather than over a WebRTC data channel, but the end result is the same:
<!DOCTYPE html>
<html>
<body>
<p id="text">Click anywhere to start streaming video</p>
<video autoplay playsinline id="video" width="800" height="450"></video>
<script>
/** @type HTMLVideoElement */
const videoElement = document.getElementById('video');
/** @type HTMLParagraphElement */
const textElement = document.getElementById('text');
async function start() {
let fileReadPointer = 0;
// Queue up video fragments to be appended to the SourceBuffer - we
// can't append them immediately as we must wait until the SourceBuffer
// has finished updating.
let fragmentQueue = [];
// File contents is fragmented MP4 (output from JMuxer) chunks.
// Each chunk is prefixed with a 4-byte chunk length.
const file = await fetch('video.dat');
const fileBytes = await file.arrayBuffer();
const CODEC_MIME_TYPE = 'video/mp4; codecs="avc1.424028"';
// Create MediaSource
const mediaSource = new MediaSource();
mediaSource.addEventListener('sourceopen', handleMediaSourceOpen);
// Assign MediaSource to <video> element
const mediaSourceUrl = URL.createObjectURL(mediaSource);
videoElement.src = mediaSourceUrl;
// Source buffer. We will create this later once the MediaSource has opened.
let sourceBuffer;
// Read next fragment from data file and append it to the fragment queue.
function receiveNextFragment() {
// Each fragment is stored as a 4-byte length header, followed by the
// actual bytes in the fragment.
if (fileReadPointer + 4 > fileBytes.byteLength) {
return;
}
// Read length from data file
const length = (new DataView(fileBytes)).getUint32(fileReadPointer);
fileReadPointer += 4;
// Read next bytes from data file
const data = new Uint8Array(fileBytes, fileReadPointer, length);
fileReadPointer += length;
fragmentQueue.push(data);
// Feed source buffer with the latest data if it's not already
// updating.
if (!sourceBuffer.updating) {
feedSourceBufferFromQueue();
}
}
function feedSourceBufferFromQueue() {
if (fragmentQueue.length === 0) {
return;
}
const nextData = fragmentQueue[0];
fragmentQueue = fragmentQueue.slice(1);
sourceBuffer.appendBuffer(nextData);
}
function handleMediaSourceOpen() {
// Create source buffer
sourceBuffer = mediaSource.addSourceBuffer(CODEC_MIME_TYPE);
sourceBuffer.addEventListener('updateend', ev => {
// If there is any more data waiting to be appended to the
// SourceBuffer, append it now.
feedSourceBufferFromQueue();
});
// Receive video fragments much more slowly than their duration, to
// cause buffer underflow.
//
// Note that the video will continue to play on Chrome as
// new frames are received, but will pause on Safari as soon as the
// buffer runs out.
setInterval(() => {
receiveNextFragment();
// Update browser text to reflect video paused state.
textElement.innerText = `Video paused: ${videoElement.paused}`;
}, 100);
}
};
// Start once the user clicks in the document (to avoid autoplay video
// issues)
let started = false;
function handleClick() {
if (started) return;
started = true;
start();
}
document.addEventListener('click', handleClick);
</script>
<style>
video {
border: 1px solid black;
}
</style>
</body>
</html>
I've found the answer, which is to explicitly set the duration to +Inf on the MediaSource
object once it has been opened:
mediaSource.addEventListener('sourceopen', () => {
mediaSource.duration = Number.POSITIVE_INFINITY;
});