ioscanvasvideorequestanimationframeglobalcompositeoperation

Mask a video overlay with globalCompositeOperation rendered in a canvas in IOS

This drives me crazy. Please help to understand the cause. I can't get a masked video to work in the canvas on iOS (Chrome and Safari). Tested on iPads and iPhones, also on many different real-devices with Sauce Labs.

I am using a transparency mask as .png (white and transparent pixels) to overlay a video with globalCompositeOperation. An android and windows devices everything is working as expected.

To rule out the cause of fabric.js, i created another test using native canvas functions.

The used .png mask The used .png mask

[Expected result] Mask is only working on android and desktop-browsers (all major browsers plus exotic ones) Mask is working on android

Not masked on all IOS devices Safari or Chrome Not masked on all IOS devices

Also: The masking with a still image instead of an image sequence (via requestAnimFrame) works perfectly. Of course, this is a common combination in many product configurators.

Changed anything to get a mimimal test case to work, without any success. Even a simple opacity value on the video sequence has no effect on ios and ipad-os devices. Tested with different video formats and tag settings. No errors found in web debugging. Anyway, the overlay video is simply rendered over the mask. What is happening here?

Searched many apple bug trackers but can't find anything about this specific issue. Every single standard for it self seems to work since years.

var canvas = new fabric.Canvas('plan', { selection: true});

canvas.objectCaching = false;

var img = "https://www.myselfie.fun/media/collection/templateselfie/frames/jungle_easy_01.png";

fabric.Image.fromURL(img, function(myImg) {
  
  myImg.crossOrigin="Anonymous";
  var img1 = myImg.set({
    originX: 'left',
    originY: 'top',
    left: 0,
    top: 0,
    scaleX: canvas.width / myImg.width,
    scaleY: canvas.height / myImg.height,
    selectable : false,
    evented: false,
  });

  img1.setCoords();
  canvas.backgroundColor = null;
  canvas.bringToFront(img1);
  canvas.add(img1);
  canvas.renderAll();

  const videoEl = document.getElementById('vid');
  videoEl.setAttribute('loop', 'loop');
 
  videoEl.setAttribute('autoplay', 'autoplay');
  videoEl.setAttribute("playsinline", true);
  videoEl.crossOrigin="Anonymous";
  videoEl.addEventListener('loadedmetadata', () => {
    const video = new fabric.Image(videoEl, {});
    video.set({
        globalCompositeOperation: "source-atop",
        originX: 'left',
        originY: 'top',
        left: 0,
        top: 0,
        scaleX: canvas.width / videoEl.width,
        scaleY: canvas.height / videoEl.height,
        lockUniScaling: true,
        centeredScaling: true,
        lockRotation: true,
        //opacity: 0.5
    });
    canvas.add(video);
    canvas.bringToFront(video);
    const render = () => {
      this.canvas.renderAll();
      this.request = fabric.util.requestAnimFrame(render);
    };
    fabric.util.requestAnimFrame(render);
  });

                  videoEl.setAttribute('src','https://www.myselfie.fun/media/collection/templateselfie/frames/test.mov');
  videoEl.load();

});

Made a test case in codepen (with fabric.js)

Second test case only with native canvas functions (Without fabric.js)

I would imagine that this is a timing issue. However, I haven't been able to find a working solution yet. I would like to know the exact cause. If there is a mistake, it must be reported. A workaround for the current implementation would of course also be very pleasant.

Update Thanks to @Kaidoo i was looking deeper again into a fabric.js workaround. I need a working version for IOS on all major browsers wird good performance and user interaction. My solution was to create a new FabricImage element outside the update loop and then to replace the fabric img.src inside the requestAnimFrame-render-loop with a fresh video.toDataURL() like so

videoEl.addEventListener('loadedmetadata', () => {
  const video = new fabric.Image(videoEl, {});
  // a bare canvas to workaround Safari's bug
  const videoRenderer = document.createElement("canvas");
  videoRenderer.width = canvas.width;
  videoRenderer.height = canvas.height;
  const ctx = videoRenderer.getContext("2d");
  const videoCanvas = new fabric.Image(videoRenderer, {});
  videoCanvas.set({
    globalCompositeOperation: "source-atop",
    originX: 'left',
    originY: 'top',
    left: 0,
    top: 0,
    lockScaling: true,
    centeredScaling: true,
    lockRotation: true
  })
  canvas.add(videoCanvas);
  canvas.bringToFront(videoCanvas);

  const render = () => {
    // render the current frame on our detached canvas
    ctx.drawImage(video.getElement(), 0, 0);

    this.canvas.renderAll();
    this.request = fabric.util.requestAnimFrame(render);
  };
  fabric.util.requestAnimFrame(render);
});

I made a new codepen with a performant working version for all major browsers also on IOS.

Solution

  • This is a Safari bug, I'll report it when I get time. Bbelow is an MCVE where this reproduces on Desktop Safari too.

    const btn = document.querySelector('button');
async function main() {
  btn.remove();
  const canvas = document.querySelector('canvas');
  const canvasContext = canvas.getContext('2d');

  const video = document.createElement('video');
  video.muted = true;
  video.autoplay = true;
  video.loop = true;
  video.playsinline = true;
  video.src = "https://upload.wikimedia.org/wikipedia/commons/transcoded/a/a4/BBH_gravitational_lensing_of_gw150914.webm/BBH_gravitational_lensing_of_gw150914.webm.180p.vp9.webm";
  await video.play();
  // draw the mask (no need to redraw it every frame)
  canvasContext.fillRect(80, 30, 100, 100);
  canvasContext.globalCompositeOperation = 'source-in';
  requestAnimationFrame(function frame() {
    canvasContext.drawImage(video, 0, 0, canvas.width, canvas.height);
    requestAnimationFrame(frame);
  });
}
btn.onclick = evt => main().catch(console.error);
    <button>Click to begin</button>
<canvas></canvas>

    You can workaround it by drawing an ImageBitmap taken from the <video>, but it's async, which is not ideal in an animation frame, and moreover, it triggers a new bug in Firefox (yeah! more reports to file...).

    const btn = document.querySelector('button');
async function main() {
  btn.remove();
  const canvas = document.querySelector('canvas');
  const canvasContext = canvas.getContext('2d');

  const video = document.createElement('video');
  video.muted = true;
  video.autoplay = true;
  video.loop = true;
  video.playsinline = true;
  video.src = "https://upload.wikimedia.org/wikipedia/commons/transcoded/a/a4/BBH_gravitational_lensing_of_gw150914.webm/BBH_gravitational_lensing_of_gw150914.webm.180p.vp9.webm";
  await video.play()

  // draw the mask (no need to redraw it every frame)
  canvasContext.fillRect(80, 30, 100, 100);
  canvasContext.globalCompositeOperation = 'source-in';
  requestAnimationFrame(async function frame() {
    const bmp = await createImageBitmap(video);
    canvasContext.drawImage(bmp, 0, 0, canvas.width, canvas.height);
    requestAnimationFrame(frame);
  });
}
btn.onclick = evt => main().catch(console.error);
    <button>Click to begin</button>
<canvas></canvas>

    So the best might be to instead use a secondary <canvas> on which you'll draw the <video> frame, and then draw that <canvas> on the visible one.

    const btn = document.querySelector('button');
async function main() {
  btn.remove();
  const canvas = document.querySelector('canvas');
  const canvasContext = canvas.getContext('2d');
  // create a detached canvas
  const videoCanvas = canvas.cloneNode();
  const videoContext = videoCanvas.getContext("2d");

  const video = document.createElement('video');
  video.muted = true;
  video.autoplay = true;
  video.loop = true;
  video.playsinline = true;
  video.src = "https://upload.wikimedia.org/wikipedia/commons/transcoded/a/a4/BBH_gravitational_lensing_of_gw150914.webm/BBH_gravitational_lensing_of_gw150914.webm.180p.vp9.webm";
  await video.play()

  // draw the mask (no need to redraw it every frame)
  canvasContext.fillRect(80, 30, 100, 100);
  canvasContext.globalCompositeOperation = 'source-in';
  requestAnimationFrame(function frame() {
    // draw on the detached canvas
    videoContext.drawImage(video, 0, 0, canvas.width, canvas.height);
    canvasContext.drawImage(videoCanvas, 0, 0);
    requestAnimationFrame(frame);
  });
}
btn.onclick = evt => main().catch(console.error);
    <button>Click to begin</button>
<canvas></canvas>

    As for the fabric.js version, it's not my forte, but it seems you can apply the same process, and create new FabricImage holding a reference to a <canvas> on which you will draw your video each frame (e.g through the .toCanvasElement() method of FabricObjects):

    var canvas = new fabric.Canvas('plan', { selection: true});

  canvas.objectCaching = false;

  var img = "https://www.myselfie.fun/media/collection/templateselfie/frames/jungle_easy_01.png";

  fabric.Image.fromURL(img, function(myImg) {

    myImg.crossOrigin="Anonymous";
    var img1 = myImg.set({
      originX: 'left',
      originY: 'top',
      left: 0,
      top: 0,
      scaleX: canvas.width / myImg.width,
      scaleY: canvas.height / myImg.height,
      selectable : false,
      evented: false,
    });

    img1.setCoords();
    canvas.backgroundColor = null;
    canvas.bringToFront(img1);
    canvas.add(img1);
    canvas.renderAll();

    const videoEl = document.getElementById('vid');
    videoEl.setAttribute('loop', 'loop');

    videoEl.setAttribute('autoplay', 'autoplay');
    videoEl.setAttribute("playsinline", true);
    videoEl.crossOrigin="Anonymous";
    videoEl.addEventListener('loadedmetadata', () => {
      const video = new fabric.Image(videoEl, {});
      video.set({
          scaleX: canvas.width / videoEl.width,
          scaleY: canvas.height / videoEl.height,
      });
      // a bare canvas to workaround Safari's bug
      const videoRenderer = document.createElement("canvas");
      videoRenderer.width = canvas.width;
      videoRenderer.height = canvas.height;
      const ctx = videoRenderer.getContext("2d");
      const videoCanvas = new fabric.Image(videoRenderer, {});
      videoCanvas.set({
        globalCompositeOperation: "source-atop",
        originX: 'left',
        originY: 'top',
        left: 0,
        top: 0,
        lockScaling: true,
        centeredScaling: true,
        lockRotation: true
      })
      canvas.add(videoCanvas);
      canvas.bringToFront(videoCanvas);

      const render = () => {
        // render the current frame on our detached canvas
        ctx.drawImage(video.toCanvasElement(), 0, 0);

        this.canvas.renderAll();
        this.request = fabric.util.requestAnimFrame(render);
      };
      fabric.util.requestAnimFrame(render);
    });

    videoEl.setAttribute('src', 'https://www.myselfie.fun/media/collection/templateselfie/frames/test.mov');
    videoEl.load();

  });
    canvas{
  border: solid 1px #000;
}
video {
  -webkit-transform-style: preserve-3d;
}
    <script src="https://cdnjs.cloudflare.com/ajax/libs/fabric.js/521/fabric.min.js"></script>
<div class="container">
  <div class="row">
    <div class="col-lg-12 col-md-12 col-xs-12">
      <canvas id="plan"  width="500" height="500"></canvas>
      <p><video 
            width="500" height="500" autoplay playsinline controls muted loop crossorigin="anonymous" id="vid" type="video/quicktime">
      </video></p>
    </div>
  </div>
</div>