[SOLVED] Detect if face is in fixed area of video frame

Detect if face is in fixed area of video frame

I am working on a project where I need to take picture of the frame automatically from live video when the face of person who is in that video is in a fixed area of that video. I am using faceapi for face detection. I tried to use mouth, nose and eyes coordinate but it's not working because those coordinates are changing and I am founding same coordinate in different place and all persons face shape are not same. Here is my js code-

const video = document.getElementById('video')

Promise.all([
  faceapi.nets.tinyFaceDetector.loadFromUri('/models'),
  faceapi.nets.faceLandmark68Net.loadFromUri('/models'),
  faceapi.nets.faceRecognitionNet.loadFromUri('/models'),
  faceapi.nets.faceExpressionNet.loadFromUri('/models')
]).then(startVideo)

function startVideo() {
  navigator.getUserMedia(
    { video: {} },
    stream => video.srcObject = stream,
    err => console.error(err)
  )
}

video.addEventListener('play', () => {
  const canvas = faceapi.createCanvasFromMedia(video)
  document.body.append(canvas)
  const displaySize = { width: video.width, height: video.height }
  faceapi.matchDimensions(canvas, displaySize)
  setInterval(async () => {
    const detections = await faceapi.detectAllFaces(video, new faceapi.TinyFaceDetectorOptions()).withFaceLandmarks().withFaceExpressions()
    const count = await faceapi
      .detectAllFaces(video, new faceapi.TinyFaceDetectorOptions())
      .withFaceLandmarks()
    if(count.length>1){
      alert("There are multiple faces in the frame. Please keep only one face");
    }  
    else{
      const resizedDetections = faceapi.resizeResults(detections, displaySize)
      canvas.getContext('2d').clearRect(0, 0, canvas.width, canvas.height)
      faceapi.draw.drawDetections(canvas, resizedDetections)
      faceapi.draw.drawFaceLandmarks(canvas, resizedDetections)
      faceapi.draw.drawFaceExpressions(canvas, resizedDetections)
      const landmarks = await faceapi.detectFaceLandmarks(video)
      const mouth = landmarks.getMouth()
      const nose = landmarks.getNose()
      const leftEye = landmarks.getLeftEye();
      console.log(mouth[0]._x, mouth[0]._y);
      console.log(nose[0]._x, nose[0]._y)    
      console.log(leftEye[0]._x, leftEye[0]._y)
      if((((mouth[0]._x<=330) && (mouth[0]._x>=280)) && ((mouth[0]._y<=310) && (mouth[0]._y>=260))) &&
      (((nose[0]._x<=320) && (nose[0]._x>=270)) && ((nose[0]._y<=220) && (nose[0]._y>=160))) &&
      (((leftEye[0]._x<=270) && (leftEye[0]._x>=210)) && ((leftEye[0]._y<=240) && (leftEye[0]._y>=180))))
      {
          alert("Click for perfect photo");        
      }
      else{
        //alert("Image not ready");
      }
    }    
  }, 100)
})

And this is my html code:

<!DOCTYPE html>
<html lang="en">
<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <meta http-equiv="X-UA-Compatible" content="ie=edge">
  <title>Document</title>
  <script defer src="face-api.min.js"></script>
  <script defer src="script.js"></script>
  <style>
    body {
      margin: 0;
      padding: 0;
      width: 100vw;
      height: 100vh;
      display: flex;
      justify-content: center;
      align-items: center;
    }

    canvas {
      position: absolute;
    }

    #box {
      width: 450px;
      height: 325px;
      position: absolute;
      background-image: url("paper.png");
      background-repeat: no-repeat;
      background-size: 450px 325px;
    }
  </style>
</head>
<body>
  <div id="box"></div>
  <video id="video" width="500" height="375" autoplay muted></video></body>
  <br/>
</body>
</html>

I put the image on the video frame because I want to click image automatically when face is in that area.

Solution

You can use detection box height, width and co-ordinates to calculate the face position and ratio of the face in the frame.

const frame_height = resizedDetections[0].detection._box._height
const frame_width = resizedDetections[0].detection._box._width
const frame_x = resizedDetections[0].detection._box._x
const frame_y = resizedDetections[0].detection._box._y