How to stop esc from closing my fullscreen video in javascript?

Currently working on a video player that has a specific styling class added to it if the video is fullscreen. If user exits using esc and not fullscreen button, the styling stays put.

/* View in fullscreen */
function openFullscreen(elem) {
  if (elem.requestFullscreen) {
    elem.requestFullscreen();
  } else if (elem.webkitRequestFullscreen) { /* Safari */
    elem.webkitRequestFullscreen();
  } else if (elem.msRequestFullscreen) { /* IE11 */
    elem.msRequestFullscreen();
  }

  video.classList.add('video-fullscreen');
}

/* Close fullscreen */
function closeFullscreen() {
  if (document.exitFullscreen) {
    document.exitFullscreen();
  } else if (document.webkitExitFullscreen) { /* Safari */
    document.webkitExitFullscreen();
  } else if (document.msExitFullscreen) { /* IE11 */
    document.msExitFullscreen();
  }

  video.classList.remove('video-fullscreen');
}

let fullscreen = false;

//Toggle fullscreen
function toggleFullScreen(){
    !fullscreen ? openFullscreen(player) : closeFullscreen();
    fullscreen = !fullscreen;
}

I tried an event listener for the esc button so I could use it to change the styling back, the first esc press closes and second does my code, very annoying:

//detect Escape key press
window.addEventListener("keydown", (e) => {
    if(e.key === "Escape" && fullscreen){
        e.preventDefault();
        closeFullscreen();
        fullscreen = !fullscreen;
   }
});

You should not try to intercept the escape keypress and derive implications for your fullscreen mode from that. Instead, you should listen for fullscreenchange events - which also won't fire if requestFullscreen fails:

To learn when other code has toggled full-screen mode on and off, you should establish listeners for the fullscreenchange event on the Document. It's also important to listen for fullscreenchange to be aware when, for example, the user manually toggles full-screen mode, or when the user switches applications, causing your application to temporarily exit full-screen mode.

document.addEventListener('fullscreenchange', (event) => {
  video.classList.toggle('video-fullscreen', document.fullscreenElement != null);
});

However, you probably don't need this at all. Instead of the .video-fullscreen class, just use the :fullscreen selector in your CSS!