React custom scrolling with ArrowKeys

I would like to create a custom scrolling animation for my comic viewer. I have recreated the sample code in a codesandbox: https://codesandbox.io/s/hardcore-microservice-kfe1q The problem I'm facing is, that the scrolling animation starts to "lag". And it isn't really possible to change the scroll speed and interval.
I'm trying to recreate a similar feeling like with this comic viewer: Example

Any help would be appreciated, I'm not even quite sure if the current example is the correct approach for this problem.

So here is one possible solution to solve this

function useKeyPress(targetKey) {
  // State for keeping track of whether key is pressed
  const [keyPressed, setKeyPressed] = useState(false);
  // If pressed key is our target key then set to true
  function downHandler(e) {
    if (e.key === targetKey) {
      setKeyPressed(true);
      e.preventDefault();
    }
  }
  // If released key is our target key then set to false
  const upHandler = (e) => {
    if (e.key === targetKey) {
      setKeyPressed(false);
    }
  };
  // Add event listeners
  useEffect(() => {
    window.addEventListener("keydown", downHandler);
    window.addEventListener("keyup", upHandler);
    // Remove event listeners on cleanup
    return () => {
      window.removeEventListener("keydown", downHandler);
      window.removeEventListener("keyup", upHandler);
    };
  }, []); // Empty array ensures that effect is only run on mount and unmount
  return keyPressed;
}

export function useUI() {
  const scrollDistance = 15;
  const ArrowUp = useKeyPress("ArrowUp");
  const ArrowDown = useKeyPress("ArrowDown");

  const k = useKeyPress("k"); // up
  const j = useKeyPress("j"); // down

  const w = useKeyPress("w"); // up
  const s = useKeyPress("s"); // down

  useEffect(() => {
    if (!(ArrowUp || ArrowDown || k || j || w || s)) {
      return null;
    }
    const scrollDirection = ArrowUp || k || w ? 1 : -1;
    let animationFrame = 0;
    let callback = () => {
      const pos = window.pageYOffset;
      window.scroll(0, pos - scrollDistance * scrollDirection);
      animationFrame = requestAnimationFrame(callback);
    };
    animationFrame = requestAnimationFrame(callback);

    return () => {
      cancelAnimationFrame(animationFrame);
    };
  }, [ArrowUp, ArrowDown, k, j, w, s]);

  return null;
}