CSS transform-origin + scale triggers offset while zooming to cursor point

I'm trying to implement zoom in/out functionality for regular div elements (not canvas) by using transform origin and scale CSS properties. Everything works as expected, except that after changing the cursor's coordinates and then resizing, there is some offset. After that, zooming in and out works fine. The issue repeats after moving the cursor. The larger the zoom level, the greater the offset. I'm having difficulty identifying the pattern and adjusting the values I pass to transform-origin.

https://stackblitz.com/edit/web-platform-teguvc?file=script.js

const container = document.querySelector('.container');
const map = document.querySelector('.map');

const scaleStep = 0.2;
let scale = 1;

container.addEventListener('wheel', (event) => {
  event.preventDefault();

  if (!event.ctrlKey) {
    return;
  }

  event.deltaY < 0 ? (scale += scaleStep) : (scale -= scaleStep);

  const originX = container.scrollLeft + event.clientX;
  const originY = container.scrollTop + event.clientY;

  map.style.transformOrigin = `${originX}px ${originY}px`;
  map.style.transform = `scale(${scale})`;
});

body * {
  box-sizing: border-box;
}

.container {
  width: 90vw;
  height: 90vh;
  border: 2px solid blue;
  padding: 10px;
}

.map {
  width: 100%;
  height: 100%;
  border: 2px solid aqua;
}

#node-1 {
  position: absolute;
  left: 50px;
  top: 50px;
}

#node-2 {
  position: absolute;
  left: 150px;
  top: 150px;
}

#node-3 {
  position: absolute;
  left: 250px;
  top: 250px;
}

<div class="container">
  <div class="map">
    <div class="nodes">
      <div id="node-1">node-1</div>
      <div id="node-2">node-2</div>
      <div id="node-3">node-3</div>
    </div>
    <div class="connections"></div>
  </div>
</div>

My goal is to get rid of this cursor "jump".

After several attepmts I ended up with this solution:

const usePanAndZoom = (
  canvasRef: RefObject<HTMLElement>,
  graphRef: RefObject<HTMLElement>
): { handleMouseDown: (event: MouseEvent) => void } => {
  let scale = 1;
  const speed = 0.2;
  const offset = { x: 0, y: 0 };
  const target = { x: 0, y: 0 };
  const coordinates = { top: 0, left: 0, x: 0, y: 0 };

  const updateScale = debounce((scale: number) => setScale(scale), 100);

  const draw = (offsetX: number, offsetY: number, scale: number) => {
    requestAnimationFrame(() => {
      if (graphRef.current) {
        graphRef.current.style.transform = `translate(${offsetX}px, ${offsetY}px) scale(${scale})`;
      }
    });
  };

  const handleZoom = (event: WheelEvent) => {
    if (!event.ctrlKey) {
      return;
    }

    event.preventDefault();

    if (graphRef && graphRef.current && canvasRef.current) {
      target.x = (event.clientX - offset.x) / scale;
      target.y = (event.clientY - offset.y) / scale;

      scale += -1 * Math.max(-1, Math.min(1, event.deltaY)) * speed * scale;

      offset.x = -target.x * scale + event.clientX;
      offset.y = -target.y * scale + event.clientY;

      draw(offset.x, offset.y, scale);
      updateScale(scale);
    }
  };

  const handleMouseDown = useCallback(
    (event: MouseEvent) => {
      // it must be left mouse button
      if (event.button !== 0) {
        return;
      }

      if (canvasRef.current) {
        coordinates.x = event.clientX;
        coordinates.y = event.clientY;
        canvasRef.current.onmousemove = mouseMoveHandler;
        canvasRef.current.onmouseup = mouseUpHandler;
      }
    },
    [scale]
  );

  const mouseMoveHandler = (event: MouseEvent) => {
    if (canvasRef.current && graphRef.current) {
      const dx = offset.x + event.clientX - coordinates.x;
      const dy = offset.y + event.clientY - coordinates.y;
      draw(dx, dy, scale);
    }
  };

  const mouseUpHandler = (event: MouseEvent) => {
    if (canvasRef.current) {
      canvasRef.current.onmousemove = null;
      canvasRef.current.onmouseup = null;

      offset.x += event.clientX - coordinates.x;
      offset.y += event.clientY - coordinates.y;
    }
  };

  useEffect(() => {
    if (graphRef.current && canvasRef.current) {
      graphRef.current.style.transformOrigin = `${canvasRef.current.scrollLeft}px ${canvasRef.current.scrollTop}px`;
    }
  }, [canvasRef.current, graphRef.current]);

  useEffect(() => {
    if (!canvasRef || !canvasRef.current) {
      return;
    }

    canvasRef.current.addEventListener('wheel', handleZoom, { passive: false });

    return () => {
      canvasRef.current?.removeEventListener('wheel', handleZoom);
    };
  }, [canvasRef]);

  return {
    handleMouseDown,
  };
};

Not sure about using requestAnimationFrame and if it does make sense to use it here, but anyway.