React + FabricJS: Custom Scrollbar - Flickering Scroll Thumbs on zoom in/out state update

I made a Custom Scrollbar for FabricJS (Since they don't provide one). Everything seems to be working fine. The thumbs of the scrollbar do change their height and width when the zoom state updates, problem is the thumbs flicker a lot when their height and width is updated. I tried debouncing and memoizing the updation logic but none seems to be working. A little help would be appreciated :)

Gif

Component:

import { useMemo, useRef, useState, useCallback, useEffect } from "react";
import { useAtomValue } from "jotai";
import { zoomAtom } from "../Atoms";
import useEditorService from "@/hooks/useEditorService";

const CustomScroll = () => {
  const zoom = useAtomValue(zoomAtom);
  const { activeEditor } = useEditorService();

  const [editorPos, setEditorPos] = useState({ x: 0, y: 0 });
  useEffect(() => {
    if (!activeEditor) return;
    setEditorPos((p) => ({
      ...p,
      x: activeEditor?.canvas.viewportTransform[4],
      y: activeEditor?.canvas.viewportTransform[5],
    }));
  }, [activeEditor?.canvas?.viewportTransform]);

  const [scrollThumbsPos, setScrollThumbsPos] = useState({
    top: 0,
    left: 0,
  });

  // =========================================================================================================

  const minFitZoom = useMemo(
    () => activeEditor?.dimensions?.height / activeEditor?.originalSize?.height,
    [activeEditor?.dimensions, activeEditor?.originalSize]
  );

  const boxYRef = useRef(null);

  const minHeight = 100;
  
  // manage height of the y thumb
  const scrollThumbHeight = useMemo(() => {
    return Math.max(
      ((5 - zoom) / 5) * activeEditor?.dimensions.height,
      minHeight
    );
  }, [zoom, activeEditor?.dimensions.height]);

  const customYScrollStyle = {
    display: minFitZoom < zoom ? "block" : "none",
  };

  // Handle Drags on Y axis
  const handleYDragStart = (e) => {
  };

  const handleYDrag = (e) => {
  };

  const handleYDragEnd = () => {
    
  };

  // =========================================================================================================

  const maxFitZoom = useMemo(
    () => activeEditor?.dimensions?.width / activeEditor?.originalSize?.width,
    [activeEditor]
  );

  const boxXRef = useRef(null);

  const minWidth = 100;

  // manage width of the x thumb
  const scrollThumbWidth = useMemo(() => {
    return Math.max(
      ((5 - zoom) / 5) * activeEditor?.dimensions.width,
      minWidth
    );
  }, [zoom, activeEditor?.dimensions.width]);

  const customXScrollStyle = {
    display: maxFitZoom < zoom ? "block" : "none",
  };

  // Handle Drags on X axis
  const handleXDragStart = (e) => {
  };

  const handleXDrag = (e) => {
  };

  const handleXDragEnd = () => {
  };

  // =========================================================================================================

  // Change Thumb Positions based on zoom
  activeEditor?.canvas.on("mouse:wheel", (opt) => {
    opt.e.preventDefault();
    const evt = opt.e;

    const pointer = activeEditor?.canvas.getPointer(evt);
    const editorHeight = activeEditor?.originalSize.height;
    const editorWidth = activeEditor?.originalSize.width;

    const newTopPercentage = (pointer.y / editorHeight) * 100;
    const scrollContainerHeight = boxYRef.current.parentElement.clientHeight;
    const newTop =
      (newTopPercentage / 100) * (scrollContainerHeight - scrollThumbHeight);
    const clampedTop = Math.min(
      Math.max(newTop, 0),
      scrollContainerHeight - scrollThumbHeight
    );

    const newLeftPercentage = (pointer.x / editorWidth) * 100;
    const scrollContainerWidth = boxXRef.current.parentElement.clientWidth;
    const newLeft =
      (newLeftPercentage / 100) * (scrollContainerWidth - scrollThumbWidth);
    const clampedLeft = Math.min(
      Math.max(newLeft, 0),
      scrollContainerWidth - scrollThumbWidth
    );

    setScrollThumbsPos((prevPos) => ({
      ...prevPos,
      top: clampedTop,
      left: clampedLeft,
    }));
  });

  return (
    <div>
      <div
        id="customYscroll"
        style={customYScrollStyle}
      >
        <div
          className="custom-y-scroll-thumb"
          style={{
            height: scrollThumbHeight,
            position: "relative",
            top: scrollThumbsPos.top,
          }}
          draggable="true"
          onDragStart={handleYDragStart}
          onDrag={handleYDrag}
          onDragEnd={handleYDragEnd}
          ref={boxYRef}
        ></div>
      </div>

      <div
        id="customXscroll"
        style={customXScrollStyle}
      >
        <div
          className="custom-x-scroll-thumb"
          style={{
            width: scrollThumbWidth,
            position: "relative",
            left: scrollThumbsPos.left,
          }}
          draggable="true"
          onDragStart={handleXDragStart}
          onDrag={handleXDrag}
          onDragEnd={handleXDragEnd}
          ref={boxXRef}
        ></div>
      </div>
    </div>
  );
};

export default CustomScroll;

Changed the useEffect to useLayoutEffect, and useState to useRef to avoid re-renders. Also turned off strict mode and it works fine.

  const scrollThumbHeight = useRef(null);
  useLayoutEffect(() => {
    const newThumbHeight = Math.max(
      ((5 - zoom) / 5) * activeEditor?.dimensions.height,
      minHeight
    );
    scrollThumbHeight.current = newThumbHeight;
  }, [zoom, activeEditor?.dimensions.height]);