Flutter ListView.separated frame drop when loading huge list

I'm trying to display a lazy list in flutter for windows. The list has approximately 2300 elements in it. The list is within a FutureBuilder, whose future is to fetch the 2300 elements from Hive database. Each element in the list is a MaterialButton with some properties. Im not getting smooth scrolling when scrolled fast. Some frames are being dropped. I have tried cacheextend and setting automatickeepalives to true. Still having the same problem. When ItemExtend is set to a large number(say 40), the scrollView works fine without frame drop. In release mode, it has better performance, but still some frames are being dropped. What is the best solution available to this problem?

//this Rawscrollbar is returned if the future is have some data
RawScrollbar(
                    isAlwaysShown: true,
                    controller: scrollControllerMLC,
                    thickness: context.percentWidth * .8,
                    radius: Radius.zero,
                    thumbColor:
                        SearchLeftContainerColors.headPoolListThumbColor,
                    child: ListView.separated(
                      padding: EdgeInsets.fromLTRB(
                          context.percentWidth * .5,
                          context.percentHeight * 0,
                          context.percentWidth * 1,
                          context.percentHeight * 1),
                      itemCount: lengthOfBoxes,
                      controller: scrollControllerMLC,
                      // addAutomaticKeepAlives: true,
                      // physics: NeverScrollableScrollPhysics(),
                      itemBuilder: (context, int index) {
                        return ListButtonMEDLC(data[index], index);
                      },
                      separatorBuilder: (BuildContext context, int index) {
                        return Divider(
                          color: SearchLeftContainerColors
                              .headsPoolSeparatorColour,
                          height: 1,
                        );
                      },
                    ));




class ListButtonMEDLC extends StatelessWidget {
  final String text;
  final int index;
  ListButtonMEDLC(this.text, this.index);
  @override
  Widget build(BuildContext context) {
    return MaterialButton(
      color:
          (context.watch<ListButtonColorChangerMEDLC>().isSelectedList[index])
              ? SearchLeftContainerColors.headPoolListSelectedColor
              : SearchLeftContainerColors.headPoolListColor,
      hoverColor:
          (context.watch<ListButtonColorChangerMEDLC>().isSelectedList[index])
              ? SearchLeftContainerColors.headPoolListSelectedColor
              : SearchLeftContainerColors.headPoolListHoverColor,
      highlightColor: SearchLeftContainerColors.headPoolListHighLightedColor,
      child: Align(
        alignment: Alignment.centerLeft,
        child: Text(
          text,
          style: TextStyle(
              fontSize: context.percentWidth * 1.1,
              color: (context
                      .watch<ListButtonColorChangerMEDLC>()
                      .isSelectedList[index])
                  ? Colors.white
                  : Colors.black),
        ),
      ),
      onPressed: () {
        context.read<ListButtonColorChangerMEDLC>().changeIsSelectedList(index);
      },
    );
  }
}

//this it the future of the future builder;
loadDrugBox() async {
  Map boxes = await DB.boxes.getBoxAsMap("drugs");
  lengthOfBoxes = boxes.length;
  return boxes;
}

//Provider
class ListButtonColorChangerMEDLC extends ChangeNotifier {
  List<bool> isSelectedList = List.generate(lengthOfBoxes, (index) => false);

  changeIsSelectedList(int indexOfSelected) {
    for (int i = 0; i < lengthOfBoxes; i++) {
      if (i == indexOfSelected) {
        isSelectedList[i] = true;
      } else
        isSelectedList[i] = false;
    }
    notifyListeners();
  }
}

Yes. I solved this issue by replacing material button which was an expensive widget with gestureDetector.