FabricJS selection handling multiple objects

I am struggling with handling the selection of multiple objects. The desired behaviour would be that each object that is clicked will be added to the current selection. Similar to holding shift-key, but also selections using the drag-options should be added to the existing selection. The current behaviour of fabricjs is creating a new selection even when pressing shift-key. In addition the selection should not be cleared when clicking a blank space on the canvas. Deselecting objects should only be possible when clicking a single object which is part of the selection (when dragging selected objects should stay selected). Or by clicking an additional button to clear the full selection (with additional user confirmation).

I tried different setups using "selection:created" and "selection:updated" but this either messed up the selection or resulted in an endless loop because modifying the selection inside the update also triggers the update again.

canvas.on("selection:updated", (event) => {
  event.selected.forEach((fabImg) => {
        if (!this.selectedImages.includes(fabImg)) {
          this.selectedImages.push(fabImg);
        }
  });
    var groupSelection = new fabric.ActiveSelection(this.selectedImages);
    canvas.setActiveObject(groupSelection);
});

Preventing the clear when clicking on the blank canvas was solved by:

var selection = [];
canvas.on("before:selection:cleared", (selected) => {
  selection = this.canvas.getActiveObjects();
});
canvas.on("selection:cleared", (event) => {
  var groupSelection = new fabric.ActiveSelection(selection);
  canvas.setActiveObject(groupSelection);
});

Just in case someone else is interested, I ended up changing 3 functions in the fabricjs code to achieve the desired behaviour:

canvas.class.js:

_shouldClearSelection: function (e, target) {
    var activeObjects = this.getActiveObjects(),
      activeObject = this._activeObject;

    return (
      (target &&
        activeObject &&
        activeObjects.length > 1 &&
        activeObjects.indexOf(target) === -1 &&
        activeObject !== target &&
        !this._isSelectionKeyPressed(e)) ||
      (target && !target.evented) ||
      (target &&
        !target.selectable &&
        activeObject &&
        activeObject !== target)
    );
  }

just removed the check if an object was clicked, to stop deselecting when clicking on blank space.

_isSelectionKeyPressed: function (e) {
    var selectionKeyPressed = false;

    if (this.selectionKey == "always") {
      return true;
    }

    if (
      Object.prototype.toString.call(this.selectionKey) === "[object Array]"
    ) {
      selectionKeyPressed = !!this.selectionKey.find(function (key) {
        return e[key] === true;
      });
    } else {
      selectionKeyPressed = e[this.selectionKey];
    }

    return selectionKeyPressed;
  }

just adding a "dummy" key called "always" to pretend always holding the shift-key. In canvas definition just add this key:

this.canvas = new fabric.Canvas("c", {
  hoverCursor: "hand",
  selection: true,
  backgroundColor: "#F0F8FF",
  selectionBorderColor: "blue",
  defaultCursor: "hand",
  selectionKey: "always",
});

And in canvas_grouping.mixin.js:

_groupSelectedObjects: function (e) {
    var group = this._collectObjects(e),
      aGroup;

    var previousSelection = this._activeObject;
    if (previousSelection) {
      if (previousSelection.type === "activeSelection") {
        var currentActiveObjects = previousSelection._objects.slice(0);
        group.forEach((obj) => {
          if (!previousSelection.contains(obj)) {
            previousSelection.addWithUpdate(obj);
          }
        });
        this._fireSelectionEvents(currentActiveObjects, e);
      } else {
        aGroup = new fabric.ActiveSelection(group.reverse(), {
          canvas: this,
        });
        this.setActiveObject(aGroup, e);
        var objects = this._activeObject._objects.slice(0);
        this._activeObject.addWithUpdate(previousSelection);
        this._fireSelectionEvents(objects, e);
      }
    } else {
      // do not create group for 1 element only
      if (group.length === 1 && !previousSelection) {
        this.setActiveObject(group[0], e);
      } else if (group.length > 1) {
        aGroup = new fabric.ActiveSelection(group.reverse(), {
          canvas: this,
        });
        this.setActiveObject(aGroup, e);
      }
    }
  }

This will extend existing groups on drag-select instead of overwriting the existing selection.