getSelection add html, but prevent nesting of html

I have a content editable field, in which I can enter and html format som text (select text and click a button to add <span class="word"></span> around it).

I use the following function:

function highlightSelection() {       
  if (window.getSelection) {
  
    let sel = window.getSelection();

    

    if (sel.rangeCount > 0) {

      if(sel.anchorNode.parentElement.classList.value) {
      

        let range = sel.getRangeAt(0).cloneRange();
        let newParent =  document.createElement('span');
        newParent.classList.add("word");
        range.surroundContents(newParent);
        sel.removeAllRanges();
        sel.addRange(range);
      } else {
        console.log("Already in span!");
        // end span - start new.
      
      }
    }
}

}

But in the case where I have:

    Hello my 
<span class="word">name is</span> 
Benny

AND I select "is" and click my button I need to prevent the html from nesting, so instead of

    Hello my 
<span class="word">name <span class="word">is</span></span> Benny

I need:

    Hello my 
<span class="word">name</span> 
<span class="word">is</span> 
Benny

I try to check the parent class, to see if it is set, but how do I prevent nested html - close span tag at caret start position, add and at the end of caret position add so the html will not nest?

It should also take into account if there are elements after selection which are included in the span:

So:

Hello my 

    <span class="word">name is Benny</span>

selecting IS again and clicking my button gives:

Hello my 

    <span class="word">name</span> 
    <span class="word">is</span> 
    <span class="word">Benny</span>

Any help is appreciated!

Thanks in advance.

Solution

One way would be to do this in multiple pass.

First you wrap your content, almost blindly like you are currently doing.
Then, you check if in this content there were some .word content. If so, you extract its content inside the new wrapper you just created.
Then, you check if your new wrapper is itself in a .word container. If so, you get the content that was before the selection and wrap it in its own new wrapper. You do the same with the content after the selection.
At this stage we may have three .word containers inside the initial one. We thus have to extract the content of the initial one, and remove it. Our three wrappers are now independent.

function highlightSelection() {       
  if (window.getSelection) {
    const sel = window.getSelection();
    if (sel.rangeCount > 0) {
      const range = sel.getRangeAt(0).cloneRange();
      if (range.collapsed) { // nothing to do
        return;
      }
      // first pass, wrap (almost) carelessly
      wrapRangeInWordSpan(range);
      // second pass, find the nested .word
      // and wrap the content before and after it in their own spans
      const inner = document.querySelector(".word .word");
      if (!inner) {
        // there is a case I couldn't identify correctly
        // when selecting two .word start to end, where the empty spans stick around
        // we remove them here
        range.startContainer.parentNode.querySelectorAll(".word:empty")
          .forEach((node) => node.remove());
        return;
      }
      const parent = inner.closest(".word:not(:scope)");
      const extractingRange = document.createRange();
      // wrap the content before
      extractingRange.selectNode(parent);
      extractingRange.setEndBefore(inner);
      wrapRangeInWordSpan(extractingRange);
      // wrap the content after
      extractingRange.selectNode(parent);
      extractingRange.setStartAfter(inner);
      wrapRangeInWordSpan(extractingRange);

      // finally, extract all the contents from parent
      // to its own parent and remove it, now that it's empty
      moveContentBefore(parent)
    }
  }
}
document.querySelector("button").onclick = highlightSelection;

function wrapRangeInWordSpan(range) {
  if (
    !range.toString().length && // empty text content
    !range.cloneContents().querySelector("img") // and not an <img>
  ) {
    return; // empty range, do nothing (collapsed may not work)
  }
  const content = range.extractContents();
  const newParent = document.createElement('span');
  newParent.classList.add("word");
  newParent.appendChild(content); 
  range.insertNode(newParent);
  // if our content was wrapping .word spans,
  // move their content in the new parent
  // and remove them now that they're empty
  newParent.querySelectorAll(".word").forEach(moveContentBefore);
}
function moveContentBefore(parent) {
  const iterator = document.createNodeIterator(parent);
  let currentNode;
  // walk through all nodes
  while ((currentNode = iterator.nextNode())) {
    // move them to the grand-parent
    parent.before(currentNode);
  }
  // remove the now empty parent
  parent.remove();
}

.word {
  display: inline-block;
  border: 1px solid red;
}
[contenteditable] {
  white-space: pre; /* leading spaces will get ignored once highlighted */
}

<button>highlight</button>
<div contenteditable
  >Hello my <span class="word">name is</span> Benny</div>

But beware, this is just a rough proof of concept, I didn't do any heavy testings and there may very well be odd cases where it will just fail (content-editable is a nightmare).
Also, this doesn't handle cases where one would copy-paste or drag & drop HTML content.