How to update nested structures in apollo cache using cache.modify

fragment commentFragment on Comment {
  id
  text
  galleryId
  commentUser {
    id
    firstName
    lastName
  }
}

fragment galleryFragment on Gallery {
  id
  path
  label
  comments {
    ...commentFragment
  }
  
}

We first retrieve the getGalleries using the following gql :

query getGalleries($filters: galleryFilterInput) {
  getGalleries(filters: $filters) {
    galleries {
      ...galleryFragment
    }
    cursor
    hasMore
  }
}

Now when the user enters a comment on a single gallery item we run the following mutation :

mutation addCommentMutation($input: addCommentInput!) {
  addComment(input: $input) {
    ...commentFragment
  }
}

Now, we were previously using refetchQueries to update the Galleries but we have now decided to use cache.modify however we are having problem with updating the galleries

  update: (cache, data: any) => {
        cache.modify({
          fields: {
            getGalleries(existing, { readField }) {
              const comment = data.data.addComment;
              const newEventRef = cache.writeFragment({
                fragment: commentFragment,
                data: comment,
                fragmentName: "commentFragment",
              });

              const index = existing.galleries.findIndex(
                aGallery => aGallery.id === comment.galleryId
              );

              if (index !== -1) {
                  const existingCommentRef = readField("comments", existing.galleries[index]) 
  as readonly Reference;
                  const newCommentsRefs = [...existingCommentRef, newRef];
                  cache.writeFragment({
                    id: "Gallery:" + readField("id", existing.galleries[index]),
                    fragment: gql`
                      fragment comments on Gallery {
                        comments {
                          ...commentFragment
                        }
                      }
                    `,
                    data: newCommentsRefs,
                  });

              }

              return existing;
            },
          },
        });
      },

I am unsure how I update the newCommentsRefs in that Gallery

update(cache, data) {

        const comment = data.data.addComment;
        cache.writeFragment({
          fragment: commentFragment,
          data: comment,
          fragmentName: "commentFragment",
        });

        const gallery: Gallery = cache.readFragment({
          id: `Gallery:${comment.galleryId}`,
          fragment: galleryFragment,
          fragmentName: 'galleryFragment'
        });
        if (gallery) {    
          const newComments = [...gallery.comments, comment]
          cache.writeFragment({
            id: `Gallery:${comment.galleryId}`,
            fragment: galleryFragment,
            fragmentName: 'galleryFragment',
            data: { ...gallery, comments: newComments }
          });
        }
      }