Remove background of image using sobel edge detection

I have a bunch of images representing coins, some of which have a noisy background (e.g. letters or different background color). I'm trying to remove the background of each coin image to leave only the coin itself but I cannot get the cv2.findContours function from OpenCV to only detect the main contour of the coin, it erases some other parts as well or it leaves some extra noise from the background.

The following is the code that I'm using, the process I'm following is:

1. Read image as numpy array from bytes object.
2. Decode it as color image.
3. Convert to gray-scale image.
4. Add Gaussian Blur to remove noise.
5. Detect edges in the image applying a sobel filter edgedetect(). Here it computes the X and Y sobels and converts to threshold by applying Otsu thresholding.
6. Computes the mean from the image and zeroes any vaalue below it to remove noise.
7. Find significant contours (findSignificantContours().
8. Creates mask from contours, inverts and removes it to get background.
9. Set mask to 255 to remove the background in the original image.

import cv2
import numpy as np
from google.colab.patches import cv2_imshow

def edgedetect(channel):
    sobelX = cv2.Sobel(channel, cv2.CV_64F, 1, 0, ksize = 3, scale = 1)
    sobelY = cv2.Sobel(channel, cv2.CV_64F, 0, 1, ksize = 3, scale = 1)
    sobel = np.hypot(sobelX, sobelY)

    sobel = cv2.convertScaleAbs(sobel)
    sobel[sobel > 255] = 255 # Some values seem to go above 255. However RGB channels has to be within 0-255

    _, sobel_binary = cv2.threshold(sobel, 0, 255, cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)

    return cv2.bitwise_not(sobel_binary)

def findSignificantContours (img, edgeImg):

    print(f'edgeimg:')
    cv2_imshow(edgeImg)
    contours, hierarchy = cv2.findContours(edgeImg, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

    # Find level 1 contours
    level1 = []
    for i, tupl in enumerate(hierarchy[0]):
        # Each array is in format (Next, Prev, First child, Parent)
        # Filter the ones without parent
        if tupl[3] == -1:
            tupl = np.insert(tupl, 0, [i])
            level1.append(tupl)

    # From among them, find the contours with large surface area.
    significant = []
    tooSmall = edgeImg.size * 5 / 100 # If contour isn't covering 5% of total area of image then it probably is too small
    for tupl in level1:
        contour = contours[tupl[0]]

        area = cv2.contourArea(contour)
        if area > tooSmall:
            significant.append([contour, area])

            # Draw the contour on the original image
            cv2.drawContours(img, [contour], 0, (0, 255, 0), 2, cv2.LINE_8)

    significant.sort(key = lambda x: x[1])

    return [x[0] for x in significant]

def remove_background(bytes_data):
    # Read image.
    image = np.asarray(bytearray(bytes_data.read()), dtype = "uint8")
    img = cv2.imdecode(image, cv2.IMREAD_COLOR)

    print(f'Original:')
    cv2_imshow(img)

    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    print(f'Gray:')
    cv2_imshow(gray)

    blurred_gray = cv2.GaussianBlur(gray, (3, 3), 0) # Remove noise.
    print(f'Blurred Gray:')
    cv2_imshow(blurred_gray)

    edgeImg = np.max( np.array([edgedetect(blurred_gray[:, :])]), axis = 0)

    mean = np.mean(edgeImg)

    # Zero any value that is less than mean. This reduces a lot of noise.
    edgeImg[edgeImg <= mean] = 0
    edgeImg_8u = np.asarray(edgeImg, np.uint8)

    # Find contours.
    significant = findSignificantContours(img, edgeImg_8u)

    # Mask.
    mask = edgeImg.copy()
    mask[mask > 0] = 0
    cv2.fillPoly(mask, significant, 255)
    mask = np.logical_not(mask) # Invert mask to get the background.

    # Remove the background.
    img[mask] = 255;

    print(f'FINAL:')
    cv2_imshow(img)

    return img

if __name__ == '__main__':
    imgUrl = 'http://images.numismatics.org/archivesimages%2Farchive%2Fschaefer_clippings_output_383_06_od.jpg/2648,1051,473,453/full/0/default.jpg'
    obvPage = requests.get(imgUrl, stream = True, verify = False, headers = header)

    img_final = remove_background(obvPage.raw)

As representation, here is the original image, as you can see it has some letters written on the right side which is what I'm trying to remove. The rest of the images are similar although some have different background color not just white.

The following image is the image of the edges after performing the edgedetect() function using the sobels.

And the last one is the final image with the 'removed' background, sadly it still contains some of the letters there and I don't know what I'm doing wrong or how could I improve my code to achieve what I want. Could someone help me with this?

Here is an example processing chain. With final filtering of the contours found based on their roundness. Certainly not perfect, but maybe a little help.

import numpy as np
import cv2

# Load an color image in grayscale
image = cv2.imread('archivesimages_archive_schaefer_clippings_output_383_06_od.jpg',0)

# blur the image
bkz = 10
blurred = cv2.blur(image, (bkz, bkz), 0)

# thresholding
(T, thresh) = cv2.threshold(blurred, 0, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU)

# Morphological filters
kernel = np.ones((5, 5), np.uint8)
thresh = cv2.erode(thresh, kernel, iterations=1)
#thresh = cv2.dilate(thresh, kernel, iterations=1)

# find contours
contours, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

# filter contours
contour_list = []
for contour in contours:
    approx = cv2.approxPolyDP(contour,0.01*cv2.arcLength(contour,True),True)
    area = cv2.contourArea(contour)
    if ((len(approx) > 8) & (area > 50000)):
        contour_list.append(contour)
print(len(contours))
print(len(contour_list))

# draw contours
thresh = cv2.cvtColor(thresh,cv2.COLOR_GRAY2RGB)
cv2.drawContours(thresh, contour_list, -1, (255,0,0), 3)

image = cv2.cvtColor(image,cv2.COLOR_GRAY2RGB)
cv2.drawContours(image, contour_list, -1, (255,0,0), 3)

# show the image
cv2.imshow('image1',thresh)
cv2.imshow('image2',image)
cv2.waitKey(0)
cv2.destroyAllWindows()