Improve signature quality extracted using OpenCV from scanned sheet paper

I extracted a signature from a scanned sheet of paper. Users know there should be only the signature on a white sheet of paper.

I implemented the signature cropping from the uploaded image using the OpenCV Java library.

I want to improve the quality of the extracted signature. I realized that the best version together with the original file is the one in greyscale (grayROI). For example, I would make the background whiter.

Could you please suggest how I can improve the final signature appearance?

The top would be converting the signature to a PNG with a transparent background.

import java.awt.image.BufferedImage;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import javax.imageio.ImageIO;

import org.apache.commons.io.FilenameUtils;
import org.opencv.core.Core;
import org.opencv.core.Mat;
import org.opencv.core.MatOfPoint;
import org.opencv.core.Point;
import org.opencv.core.Rect;
import org.opencv.core.Scalar;
import org.opencv.core.Size;
import org.opencv.imgcodecs.Imgcodecs;
import org.opencv.imgproc.Imgproc;
import org.opencv.utils.Converters;

import nu.pattern.OpenCV;


    public static synchronized void extractSignature(File signature) {

        if (signature == null)
            return;

        String fileExtension = FilenameUtils.getExtension(signature.getName());

        // check if the file extension is correct!
        if (!fileExtension.toLowerCase().equals("jpg") && !fileExtension.toLowerCase().equals("jpeg") && !fileExtension.toLowerCase().equals("png")) {
            return;
        }
        
        try {
            
            if (!isLoaded) {
                OpenCV.loadLocally();
                isLoaded = true;
            }
            
        } catch (Exception e) {
            System.out.println(e.toString());
        }

        // Load image
        Mat image = Imgcodecs.imread(signature.getPath());
        Mat imageOriginal = image.clone();

        // Convert image to HSV color space
        Mat hsv = new Mat();
        Imgproc.cvtColor(image, hsv, Imgproc.COLOR_BGR2HSV);

        // Define lower and upper bounds for color threshold
        Scalar lower = new Scalar(90, 38, 0);
        Scalar upper = new Scalar(145, 255, 255);

        // Threshold the HSV image to get only desired colors
        Mat mask = new Mat();
        Core.inRange(hsv, lower, upper, mask);

        // Find contours
        List<MatOfPoint> contours = new ArrayList<>();
        Mat hierarchy = new Mat();
        Imgproc.findContours(mask, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);

        // Combine all contours into one and get bounding box
        MatOfPoint allContours = new MatOfPoint();
        for (MatOfPoint contour : contours) {
            List<Point> pts = contour.toList();
            allContours.push_back(new MatOfPoint(Converters.vector_Point_to_Mat(pts)));
        }

        Rect boundingBox = Imgproc.boundingRect(allContours);

        
        // Add 5 pixels to each dimension of the bounding box
        int padding = 10;
        int x = Math.max(boundingBox.x - padding, 0);
        int y = Math.max(boundingBox.y - padding, 0);
        int width = Math.min(boundingBox.width + 2 * padding, image.cols() - x);
        int height = Math.min(boundingBox.height + 2 * padding, image.rows() - y);
        Rect paddedBoundingBox = new Rect(x, y, width, height);

        
        // Extract ROI
        //Mat ROI = new Mat(imageOriginal, boundingBox);
        Mat ROI = new Mat(imageOriginal, paddedBoundingBox);
        
        
        // Convert ROI to grayscale
        Mat grayROI = new Mat();
        Imgproc.cvtColor(ROI, grayROI, Imgproc.COLOR_BGR2GRAY);
        
        // Apply histogram equalization to improve contrast
        Mat equalizedROI = new Mat();
        Imgproc.equalizeHist(grayROI, equalizedROI);
        
        // Apply adaptive thresholding to binarize the image
        Mat binaryROI = new Mat();
        Imgproc.adaptiveThreshold(grayROI, binaryROI, 255, Imgproc.ADAPTIVE_THRESH_GAUSSIAN_C, Imgproc.THRESH_BINARY, 11, 2);

        // Apply morphological transformations to improve the signature appearance
        Mat morphROI = new Mat();
        Mat kernel = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(2, 2));
        Imgproc.morphologyEx(binaryROI, morphROI, Imgproc.MORPH_CLOSE, kernel);
        Imgproc.morphologyEx(morphROI, morphROI, Imgproc.MORPH_OPEN, kernel);


        // Save and display images
        Imgcodecs.imwrite(FileUtils.getSignaturesFolder().getAbsolutePath() + File.separator + signature.getName(), ROI);
        Imgcodecs.imwrite(FileUtils.getSignaturesFolder().getAbsolutePath() + File.separator + "morphROI.jpg", morphROI);
        Imgcodecs.imwrite(FileUtils.getSignaturesFolder().getAbsolutePath() + File.separator + "grayROI.jpg", grayROI);
        Imgcodecs.imwrite(FileUtils.getSignaturesFolder().getAbsolutePath() + File.separator + "binaryROI.jpg", binaryROI);
        Imgcodecs.imwrite(FileUtils.getSignaturesFolder().getAbsolutePath() + File.separator + "equalizedROI.jpg", equalizedROI);
        
        return;
    }

Here an example, from this original image:

I get this result:

HSV is too much related to the background, so it is not a good choice.

Eventually, I moved to a solution using MSER regions. My final solution seems to work accurately for all my test cases.

These are the main steps implemented:

1. Convert the image to gray
2. Detected MSER regions
3. Calculated the region’s centroid
4. Added two thresholds one vertical and one horizontal (to skip regions not related to the signature)
5. Created a bounding box with all the regions respecting the threshold distances with the centroid
6. Extracted the ROI using the bbox

It works with scanned documents with only a signature to be cropped and stored.

public static synchronized void extractSignature(File signature) {

        if (signature == null)
            return;

        String fileExtension = FilenameUtils.getExtension(signature.getName());

        // check if the file extension is correct!
        if (!fileExtension.toLowerCase().equals("jpg") && !fileExtension.toLowerCase().equals("jpeg") && !fileExtension.toLowerCase().equals("png")) {
            return;
        }

        // Load OpenCV library
        try {

            if (!isLoaded) {
                // System.loadLibrary(Core.NATIVE_LIBRARY_NAME); // not working
                // OpenCV.loadShared(); // not working
                OpenCV.loadLocally();
                isLoaded = true;
            }

        } catch (Exception e) {
            System.out.println(e.toString());
        }

        // Load image from file
        Mat image = Imgcodecs.imread(signature.getPath());
        // Make a copy of the original image
        Mat originalImage = image.clone();

        // Convert the image to grayscale
        Mat gray = new Mat();
        Imgproc.cvtColor(image, gray, Imgproc.COLOR_BGR2GRAY);

        // Create MSER and Detect MSER regions
        MSER mser = MSER.create();
        List<MatOfPoint> regions = new ArrayList<>();
        MatOfRect bboxes = new MatOfRect();
        mser.detectRegions(gray, regions, bboxes);

        // Calculate the Centroid of all MSER regions
        int totalX = 0;
        int totalY = 0;
        int totalPoints = 0;
        for (MatOfPoint region : regions) {
            Point[] points = region.toArray();
            for (Point point : points) {
                totalX += point.x;
                totalY += point.y;
                totalPoints++;
            }
        }

        Point centroid = new Point(totalX / totalPoints, totalY / totalPoints);


        // Draw the Centroid
        Mat centroidM = image.clone();
        Imgproc.circle(centroidM, centroid, 5, new Scalar(255, 0, 0), -1); // Draw the Centroid as a blue dot

        
        // Define vertical and horizontal distance thresholds (adjust as needed)
        double verticalThreshold = 200.0; // Vertical distance threshold
        double horizontalThreshold = 700.0; // Horizontal distance threshold

        // Find the bounding box that covers filtered MSER regions
        int minX = Integer.MAX_VALUE;
        int minY = Integer.MAX_VALUE;
        int maxX = Integer.MIN_VALUE;
        int maxY = Integer.MIN_VALUE;

        for (MatOfPoint region : regions) {
            Point[] points = region.toArray();
            for (Point point : points) {
                double verticalDistance = Math.abs(point.y - centroid.y);
                double horizontalDistance = Math.abs(point.x - centroid.x);

                if (verticalDistance <= verticalThreshold && horizontalDistance <= horizontalThreshold) {
                    if (point.x < minX) minX = (int) point.x;
                    if (point.y < minY) minY = (int) point.y;
                    if (point.x > maxX) maxX = (int) point.x;
                    if (point.y > maxY) maxY = (int) point.y;
                }
            }
        }

        // Draw the bounding box if valid points were found
        if (minX < Integer.MAX_VALUE && minY < Integer.MAX_VALUE && maxX > Integer.MIN_VALUE && maxY > Integer.MIN_VALUE) {
            Imgproc.rectangle(image, new Point(minX, minY), new Point(maxX, maxY), new Scalar(0, 255, 0), 2);
        }

        // Draw convex hulls
        Mat grayImg = image.clone();
        for (MatOfPoint region : regions) {
            MatOfInt hullIndices = new MatOfInt();
            Imgproc.convexHull(region, hullIndices);

            // Convert hull indices to points
            Point[] regionArray = region.toArray();
            List<Point> hullPoints = new ArrayList<>();
            for (int index : hullIndices.toArray()) {
                hullPoints.add(regionArray[index]);
            }
            MatOfPoint hull = new MatOfPoint();
            hull.fromList(hullPoints);

            // Draw the hull on the image
            List<MatOfPoint> hulls = new ArrayList<>();
            hulls.add(hull);
            Imgproc.polylines(grayImg, hulls, true, new Scalar(0, 0, 255), 2);
        }

        // Extract ROI, crop the original image using the bounding box
        Rect ROI = new Rect(minX, minY, maxX - minX, maxY - minY);
        Mat croppedImage = new Mat(originalImage, ROI);

        // Convert ROI to grayscale
        Mat grayROI = new Mat();
        Imgproc.cvtColor(croppedImage, grayROI, Imgproc.COLOR_BGR2GRAY);

        Imgproc.threshold(croppedImage, croppedImage, 150, 255, Imgproc.THRESH_BINARY);

        // Save and display images
        Imgcodecs.imwrite(FileUtils.getSignaturesFolder().getAbsolutePath() + File.separator + "image_with_bbox.jpg", image);
        Imgcodecs.imwrite(FileUtils.getSignaturesFolder().getAbsolutePath() + File.separator + signature.getName(), croppedImage);
        Imgcodecs.imwrite(FileUtils.getSignaturesFolder().getAbsolutePath() + File.separator + "grayROI.jpg", grayROI);
        Imgcodecs.imwrite(FileUtils.getSignaturesFolder().getAbsolutePath() + File.separator + "last.jpg", grayImg);
        Imgcodecs.imwrite(FileUtils.getSignaturesFolder().getAbsolutePath() + File.separator + "centroid.jpg", centroidM);

        return;
    }