javaopencvjavacv

How to get rid of skewness in scanned form?


I have to build a program that takes in skewed forms (images that have been scanned) for image processing. The first step is to get rid of the skeweness. I'm successfully getting the contours of the image, and I'm attempting to do a four_point_transform as presented in this post Remove top section of image above border line to detect text document . However, my code is failing due to:

Error

java.lang.RuntimeException: OpenCV(4.4.0) C:\projects\javacpp-presets\opencv\cppbuild\windows-x86_64\opencv-4.4.0\modules\imgproc\src\imgwarp.cpp:3391: error: (-215:Assertion failed) src.checkVector(2, CV_32F) == 4 && dst.checkVector(2, CV_32F) == 4 in function 'cv::getPerspectiveTransform

Code

  protected  static void fixSkeweness(Mat mat){

        Mat mask = new Mat();
        Mat gray = new Mat();
        Mat denoised = new Mat();
        Mat bin = new Mat();
        Mat hierarchy = new Mat();
        MatVector contours = new MatVector();

        cvtColor(mat, gray, COLOR_BGR2GRAY);
        //Normalize
        GaussianBlur(gray, denoised, new Size(5, 5), 0);
        threshold(denoised, mask, 0, 255, THRESH_BINARY_INV | THRESH_OTSU);
        normalize(gray, gray, 0, 255, NORM_MINMAX, -1, mask);
        // Convert image to binary
        threshold(gray, bin, 150, 255, THRESH_BINARY);
        // Find contours
        findContours(bin, contours, hierarchy, RETR_TREE, CHAIN_APPROX_NONE);
        long contourCount = contours.size();
        System.out.println("Countour count " + contourCount);

        double maxArea = 0;
        int maxAreaId = 0;
        for (int i = 0; i < contourCount; ++i) {
            // Calculate the area of each contour
            Mat contour = contours.get(i);
            double area = contourArea(contour);
            if(area > maxArea){
                maxAreaId = i;
                maxArea = area;
            }

        }


        Double peri = arcLength(contours.get(maxAreaId), true);
        Mat newcontour = new Mat();
        approxPolyDP(contours.get(maxAreaId), newcontour,0.02 * peri, true);
        Mat result = new Mat();
        getPerspectiveTransform(newcontour.reshape(4,2), result);
        imwrite("src/test/resources/isDataPage/fourPointTransform.jpg", result);

    }

The line of code that is failing is:

getPerspectiveTransform(newcontour.reshape(4,2), result);

Can I get some help to get this working, please?

Example Image:

example form

Working code as per suggested answer

protected static Mat findBiggestContour(Mat mat){
    Mat mask = new Mat();
    Mat gray = new Mat();
    Mat denoised = new Mat();
    Mat bin = new Mat();
    Mat hierarchy = new Mat();
    MatVector contours = new MatVector();

    //Pre-process image
    cvtColor(mat, gray, COLOR_BGR2GRAY);
    threshold(gray, bin, 0, 255, THRESH_BINARY_INV + THRESH_OTSU);
    findContours(bin, contours, hierarchy, RETR_TREE, CHAIN_APPROX_SIMPLE);


    double maxArea = 0;
    int maxAreaId = 0;

    for (int i = 0; i < contours.size(); ++i) {

        // Calculate the area of each contour
        Mat contour = contours.get(i);
        double area = contourArea(contour);

        if(area > 5000 && i!=0){
            maxAreaId = i;
            maxArea = area;
        }

    }

    //Get Min Area Rect and inverse it
    RotatedRect rect = minAreaRect(contours.get(maxAreaId));
    float newAngle = rect.angle();

    if (rect.angle() < 45){
        newAngle = newAngle + 90;
    }
    RotatedRect angle =rect.angle( newAngle);

    int h = mat.size().height();
    int w = mat.size().width();

    int centerW =  w/2;
    int centerH = h/2;
    
    //find rotation matrix and apply it woohoo
    Point2f center = new Point2f(centerW, centerH);
    Mat m = getRotationMatrix2D(center, angle.angle(), 1.0);
    Mat rotated = new Mat();

    warpAffine(mat,rotated,m, new Size(w, h),INTER_CUBIC,BORDER_REPLICATE,new Scalar(10,10));
    imwrite("src/test/resources/tmp2/rotrated.png",rotated);

    return rotated;
}

Solution

  • getPerspectiveTransform() is working some other way (see my comment). However, I found minAreaRect() as more suitable method here. I have no prepared java enviroment so here is python code. I hope you will have no difficulties while converting it.

    img = cv2.imread('images/form.png')
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    
    # some preprocessing as you did
    # your src image is pretty clean though, and if they all are like that,
    #   I wouldn't use blur as it makes form borders less obvious
    # gray = cv2.blur(gray, (5, 5))
    thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1]
    
    # find the largest contour assuming it will be some nice rectangle
    ctrs, hier = cv2.findContours(thresh, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
    largest_ctr_idx = max(range(len(ctrs)), key=lambda i: cv2.contourArea(ctrs[i]))
    
    # get the contour's rotation angle
    angle = cv2.minAreaRect(ctrs[largest_ctr_idx])[-1]
    if angle < -45:
        angle += 90
    
    # find rotation matrix and apply it woohoo
    h, w = img.shape[:2]
    center = (w // 2, h // 2)
    m = cv2.getRotationMatrix2D(center, angle, 1.0)
    rotated = cv2.warpAffine(img, m, (w, h), flags=cv2.INTER_CUBIC,
        borderMode=cv2.BORDER_REPLICATE)
    

    Found contour:

    enter image description here

    Deskewed image:

    enter image description here