Python - Detect a QR code from an image and crop using OpenCV

Question

I'm working on a project using Python(3.7) and OpenCV in which I have an Image(captured using the camera) of a document with a QR code placed on it.

This QR code has 6 variables respectively as:

1. Size of QR code image

2. Top

3. Right

4. Bottom

5. Left

6. Unit

---

Latest Update:

Here are the steps I need to perform in the same order:

1. Detect the qr code & decode it to read size values
2. So, if the size of QR-code(image) is not equal to the size which is mentioned inside it then scale the image to equal both size values.
3. Then crop the image towards all sides from QR code image according to the values mentioned inside qr code.

I have tried this code:

def decodeAndCrop(inputImage):
    print(str(inputImage))
    image = cv2.imread(str(inputImage))
    qrCodeDetector = cv2.QRCodeDetector()
    decodedText, points, _ = qrCodeDetector.detectAndDecode(image)
    qr_data = decodedText.split(",")
    print("qr data from fucntion: {}".format(qr_data))
    if points is not None:
        pts = len(points)
    # print(pts)
    for i in range(pts):
        nextPointIndex = (i + 1) % pts
        if str(inputImage) == "scaled_img.jpg":
            cv2.line(
                image,
                tuple(points[i][0]),
                tuple(points[nextPointIndex][0]),
                (255, 0, 0),
                5,
            )
        print(points[i][0])
        width = int(
            math.sqrt(
                (points[0][0][0] - points[1][0][0]) ** 2
                + (points[0][0][1] - points[1][0][1]) ** 2
            )
        )
        height = int(
            math.sqrt(
                (points[1][0][0] - points[2][0][0]) ** 2
                + (points[1][0][1] - points[2][0][1]) ** 2
            )
        )
        print("height and width after scaling: {} {}".format(height, width))
        if not str(inputImage) == "scaled_img.jpg":
            scaled_img = None
            if width == qr_data[0] and height == qr_data[0]:
                print("Sizes are equal")
                # Add the extension values to points and crop
                y = int(points[0][0][1]) - int(qr_data[1])
                x = int(points[0][0][0]) - int(qr_data[4])
                roi = image[
                    y : y + height + int(qr_data[3]), x : x + width + int(qr_data[2])
                ]
                scaled_img = cv2.imwrite("scaled_img.jpg", roi)
                return scaled_img
            else:
                print(
                    "Width and height  "
                    + str(width)
                    + "x"
                    + str(height)
                    + "  not equal to "
                    + str(qr_data[0])
                    + "x"
                    + str(qr_data[0])
                )
                if height > int(qr_data[0]):
                    scale_width = int(width) - int(qr_data[0])
                    scale_height = int(height) - int(qr_data[0])
                    print(f"scaled width: {scale_width} scaled height: {scale_height}")
                    dimension = (scale_width, scale_height)
                    scaled_img = cv2.resize(
                        image, dimension, interpolation=cv2.INTER_AREA
                    )
                    print("new img dims: {}".format(scaled_img.shape))
                    cv2.imshow("scaled image:", scaled_img)
                    cv2.imwrite("scaled_img.jpg", scaled_img)
                elif height < int(qr_data[0]):
                    scale_width = int(qr_data[0]) - width
                    scale_height = int(qr_data[0] - height)
                    print(f"scaled width: {scale_width} scaled height: {scale_height}")
                    dimension = (scale_width, scale_height)
                    scaled_img = cv2.resize(
                        image, dimension, interpolation=cv2.INTER_AREA
                    )
                    print("new img dims: {}".format(scaled_img.shape))
                    cv2.imshow("scaled image:", scaled_img)
                    cv2.imwrite("scaled_img.jpg", scaled_img)
                    cv2.imshow("final output:", roi)
                return scaled_img

        else:
            y = int(points[0][0][1]) - int(qr_data[1])
            x = int(points[0][0][0]) - int(qr_data[4])
            print(" x and y")
            print(x)
            print(y)
            roi = image[
                y : y + height + int(qr_data[3]), x : x + width + int(qr_data[2])
            ]
            final_img = cv2.imwrite("finalized_image.jpg", roi)
            cv2.imshow("finalized image:", final_img)
            return final_img


if __name__ == "__main__":
    image_to_crop = decodeAndCrop("example_input_1.jpg")
    final_image = decodeAndCrop("scaled_img.jpg")
    cv2.imshow("Cropped:", image_to_crop)
    # cv2.imshow("Final: ", final_image)
    cv2.waitKey(0)
    cv2.destroyAllWindows()

---

The code above gives an error as: final_img = cv2.imwrite("finalized_image.jpg", roi) cv2.error: OpenCV(4.2.0) /Users/travis/build/skvark/opencv-python/opencv/modules/imgcodecs/src/loadsave.cpp:715: error: (-215:Assertion failed) !_img.empty() in function 'imwrite'

---

End of Latest Update:

---

An example decoded information of a QR code is as: 100, 20, 40, 60, 20, px

Now, I need to detect the QR code from this document image and in the first step I need to compare the size of QR code in captured image of document with the size which is mentioned in the decoded information for example if in the captured image the size of the QR image is 90X90px and the size from decoded info is 100X100px we need to compare that.

Then, in the second step I have to crop the complete image by using the Top, Right, Bottom & Left variables accordingly. According to the above example we need to crop the image from the position of detected QR code to 20px Top, 40px Right, 60px Bottom and 20px Right. I have added an example Image below.

I have done to decode the QR code information but how can I take the detected QR code area as a seprate image and compare it's size with the mentioned size and then crop the Image accordingly?

Here's what I have tried so far:

import cv2

image = cv2.imread('/Users/abdul/PycharmProjects/QScanner/images/second.jpg')

qrCodeDetector = cv2.QRCodeDetector()
decodedText, points, _ = qrCodeDetector.detectAndDecode(image)
qr_data = decodedText.split(',')
qr_size = qr_data[0]
top = qr_data[1]
right = qr_data[2]
bottom = qr_data[3]
left = qr_data[4]

print(f'Size: {qr_size}' + str(qr_data[5]))
print(f'Top: {top}')
print(f'Right: {right}')
print(f'Bottom: {bottom}')
print(f'Left: {left}')
if points is not None:
    pts = len(points)
    print(pts)
    for i in range(pts):
        nextPointIndex = (i+1) % pts
        cv2.line(image, tuple(points[i][0]), tuple(points[nextPointIndex][0]), (255,0,0), 5)
        print(points[i][0])
    print(decodedText)    
    cv2.imshow("Image", image)
    cv2.waitKey(0)
    cv2.destroyAllWindows()
else:
    print("QR code not detected")

Here's an example Image:

and here's a sample of input image:

Indifference

In your example, ıt seems you are getting the image, i couldnt exactly understand what the problem is – Mimetic

@YunusTemurlenk there's two things actually I need achieve: 1): Crop the QR code and compare it with the size mentioned inside this 2): Crop the image according to the values for Top, Right, Bottom & Left – Indifference

The first image I mentioned above is the desired thing. – Indifference

You already have the detected rectangle points. So by subtracting the points x and y values, you can get the size of detected QR. Then you can also manipulate the crop image point according to those points. – Mimetic

@YunusTemurlenk can you put a code example, please! – Indifference

I think the distances (units) from the borders are kinda messed up. When the qr is 90*90px, I would expect other distances to be significantly higher. For example from top, it should be around 250px or so. Also, the edge should start from the border of the image, not the margins of the text, I am not sure if you draw that way or it is calculating from the margins. – Intoxicating

@Intoxicating the sample image is not scald 100% correctly, it's just a sample but I think in real scenario the qr codes will be placed with proper measurement, so we can build a working solution with this sample which can be modified latter. – Indifference

@Intoxicating we just need to crop the image by using the provided data for example it should crop 20px towards top from the QR code position. – Indifference

If this values are correct, what is the problem with : im[20:110, 20:110].copy()? – Intoxicating

@Intoxicating I couldn't got your point, can you elaborate it, please! – Indifference

I initially misunderstood the problem. I think you just want to crop out the margins. There are multiple issues here. 1) If the image is rotated with an angle \theta, 2) if the sheet is one a plane. (i.e., in the images, the upper line doesn't seem to be linear. But it should not be a big deal.) 3) The black borders. Will you always have those or may it be a different backgroud? This is important because without cropping out those, you won't be able to get a reasonable result. – Intoxicating

@AbdulRehman did you ever get this working how you wanted? I'm working on a slightly similar project now – Pelting

Answer 1

Here's a simple approach using thresholding, morphological operations, and contour filtering.

1. Obtain binary image. Load image, grayscale, Gaussian blur, Otsu's threshold

2. Connect individual QR contours. Create a rectangular structuring kernel with cv2.getStructuringElement() then perform morphological operations with cv2.MORPH_CLOSE.

3. Filter for QR code. Find contours and filter using contour approximation, contour area, and aspect ratio.

---

Detected QR code

Extracted QR code

From here you can compare the QR code with your reference information

Code

import cv2
import numpy as np

# Load imgae, grayscale, Gaussian blur, Otsu's threshold
image = cv2.imread('1.jpg')
original = image.copy()
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
blur = cv2.GaussianBlur(gray, (9,9), 0)
thresh = cv2.threshold(blur, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1]

# Morph close
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5,5))
close = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, kernel, iterations=2)

# Find contours and filter for QR code
cnts = cv2.findContours(close, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cnts = cnts[0] if len(cnts) == 2 else cnts[1]
for c in cnts:
    peri = cv2.arcLength(c, True)
    approx = cv2.approxPolyDP(c, 0.04 * peri, True)
    x,y,w,h = cv2.boundingRect(approx)
    area = cv2.contourArea(c)
    ar = w / float(h)
    if len(approx) == 4 and area > 1000 and (ar > .85 and ar < 1.3):
        cv2.rectangle(image, (x, y), (x + w, y + h), (36,255,12), 3)
        ROI = original[y:y+h, x:x+w]
        cv2.imwrite('ROI.png', ROI)

cv2.imshow('thresh', thresh)
cv2.imshow('close', close)
cv2.imshow('image', image)
cv2.imshow('ROI', ROI)
cv2.waitKey()     

Hyperbaric

Your code would detect any image with a box-like contour and return it as ROI. Even if there is a diagram on the page as well as a QR code on the same page, chances are that the code returns the box-like diagram as the ROI – Sn

Answer 2

I got the width and height data using points and compare it with the qr_data size. Then cropped the QR according to needed.

import cv2
import math  

image = cv2.imread('/ur/image/directory/qr.jpg')

qrCodeDetector = cv2.QRCodeDetector()
decodedText, points, _ = qrCodeDetector.detectAndDecode(image)
qr_data = decodedText.split(',')
qr_size = qr_data[0]
top = qr_data[1]
right = qr_data[2]
bottom = qr_data[3]
left = qr_data[4]

if points is not None:
    pts = len(points)
    print(pts)
    for i in range(pts):
        nextPointIndex = (i+1) % pts
        cv2.line(image, tuple(points[i][0]), tuple(points[nextPointIndex][0]), (255,0,0), 5)
        print(points[i][0])

    width = int(math.sqrt((points[0][0][0]-points[1][0][0])**2 + (points[0][0][1]-points[1][0][1])**2))
    height = int(math.sqrt((points[1][0][0]-points[2][0][0])**2 + (points[1][0][1]-points[2][0][1])**2))

    # Compare the size
    if(width==qr_data[0] and height==qr_data[0]):
        print("Sizes are equal")
    else:
        print("Width and height  " + str(width) + "x" +  str(height) + "  not equal to " 
        + str(qr_data[0]) + "x" + str(qr_data[0]))

    # Add the extension values to points and crop
    y = int(points[0][0][1]) - int(qr_data[1])
    x = int(points[0][0][0]) - int(qr_data[4])
    roi = image[y:y+height + int(qr_data[3]), x:x+width + int(qr_data[2])]
    print(decodedText)    
    cv2.imshow("Image", image)
    cv2.imshow("Crop", roi)
    cv2.waitKey(0)
    cv2.destroyAllWindows()
else:
    print("QR code not detected")

Result:

Mimetic

is it cropping the full image? or only the qr code? – Indifference

Crop of qr by extending 20px to top,20 px to left, 40 px to right and 60 px to bottom of original qr rectangle – Mimetic

actually, we have to crop the image (mean sroundings) of qr rectangle. – Indifference

This is also crop of qr code. You should be more clear about what you need. – Mimetic

I think I should try it with a different image first, let me try it out with a proper image. – Indifference

I have added these values to the QR code 200,500,80,500,150,px so it should crop 500px towards top and 80px towards right... and so on but the ROI image is even cut down the qr code from right side mean in cropped image I have the half qr code which is cropped from right side. – Indifference

I just use left top point of qr rectangle to crop. If the point you used while cropping is not left top one, it may crop wrongly. Just be sure about it. Otherwise my code shouldn't crop half or smaller than the qr size – Mimetic

I have update the question, take a look at the Latest Update section in the question, please! – Indifference

I have adjusted your code in the form of a function, see updated section in the question. Now it's giving another error as:cv2.error: OpenCV(4.2.0) /Users/travis/build/skvark/opencv-python/opencv/modules/imgcodecs/src/loadsave.cpp:715: error: (-215:Assertion failed) !_img.empty() in function 'imwrite' can you help me with that? – Indifference

@AbdulRehman This king of error occurs cos of there is no image. the image roi doesnt exist or dimesions are width:0 and height:0 . To be sure, firstly try to show roi with imshow – Mimetic

You can see it under Latest Update section and in code it’s just above the main section where I’m trying to write finalezed_img – Indifference

Answer 3

So, you mainly have 3 problems here.

1. If the image is rotated with an angle \theta,
2. If the sheet is one a plane. (i.e., in the images, the upper line doesn't seem to be linear. But it should not be a big deal.)
3. The black borders. Will you always have those or may it be a different background? This is important because without cropping out those, you won't be able to get a reasonable result.

I improved your code a little bit and removed the border pixels:

import cv2
import matplotlib.pyplot as plt    
import math
import numpy as np

image = cv2.imread('/Users/samettaspinar/Public/im.jpg')

qrCodeDetector = cv2.QRCodeDetector()
decodedText, points, _ = qrCodeDetector.detectAndDecode(image)
qr_data = decodedText.split(',')
qr_size = int(qr_data[0])
top = int(qr_data[1])
right = int(qr_data[2])
bottom = int(qr_data[3])
left = int(qr_data[4])

print(f'Size: {qr_size}' + str(qr_data[5]))
print(f'Top: {top}')
print(f'Right: {right}')
print(f'Bottom: {bottom}')
print(f'Left: {left}')

plt.imshow(image)
plt.show()

dists = [] #This is for estimating distances between corner points.
           #I will average them to find ratio of pixels in image vs qr_size  
           #in the optimal case, all dists should be equal

if points is not None:
    pts = len(points)
    for i in range(pts):
        p1 = points[i][0]
        p2 = points[(i+1) % pts][0]

        dists.append(math.sqrt((p1[0]-p2[0])**2 + (p1[1]-p2[1])**2))

        print('line', tuple(p1), tuple(p2))
        image = cv2.line(image, tuple(p1), tuple(p2), (255,0,0), 5)
else:
    print("QR code not detected")

print('distances: ', dists)


# Remove the black border pixels. I had a simple idea for this
# Get the average intensity of the gray image
# If count the row average of the first half that are less than intensity/2. 
# It approx gives number of black borders on the left. etc.  
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
inten = np.mean(gray)

x = np.mean(gray, axis=0) # finds the vertical average
y = np.mean(gray, axis=1) # finds horizontal average

bl_left = np.sum([x[:int(col/2)] < inten/2])
bl_right = np.sum([x[int(col/2)+1:] < inten/2])

bl_top = np.sum([y[:int(row/2)] < inten/2])
bl_bottom = np.sum([y[int(row/2)+1:] < inten/2])

print('black margins: ', bl_left, bl_right, bl_top, bl_bottom)

# Estimate how many pixel you will crop out
ratio = np.mean(dists)/ int(qr_size)
print('actual px / qr_size in px: ', ratio)

row,col,dim = image.shape

top, left, right, bottom = int(top*ratio), int(left*ratio), int(right*ratio), int(bottom*ratio)
top += bl_top
left += bl_left
right += bl_right
bottom += bl_bottom

print('num pixels to be cropped: ', top, left, right, bottom)

gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
image2 = image[top:row-bottom, left:col-right, :]

plt.imshow(image2)
plt.show()

Notice that I ignored the rotation issue. If there is rotation, you can find the angle by calculating the tangents/arctan where I calculated the distances.

Intoxicating

Answer 4

For QR detection and parsing

import cv2
import sys

filename = sys.argv[1]

# read the QRCODE image
#in case if QR code is not black/white it is better to convert it into grayscale
img = cv2.imread(filename, 0)# Zero means grayscale
img_origin = cv2.imread(filename)

# initialize the cv2 QRCode detector
detector = cv2.QRCodeDetector()

# detect and decode
data, bbox, straight_qrcode = detector.detectAndDecode(img)

# if there is a QR code
if bbox is not None:
    print(f"QRCode data:\n{data}")
    # display the image with lines
    # length of bounding box
    n_lines = len(bbox[0])#Cause bbox = [[[float, float]]], we need to convert fload into int and loop over the first element of array
    bbox1 = bbox.astype(int) #Float to Int conversion
    for i in range(n_lines):
        # draw all lines
        point1 = tuple(bbox1[0, [i][0]])
        point2 = tuple(bbox1[0, [(i+1) % n_lines][0]])
        cv2.line(img_origin, point1, point2, color=(255, 0, 0), thickness=2)


    # display the result
    cv2.imshow("img", img)
    cv2.waitKey(0)
    cv2.destroyAllWindows()
    
else:
    print("QR code not detected")

Kalb

Hi, it would be great if you could explain what your code does. It does make your answer a lot better and easier to understand for the rest of us! – Warfeld