Fast template matching using Pyramids in Python

Question

I am trying to implement in Python the following C++ code: https://opencv-code.com/tutorials/fast-template-matching-with-image-pyramid/

If you check the C++ code, you'll see this loop:

for (int i = 0; i < contours.size(); i++)
{
    cv::Rect r = cv::boundingRect(contours[i]);
    cv::matchTemplate(
        ref(r + (tpl.size() - cv::Size(1,1))), 
        tpl, 
        res(r), 
        CV_TM_CCORR_NORMED
    );
}

My Python code:

for i in range(0, np.size(contours)-1):
    x, y, w, h = cv.boundingRect(contours[i][0])
    tpl_X = curr_template.shape[1]-1
    tpl_Y = curr_template.shape[0]-1

    result[y:h, x:w] = cv.matchTemplate(
                           curr_image[y:h+tpl_Y, x:w+tpl_X],
                           curr_template, cv.TM_CCORR_NORMED)

And something is wrong as I keep getting: ValueError: could not broadcast input array from shape (53,51) into shape (52,52)

This numbers (53, 51) (52,52) may change as I modify just a little the coordinates on result or in curr_image, but that's not the right answer.

This is my current code:

import cv2 as cv
import numpy as np
import argparse
import os

"""
This script performs a fast template matching algorithm using the OpenCV
function matchTemplate plus an approximation through pyramid construction to
improve it's performance on large images.
"""

def buildPyramid(image, max_level):

    results = [image]
    aux = image

    for i in range(0,max_level):
        aux = cv.pyrDown(aux)
        results = [aux] + results

    return results


def temp_match(input, template, max_level):

    results = []

    source_pyr = buildPyramid(input, max_level)
    template_pyr = buildPyramid(template, max_level)

    for lvl in range(0, int(max_level), 1):

        curr_image = source_pyr[lvl]
        curr_template = template_pyr[lvl]

        dX = curr_image.shape[1] + 1 - curr_template.shape[1]
        dY = curr_image.shape[0] + 1 - curr_template.shape[0]

        result = np.zeros([dX, dY])


        #On the first level performs regular template matching.
        if lvl == 0:
            result = cv.matchTemplate(curr_image, curr_template,
                                      cv.TM_CCORR_NORMED)

        #On every other level, perform pyramid transformation and template
        #matching on the predefined ROI areas, obtained using the result of the
        #previous level.
        else:
            mask = cv.pyrUp(r)

            mask8u = cv.inRange(mask, 0, 255)
            contours = cv.findContours(mask8u, cv.RETR_EXTERNAL,
                                       cv.CHAIN_APPROX_NONE)

            #Uses contours to define the region of interest and perform TM on
            #the areas.

            for i in range(0, np.size(contours)-1):
                x, y, w, h = cv.boundingRect(contours[i][0])
                tpl_X = curr_template.shape[1]
                tpl_Y = curr_template.shape[0]

                #result = cv.matchTemplate(curr_image, curr_template,
                #                          cv.TM_CCORR_NORMED)

                result[y:y+h, x:x+w] = cv.matchTemplate(
                                curr_image[y:y+h+tpl_Y, x:x+w+tpl_X],
                                curr_template, cv.TM_CCORR_NORMED)

        T, r = cv.threshold(result, 0.94, 1., cv.THRESH_TOZERO)
        cv.imshow("test", r)
        cv.waitKey()
        results.append(r)
    return results


def ftm_pyramid(input_file, template_file, max_level = 5):

    if file_exists(input_file) == False:
        raise IOError("Input file not found.")

    if file_exists(template_file) == False:
        raise IOError("Input file not found.")

    img = cv.imread(input_file)
    tpl = cv.imread(template_file)

    image = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
    template = cv.cvtColor(tpl, cv.COLOR_BGR2GRAY)

    tm_results = temp_match(image, template, max_level)

    c = 0
    flag = False

    while flag == False and c < np.size(tm_results):
        current = tm_results[c]
        min_val, max_val, min_loc, max_loc = cv.minMaxLoc(current)
        if max_val > 0.9:
            cv.rectangle(img,
                        max_loc,
                        (max_loc[0] + template.shape[1],
                         max_loc[1] + template.shape[0]),
                        (0,0,255), 2)
        else:
            flag = True

        c = c+1

    cv.imshow("Result", img)
    cv.waitKey()
    return 0


# Auxiliary methods

def file_exists(input_file):
    """
    :param input_file: path to the input file
    :return: true or false wether the file exists or not.
    """
    if input_file == '':
        raise ValueError("The input file can't be ''.")
    if input_file == None:
        raise ValueError("The input file can't be a None object")

    return os.path.isfile(input_file)


if __name__ == '__main__':
    #CLI arguments
    ap = argparse.ArgumentParser()
    ap.add_argument("-i", "--input", required="True",
                    help="Path to the input image.")
    ap.add_argument("-t", "--template", required="True",
                    help="Path to the template image.")
    ap.add_argument("-l", "--levels", help="Number of levels of the pyramid.")
    args = vars(ap.parse_args())

    #Loading values
    input_file = args["input"]
    template = args["template"]
    max_lvl = args["levels"]

    if max_lvl == None:
        max_lvl = 5

    ftm_pyramid(input_file, template, max_lvl)

Any help will be really appreciated!

Mckissick

Answer 1

Do image template matching in image pyramid from coarse to fine, basic idea in many domain.

There is something wrong in your code, I rewrite the code while refering the original CPP code and your Python code.

---

This is the referer image and the template image:

This is the result:

My code is here, feel free to have a test.

#!/usr/bin/python3
# 2017.10.04 14:50:50 CST START
# 2017.10.04 17:32:39 CST FINISH

import cv2
import numpy as np
import argparse
import os

def fileExists(filename):
    """Judge wether the file exists!
    """
    if filename in ('', None):
        raise ValueError("The input file can't be '' or None.")
    return os.path.isfile(filename)

def buildPyramid(image, maxleval):
    """Build image pyramid for level [0,...,maxlevel]
    """
    imgpyr = [image]
    aux = image
    for i in range(0,maxleval):
        aux = cv2.pyrDown(aux)
        imgpyr.append(aux)

    imgpyr.reverse()
    return imgpyr


def fastTemplateMatchPyramid(src_refimg, src_tplimg, maxleval):
    """Do fast template matching using matchTemplate plus an approximation
    through pyramid construction to improve it's performance on large images.
    """
    results = []

    ## Change BGR to Grayscale
    gray_refimg = cv2.cvtColor(src_refimg, cv2.COLOR_BGR2GRAY)
    gray_tplimg = cv2.cvtColor(src_tplimg, cv2.COLOR_BGR2GRAY)

    ## Build image pyramid
    refimgs = buildPyramid(gray_refimg, maxleval)
    tplimgs = buildPyramid(gray_tplimg, maxleval)

    ## Do template match
    for idx in range(0, maxleval+1):
        refimg = refimgs[idx]
        tplimg = tplimgs[idx]

        # On the first level performs regular template matching.
        # On every other level, perform pyramid transformation and template matching
        # on the predefined ROI areas, obtained using the result of the previous level.
        # Uses contours to define the region of interest and perform TM on the areas.
        if idx == 0:
            result = cv2.matchTemplate(refimg, tplimg, cv2.TM_CCORR_NORMED)
        else:
            mask = cv2.pyrUp(threshed)
            mask8u = cv2.inRange(mask, 0, 255)
            contours = cv2.findContours(mask8u, cv2.RETR_EXTERNAL,  cv2.CHAIN_APPROX_NONE)[-2]

            tH, tW = tplimg.shape[:2]
            for cnt in contours:
                x, y, w, h = cv2.boundingRect(cnt)
                src = refimg[y:y+h+tH, x:x+w+tW]
                result = cv2.matchTemplate(src, tplimg, cv2.TM_CCORR_NORMED)

        T, threshed = cv2.threshold(result, 0.90, 1., cv2.THRESH_TOZERO)
        results.append(threshed)

    return threshed
    #return results


def fastTemplateMatch(refname, tplname, maxleval = 5):
    """Fast template match.
    """
    ## Read the image pairs.
    if fileExists(refname) == False:
        raise IOError("Input file not found.")
    if fileExists(tplname) == False:
        raise IOError("Input file not found.")

    refimg = cv2.imread(refname)
    tplimg = cv2.imread(tplname)
    cv2.imwrite("cat.png",refimg)

    ## Call fastTemplateMatchInPyramid()
    result = fastTemplateMatchPyramid(refimg, tplimg, maxleval)

    ## Analysis the result
    minval, maxval, minloc, maxloc = cv2.minMaxLoc(result)
    if maxval > 0.9:
        pt1 = maxloc
        pt2 = (maxloc[0] + tplimg.shape[1], maxloc[1] + tplimg.shape[0])
        print("Found the template region: {} => {}".format(pt1,pt2))
        dst = refimg.copy()
        cv2.rectangle(dst, pt1, pt2, (0,255,0), 2)
        cv2.imshow("Result", dst)
        cv2.imwrite("template_matching_result.png",dst)
        cv2.waitKey()
    else:
        print("Cannot find the template in the origin image!")


if __name__ == '__main__':
    ## CLI arguments
    """
    ap = argparse.ArgumentParser()
    ap.add_argument("-r", "--referer", required="True",
                    help="Path to the referer image.")
    ap.add_argument("-t", "--template", required="True",
                    help="Path to the template image.")
    ap.add_argument("-l", "--levels", help="Number of levels of the pyramid.")
    args = vars(ap.parse_args())

    ## Loading values
    refname = args["referer"]
    tplname = args["template"]
    maxlevel = args["levels"]
    """
    ## Set parmeters
    refname = "/home/auss/Pictures/cat.jpg"
    tplname = "cat_face.png"
    maxlevel = 5

    ## call the function
    fastTemplateMatch(refname, tplname, maxlevel)

Bowstring

I don't understand how this is supposed to work, when looping on contours you overwrite result with the TM of a small patch instead of updating the same patch in result. Have you benched the code vs non pyramidal TM? – Mechanism