How to detect multiple objects with OpenCV in C++?

Question

I got inspiration from this answer here, which is a Python implementation, but I need C++, that answer works very well, I got the thought is that: detectAndCompute to get keypoints, use kmeans to segment them to clusters, then for each cluster do matcher->knnMatch with each's descriptors, then do the other stuffs like the common single detecting method. The main problem is, how to provide descriptors for each cluster's matcher->knnMatch process? I thought we should set value of the other keypoints corresponding descriptor to 0(useless), am I right? And got some problems in my trying:

1. how to estimate cluster count for kmeans?
2. Why can create Mat array for clusters like this Mat descriptors_scene_clusters[3] = { Mat(descriptors_scene.rows, descriptors_scene.cols, CV_8U, Scalar(0)) };?

Very appreciate any help, pls!

---

#include <stdio.h>
#include <iostream>
#include <opencv2/core/core.hpp>
#include <opencv2/features2d/features2d.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/calib3d/calib3d.hpp>
#include <opencv2/xfeatures2d.hpp>

using namespace cv;
using namespace cv::xfeatures2d;

#define MIN_MATCH_COUNT 10

int main()
{
    Mat img_object = imread("./2.PNG", IMREAD_GRAYSCALE);
    Mat img_scene = imread("./1.PNG", IMREAD_GRAYSCALE);

    Ptr<ORB> detector = ORB::create();
    std::vector<KeyPoint> keypoints_object, keypoints_scene;
    Mat descriptors_object, descriptors_scene;
    detector->detectAndCompute(img_object, cv::Mat(), keypoints_object, descriptors_object);
    detector->detectAndCompute(img_scene, cv::Mat(), keypoints_scene, descriptors_scene);


    std::cout << descriptors_scene.row(0) << "\n";
    std::cout << descriptors_scene.cols << "\n";


    std::vector<Point2f> keypoints_scene_points_;
    for (int i=0; i<keypoints_scene.size(); i++) {
        keypoints_scene_points_.push_back(keypoints_scene[i].pt);
    }
    Mat keypoints_scene_points(keypoints_scene_points_);

    Mat labels;
    int estimate_cluster_count = 3; // estimated ??????????
    kmeans(keypoints_scene_points, estimate_cluster_count, labels, TermCriteria(TermCriteria::EPS + TermCriteria::COUNT, 10, 1.0), 3, KMEANS_RANDOM_CENTERS);

    std::cout << "==================================111111\n";

    Mat descriptors_scene_clusters[3] = { Mat(descriptors_scene.rows, descriptors_scene.cols, CV_8U, Scalar(0)) };

    std::cout << "==================================111111------\n";

    for (int i=0; i<labels.rows; i++) {
        int clusterIndex = labels.at<int>(i);
        Point2f pt = keypoints_scene_points.at<Point2f>(i);
        descriptors_scene_clusters[clusterIndex].at<uchar>(pt) = descriptors_scene.at<uchar>(pt);  // ?????? error
    }

    std::cout << descriptors_scene_clusters[0] << "\n";
    std::cout << "==================================22222222\n";
    // return 0;

    Mat img_matches = img_scene;
    std::vector<DMatch> all_good_matches;
    for (int i=0; i<estimate_cluster_count; i++) {
        std::cout << "==================================33333\n";

        Ptr<flann::IndexParams> indexParams = makePtr<flann::KDTreeIndexParams>(5);
        Ptr<flann::SearchParams> searchParams = makePtr<flann::SearchParams>(50);
        Ptr<FlannBasedMatcher> matcher = makePtr<FlannBasedMatcher>(indexParams, searchParams);
        // BFMatcher matcher;
        std::vector<std::vector<DMatch>> matches;

        std::cout << "==================================444444\n";
        matcher->knnMatch(descriptors_object, descriptors_scene_clusters[i], matches, 2);
        std::cout << "==================================555555\n";
        std::vector<DMatch> good_matches;

        for (auto &match : matches) {
            if (match[0].distance < 0.7 * match[1].distance) {
                good_matches.push_back(match[0]);
            }
        }

        all_good_matches.insert(all_good_matches.end(), good_matches.begin(), good_matches.end());

        std::cout << "==================================66666\n";

        if (good_matches.size() > MIN_MATCH_COUNT) {

            //-- Localize the object
            std::vector<Point2f> obj;
            std::vector<Point2f> scene;

            for (auto &match : good_matches) {
                //-- Get the keypoints from the good matches
                obj.push_back(keypoints_object[match.queryIdx].pt);
                scene.push_back(keypoints_scene[match.trainIdx].pt);
            }

            Mat H = findHomography(obj, scene, RANSAC);

            //-- Get the corners from the image_1 ( the object to be "detected" )
            std::vector<Point2f> obj_corners(4);
            obj_corners[0] = cvPoint(0, 0);
            obj_corners[1] = cvPoint(img_object.cols, 0);
            obj_corners[2] = cvPoint(img_object.cols, img_object.rows);
            obj_corners[3] = cvPoint(0, img_object.rows);
            std::vector<Point2f> scene_corners(4);

            perspectiveTransform(obj_corners, scene_corners, H);

            //-- Draw lines between the corners (the mapped object in the scene - image_2 )
            line(img_matches, scene_corners[0] + Point2f(img_object.cols, 0),
                 scene_corners[1] + Point2f(img_object.cols, 0), Scalar(0, 255, 0), 4);
            line(img_matches, scene_corners[1] + Point2f(img_object.cols, 0),
                 scene_corners[2] + Point2f(img_object.cols, 0), Scalar(0, 255, 0), 4);
            line(img_matches, scene_corners[2] + Point2f(img_object.cols, 0),
                 scene_corners[3] + Point2f(img_object.cols, 0), Scalar(0, 255, 0), 4);
            line(img_matches, scene_corners[3] + Point2f(img_object.cols, 0),
                 scene_corners[0] + Point2f(img_object.cols, 0), Scalar(0, 255, 0), 4);

            print(scene_corners);
        }
    }

    drawMatches(img_object, keypoints_object, img_scene, keypoints_scene,
                    all_good_matches, img_matches, Scalar::all(-1), Scalar::all(-1),
                    std::vector<char>(), DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS);


    //-- Show detected matches
    imshow("Good Matches & Object detection", img_matches);

    waitKey(0);
    return 0;
}

Curet

Try this question, it has some theory in the answers, hope it will help – Toner

Can you attach some sample images and expected output as well ? – Jennet

@Jennet I've attached the images, pls take a look, thank you! – Curet

Your question is about detecting multiple objects? So do you want to detect if the given screenshot contains the desired set of icons? – Jennet

@ZdaR, yes, I need to detect multiple objects from a given scene. – Curet

Do you strictly want to solution in domain of application icons? or the objects can be other things like box, tiger, etc ? – Jennet

@ZdaR, not only application icons, but might be any type of things. Say in short, I need to find all locations of a small image from a big image. – Curet

See the nature of solution for this kind of problem depends upon your use case. I asking these questions because, in case of app icons, we can simply use template matching to detect all the desired icons, because the input image has no rotation, scaling, contrast variation etc. But if we want a solution for images in wild where the objets are not completely visible, color may be different etc., then we need to use some more complex tenchniques. So if possible, try to describe the exact use case for an accurate solution. – Jennet

@Jennet Different colors, rotations, sizes need to be considered, the image is just an example, not icons only. – Curet

Why don't you want to apply the python solution that you are referencing? This is a really simple and stable method and, as you've stated yourself, works as expected. Does it lack something that you need? – Cherycherye

@Cherycherye I need c++ to run on mobile devices. – Curet

Ok, I've read your comments to the topic you've referenced. It would be easier if you just wrote here that you need a C++ implementation of MeanShift. What stops you from translating this function to C++? Complete source code of the function is open for anyone to see and linked to in Scikit docs: github.com/scikit-learn/scikit-learn/blob/bac89c2/sklearn/… – Cherycherye

Why not simply use a CNN ? Here there is a tutorial with implementation : learnopencv.com/tag/mask-rcnn – Cacophony

Answer 1

You can use a simple clusterer to either directly compute the clusters, find the number of clusters and / or the cluster centers initialization for kmeans. Here below a possible implementation of an agglomerative clusterer, which groups together points closer to a specified distance - see parameter dist in constructor. In your case dist can be the biggest distance between keypoints in the small image.

Header file:

class PointsClusterer {
public:
     PointsClusterer (int dist, std::vector <cv::Point2f> points);
     bool cluster();
     std::map <int, std::vector <cv::Point2f>> getClusters ();

private:

    double distance (int i, int j);
    void merge (int i, int j);

private:

    std::vector <cv::Point2f> m_Points;
    std::map <int, std::vector <int>> m_Clusters;
    std::map <int, cv::Point2f> m_Sums;
    int m_dist = 0;
};

Cpp file:

PointsClusterer::PointsClusterer (int dist, std::vector <cv::Point2f> points) :
m_dist(dist), m_Points (points)
{}

bool PointsClusterer::cluster()
{
//initialization
    for (int i = 0; i < m_Points.size(); ++i)
    {
        clusters[i] = std::vector<int>(1, i);
        sum_clusters[i] = m_Points[i];
    }

    bool still_merge = true;
    //Merge clusters
    while (still_merge)
    {
        still_merge = false;
        bool break_i = false;

        for (int i=0; i < m_Clusters.size () && !break_i ;++i)
        for (int j=i+1; j < m_Clusters.size ();++j)
        {
            if (distance(i, j) < m_dist)
            {
                merge(i, j);
                break_i = true;
                still_merge = true;
                break;
            }
        }
    }
//final conversion to std::map <int, std::vector <cv::Point2f>> is missing

}


void PointsClusterer::merge(int i, int j)
{
    auto it = m_Clusters.begin();
    auto iti = it+i;
    auto itj = it+j;

    for (val : itj->second)
    {
         iti->second.push_back(val);
         m_Sums[iti->first]+=m_Points[val];
    }
    m_Clusters.erase(itj);
}

double PointsClusterer::distance(int i, int j)
{
    auto it = m_Clusters.begin();
    auto iti = it + i;
    auto itj = it + j;
    auto vali = m_Sums[iti->first] / iti->second.size();
    auto valj = m_Sums[itj->first] / itj->second.size();
    return cv::norm(vali - valj);
}

The implementation of the cluster method was oversimplified such that it is obvious how it works. Its performance can be improved, but I believe that is beyond the scope of your question.

Foochow

Answer 2

I don't know a solution to your problem, but the following might help answer the questions you've asked.

In the comments it says that you might need an implementation of meanshift, which opencv already has. Here an example, here the documentation with a tutorial.

1. The clusterCount for kmeansis the number of clusters you want to create link. I don't know how to estimate the number you want to create, but I guess you could know.

2. You initialize descriptors_scene_clusters only with one element:

Mat descriptors_scene_clusters[3] = { Mat(descriptors_scene.rows, descriptors_scene.cols, CV_8U, Scalar(0)) };

And when you iterate over it:

for (int i=0; i<labels.rows; i++) {
    int clusterIndex = labels.at<int>(i);
    Point2f pt = keypoints_scene_points.at<Point2f>(i);
    descriptors_scene_clusters[clusterIndex].at<uchar>(pt) = descriptors_scene.at<uchar>(pt);  // ?????? error
}

clusterIndex is 2 and you access an uninitialized element in the array, which results in the EXEC_BAD_ACCESS error.

I hope this helps for further investigation!

Barcroft