I am attempting to use OpenCV to take stereo image pairs...ie a left and a right image of the same subject...and then correct them for rotation and translation without knowing any of the properties of the camera. Once the images are corrected I should be able to display them to the user.
So far I have merged two demo programs from the OpenCV samples directory, badly for the moment...I will clean the code up and arrange it more nicely when I get it working...and it seems to be working, however when I attempt to display the results the program crashes with a debug error. In the command window it says "OpenCV Error: Assertion failed (scn ==1 && (dcn == 3 || dcn == 4)) in unknown function in file ........\opencv\modules\imgproc\src\color.cpp, line 2453"
Commenting out various parts of the code to display the results just results in different OpenCV Errors. Here's my code. If anyone can help I will love you forever.
#include "stdafx.h"
#include "opencv2/highgui/highgui.hpp"
#include "opencv2/calib3d/calib3d.hpp"
#include "opencv2/imgproc/imgproc.hpp"
#include "opencv2/features2d/features2d.hpp"
#include <iostream>
using namespace cv;
using namespace std;
void help(char** argv)
{
cout << "\nThis program demonstrates keypoint finding and matching between 2 images using features2d framework.\n"
<< "Example of usage:\n"
<< argv[0] << " [detectorType] [descriptorType] [image1] [image2] [ransacReprojThreshold]\n"
<< "\n"
<< "Matches are filtered using homography matrix if ransacReprojThreshold>=0\n"
<< "Example:\n"
<< "./descriptor_extractor_matcher SURF SURF cola1.jpg cola2.jpg 3\n"
<< "\n"
<< "Possible detectorType values: see in documentation on createFeatureDetector().\n"
<< "Possible descriptorType values: see in documentation on createDescriptorExtractor().\n" << endl;
}
const string winName = "rectified";
void crossCheckMatching( Ptr<DescriptorMatcher>& descriptorMatcher,
const Mat& descriptors1, const Mat& descriptors2,
vector<DMatch>& filteredMatches12, int knn=1 )
{
filteredMatches12.clear();
vector<vector<DMatch> > matches12, matches21;
descriptorMatcher->knnMatch( descriptors1, descriptors2, matches12, knn );
descriptorMatcher->knnMatch( descriptors2, descriptors1, matches21, knn );
for( size_t m = 0; m < matches12.size(); m++ )
{
bool findCrossCheck = false;
for( size_t fk = 0; fk < matches12[m].size(); fk++ )
{
DMatch forward = matches12[m][fk];
for( size_t bk = 0; bk < matches21[forward.trainIdx].size(); bk++ )
{
DMatch backward = matches21[forward.trainIdx][bk];
if( backward.trainIdx == forward.queryIdx )
{
filteredMatches12.push_back(forward);
findCrossCheck = true;
break;
}
}
if( findCrossCheck ) break;
}
}
}
void doIteration( const Mat& leftImg, Mat& rightImg,
vector<KeyPoint>& keypoints1, const Mat& descriptors1,
Ptr<FeatureDetector>& detector, Ptr<DescriptorExtractor>& descriptorExtractor,
Ptr<DescriptorMatcher>& descriptorMatcher,
double ransacReprojThreshold )
{
assert( !leftImg.empty() );
Mat H12;
assert( !rightImg.empty()/* && rightImg.cols==leftImg.cols && rightImg.rows==leftImg.rows*/ );
cout << endl << "< Extracting keypoints from second image..." << endl;
vector<KeyPoint> keypoints2;
detector->detect( rightImg, keypoints2 );
cout << keypoints2.size() << " points" << endl << ">" << endl;
cout << "< Computing descriptors for keypoints from second image..." << endl;
Mat descriptors2;
descriptorExtractor->compute( rightImg, keypoints2, descriptors2 );
cout << ">" << endl;
cout << "< Matching descriptors..." << endl;
vector<DMatch> filteredMatches;
crossCheckMatching( descriptorMatcher, descriptors1, descriptors2, filteredMatches, 1 );
cout << ">" << endl;
vector<int> queryIdxs( filteredMatches.size() ), trainIdxs( filteredMatches.size() );
for( size_t i = 0; i < filteredMatches.size(); i++ )
{
queryIdxs[i] = filteredMatches[i].queryIdx;
trainIdxs[i] = filteredMatches[i].trainIdx;
}
cout << "< Computing homography (RANSAC)..." << endl;
vector<Point2f> points1; KeyPoint::convert(keypoints1, points1, queryIdxs);
vector<Point2f> points2; KeyPoint::convert(keypoints2, points2, trainIdxs);
H12 = findHomography( Mat(points1), Mat(points2), CV_RANSAC, ransacReprojThreshold );
cout << ">" << endl;
//Mat drawImg;
if( !H12.empty() ) // filter outliers
{
vector<char> matchesMask( filteredMatches.size(), 0 );
vector<Point2f> points1; KeyPoint::convert(keypoints1, points1, queryIdxs);
vector<Point2f> points2; KeyPoint::convert(keypoints2, points2, trainIdxs);
Mat points1t; perspectiveTransform(Mat(points1), points1t, H12);
for( size_t i1 = 0; i1 < points1.size(); i1++ )
{
if( norm(points2[i1] - points1t.at<Point2f>((int)i1,0)) < 4 ) // inlier
matchesMask[i1] = 1;
}
/* draw inliers
drawMatches( leftImg, keypoints1, rightImg, keypoints2, filteredMatches, drawImg, CV_RGB(0, 255, 0), CV_RGB(0, 0, 255), matchesMask, 2 ); */
}
Size imageSize = leftImg.size();
Mat F = findFundamentalMat(Mat(points1), Mat(points2), FM_8POINT, 0, 0);
Mat H1, H2;
stereoRectifyUncalibrated(Mat(points1), Mat(points2), F, imageSize, H1, H2, 3);
Mat cameraMatrix[2], distCoeffs[2], R1, R2, P1, P2, rmap[2][2];
cameraMatrix[0] = Mat::eye(3, 3, CV_64F);
cameraMatrix[1] = Mat::eye(3, 3, CV_64F);
R1 = cameraMatrix[0].inv()*H1*cameraMatrix[0];
R2 = cameraMatrix[1].inv()*H2*cameraMatrix[1];
P1 = cameraMatrix[0];
P2 = cameraMatrix[1];
initUndistortRectifyMap(cameraMatrix[0], distCoeffs[0], R1, P1, imageSize, CV_16SC2, rmap[0][0], rmap[0][1]);
initUndistortRectifyMap(cameraMatrix[1], distCoeffs[1], R2, P2, imageSize, CV_16SC2, rmap[1][0], rmap[1][1]);
Mat canvas, img;
double sf;
int i, j, w, h;
sf = 600./MAX(imageSize.width, imageSize.height);
w = cvRound(imageSize.width*sf);
h = cvRound(imageSize.height*sf);
canvas.create(h, w*2, CV_8UC3);
for (i = 0; i < 2; i++)
{
if (i == 0)
img = leftImg;
else
img = rightImg;
Mat rimg, cimg;
remap(img, rimg, rmap[i][0], rmap[i][1], CV_INTER_LINEAR);
cvtColor(rimg, cimg, CV_GRAY2BGR);
Mat canvasPart = canvas(Rect(w*i, 0, w, h));
resize(cimg, canvasPart, canvasPart.size(), 0, 0, CV_INTER_AREA);
}
for( j = 0; j < canvas.rows; j += 16 )
{
line(canvas, Point(0, j), Point(canvas.cols, j), Scalar(0, 255, 0), 1, 8);
}
imshow(winName, canvas);
}
int main(int argc, char** argv)
{
if( argc != 6 )
{
help(argv);
return -1;
}
double ransacReprojThreshold = atof(argv[5]);
cout << "< Creating detector, descriptor extractor and descriptor matcher ..." << endl;
Ptr<FeatureDetector> detector = FeatureDetector::create( argv[1] );
Ptr<DescriptorExtractor> descriptorExtractor = DescriptorExtractor::create( argv[2] );
Ptr<DescriptorMatcher> descriptorMatcher = DescriptorMatcher::create("FlannBased");
cout << ">" << endl;
if( detector.empty() || descriptorExtractor.empty() || descriptorMatcher.empty() )
{
cout << "Can not create detector or descriptor extractor or descriptor matcher of given types" << endl;
return -1;
}
cout << "< Reading the images..." << endl;
Mat leftImg = imread( argv[3] );
Mat rightImg = imread( argv[4] );
cout << ">" << endl;
if( leftImg.empty() || ( rightImg.empty()) )
{
cout << "Can not read images" << endl;
return -1;
}
cout << endl << "< Extracting keypoints from first image..." << endl;
vector<KeyPoint> keypoints1;
detector->detect( leftImg, keypoints1 );
cout << keypoints1.size() << " points" << endl << ">" << endl;
cout << "< Computing descriptors for keypoints from first image..." << endl;
Mat descriptors1;
descriptorExtractor->compute( leftImg, keypoints1, descriptors1 );
cout << ">" << endl;
namedWindow(winName, CV_WINDOW_NORMAL);
doIteration( leftImg, rightImg, keypoints1, descriptors1,
detector, descriptorExtractor, descriptorMatcher,
ransacReprojThreshold );
for(;;)
{
char c = (char)waitKey(0);
if( c == '\x1b' ) // esc
{
cout << "Exiting ..." << endl;
return 0;
}
}
waitKey(0);
return 0;
}
The main focus should probably be around the doIteration method, but I've put the rest of it in there so you can see exactly what is going on.