OpenCV StereoSGBM gives really bad disparity map

Question

these is my calibration code:

  void calibrate()
 {
    int numBoards = 10;
    int board_w = 6;
    int board_h = 9;
    Size board_sz = Size(board_w, board_h);
    int board_n = board_w*board_h;
    vector<vector<Point3f> > object_points;
    vector<vector<Point2f> > imagePoints1, imagePoints2;
    vector<Point2f> corners1, corners2;
    vector<Point3f> obj;
    for (int j=0; j<board_n; j++)
    {
        obj.push_back(Point3f(j/board_w, j%board_w, 0.0f));
    }
    Mat img1, img2, gray1, gray2;
    VideoCapture cap1(0);
    VideoCapture cap2(1);
    int success = 0, k = 0;
    bool found1 = false, found2 = false;
    namedWindow("left 1");
    namedWindow("right 1");
    while (success < numBoards)
    {
        cap1 >> img1;
        cap2 >> img2;

        cvtColor(img1, gray1, CV_BGR2GRAY);
        cvtColor(img2, gray2, CV_BGR2GRAY);
        found1 = findChessboardCorners(img1, board_sz, corners1, CV_CALIB_CB_ADAPTIVE_THRESH | CV_CALIB_CB_FILTER_QUADS);
        found2 = findChessboardCorners(img2, board_sz, corners2, CV_CALIB_CB_ADAPTIVE_THRESH | CV_CALIB_CB_FILTER_QUADS);
        if (found1)
        {
            cornerSubPix(gray1, corners1, Size(11, 11), Size(-1, -1), TermCriteria(CV_TERMCRIT_EPS | CV_TERMCRIT_ITER, 30, 0.1));
            drawChessboardCorners(gray1, board_sz, corners1, found1);
        }
        if (found2)
        {
            cornerSubPix(gray2, corners2, Size(11, 11), Size(-1, -1), TermCriteria(CV_TERMCRIT_EPS | CV_TERMCRIT_ITER, 30, 0.1));
            drawChessboardCorners(gray2, board_sz, corners2, found2);
        }
        imshow("left 1", img1);
        imshow("right 1", img2);
        k = cv::waitKey(2);
        if(found1) cout<<"found 1"<<endl;
        if(found2) cout<<"found 2"<<endl;
        if(!found1 && !found2) cout<<"no"<<endl;

        if ( found1  && found2)
        {
            imagePoints1.push_back(corners1);
            imagePoints2.push_back(corners2);
            object_points.push_back(obj);
            printf ("Corners stored\n");
            success++;
            if (success >= numBoards)
            {
                break;
            }
        }
    }
    destroyAllWindows();
    printf("Starting Calibration\n");
    Mat CM1 = Mat(3, 3, CV_64FC1);
    Mat CM2 = Mat(3, 3, CV_64FC1);
    Mat D1, D2;
    Mat R, T, E, F;
    stereoCalibrate(object_points, imagePoints1, imagePoints2,
    CM1, D1, CM2, D2, img1.size(), R, T, E, F,
    cvTermCriteria(CV_TERMCRIT_ITER+CV_TERMCRIT_EPS, 100, 1e-5),
    CV_CALIB_SAME_FOCAL_LENGTH | CV_CALIB_ZERO_TANGENT_DIST);
    FileStorage fs1("mystereocalib.yml", FileStorage::WRITE);
    fs1 << "CM1" << CM1;
    fs1 << "CM2" << CM2;
    fs1 << "D1" << D1;
    fs1 << "D2" << D2;
    fs1 << "R" << R;
    fs1 << "T" << T;
    fs1 << "E" << E;
    fs1 << "F" << F;
    printf("Done Calibration\n");
    printf("Starting Rectification\n");
    Mat R1, R2, P1, P2, Q;
    stereoRectify(CM1, D1, CM2, D2, img1.size(), R, T, R1, R2, P1, P2, Q);
    fs1 << "R1" << R1;
    fs1 << "R2" << R2;
    fs1 << "P1" << P1;
    fs1 << "P2" << P2;
    fs1 << "Q" << Q;
    printf("Done Rectification\n");
    printf("Applying Undistort\n");
    Mat map1x, map1y, map2x, map2y;
    Mat imgU1, imgU2;
    initUndistortRectifyMap(CM1, D1, R1, P1, img1.size(), CV_32FC1, map1x, map1y);
    initUndistortRectifyMap(CM2, D2, R2, P2, img2.size(), CV_32FC1, map2x, map2y);
    printf("Undistort complete\n");

    cap1.release();
    cap2.release();
    return;
 }

And this is my disparity map code with different params:

    int main(int argc, char* argv[])
{
    //calibrate();

    Mat img1, img2;

    VideoCapture cap1 = VideoCapture(0);
    VideoCapture cap2 = VideoCapture(1);

    cap1 >> img1;
    cap2 >> img2;

    Mat Q;
    FileStorage fs("mystereocalib.yml", FileStorage::READ);
    fs["Q"] >> Q;

    Mat CM1, CM2, D1, D2, P1, P2, R1, R2;

    fs["CM1"] >> CM1;
    fs["CM2"] >> CM2;
    fs["D1"] >> D1;
    fs["D2"] >> D2;
    fs["P1"] >> P1;
    fs["P2"] >> P2;
    fs["R1"] >> R1;
    fs["R2"] >> R2;

    fs.release();

    Mat map1x, map1y, map2x, map2y;
    Mat imgU1, imgU2;
    initUndistortRectifyMap(CM1, D1, R1, P1, img1.size(), CV_32FC1, map1x, map1y);
    initUndistortRectifyMap(CM2, D2, R2, P2, img2.size(), CV_32FC1, map2x, map2y);


    while(1) 
    {
    cap1 >> img1;
    cap2 >> img2;

    imshow("img1", img1);
    imshow("img2", img2);
    cv::waitKey(1);

    remap(img1, imgU1, map1x, map1y, INTER_LINEAR, BORDER_CONSTANT, Scalar());
    remap(img2, imgU2, map2x, map2y, INTER_LINEAR, BORDER_CONSTANT, Scalar());


    Mat g1,g2, disp, disp8;

    cvtColor(imgU1, g1, CV_BGR2GRAY);
    cvtColor(imgU2, g2, CV_BGR2GRAY);

    /*StereoSGBM sbm;
    sbm.SADWindowSize = 5;
    sbm.numberOfDisparities = 144;
    sbm.preFilterCap = 63;
    sbm.minDisparity = -39;
    sbm.uniquenessRatio = 10;
    sbm.speckleWindowSize = 100;
    sbm.speckleRange = 32;
    sbm.disp12MaxDiff = 2;
    sbm.fullDP = true;
    sbm.P1 = 216;
    sbm.P2 = 864;
    sbm(g1, g2, disp);*/

    int sadSize = 3;
    StereoSGBM sbm;
    sbm.SADWindowSize = sadSize;
    sbm.numberOfDisparities = 128;//144;
    sbm.preFilterCap = 63;
    sbm.minDisparity = 0; //-39;
    sbm.uniquenessRatio = 10;
    sbm.speckleWindowSize = 100;
    sbm.speckleRange = 32;
    sbm.disp12MaxDiff = 1;
    sbm.fullDP = true;
    sbm.P1 = sadSize*sadSize*4;
    sbm.P2 = sadSize*sadSize*32;
    sbm(g1, g2, disp);

    //StereoSGBM sgbm;
    //sgbm.SADWindowSize = 5;
    //sgbm.numberOfDisparities = 192;
    //sgbm.preFilterCap = 4;
    //sgbm.minDisparity = -64;
    //sgbm.uniquenessRatio = 1;
    //sgbm.speckleWindowSize = 150;
    //sgbm.speckleRange = 2;
    //sgbm.disp12MaxDiff = 10;
    //sgbm.fullDP = false;
    //sgbm.P1 = 600;
    //sgbm.P2 = 2400;
    //sgbm(g1, g2, disp);


    //StereoBM sbm;
    //sbm.state->SADWindowSize = 9;
    //sbm.state->numberOfDisparities = 112;
    //sbm.state->preFilterSize = 5;
    //sbm.state->preFilterCap = 61;
    //sbm.state->minDisparity = -39;
    //sbm.state->textureThreshold = 507;
    //sbm.state->uniquenessRatio = 0;
    //sbm.state->speckleWindowSize = 0;
    //sbm.state->speckleRange = 8;
    //sbm.state->disp12MaxDiff = 1;
    //sbm(g1, g2, disp);


    normalize(disp, disp8, 0, 255, CV_MINMAX, CV_8U);
    //disp.convertTo(disp8, CV_8U);

    imshow("disp8", disp8);


    }

    waitKey(0);

    return 0;
}

I'm new in OpenCV and image processing, what I'm doing wrong?

Sleep

stereoCalibrate() returns an err value, please check, if your calibration already was bad. – Mechanize

@Mechanize Ok, thanks I'll check. Maybe numBoards=10 is'not enough for good calibaration? – Sleep

unfortunately, that's the 'experimental' part of it. try between 10 and 30 boards, see to get the err as low as possible – Mechanize

@berak: and what is acceptable err value? – Sleep

as small as possible. anything >0.5 will be terrible – Mechanize

@berak: I see, I get error 1.75 and I think my before calibration was worse. – Sleep

So calibration is so important for stereoSGMB, everything else I do right? – Sleep

yes, very much so (also, getting this right will be painful..) – Mechanize

@berak: Ok, thanks. Please could you also advice parametrs for SBGM? – Sleep

sorry, but no, i can't . those also heavily depend on your images. – Mechanize

Answer 1

Try to use this calibration file - https://github.com/foxymop/3DPoseEstimation/blob/master/src/out_camera_data.yml I was trying to calibrate my camera as well, but getting small error is really hard and using this configurations gave me better results than trying to find parameters of my camera (most likely my camera is similar to the one from calibration file). Eventually you may try to not use calibration file at all.

Artistic

sorry but you did not get the main part of it: the calibration file is the output of the calibration of the actual camera, you can't arbitrarily exchange data files there. – Mechanize

please delete this answer, it's not valid. it was pure chance, that your 'solution' worked in your single case. – Mechanize

I know about that and as i wrote in my post i've had similar problem - calibrating my camera gave really poor results so i've used this calibration file and it worked much better than any of my attempts. I know that it is strange, but just give it a try, maybe it will work for you as well. Also don't forget that lot of cameras are really similar to each other (or at least their lenses are similar). Note the last sentence of my post as well. – Artistic

again, pure chance. (not a valid option) – Mechanize

Pure chance, but you can't tell whether it will work for you or not until you try - maybe you camera is similar to this one as well (you didn't specified what is your cam, so we can only guess). Also try without any calibration file - bad calibration file definitely will cause really bad results, no calibration file at least will let you check whether you StereoSGBM code is correct. – Artistic

@cyriel: it was strange for me too, but I want to try it. But I need calibration for 2 cameras, and also transformation and rotation matrixes between them. – Sleep