c++ - 使用掩码 RCNN 从运动结构中重建对象

Question

我正在从运动的结构中进行对象重建。目前的情况是，我正在获取汽车的多个视图，并为该对象应用蒙版 rcnn 以移除背景，因为我只希望该对象重建并拥有一个干净的对象。

1. 我目前的问题是对象没有完全重建。
2. 我从 Mask RCNN 获得的掩码并不总是具有固定大小以供 SFM 工作
3. 场景重建对象中仍然存在背景噪声
4. 当我仅使用从不同视图获得的蒙版时，相机参数会混乱，如何解决？

以下是一些结果：

原始图像当然有它的多个视图

我用于 SFM 的掩码 RCNN 结果

这是 SFM 的结果

// Draw the predicted bounding box, colorize and show the mask on the image
void drawBox(Mat& frame, int classId, float conf, Rect box, Mat& objectMask, std::vector<Mat> &contours_images)
{
    //Draw a rectangle displaying the bounding box
    //rectangle(frame, Point(box.x, box.y), Point(box.x + box.width, box.y + box.height), Scalar(255, 178, 50), 3);

    //Get the label for the class name and its confidence
    string label = format("%.2f", conf);
    if (!classes.empty())
    {
        CV_Assert(classId < (int)classes.size());
        label = classes[classId] + ":" + label;
    }

    //Display the label at the top of the bounding box
    int baseLine;
    Size labelSize = getTextSize(label, FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine);
    box.y = max(box.y, labelSize.height);
    //rectangle(frame, Point(box.x, box.y - round(1.5 * labelSize.height)), Point(box.x + round(1.5 * labelSize.width), box.y + baseLine), Scalar(255, 255, 255), FILLED);
    //putText(frame, label, Point(box.x, box.y), FONT_HERSHEY_SIMPLEX, 0.75, Scalar(0, 0, 0), 1);

    // Resize the mask, threshold, color and apply it on the image

   // Scalar color = colors[classId % colors.size()];

    // Resize the mask, threshold, color and apply it on the image
    resize(objectMask, objectMask, Size(box.width, box.height));
    Mat mask = (objectMask > maskThreshold);
    //Mat coloredRoi = (0.3 * color + 0.7 * frame(box));
   // coloredRoi.convertTo(coloredRoi, CV_8UC3);
    Mat coloredRoi(frame(box));
    // Draw the contours on the image
    vector<Mat> contours;
    Mat hierarchy;
    mask.convertTo(mask, CV_8U);
    findContours(mask, contours, hierarchy, RETR_CCOMP, cv::CHAIN_APPROX_NONE);
    //drawContours(coloredRoi, contours, -1, color, 5, LINE_8, hierarchy, 100);
  //  coloredRoi.copyTo(frame(box), mask);
    Mat outframe;
    coloredRoi.copyTo(outframe, mask);
    cv::resize(outframe, outframe, cv::Size(400, 400));
    contours_images.push_back(outframe);
   // imshow("outframe", outframe);
   // waitKey(0);
 }

    // For each frame, extract the bounding box and mask for each detected object
    void postprocess(Mat& frame, const vector<Mat>& outs, vector<Mat> & maskes)
    {
        Mat outDetections = outs[0];
        Mat outMasks = outs[1];
    
        // Output size of masks is NxCxHxW where
        // N - number of detected boxes
        // C - number of classes (excluding background)
        // HxW - segmentation shape
        const int numDetections = outDetections.size[2];
        const int numClasses = outMasks.size[1];
    
        outDetections = outDetections.reshape(1, outDetections.total() / 7);
        for (int i = 0; i < numDetections; ++i)
        {
            float score = outDetections.at<float>(i, 2);
            if (score > confThreshold)
            {
                // Extract the bounding box
                int classId = static_cast<int>(outDetections.at<float>(i, 1));
                int left = static_cast<int>(frame.cols * outDetections.at<float>(i, 3));
                int top = static_cast<int>(frame.rows * outDetections.at<float>(i, 4));
                int right = static_cast<int>(frame.cols * outDetections.at<float>(i, 5));
                int bottom = static_cast<int>(frame.rows * outDetections.at<float>(i, 6));
    
                left = max(0, min(left, frame.cols - 1));
                top = max(0, min(top, frame.rows - 1));
                right = max(0, min(right, frame.cols - 1));
                bottom = max(0, min(bottom, frame.rows - 1));
                Rect box = Rect(left, top, right - left + 1, bottom - top + 1);
    
                // Extract the mask for the object
                Mat objectMask(outMasks.size[2], outMasks.size[3], CV_32F, outMasks.ptr<float>(i, classId));
    
                // Draw bounding box, colorize and show the mask on the image
                drawBox(frame, classId, score, box, objectMask, maskes);
    
            }
        }
    }