使用光流进行人脸标记和稳定化

Question

我在窗口显示面和一些特殊点时编写程序（68）。 我使用 Haar casscade 和 FaceLandmarkLBF。我的程序有问题。当面部有稳定的位置时，面部点会抖动（抖动）。我该如何解决？谢谢。

#include <iostream>

#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/objdetect/objdetect.hpp>

#include <opencv2/face.hpp>

using cv::Scalar;
using cv::Point;

int main(int argc, char** argv)
{
    cv::CascadeClassifier faceDetector("haarcascade_frontalface_alt2.xml");

    cv::Ptr<cv::face::Facemark>facemark = cv::face::FacemarkLBF::create();

    facemark->loadModel("lbfmodel.yaml");

    cv::VideoCapture vc(0);

    while (true)
    {
        cv::Mat frame, gray;

        vc.read(frame);

        cv::cvtColor(frame, gray, cv::COLOR_BGR2GRAY);
        //
        std::vector<cv::Rect> faces;

        faceDetector.detectMultiScale(gray, faces);

        std::vector< std::vector<cv::Point2f> > landmarks;

        bool success = facemark->fit(frame, faces, landmarks);
        for (size_t i = 0; i < landmarks.size(); i++)
        {
            for (size_t j = 0; j < landmarks[i].size(); j++)
            {
                cv::circle(frame, cv::Point(landmarks[i][j].x, landmarks[i][j].y), 2, Scalar(255, 0, 0), 2);
            }
        }
        cv::imshow("1", frame);

        if ((char)cv::waitKey(20) == 27)
            break;

    }

    return 0;
}

我看到了@Nuzhny 链接：lkdemo.cpp。对我来说，并非一切都清楚。 我重写了我的代码，但没有任何改变：

#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/objdetect/objdetect.hpp>

#include "opencv2/video/tracking.hpp"

#include <opencv2/face.hpp>

int main(int argc, char** argv)
{
    cv::CascadeClassifier faceDetector("haarcascade_frontalface_alt2.xml");

    cv::Ptr<cv::face::Facemark>facemark = cv::face::FacemarkLBF::create();

    facemark->loadModel("lbfmodel.yaml");

    cv::VideoCapture vc(0);

    cv::Mat gray, prevGray, image, frame;
    cv::Size subPixWinSize(10, 10), winSize(64, 64);
    cv::TermCriteria termcrit(cv::TermCriteria::COUNT | cv::TermCriteria::EPS, 20, 0.03);
    std::vector<uchar> status;
    std::vector<float> err;

    std::vector<cv::Point2f> oldLandmarks;
    std::vector< std::vector<cv::Point2f> > landmarks;

    bool b = true;
    while (true)
    {

        vc.read(frame);

        cv::cvtColor(frame, gray, cv::COLOR_BGR2GRAY);

        std::vector<cv::Rect> faces;

        faceDetector.detectMultiScale(gray, faces);

        bool success = facemark->fit(frame, faces, landmarks);

        if (!success)
        {
            cv::imshow("1", frame);
            continue;
        }
        if (oldLandmarks.empty())
            oldLandmarks = landmarks.front();

        if (prevGray.empty())
            gray.copyTo(prevGray);

        calcOpticalFlowPyrLK(prevGray, gray, landmarks.front(), oldLandmarks, status, err, winSize, 3, termcrit, cv::OPTFLOW_LK_GET_MIN_EIGENVALS, 0.001);
        for (size_t i = 0; i < oldLandmarks.size(); i++)
        {
            cv::circle(frame, cv::Point(oldLandmarks[i].x, oldLandmarks[i].y), 2, cv::Scalar(255, 0, 0), 2);
        }

        cv::imshow("1", frame);

        std::swap(oldLandmarks, landmarks.front());
        cv::swap(prevGray, gray);

        if ((char)cv::waitKey(20) == 27)
            break;
    }

    return 0;
}

Answer 1

仅 LK 跟踪可能还不够。我正在编写一些简单的应用程序，用于在 LK 之后使用线性卡尔曼滤波器校正地标（EDIT 2 - 删除上一个地标）：

#include <opencv2/opencv.hpp>
#include <opencv2/face.hpp>

///
class PointState
{
public:
    PointState(cv::Point2f point)
        :
        m_point(point),
        m_kalman(4, 2, 0, CV_64F)
    {
        Init();
    }

    void Update(cv::Point2f point)
    {
        cv::Mat measurement(2, 1, CV_64FC1);
        if (point.x < 0 || point.y < 0)
        {
            Predict();
            measurement.at<double>(0) = m_point.x;  //update using prediction
            measurement.at<double>(1) = m_point.y;

            m_isPredicted = true;
        }
        else
        {
            measurement.at<double>(0) = point.x;  //update using measurements
            measurement.at<double>(1) = point.y;

            m_isPredicted = false;
        }

        // Correction
        cv::Mat estimated = m_kalman.correct(measurement);
        m_point.x = static_cast<float>(estimated.at<double>(0));   //update using measurements
        m_point.y = static_cast<float>(estimated.at<double>(1));

        Predict();
    }

    cv::Point2f GetPoint() const
    {
        return m_point;
    }

    bool IsPredicted() const
    {
        return m_isPredicted;
    }

private:
    cv::Point2f m_point;
    cv::KalmanFilter m_kalman;

    double m_deltaTime = 0.2;
    double m_accelNoiseMag = 0.3;

    bool m_isPredicted = false;

    void Init()
    {
        m_kalman.transitionMatrix = (cv::Mat_<double>(4, 4) <<
            1, 0, m_deltaTime, 0,
            0, 1, 0, m_deltaTime,
            0, 0, 1, 0,
            0, 0, 0, 1);

        m_kalman.statePre.at<double>(0) = m_point.x; // x
        m_kalman.statePre.at<double>(1) = m_point.y; // y

        m_kalman.statePre.at<double>(2) = 1; // init velocity x
        m_kalman.statePre.at<double>(3) = 1; // init velocity y

        m_kalman.statePost.at<double>(0) = m_point.x;
        m_kalman.statePost.at<double>(1) = m_point.y;

        cv::setIdentity(m_kalman.measurementMatrix);

        m_kalman.processNoiseCov = (cv::Mat_<double>(4, 4) <<
            pow(m_deltaTime, 4.0) / 4.0, 0, pow(m_deltaTime, 3.0) / 2.0, 0,
            0, pow(m_deltaTime, 4.0) / 4.0, 0, pow(m_deltaTime, 3.0) / 2.0,
            pow(m_deltaTime, 3.0) / 2.0, 0, pow(m_deltaTime, 2.0), 0,
            0, pow(m_deltaTime, 3.0) / 2.0, 0, pow(m_deltaTime, 2.0));


        m_kalman.processNoiseCov *= m_accelNoiseMag;

        cv::setIdentity(m_kalman.measurementNoiseCov, cv::Scalar::all(0.1));

        cv::setIdentity(m_kalman.errorCovPost, cv::Scalar::all(.1));
    }

    cv::Point2f Predict()
    {
        cv::Mat prediction = m_kalman.predict();
        m_point.x = static_cast<float>(prediction.at<double>(0));
        m_point.y = static_cast<float>(prediction.at<double>(1));
        return m_point;
    }
};

///
void TrackPoints(cv::Mat prevFrame, cv::Mat currFrame,
const std::vector<cv::Point2f>& currLandmarks,
std::vector<PointState>& trackPoints)
{
    // Lucas-Kanade
    cv::TermCriteria termcrit(cv::TermCriteria::COUNT | cv::TermCriteria::EPS, 30, 0.01);
    cv::Size winSize(7, 7);

    std::vector<uchar> status(trackPoints.size(), 0);
    std::vector<float> err;
    std::vector<cv::Point2f> newLandmarks;

    std::vector<cv::Point2f> prevLandmarks;
    std::for_each(trackPoints.begin(), trackPoints.end(), [&](const PointState& pts) { prevLandmarks.push_back(pts.GetPoint()); });

    cv::calcOpticalFlowPyrLK(prevFrame, currFrame, prevLandmarks, newLandmarks, status, err, winSize, 3, termcrit, 0, 0.001);

    for (size_t i = 0; i < status.size(); ++i)
    {
        if (status[i])
        {
            trackPoints[i].Update((newLandmarks[i] + currLandmarks[i]) / 2);
        }
        else
        {
            trackPoints[i].Update(currLandmarks[i]);
        }
    }
}

///
int main(int argc, char** argv)
{
    cv::CascadeClassifier faceDetector("haarcascade_frontalface_alt2.xml");

    cv::Ptr<cv::face::Facemark> facemark = cv::face::FacemarkLBF::create();    

    facemark->loadModel("lbfmodel.yaml");

    cv::VideoCapture cam(0, cv::CAP_DSHOW);

    cv::namedWindow("Facial Landmark Detection", cv::WINDOW_NORMAL);

    cv::Mat frame;
    cv::Mat currGray;
    cv::Mat prevGray;

    std::vector<PointState> trackPoints;
    trackPoints.reserve(68);

    while (cam.read(frame))
    {
        std::vector<cv::Rect> faces;
        cv::cvtColor(frame, currGray, cv::COLOR_BGR2GRAY);

        faceDetector.detectMultiScale(currGray, faces, 1.1, 3, cv::CASCADE_FIND_BIGGEST_OBJECT);

        std::vector<std::vector<cv::Point2f>> landmarks;

        bool success = facemark->fit(frame, faces, landmarks);

        if (success)
        {
            if (prevGray.empty())
            {
                trackPoints.clear();

                for (cv::Point2f lp : landmarks[0])
                {
                    trackPoints.emplace_back(lp);
                }
            }
            else
            {
                if (trackPoints.empty())
                {
                    for (cv::Point2f lp : landmarks[0])
                    {
                        trackPoints.emplace_back(lp);
                    }
                }
                else
                {
                    TrackPoints(prevGray, currGray, landmarks[0], trackPoints);
                }
            }

            for (const PointState& tp : trackPoints)
            {
                cv::circle(frame, tp.GetPoint(), 3, tp.IsPredicted() ? cv::Scalar(0, 0, 255) : cv::Scalar(0, 255, 0), cv::FILLED);
            }

            for (cv::Point2f lp : landmarks[0])
            {
                cv::circle(frame, lp, 2, cv::Scalar(255, 0, 255), cv::FILLED);
            }
        }

        cv::imshow("Facial Landmark Detection", frame);
        if (cv::waitKey(1) == 27)
            break;

        prevGray = currGray;
    }
    return 0;
}

所以，在 LK+Kalman 之后，margenta 点 - 原始地标和绿色点 - 更正：result video。

您可以使用 2 个常量更改卡尔曼选项：

double m_deltaTime = 0.2;
double m_accelNoiseMag = 0.3;

这是延迟和噪音。