devbox
不正经
这个屌丝很懒，什么也没留下！
关注作者
热门标签
热门文章
当前位置:   article > 正文

探索OpenCV的光流算法视频超分与源码实现

作者：不正经 | 2024-02-13 13:39:41

赞

踩

视频超分

在OpenCV4.0以后，视频图像超分模块已经迁移到opencv_contrib独立仓库。在视频超分有两种形式：结合光流算法实现超分、使用CNN卷积神经网络实现超分。在这里主要探索光流算法实现视频超分，然后进一步分析源码实现。

一、视频超分示例

1、光流算法选择

OpenCV提供多种光流算法：farneback、tvl1、brox、pyrlk。同时支持GPU加速，示例代码如下：

  1. static Ptr<cv::superres::DenseOpticalFlowExt> createOptFlow(const string& name, bool useGpu)
  2. {
  3.     if (name == "farneback")
  4.     {
  5.         if (useGpu) // 开启GPU加速，使用CUDA实现
  6.             return cv::superres::createOptFlow_Farneback_CUDA();
  7.         else
  8.             return cv::superres::createOptFlow_Farneback();
  9.     }
  10.     else if (name == "tvl1")
  11.     {
  12.         if (useGpu)
  13.             return cv::superres::createOptFlow_DualTVL1_CUDA();
  14.         else
  15.             return cv::superres::createOptFlow_DualTVL1();
  16.     }
  17.     else if (name == "brox")
  18.         return cv::superres::createOptFlow_Brox_CUDA();
  19.     else if (name == "pyrlk")
  20.         return cv::superres::createOptFlow_PyrLK_CUDA();
  21.     else
  22.         cerr << "Incorrect Optical Flow algorithm - " << name << endl;
  23.  
  24.     return Ptr<cv::superres::DenseOpticalFlowExt>();
  25. }

2、创建超分实例

根据是否使用CUDA进行GPU加速，来创建视频超分实例：

  1. Ptr<SuperResolution> createSuperResolution(bool useCuda)
  2. {
  3.     if (useCuda)
  4.         return createSuperResolution_BTVL1_CUDA();
  5.     else
  6.         return createSuperResolution_BTVL1();
  7. }

3、执行视频超分

在创建光流算法、视频超分实例后，设置相关参数，执行视频超分操作。示例代码如下：

  1. #include <iostream>
  2. #include <string>
  3. #include <ctype.h>
  4.  
  5. #include "opencv2/core.hpp"
  6. #include "opencv2/highgui.hpp"
  7. #include "opencv2/imgproc.hpp"
  8. #include "opencv2/superres.hpp"
  9. #include "opencv2/superres/optical_flow.hpp"
  10. #include "opencv2/opencv_modules.hpp"
  11.  
  12. using namespace std;
  13. using namespace cv;
  14. using namespace cv::superres;
  15.  
  16.  
  17. int main(int argc, const char* argv[])
  18. {
  19.     CommandLineParser cmd(argc, argv,
  20.         "{ v video      |           | Input video (mandatory)}"
  21.         "{ o output     |           | Output video }"
  22.         "{ s scale      | 4         | Scale factor }"
  23.         "{ i iterations | 180       | Iteration count }"
  24.         "{ t temporal   | 4         | Radius of the temporal search area }"
  25.         "{ f flow       | farneback | Optical flow algorithm (farneback, tvl1, brox, pyrlk) }"
  26.         "{ g gpu        | false     | CPU as default device, cuda for CUDA }"
  27.     );
  28.     // 解析参数
  29.     const string inputVideoName  = cmd.get<string>("video");
  30.     const string outputVideoName = cmd.get<string>("output");
  31.     const int scale              = cmd.get<int>("scale");
  32.     const int iterations         = cmd.get<int>("iterations");
  33.     const int temporalAreaRadius = cmd.get<int>("temporal");
  34.     const string optFlow         = cmd.get<string>("flow");
  35.     string gpuOption             = cmd.get<string>("gpu");
  36.     bool useCuda                 = gpuOption.compare("cuda") == 0;
  37.     // 创建视频超分实例
  38.     Ptr<SuperResolution> superRes = createSuperResolution(useCuda);
  39.     // 创建光流算法
  40.     Ptr<cv::superres::DenseOpticalFlowExt> of = createOptFlow(optFlow, useCuda);
  41.     // 设置光流算法、超分倍数、迭代次数、半径系数
  42.     superRes->setOpticalFlow(of);
  43.     superRes->setScale(scale);
  44.     superRes->setIterations(iterations);
  45.     superRes->setTemporalAreaRadius(temporalAreaRadius);
  46.     // 读取视频帧
  47.     Ptr<FrameSource> frameSource;
  48.     if (useCuda)
  49.     {
  50.         try
  51.         {
  52.             frameSource = createFrameSource_Video_CUDA(inputVideoName);
  53.             Mat frame;
  54.             frameSource->nextFrame(frame);
  55.         }
  56.         catch (const cv::Exception&)
  57.         {
  58.             frameSource.release();
  59.         }
  60.     }
  61.     if (!frameSource)
  62.         frameSource = createFrameSource_Video(inputVideoName);
  63.  
  64.     // 跳过第一帧
  65.     Mat frame;
  66.     frameSource->nextFrame(frame);
  67.     // 设置输入源
  68.     superRes->setInput(frameSource);
  69.  
  70.     VideoWriter writer;
  71.  
  72.     for (int i = 0;; ++i)
  73.     {
  74.         Mat result;
  75.         // 执行视频图像超分
  76.         superRes->nextFrame(result);
  77.  
  78.         if (result.empty())
  79.             break;
  80.  
  81.         imshow("Super Resolution", result);
  82.  
  83.         if (waitKey(1000) > 0)
  84.             break;
  85.  
  86.         // 视频帧超分结果写到输出文件
  87.         if (!outputVideoName.empty())
  88.         {
  89.             if (!writer.isOpened())
  90.                 writer.open(outputVideoName, VideoWriter::fourcc('X', 'V', 'I', 'D'), 25.0, result.size());
  91.             writer << result;
  92.         }
  93.     }
  94.  
  95.     return 0;
  96. }

二、视频超分源码

1、构造函数

视频图像超分的源码在opencv_contrib/modules/superres中，创建超分实例的函数位于btv_l1.cpp，其实是创建BTVL1()构造函数的智能指针：

  1. Ptr<cv::superres::SuperResolution> cv::superres::createSuperResolution_BTVL1()
  2. {
  3.     return makePtr<BTVL1>();
  4. }

2、超分入口代码

而nextFrame()是执行超分的函数，位于super_resolution.cpp：

  1. void cv::superres::SuperResolution::nextFrame(OutputArray frame)
  2. {
  3.     isUmat_ = frame.isUMat() && cv::ocl::useOpenCL();
  4.  
  5.     if (firstCall_)
  6.     {
  7.         initImpl(frameSource_);
  8.         firstCall_ = false;
  9.     }
  10.  
  11.     processImpl(frameSource_, frame);
  12. }

可以看到内部调用processImpl()函数来执行超分：

  1.     void BTVL1::processImpl(Ptr<FrameSource>& frameSource, OutputArray _output)
  2.     {
  3.         if (outPos_ >= storePos_)
  4.         {
  5.             _output.release();
  6.             return;
  7.         }
  8.         // 读取下一个视频帧
  9.         readNextFrame(frameSource);
  10.         // 处理视频帧
  11.         if (procPos_ < storePos_)
  12.         {
  13.             ++procPos_;
  14.             processFrame(procPos_);
  15.         }
  16.         ++outPos_;
  17.         // 调用ocl_processImpl函数执行超分
  18.         CV_OCL_RUN(isUmat_,
  19.                    ocl_processImpl(frameSource, _output))
  20.  
  21.         const Mat& curOutput = at(outPos_, outputs_);
  22.  
  23.         if (_output.kind() < _InputArray::OPENGL_BUFFER || _output.isUMat())
  24.             curOutput.convertTo(_output, CV_8U);
  25.         else
  26.         {
  27.             curOutput.convertTo(finalOutput_, CV_8U);
  28.             arrCopy(finalOutput_, _output);
  29.         }
  30.     }

3、光流检测运动矢量

接着看readNextFrame()的实现，调用光流检测算法来计算运动矢量：

  1. void BTVL1::readNextFrame(Ptr<FrameSource>& frameSource)
  2.     {
  3.         frameSource->nextFrame(curFrame_);
  4.         if (curFrame_.empty())
  5.             return;
  6.  
  7.         ++storePos_;
  8.  
  9.         CV_OCL_RUN(isUmat_,
  10.                    ocl_readNextFrame(frameSource))
  11.  
  12.         curFrame_.convertTo(at(storePos_, frames_), CV_32F);
  13.         // 结合上一帧和当前帧计算运动矢量
  14.         if (storePos_ > 0)
  15.         {
  16.             opticalFlow_->calc(prevFrame_, curFrame_, at(storePos_ - 1, forwardMotions_));
  17.             opticalFlow_->calc(curFrame_, prevFrame_, at(storePos_, backwardMotions_));
  18.         }
  19.  
  20.         curFrame_.copyTo(prevFrame_);
  21.     }

4、处理超分

processFrame()函数负责处理视频帧：

  1. void BTVL1::processFrame(int idx)
  2.     {
  3.         CV_OCL_RUN(isUmat_,
  4.                    ocl_processFrame(idx))
  5.  
  6.         const int startIdx = std::max(idx - temporalAreaRadius_, 0);
  7.         const int procIdx = idx;
  8.         const int endIdx = std::min(startIdx + 2 * temporalAreaRadius_, storePos_);
  9.  
  10.         const int count = endIdx - startIdx + 1;
  11.  
  12.         srcFrames_.resize(count);
  13.         srcForwardMotions_.resize(count);
  14.         srcBackwardMotions_.resize(count);
  15.  
  16.         int baseIdx = -1;
  17.  
  18.         for (int i = startIdx, k = 0; i <= endIdx; ++i, ++k)
  19.         {
  20.             if (i == procIdx)
  21.                 baseIdx = k;
  22.  
  23.             srcFrames_[k] = at(i, frames_);
  24.  
  25.             if (i < endIdx)
  26.                 srcForwardMotions_[k] = at(i, forwardMotions_);
  27.             if (i > startIdx)
  28.                 srcBackwardMotions_[k] = at(i, backwardMotions_);
  29.         }
  30.         // 根据前后方向的运动矢量来处理视频帧
  31.         process(srcFrames_, at(idx, outputs_), srcForwardMotions_, srcBackwardMotions_, baseIdx);
  32.     }
  33. }

在计算得到前后方向的运动矢量后，调用BTVL1_Base基类的process()函数来处理：

  1.     void BTVL1_Base::process(InputArrayOfArrays _src, OutputArray _dst, InputArrayOfArrays _forwardMotions,
  2.                              InputArrayOfArrays _backwardMotions, int baseIdx)
  3.     {
  4.         CV_OCL_RUN(_src.isUMatVector() && _dst.isUMat() && _forwardMotions.isUMatVector() &&
  5.                    _backwardMotions.isUMatVector(),
  6.                    ocl_process(_src, _dst, _forwardMotions, _backwardMotions, baseIdx))
  7.  
  8.         std::vector<Mat> & src = *(std::vector<Mat> *)_src.getObj(),
  9.                 & forwardMotions = *(std::vector<Mat> *)_forwardMotions.getObj(),
  10.                 & backwardMotions = *(std::vector<Mat> *)_backwardMotions.getObj();
  11.  
  12.         // 更新运动模糊(高斯滤波)、 btv权重
  13.         if (blurKernelSize_ != curBlurKernelSize_ || blurSigma_ != curBlurSigma_ || src[0].type() != curSrcType_)
  14.         {
  15.             //filter_ = createGaussianFilter(src[0].type(), Size(blurKernelSize_, blurKernelSize_), blurSigma_);
  16.             curBlurKernelSize_ = blurKernelSize_;
  17.             curBlurSigma_ = blurSigma_;
  18.             curSrcType_ = src[0].type();
  19.         }
  20.  
  21.         if (btvWeights_.empty() || btvKernelSize_ != curBtvKernelSize_ || alpha_ != curAlpha_)
  22.         {
  23.             calcBtvWeights(btvKernelSize_, alpha_, btvWeights_);
  24.             curBtvKernelSize_ = btvKernelSize_;
  25.             curAlpha_ = alpha_;
  26.         }
  27.  
  28.         // 计算相对运动
  29.         calcRelativeMotions(forwardMotions, backwardMotions, lowResForwardMotions_, lowResBackwardMotions_, baseIdx, src[0].size());
  30.         // 对运动矢量进行放大
  31.         upscaleMotions(lowResForwardMotions_, highResForwardMotions_, scale_);
  32.         upscaleMotions(lowResBackwardMotions_, highResBackwardMotions_, scale_);
  33.  
  34.         forwardMaps_.resize(highResForwardMotions_.size());
  35.         backwardMaps_.resize(highResForwardMotions_.size());
  36.         for (size_t i = 0; i < highResForwardMotions_.size(); ++i)
  37.             buildMotionMaps(highResForwardMotions_[i], highResBackwardMotions_[i], forwardMaps_[i], backwardMaps_[i]);
  38.  
  39.  
  40.         const Size lowResSize = src[0].size();
  41.         const Size highResSize(lowResSize.width * scale_, lowResSize.height * scale_);
  42.  
  43.         resize(src[baseIdx], highRes_, highResSize, 0, 0, INTER_CUBIC);
  44.  
  45.         diffTerm_.create(highResSize, highRes_.type());
  46.         a_.create(highResSize, highRes_.type());
  47.         b_.create(highResSize, highRes_.type());
  48.         c_.create(lowResSize, highRes_.type());
  49.  
  50.         for (int i = 0; i < iterations_; ++i)
  51.         {
  52.             diffTerm_.setTo(Scalar::all(0));
  53.  
  54.             for (size_t k = 0; k < src.size(); ++k)
  55.             {
  56.                 // a = M * Ih
  57.                 remap(highRes_, a_, backwardMaps_[k], noArray(), INTER_NEAREST);
  58.                 // 高斯模糊 b = HM * Ih
  59.                 GaussianBlur(a_, b_, Size(blurKernelSize_, blurKernelSize_), blurSigma_);
  60.                 // c = DHM * Ih
  61.                 resize(b_, c_, lowResSize, 0, 0, INTER_NEAREST);
  62.  
  63.                 diffSign(src[k], c_, c_);
  64.  
  65.                 // 超分运算 a = Dt * diff
  66.                 upscale(c_, a_, scale_);
  67.                 // b = HtDt * diff
  68.                 GaussianBlur(a_, b_, Size(blurKernelSize_, blurKernelSize_), blurSigma_);
  69.                 // a = MtHtDt * diff
  70.                 remap(b_, a_, forwardMaps_[k], noArray(), INTER_NEAREST);
  71.  
  72.                 add(diffTerm_, a_, diffTerm_);
  73.             }
  74.  
  75.             if (lambda_ > 0)
  76.             {
  77.                 calcBtvRegularization(highRes_, regTerm_, btvKernelSize_, btvWeights_, ubtvWeights_);
  78.                 addWeighted(diffTerm_, 1.0, regTerm_, -lambda_, 0.0, diffTerm_);
  79.             }
  80.  
  81.             addWeighted(highRes_, 1.0, diffTerm_, tau_, 0.0, highRes_);
  82.         }
  83.  
  84.         Rect inner(btvKernelSize_, btvKernelSize_, highRes_.cols - 2 * btvKernelSize_, highRes_.rows - 2 * btvKernelSize_);
  85.         highRes_(inner).copyTo(_dst);
  86.     }

我们再看下upscaleMotions()函数的实现：

  1. void upscaleMotions(InputArrayOfArrays _lowResMotions, OutputArrayOfArrays _highResMotions, int scale)
  2.     {
  3.         CV_OCL_RUN(_lowResMotions.isUMatVector() && _highResMotions.isUMatVector(),
  4.                    ocl_upscaleMotions(_lowResMotions, _highResMotions, scale))
  5.  
  6.         std::vector<Mat> & lowResMotions = *(std::vector<Mat> *)_lowResMotions.getObj(),
  7.                 & highResMotions = *(std::vector<Mat> *)_highResMotions.getObj();
  8.  
  9.         highResMotions.resize(lowResMotions.size());
  10.  
  11.         for (size_t i = 0; i < lowResMotions.size(); ++i)
  12.         {
  13.             // 三次样条差值
  14.             resize(lowResMotions[i], highResMotions[i], Size(), scale, scale, INTER_CUBIC);
  15.             // 运动矢量与缩放因子相乘
  16.             multiply(highResMotions[i], Scalar::all(scale), highResMotions[i]);
  17.         }
  18.     }

接着是upscale()超分函数的实现：

  1.     void upscale(InputArray _src, OutputArray _dst, int scale)
  2.     {
  3.         int cn = _src.channels();
  4.         CV_Assert( cn == 1 || cn == 3 || cn == 4 );
  5.  
  6.         CV_OCL_RUN(_dst.isUMat(),
  7.                    ocl_upscale(_src, _dst, scale))
  8.  
  9.         typedef void (*func_t)(InputArray src, OutputArray dst, int scale);
  10.         static const func_t funcs[] =
  11.         {
  12.             0, upscaleImpl<float>, 0, upscaleImpl<Point3f>, upscaleImpl<Point4f>
  13.         };
  14.  
  15.         const func_t func = funcs[cn];
  16.         CV_Assert(func != 0);
  17.         func(_src, _dst, scale);
  18.     }

最终是调用upscaleImpl()进行超分运算，属于模板函数：

  1.     template <typename T>
  2.     void upscaleImpl(InputArray _src, OutputArray _dst, int scale)
  3.     {
  4.         Mat src = _src.getMat();
  5.         _dst.create(src.rows * scale, src.cols * scale, src.type());
  6.         _dst.setTo(Scalar::all(0));
  7.         Mat dst = _dst.getMat();
  8.  
  9.         for (int y = 0, Y = 0; y < src.rows; ++y, Y += scale)
  10.         {
  11.             const T * const srcRow = src.ptr<T>(y);
  12.             T * const dstRow = dst.ptr<T>(Y);
  13.  
  14.             for (int x = 0, X = 0; x < src.cols; ++x, X += scale)
  15.                 dstRow[X] = srcRow[x];
  16.         }
  17.     }

声明：本文内容由网友自发贡献，不代表【wpsshop博客】立场，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：https://www.wpsshop.cn/w/不正经/article/detail/80127
推荐阅读
相关标签

Copyright © 2003-2013 www.wpsshop.cn 版权所有，并保留所有权利。

  

闽ICP备14008679号