devbox
羊村懒王
这个屌丝很懒，什么也没留下！
关注作者
热门标签
热门文章
当前位置:   article > 正文

Android-Opencv-Ncnn实现图片人像换背景-代码实施

作者：羊村懒王 | 2024-03-22 03:40:10

赞

踩

Android-Opencv-Ncnn实现图片人像换背景-代码实施

上篇位置里提供了实现图片人像换背景相关的资源，这篇我们来看看代码相关的内容。

这里主要使用的是字节跳动开源的rvm，结合腾讯开源的ncnn和opencv来实现，在上篇展示的代码结构图可以看到rvm相关的几个文件，使用jni在android里加载assets里面的rvm文件，使用这些文件结合ncnn来实现人像相关的识别与抠像，人像抠图结果是有纯色背景的，最关键的还是对结果图进行透明化的设置，实现人像抠图透明化后使用opencv来实现抠图人像换背景，这是大体的一个思路。

1、nanodet.h

 这个文件里主要定义相关方法，可以看到有加载rvm、制作人像抠图换背景、开放到android端的native等方法。

  1. // Tencent is pleased to support the open source community by making ncnn available.
  2. //
  3. // Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
  4. //
  5. // Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
  6. // in compliance with the License. You may obtain a copy of the License at
  7. //
  8. // https://opensource.org/licenses/BSD-3-Clause
  9. //
  10. // Unless required by applicable law or agreed to in writing, software distributed
  11. // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
  12. // CONDITIONS OF ANY KIND, either express or implied. See the License for the
  13. // specific language governing permissions and limitations under the License.
  14.  
  15. #ifndef NANODET_H
  16. #define NANODET_H
  17.  
  18. #include <jni.h>
  19.  
  20. #include <opencv2/core/core.hpp>
  21.  
  22. #include <net.h>
  23.  
  24. struct Object
  25. {
  26.     cv::Rect_<float> rect;
  27.     cv::Mat mask;
  28.     int label;
  29.     float prob;
  30. };
  31.  
  32. class NanoDet
  33. {
  34. public:
  35.     NanoDet();
  36.  
  37.     int load(const char* modeltype, int target_size, const float* mean_vals, const float* norm_vals, bool use_gpu = false);
  38.  
  39.     int load(AAssetManager* mgr, const char* modeltype, int target_size, const float* mean_vals, const float* norm_vals, bool use_gpu = false);
  40.  
  41.     int detect(const cv::Mat& rgb, std::vector<Object>& objects, float prob_threshold = 0.4f, float nms_threshold = 0.5f);
  42.  
  43.     int draw(cv::Mat& rgb,cv::Mat& e, const std::vector<Object>& objects);
  44.  
  45.     int detect_rvm(const cv::Mat& bgr, cv::Mat& pha, cv::Mat& fgr);
  46.  
  47.     void draw_objects(const cv::Mat& bgr,cv::Mat& e, const cv::Mat& fgr, const cv::Mat& pha);
  48.  
  49.     void imageOverlayD(const cv::Mat&people, cv::Mat&background, int x, int y);
  50.  
  51.     int addAlpha(cv::Mat& src, cv::Mat& dst, cv::Mat& alpha);
  52.  
  53.     cv::Mat createAlpha(cv::Mat& src);
  54.  
  55.     int imageHumanMatting(const char* imagepath);
  56.  
  57.     int imageQualityChange(cv::Mat& src,float strength);
  58.  
  59.     int imageQualityChange(JNIEnv *,jclass,jstring imagepath,float strength);
  60.  
  61. private:
  62.  
  63.     void matting(cv::Mat &rgb, cv::Mat &mask, cv::Mat &fgr);
  64.     ncnn::Net faceseg;
  65.  
  66.     ncnn::Mat r1i, r2i, r3i, r4i;
  67.     ncnn::Mat r4o, r3o, r2o, r1o;
  68.     int is_first;
  69.     int target_size;
  70.     float mean_vals[3];
  71.     float norm_vals[3];
  72.  
  73.  
  74.     ncnn::UnlockedPoolAllocator blob_pool_allocator;
  75.     ncnn::PoolAllocator workspace_pool_allocator;
  76. };
  77.  
  78. #endif // NANODET_H

2、实现加载android assets中rvm。

  1. int
  2. NanoDet::load(AAssetManager *mgr, const char *modeltype, int _target_size, const float *_mean_vals,
  3.               const float *_norm_vals, bool use_gpu) {
  4.     faceseg.clear();
  5.     blob_pool_allocator.clear();
  6.     workspace_pool_allocator.clear();
  7.  
  8.     ncnn::set_cpu_powersave(2);
  9.     ncnn::set_omp_num_threads(ncnn::get_big_cpu_count());
  10.  
  11.     faceseg.opt = ncnn::Option();
  12.  
  13. #if NCNN_VULKAN
  14.     faceseg.opt.use_vulkan_compute = use_gpu;
  15. #endif
  16.  
  17.     faceseg.opt.num_threads = ncnn::get_big_cpu_count();
  18.     faceseg.opt.blob_allocator = &blob_pool_allocator;
  19.     faceseg.opt.workspace_allocator = &workspace_pool_allocator;
  20.     char parampath[256];
  21.     char modelpath[256];
  22.     sprintf(parampath, "%s.param", modeltype);
  23.     sprintf(modelpath, "%s.bin", modeltype);
  24.  
  25. //    faceseg.load_param(mgr, parampath);
  26. //    faceseg.load_model(mgr, modelpath);
  27.  
  28.  
  29.     faceseg.load_param(mgr, parampath);
  30.     faceseg.load_model(mgr, modelpath);
  31.  
  32.     target_size = _target_size;
  33.     mean_vals[0] = _mean_vals[0];
  34.     mean_vals[1] = _mean_vals[1];
  35.     mean_vals[2] = _mean_vals[2];
  36.     norm_vals[0] = _norm_vals[0];
  37.     norm_vals[1] = _norm_vals[1];
  38.     norm_vals[2] = _norm_vals[2];
  39.  
  40.     if (target_size == 512) {
  41.         r1i = ncnn::Mat(128, 128, 16);
  42.         r2i = ncnn::Mat(64, 64, 20);
  43.         r3i = ncnn::Mat(32, 32, 40);
  44.         r4i = ncnn::Mat(16, 16, 64);
  45.     } else {
  46.         r1i = ncnn::Mat(160, 120, 16);
  47.         r2i = ncnn::Mat(80, 60, 20);
  48.         r3i = ncnn::Mat(40, 30, 40);
  49.         r4i = ncnn::Mat(20, 15, 64);
  50.     }
  51.     r1i.fill(0.0f);
  52.     r2i.fill(0.0f);
  53.     r3i.fill(0.0f);
  54.     r4i.fill(0.0f);
  55.  
  56.     return 0;
  57. }

3、检测rvm。

  1. int NanoDet::detect_rvm(const cv::Mat &bgr, cv::Mat &pha, cv::Mat &fgr) {
  2.     const float downsample_ratio = 0.5f;
  3.     const int target_width = 512;
  4.     const int target_height = 512;
  5.  
  6.     faceseg.opt.use_vulkan_compute = false;
  7.     //original pretrained model from https://github.com/PeterL1n/RobustVideoMatting
  8.     //ncnn model https://pan.baidu.com/s/11iEY2RGfzWFtce8ue7T3JQ password: d9t6
  9. //    net.load_param("/data/data/com.tencent.ncnnbodyseg/files/rvm_512.param");
  10. //    net.load_model("/data/data/com.tencent.ncnnbodyseg/files/rvm_512.bin");
  11.  
  12.     //if you use another input size,pleaze change input shape
  13.     ncnn::Mat r1i = ncnn::Mat(128, 128, 16);
  14.     ncnn::Mat r2i = ncnn::Mat(64, 64, 20);
  15.     ncnn::Mat r3i = ncnn::Mat(32, 32, 40);
  16.     ncnn::Mat r4i = ncnn::Mat(16, 16, 64);
  17.     r1i.fill(0.0f);
  18.     r2i.fill(0.0f);
  19.     r3i.fill(0.0f);
  20.     r4i.fill(0.0f);
  21.  
  22.     ncnn::Extractor ex = faceseg.create_extractor();
  23.     const float mean_vals1[3] = {123.675f, 116.28f, 103.53f};
  24.     const float norm_vals1[3] = {0.01712475f, 0.0175f, 0.01742919f};
  25.     const float mean_vals2[3] = {0, 0, 0};
  26.     const float norm_vals2[3] = {1 / 255.0, 1 / 255.0, 1 / 255.0};
  27.     ncnn::Mat ncnn_in2 = ncnn::Mat::from_pixels_resize(bgr.data, ncnn::Mat::PIXEL_BGR2RGB, bgr.cols,
  28.                                                        bgr.rows, target_width, target_height);
  29.     ncnn::Mat ncnn_in1 = ncnn::Mat::from_pixels_resize(bgr.data, ncnn::Mat::PIXEL_BGR2RGB, bgr.cols,
  30.                                                        bgr.rows, target_width * downsample_ratio,
  31.                                                        target_height * downsample_ratio);
  32.  
  33.     ncnn_in1.substract_mean_normalize(mean_vals1, norm_vals1);
  34.     ncnn_in2.substract_mean_normalize(mean_vals2, norm_vals2);
  35.  
  36.     ex.input("src1", ncnn_in1);
  37.     ex.input("src2", ncnn_in2);
  38.     ex.input("r1i", r1i);
  39.     ex.input("r2i", r2i);
  40.     ex.input("r3i", r3i);
  41.     ex.input("r4i", r4i);
  42.  
  43.     //if use video matting,these output will be input of next infer
  44.     ex.extract("r4o", r4i);
  45.     ex.extract("r3o", r3i);
  46.     ex.extract("r2o", r2i);
  47.     ex.extract("r1o", r1i);
  48.  
  49.     ncnn::Mat pha_;
  50.     ex.extract("pha", pha_);
  51.     ncnn::Mat fgr_;
  52.     ex.extract("fgr", fgr_);
  53.  
  54.     cv::Mat cv_pha = cv::Mat(pha_.h, pha_.w, CV_32FC1, (float *) pha_.data);
  55.     cv::Mat cv_fgr = cv::Mat(fgr_.h, fgr_.w, CV_32FC3);
  56.     float *fgr_data = (float *) fgr_.data;
  57.     for (int i = 0; i < fgr_.h; i++) {
  58.         for (int j = 0; j < fgr_.w; j++) {
  59.             cv_fgr.at<cv::Vec3f>(i, j)[2] = fgr_data[0 * fgr_.h * fgr_.w + i * fgr_.w + j];
  60.             cv_fgr.at<cv::Vec3f>(i, j)[1] = fgr_data[1 * fgr_.h * fgr_.w + i * fgr_.w + j];
  61.             cv_fgr.at<cv::Vec3f>(i, j)[0] = fgr_data[2 * fgr_.h * fgr_.w + i * fgr_.w + j];
  62.         }
  63.     }
  64.  
  65.     cv_pha.copyTo(pha);
  66.     cv_fgr.copyTo(fgr);
  67.  
  68.     return 0;
  69. }

4、人像抠图并实现结果透明化和画质增强。

  1.  
  2. void NanoDet::draw_objects(const cv::Mat &bgr, cv::Mat &e, const cv::Mat &fgr, const cv::Mat &pha) {
  3.     cv::Mat fgr8U;
  4.     fgr.convertTo(fgr8U, CV_8UC3, 255.0, 0);
  5.     cv::Mat pha8U;
  6.     pha.convertTo(pha8U, CV_8UC1, 255.0, 0);
  7.  
  8.     if (bgr.size().empty()) {
  9.  
  10.         return;
  11.     }
  12.  
  13.     cv::Mat comp;
  14.     cv::resize(bgr, comp, pha.size(), 0, 0, 1);
  15.     for (int i = 0; i < pha8U.rows; i++) {
  16.         for (int j = 0; j < pha8U.cols; j++) {
  17.             uchar data = pha8U.at<uchar>(i, j);
  18.             float alpha = (float) data / 255;
  19.             comp.at<cv::Vec3b>(i, j)[0] =
  20.                     fgr8U.at<cv::Vec3b>(i, j)[0] * alpha + (1 - alpha) * 0;//155
  21.             comp.at<cv::Vec3b>(i, j)[1] =
  22.                     fgr8U.at<cv::Vec3b>(i, j)[1] * alpha + (1 - alpha) * 0;//255
  23.             comp.at<cv::Vec3b>(i, j)[2] =
  24.                     fgr8U.at<cv::Vec3b>(i, j)[2] * alpha + (1 - alpha) * 0;//120
  25.         }
  26.     }
  27.  
  28.     // cv::imshow("pha", pha8U);
  29.     //cv::imshow("fgr", fgr8U);
  30.     //cv::Mat img1_t1(comp, cv::Rect(0, 0, comp.cols, comp.rows));
  31.     // toPng(comp, img1_t1, 0);
  32.     // cv::imshow("comp", comp);
  33.  
  34.  
  35.  
  36.     cv::Mat img_alpha_0;
  37.  
  38.  
  39.     //
  40.     cv::resize(comp, comp, cv::Size(bgr.cols, bgr.rows), 0, 0, cv::INTER_LINEAR);
  41.     toPng(comp, img_alpha_0, 0);
  42.  
  43.  
  44.  
  45.     cv::Mat alpha = cv::Mat::zeros(img_alpha_0.rows, img_alpha_0.cols, CV_8UC1);
  46.     cv::Mat gray = cv::Mat::zeros(img_alpha_0.rows, img_alpha_0.cols, CV_8UC1);
  47.  
  48.     cv::cvtColor(img_alpha_0, gray, cv::COLOR_RGB2GRAY);
  49.  
  50.     for (int i = 0; i < img_alpha_0.rows; i++) {
  51.         for (int j = 0; j < img_alpha_0.cols; j++) {
  52.             alpha.at<uchar>(i, j) = gray.at<uchar>(i, j);
  53.         }
  54.     }
  55.  
  56.     cv::Mat dst = cv::Mat(img_alpha_0.rows, img_alpha_0.cols, CV_8UC4);
  57.  
  58.     std::vector<cv::Mat> srcChannels;
  59.     std::vector<cv::Mat> dstChannels;
  60.     //分离通道
  61.     cv::split(img_alpha_0, srcChannels);
  62.  
  63.     dstChannels.push_back(srcChannels[0]);
  64.     dstChannels.push_back(srcChannels[1]);
  65.     dstChannels.push_back(srcChannels[2]);
  66.     //添加透明度通道
  67.     dstChannels.push_back(alpha);
  68.     //合并通道
  69.     cv::merge(dstChannels, dst);
  70.  
  71.  
  72.  
  73.     imwrite("/storage/emulated/0/DCIM/Camera/aaaac.png", dst);
  74.  
  75.     cv::Mat ed = cv::imread("/storage/emulated/0/DCIM/Camera/aaaac.png",-1);
  76.  
  77.     cv::Mat ebg = cv::imread("/storage/emulated/0/DCIM/Camera/hbg.jpg");
  78.  
  79.  
  80.     imageOverlayD(ed, ebg, 550, 667);
  81.  
  82.  
  83.     imwrite("/storage/emulated/0/DCIM/Camera/aaaacdd.jpg", ebg);
  84.  
  85.  
  86.     Mat dstT;
  87.  
  88.     cv::GaussianBlur(ebg, dstT, cv::Size(0, 0), 9);
  89.     cv::addWeighted(ebg, 1.5, dstT, -0.335, 0, dstT);
  90.  
  91.     imwrite("/storage/emulated/0/DCIM/Camera/aaaactt.jpg", dstT);
  92.  
  93.     cv::waitKey(0);
  94. }
  95.  
  96. // 图像叠加函数的实现
  97. void NanoDet::imageOverlayD(const Mat &people, Mat &background, int x, int y)
  98. {
  99.     int channelNum = 3; // rgb通道数为3
  100.     int alpha = 1;  // alpha值表示图像透明度，0代表完全透明，1代表完全不透明
  101.  
  102.     for (int i = 0; i < people.rows; i++)
  103.     {
  104.         for(int j = 0; j < people.cols * 3; j += 3)
  105.         {
  106.             alpha = people.ptr<uchar>(i)[j / 3*4 + 3];
  107.  
  108.             if(alpha != 0)
  109.             {
  110.                 for (int k = 0; k < 3; k++)
  111.                 {
  112.                     // jpg不存在alpha通道，所以给读取出来的数组增加一个alpha的维度
  113.                     if( (i+y < background.rows) && (i+y>=0) &&
  114.                         ((j+x*3) / 3*3 + k < background.cols*3) && ((j+x*3) / 3*3 + k >= 0) &&
  115.                         (i/channelNum*4 + k < background.cols*4) && (j/channelNum*4 + k >=0) )
  116.                     {
  117.                         background.ptr<uchar>(i+y)[(j+x*channelNum) / channelNum*channelNum + k] = people.ptr<uchar>(i)[(j) / channelNum*4 + k];
  118.                     }
  119.                 }
  120.             }
  121.         }
  122.     }
  123. }

5、jni native实现-加载android assets rvm文件

  1. JNIEXPORT jboolean JNICALL Java_com_tencent_ncnnbodyseg_NcnnBodyseg_loadModel(JNIEnv* env, jobject thiz, jobject assetManager, jint modelid, jint cpugpu)
  2. {
  3.     if (modelid < 0 || modelid > 6 || cpugpu < 0 || cpugpu > 1)
  4.     {
  5.         return JNI_FALSE;
  6.     }
  7.  
  8.     AAssetManager* mgr = AAssetManager_fromJava(env, assetManager);
  9.  
  10.     __android_log_print(ANDROID_LOG_DEBUG, "ncnn", "loadModel %p", mgr);
  11.  
  12.     const char* modeltypes[] =
  13.     {
  14.         "rvm-512",
  15.         "rvm-640",
  16.     };
  17.  
  18.     const int target_sizes[] =
  19.     {
  20.         512,
  21.         640,
  22.     };
  23.  
  24.     const float mean_vals[][3] =
  25.     {
  26.         {123.675f, 116.28f,  103.53f},
  27.         {123.675f, 116.28f,  103.53f},
  28.     };
  29.  
  30.     const float norm_vals[][3] =
  31.     {
  32.         {0.01712475f, 0.0175f, 0.01742919f},
  33.         {0.01712475f, 0.0175f, 0.01742919f},
  34.     };
  35.  
  36.     const char* modeltype = modeltypes[(int)modelid];
  37.     int target_size = target_sizes[(int)modelid];
  38.     bool use_gpu = (int)cpugpu == 1;
  39.  
  40.     // reload
  41.     {
  42.         ncnn::MutexLockGuard g(lock);
  43.  
  44.         if (use_gpu && ncnn::get_gpu_count() == 0)
  45.         {
  46.             // no gpu
  47.             delete g_nanodet;
  48.             g_nanodet = 0;
  49.         }
  50.         else
  51.         {
  52.             if (!g_nanodet)
  53.                 g_nanodet = new NanoDet;
  54.             g_nanodet->load(mgr, modeltype, target_size, mean_vals[(int)modelid], norm_vals[(int)modelid], use_gpu);
  55.         }
  56.     }
  57.  
  58.     return JNI_TRUE;
  59. }

 6、jni native实现-执行人像抠图换背景

  1.  
  2. JNIEXPORT jint JNICALL Java_com_yiachang_keepmemo_jni_KeepOpencvNcnn_imageQualityChange(JNIEnv *env, jclass clazz,  jstring imagepath,jfloat strength)
  3. {
  4.  
  5.  
  6.     std::vector<Object> objects;
  7.  
  8.     std::string path = (char *) env->GetStringUTFChars(imagepath, 0);
  9.  
  10.     cv::Mat m = cv::imread(path);
  11.  
  12.     int result = g_nanodet->imageQualityChange(m, strength);
  13.  
  14.     jmethodID methodID = env->GetStaticMethodID(clazz, "onExecuted", "(I)V");
  15.  
  16.     //调用该java方法
  17.     env->CallStaticVoidMethod(clazz, methodID,result);
  18.  
  19.     return 0;
  20.  
  21. }

 7、android 端native相关接口

  1. public class KeepOpencvNcnn
  2. {
  3.     public native boolean loadModel(AssetManager mgr, int modelid, int cpugpu);
  4.     public native int imageHumanMatting(String imagepath);
  5.     public static native int imageQualityChange(String imagepath,float strength);
  6.  
  7.  
  8.     static {
  9.         System.loadLibrary("keepOpencvNcnn");
  10.     }
  11.  
  12.  
  13.     public static void exec(String imagepath,float strength, KoNListener listener)
  14.     {
  15.         mKoNListener = listener;
  16.  
  17.         imageQualityChange(imagepath, strength);
  18.  
  19.     }
  20.  
  21.  
  22.     public static void onExecuted(int n) {
  23.  
  24.         if(n == -1){
  25.  
  26.             if(mKoNListener != null){
  27.                 mKoNListener.onFail();
  28.             }
  29.  
  30.             return;
  31.         }
  32.  
  33.         if(mKoNListener != null){
  34.             mKoNListener.onFinish();
  35.         }
  36.  
  37.     }
  38.  
  39.     static KoNListener mKoNListener;
  40.     /**
  41.      * 回调监听
  42.      */
  43.     public interface KoNListener {
  44.  
  45.         /**
  46.          *
  47.          */
  48.         void onFinish();
  49.  
  50.         /**
  51.          *
  52.          */
  53.         void onFail();
  54.  
  55.     }
  56.  
  57.  
  58. }

 以上是所有相关的代码内容了，特别要注意里面相关的图片文件路径，如果使用相关代码的话需要准备相关路径的图片，或者替换里面的图片路径。

重点摘要：

纯背景图片透明化：网上很多事针对PNG图片进行的纯背景图片透明化，以下代码可以实现jpg、png纯背景图片的透明化。

  1.     cv::Mat dst = cv::Mat(img_alpha_0.rows, img_alpha_0.cols, CV_8UC4);
  2.  
  3.     std::vector<cv::Mat> srcChannels;
  4.     std::vector<cv::Mat> dstChannels;
  5.     //分离通道
  6.     cv::split(img_alpha_0, srcChannels);
  7.  
  8.     dstChannels.push_back(srcChannels[0]);
  9.     dstChannels.push_back(srcChannels[1]);
  10.     dstChannels.push_back(srcChannels[2]);
  11.     //添加透明度通道
  12.     dstChannels.push_back(alpha);
  13.     //合并通道
  14.     cv::merge(dstChannels, dst);

图片画质增强：网上也有很多关于对比度、亮度、饱和度等图片画质增强的文章，这里是使用高斯和权重来实现图片画质增强的，不过以下方法可以实现图片画质调整，区间范围是[-1,1]，在开放的进度范围是[0,1]，也就是说android端可以调整0~1范围的进度(SeekBar)，来实现图片画质范围[-1,1]的调整，当然朋友们可以使用以下代码进行调整，实现自己合适的画质区间。

  1.  
  2. int NanoDet::imageQualityChange(cv::Mat& src,float strength) {
  3.  
  4.  
  5.     Mat dstT;
  6.  
  7.     if (src.size().empty()) {
  8.  
  9.         return -1;
  10.     }
  11.  
  12.     cv::GaussianBlur(src, dstT, cv::Size(0, 0), 9);
  13.     cv::addWeighted(src, 1.5, dstT, -1+strength , 0, dstT);//0~1[-1~1]
  14.  
  15.     imwrite("/storage/emulated/0/DCIM/Camera/aaaactt.jpg", dstT);
  16.  
  17.     cv::waitKey(0);
  18.  
  19.     return 0;
  20. }

本文内容由网友自发贡献，转载请注明出处：https://www.wpsshop.cn/w/羊村懒王/article/detail/285055
推荐阅读
相关标签

Copyright © 2003-2013 www.wpsshop.cn 版权所有，并保留所有权利。

  

闽ICP备14008679号