devbox
寸_铁
这个屌丝很懒，什么也没留下！
关注作者
热门标签
热门文章
当前位置:   article > 正文

android 使用 onnxruntime 部署 yolov5_face_landmark 人脸检测_onnxruntime android

作者：寸_铁 | 2024-06-22 17:04:26

赞

踩

onnxruntime android

下面是使用 opencv-camera，实时处理区域内人脸检测 android 推理 demo。

首先是整合 opcv-camera 进去：

为了方便直接将整个 opencv-android-sdk 全部导入：

 然后在原来的项目模块app中添加 opencv的 java 相关依赖，主要添加红色两行：
app/build.grandle

  1. dependencies {
  2.     implementation fileTree(dir: 'libs', include: ['*.jar'])
  3.     implementation 'androidx.appcompat:appcompat:1.4.1'
  4.     implementation 'com.google.android.material:material:1.5.0'
  5.     implementation 'androidx.constraintlayout:constraintlayout:2.1.3'
  6.     testImplementation 'junit:junit:4.13.2'
  7.     androidTestImplementation 'androidx.test.ext:junit:1.1.3'
  8.     androidTestImplementation 'androidx.test.espresso:espresso-core:3.4.0'
  9.     implementation project(':opencvsdk')
  10. }

最后在项目中要使用opencv的地方加载jni库，可以添加到 MainActivity 中：

System.loadLibrary("opencv_java4"); 或者 OpenCVLoader.initDebug();

要使用 opencv-camera，MainActivity 继承 CameraActivity，然后在回调函数中获取每一帧进行处理，比如下面对每一帧添加识别区域边框：

  1.    // 获取每一帧回调数据
  2.     private CameraBridgeViewBase.CvCameraViewListener2 cameraViewListener2 = new CameraBridgeViewBase.CvCameraViewListener2() {
  3.         @Override
  4.         public void onCameraViewStarted(int width, int height) {
  5.             System.out.println("开始预览 width="+width+",height="+height);
  6.             // 预览界面是 640*480,模型输入时 320*320,计算识别区域坐标
  7.             int detection_x1 = (640 - OnnxUtil.w)/2;
  8.             int detection_x2 = (640 - OnnxUtil.w)/2 + OnnxUtil.w;
  9.             int detection_y1 = (480 - OnnxUtil.h)/2;
  10.             int detection_y2 = (480 - OnnxUtil.h)/2 + OnnxUtil.h;;
  11.             System.out.println("识别区域:"+"("+detection_x1+","+detection_y1+")"+"("+detection_x2+","+detection_y2+")");
  12.             // 缓存识别区域两个点
  13.             detection_p1 = new Point(detection_x1,detection_y1);
  14.             detection_p2 = new Point(detection_x2,detection_y2);
  15.             detection_box_color = new Scalar(255, 0, 0);
  16.             detection_box_tickness = 2;
  17.         }
  18.         @Override
  19.         public void onCameraViewStopped() {}
  20.         @Override
  21.         public Mat onCameraFrame(CameraBridgeViewBase.CvCameraViewFrame frame) {
  22.  
  23.             // 获取 cv::Mat
  24.             Mat mat = frame.rgba();
  25.  
  26.             // 标注识别区域
  27.             Imgproc.rectangle(mat, detection_p1, detection_p2,detection_box_color,detection_box_tickness);
  28.  
  29.             return mat;
  30.         }
  31.     };

在界面中开启预览：

  1. ui资源：
  2.   <org.opencv.android.JavaCamera2View
  3.             android:id="@+id/camera_view"
  4.             app:layout_constraintTop_toTopOf="parent"
  5.             app:layout_constraintLeft_toLeftOf="parent"
  6.             android:layout_width="match_parent"
  7.             android:layout_height="match_parent">
  8.  
  9.     </org.opencv.android.JavaCamera2View>
  10. java开启预览：
  11.     private BaseLoaderCallback baseLoaderCallback = new BaseLoaderCallback(this) {
  12.         @Override
  13.         public void onManagerConnected(int status) {
  14.             switch (status) {
  15.                 case LoaderCallbackInterface.SUCCESS: {
  16.                     if (camera2View != null) {
  17.                         // 设置前置还是后置摄像头 0后置 1前置
  18.                         camera2View.setCameraIndex(cameraId);
  19.                         // 注册每一帧回调
  20.                         camera2View.setCvCameraViewListener(cameraViewListener2);
  21.                         // 显示/关闭 帧率  disableFpsMeter/enableFpsMeter
  22.                         // 要修改字体和颜色直接修改 FpsMeter 类即可
  23.                         camera2View.enableFpsMeter();
  24.                         // 设置视图宽高和模型一致减少resize操作,模型输入一般尺寸不大,这样相机渲染fps会更高
  25.                         camera2View.setMaxFrameSize(win_w,win_h);
  26.                         // 开启
  27.                         camera2View.enableView();
  28.                     }
  29.                 }
  30.                 break;
  31.                 default:
  32.                     super.onManagerConnected(status);
  33.                     break;
  34.             }
  35.         }
  36.     };

下面是全部推理 MainActivity 代码：

  1. package com.example.camera_opencv;
  2.  
  3.  
  4. import android.content.pm.ActivityInfo;
  5. import android.os.Bundle;
  6. import android.view.WindowManager;
  7. import com.example.camera_opencv.databinding.ActivityMainBinding;
  8. import org.opencv.android.*;
  9. import org.opencv.core.Mat;
  10. import org.opencv.core.Point;
  11. import org.opencv.core.Scalar;
  12. import org.opencv.imgproc.Imgproc;
  13.  
  14. import java.util.Arrays;
  15. import java.util.List;
  16.  
  17. public class MainActivity extends CameraActivity{
  18.  
  19.     // 动态库
  20.     static {
  21.         // 我们自己的jni
  22.         System.loadLibrary("camera_opencv");
  23.         // 新加的 opencv 的jni
  24.         System.loadLibrary("opencv_java4");
  25.     }
  26.  
  27.     private ActivityMainBinding binding;
  28.  
  29.     // 预览界面
  30.     private JavaCamera2View camera2View;
  31.  
  32.     // 相机编号 0后置 1前置
  33.     private int cameraId = 1;
  34.  
  35.     // 设置预览界面宽高,在次宽高基础上限制识别区域
  36.     private int win_w = 640;
  37.     private int win_h = 480;
  38.  
  39.     // 识别区域两个点
  40.     private Point detection_p1;
  41.     private Point detection_p2;
  42.     private Scalar detection_box_color;
  43.     private int detection_box_tickness;
  44.  
  45.     @Override
  46.     protected void onCreate(Bundle savedInstanceState) {
  47.  
  48.         super.onCreate(savedInstanceState);
  49.         binding = ActivityMainBinding.inflate(getLayoutInflater());
  50.         setContentView(binding.getRoot());
  51.  
  52.         // 加载模型
  53.         OnnxUtil.loadModule(getAssets());
  54.  
  55.         // 强制横屏
  56.         setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_LANDSCAPE);
  57.         // 隐藏上方状态栏
  58.         getWindow().setFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN, WindowManager.LayoutParams.FLAG_FULLSCREEN);
  59.         // 预览界面
  60.         camera2View = findViewById(R.id.camera_view);
  61.     }
  62.  
  63.     @Override
  64.     protected List<? extends CameraBridgeViewBase> getCameraViewList() {
  65.         return Arrays.asList(camera2View);
  66.     }
  67.  
  68.  
  69.     @Override
  70.     public void onPause() {
  71.         super.onPause();
  72.         if (camera2View != null) {
  73.             // 关闭预览
  74.             camera2View.disableView();
  75.         }
  76.     }
  77.  
  78.     @Override
  79.     public void onResume() {
  80.         super.onResume();
  81.         if (OpenCVLoader.initDebug()) {
  82.             baseLoaderCallback.onManagerConnected(LoaderCallbackInterface.SUCCESS);
  83.         } else {
  84.             OpenCVLoader.initAsync(OpenCVLoader.OPENCV_VERSION, this, baseLoaderCallback);
  85.         }
  86.     }
  87.  
  88.     // 获取每一帧回调数据
  89.     private CameraBridgeViewBase.CvCameraViewListener2 cameraViewListener2 = new CameraBridgeViewBase.CvCameraViewListener2() {
  90.         @Override
  91.         public void onCameraViewStarted(int width, int height) {
  92.             System.out.println("开始预览 width="+width+",height="+height);
  93.             // 预览界面是 640*480,模型输入时 320*320,计算识别区域坐标
  94.             int detection_x1 = (640 - OnnxUtil.w)/2;
  95.             int detection_x2 = (640 - OnnxUtil.w)/2 + OnnxUtil.w;
  96.             int detection_y1 = (480 - OnnxUtil.h)/2;
  97.             int detection_y2 = (480 - OnnxUtil.h)/2 + OnnxUtil.h;;
  98.             System.out.println("识别区域:"+"("+detection_x1+","+detection_y1+")"+"("+detection_x2+","+detection_y2+")");
  99.             // 缓存识别区域两个点
  100.             detection_p1 = new Point(detection_x1,detection_y1);
  101.             detection_p2 = new Point(detection_x2,detection_y2);
  102.             detection_box_color = new Scalar(255, 0, 0);
  103.             detection_box_tickness = 2;
  104.         }
  105.         @Override
  106.         public void onCameraViewStopped() {}
  107.         @Override
  108.         public Mat onCameraFrame(CameraBridgeViewBase.CvCameraViewFrame frame) {
  109.  
  110.             // 获取 cv::Mat
  111.             Mat mat = frame.rgba();
  112.  
  113.             // 标注识别区域
  114.             Imgproc.rectangle(mat, detection_p1, detection_p2,detection_box_color,detection_box_tickness);
  115.  
  116.             // 推理并标注
  117.             OnnxUtil.inference(mat,detection_p1,detection_p2);
  118.  
  119.             return mat;
  120.         }
  121.     };
  122.  
  123.     // 开启预览
  124.     private BaseLoaderCallback baseLoaderCallback = new BaseLoaderCallback(this) {
  125.         @Override
  126.         public void onManagerConnected(int status) {
  127.             switch (status) {
  128.                 case LoaderCallbackInterface.SUCCESS: {
  129.                     if (camera2View != null) {
  130.                         // 设置前置还是后置摄像头 0后置 1前置
  131.                         camera2View.setCameraIndex(cameraId);
  132.                         // 注册每一帧回调
  133.                         camera2View.setCvCameraViewListener(cameraViewListener2);
  134.                         // 显示/关闭 帧率  disableFpsMeter/enableFpsMeter
  135.                         // 要修改字体和颜色直接修改 FpsMeter 类即可
  136.                         camera2View.enableFpsMeter();
  137.                         // 设置视图宽高和模型一致减少resize操作,模型输入一般尺寸不大,这样相机渲染fps会更高
  138.                         camera2View.setMaxFrameSize(win_w,win_h);
  139.                         // 开启
  140.                         camera2View.enableView();
  141.                     }
  142.                 }
  143.                 break;
  144.                 default:
  145.                     super.onManagerConnected(status);
  146.                     break;
  147.             }
  148.         }
  149.     };
  150.  
  151. }

onnx 模型加载和推理代码：
使用的微软onnx推理框架：

implementation 'com.microsoft.onnxruntime:onnxruntime-android:latest.release'
implementation 'com.microsoft.onnxruntime:onnxruntime-extensions-android:latest.release'

  1. package com.example.camera_opencv;
  2.  
  3. import ai.onnxruntime.*;
  4. import android.content.res.AssetManager;
  5. import org.opencv.core.*;
  6. import org.opencv.dnn.Dnn;
  7. import org.opencv.imgproc.Imgproc;
  8.  
  9. import java.io.ByteArrayOutputStream;
  10. import java.io.InputStream;
  11. import java.nio.FloatBuffer;
  12. import java.util.*;
  13.  
  14. public class OnnxUtil {
  15.  
  16.     // onnxruntime 环境
  17.     public static OrtEnvironment env;
  18.     public static OrtSession session;
  19.  
  20.     // 模型输入
  21.     public static int w = 0;
  22.     public static int h = 0;
  23.     public static int c = 3;
  24.  
  25.     // 标注颜色
  26.     public static Scalar green = new Scalar(0, 255, 0);
  27.     public static int tickness = 2;
  28.  
  29.     // 模型加载
  30.     public static void loadModule(AssetManager assetManager){
  31.  
  32.         // 下面包含了多个模型
  33.         // yolov5face-blazeface-640x640.onnx   3.4Mb
  34.         // yolov5face-l-640x640.onnx   181Mb
  35.         // yolov5face-m-640x640.onnx   	83Mb
  36.         // yolov5face-n-0.5-320x320.onnx   2.5Mb
  37.         // yolov5face-n-0.5-640x640.onnx   4.6Mb
  38.         // yolov5face-n-640x640.onnx   9.5Mb
  39.         // yolov5face-s-640x640.onnx   30Mb
  40.  
  41.         w = 320;
  42.         h = 320;
  43.         c = 3;
  44.  
  45.         try {
  46.             // 模型输入:  input -> [1, 3, 320, 320] -> FLOAT
  47.             // 模型输出:  output -> [1, 6300, 16] -> FLOAT
  48.             InputStream inputStream = assetManager.open("yolov5face-n-0.5-320x320.onnx");
  49.             ByteArrayOutputStream buffer = new ByteArrayOutputStream();
  50.             int nRead;
  51.             byte[] data = new byte[1024];
  52.             while ((nRead = inputStream.read(data, 0, data.length)) != -1) {
  53.                 buffer.write(data, 0, nRead);
  54.             }
  55.             buffer.flush();
  56.             byte[] module = buffer.toByteArray();
  57.             System.out.println("开始加载模型");
  58.             env = OrtEnvironment.getEnvironment();
  59.             session = env.createSession(module, new OrtSession.SessionOptions());
  60.             session.getInputInfo().entrySet().stream().forEach(n -> {
  61.                 String inputName = n.getKey();
  62.                 NodeInfo inputInfo = n.getValue();
  63.                 long[] shape = ((TensorInfo) inputInfo.getInfo()).getShape();
  64.                 String javaType = ((TensorInfo) inputInfo.getInfo()).type.toString();
  65.                 System.out.println("模型输入:  "+inputName + " -> " + Arrays.toString(shape) + " -> " + javaType);
  66.             });
  67.             session.getOutputInfo().entrySet().stream().forEach(n -> {
  68.                 String outputName = n.getKey();
  69.                 NodeInfo outputInfo = n.getValue();
  70.                 long[] shape = ((TensorInfo) outputInfo.getInfo()).getShape();
  71.                 String javaType = ((TensorInfo) outputInfo.getInfo()).type.toString();
  72.                 System.out.println("模型输出:  "+outputName + " -> " + Arrays.toString(shape) + " -> " + javaType);
  73.             });
  74.         } catch (Exception e) {
  75.             e.printStackTrace();
  76.         }
  77.  
  78.     }
  79.  
  80.  
  81.     // 模型推理,输入原始图片和识别区域两个点
  82.     public static void inference(Mat mat,Point detection_p1,Point detection_p2){
  83.  
  84.         int px = Double.valueOf(detection_p1.x).intValue();
  85.         int py = Double.valueOf(detection_p1.y).intValue();
  86.  
  87.         // 提取rgb(chw存储)并做归一化,也就是 rrrrr bbbbb ggggg
  88.         float[] chw = new float[c*h*w];
  89.         // 像素点索引
  90.         int index = 0;
  91.         for(int j=0 ; j<h ; j++){
  92.             for(int i=0 ; i<w ; i++){
  93.                 // 第j行,第i列,根据识别区域p1得到xy坐标的偏移,直接加就行
  94.                 double[] rgb = mat.get(j+py,i+px);
  95.                 // 缓存到 chw 中,mat 是 rgba 数据对应的下标 2103
  96.                 chw[index] = (float)(rgb[2]/255);//r
  97.                 chw[index + w * h * 1 ] = (float)(rgb[1]/255);//G
  98.                 chw[index + w * h * 2 ] = (float)(rgb[0]/255);//b
  99.                 index ++;
  100.             }
  101.         }
  102.  
  103.         // 创建张量并进行推理
  104.         try {
  105.  
  106.             OnnxTensor tensor = OnnxTensor.createTensor(env, FloatBuffer.wrap(chw), new long[]{1,c,h,w});
  107.             OrtSession.Result output = session.run(Collections.singletonMap("input", tensor));
  108.             float[][] out = ((float[][][])(output.get(0)).getValue())[0];
  109.  
  110.             ArrayList<float[]> datas = new ArrayList<>();
  111.  
  112.             for(int i=0;i<out.length;i++){
  113.  
  114.                 float[] data = out[i];
  115.  
  116.                 float score1 = data[4]; // 边框置信度
  117.                 float score2 = data[15];// 人脸置信度
  118.                 if( score1 >= 0.2 && score2>= 0.2){
  119.                     // xywh 转 x1y1x2y2
  120.                     float xx = data[0];
  121.                     float yy = data[1];
  122.                     float ww = data[2];
  123.                     float hh = data[3];
  124.                     float[] xyxy = xywh2xyxy(new float[]{xx,yy,ww,hh},w,h);
  125.                     data[0] = xyxy[0];
  126.                     data[1] = xyxy[1];
  127.                     data[2] = xyxy[2];
  128.                     data[3] = xyxy[3];
  129.                     datas.add(data);
  130.                 }
  131.             }
  132.  
  133.             // nms
  134.             ArrayList<float[]> datas_after_nms = new ArrayList<>();
  135.             while (!datas.isEmpty()){
  136.                 float[] max = datas.get(0);
  137.                 datas_after_nms.add(max);
  138.                 Iterator<float[]> it = datas.iterator();
  139.                 while (it.hasNext()) {
  140.                     // nsm阈值
  141.                     float[] obj = it.next();
  142.                     double iou = calculateIoU(max,obj);
  143.                     if (iou > 0.5f) {
  144.                         it.remove();
  145.                     }
  146.                 }
  147.             }
  148.  
  149.             // 标注
  150.             datas_after_nms.stream().forEach(n->{
  151.  
  152.                 // x y w h score  中心点坐标和分数
  153.                 // x y 关键点坐标
  154.                 // x y 关键点坐标
  155.                 // x y 关键点坐标
  156.                 // x y 关键点坐标
  157.                 // x y 关键点坐标
  158.                 // cls_conf 人脸置信度
  159.  
  160.                 // 画边框和关键点需要添加偏移
  161.                 int x1 = Float.valueOf(n[0]).intValue() + px;
  162.                 int y1 = Float.valueOf(n[1]).intValue() + py;
  163.                 int x2 = Float.valueOf(n[2]).intValue() + px;
  164.                 int y2 = Float.valueOf(n[3]).intValue() + py;
  165.                 Imgproc.rectangle(mat, new Point(x1, y1), new Point(x2, y2), green, tickness);
  166.  
  167.                 float point1_x = Float.valueOf(n[5]).intValue() + px;// 关键点1
  168.                 float point1_y = Float.valueOf(n[6]).intValue() + py;//
  169.                 float point2_x = Float.valueOf(n[7]).intValue() + px;// 关键点2
  170.                 float point2_y = Float.valueOf(n[8]).intValue() + py;//
  171.                 float point3_x = Float.valueOf(n[9]).intValue() + px;// 关键点3
  172.                 float point3_y = Float.valueOf(n[10]).intValue() + py;//
  173.                 float point4_x = Float.valueOf(n[11]).intValue() + px;// 关键点4
  174.                 float point4_y = Float.valueOf(n[12]).intValue() + py;//
  175.                 float point5_x = Float.valueOf(n[13]).intValue() + px;// 关键点5
  176.                 float point5_y = Float.valueOf(n[14]).intValue() + py;//
  177.  
  178.                 Imgproc.circle(mat, new Point(point1_x, point1_y), 1, green, tickness);
  179.                 Imgproc.circle(mat, new Point(point2_x, point2_y), 1, green, tickness);
  180.                 Imgproc.circle(mat, new Point(point3_x, point3_y), 1, green, tickness);
  181.                 Imgproc.circle(mat, new Point(point4_x, point4_y), 1, green, tickness);
  182.                 Imgproc.circle(mat, new Point(point5_x, point5_y), 1, green, tickness);
  183.  
  184.             });
  185.  
  186.         }
  187.         catch (Exception e){
  188.             e.printStackTrace();
  189.         }
  190.     }
  191.  
  192.  
  193.     // 中心点坐标转 xin xmax ymin ymax
  194.     public static float[] xywh2xyxy(float[] bbox,float maxWidth,float maxHeight) {
  195.         // 中心点坐标
  196.         float x = bbox[0];
  197.         float y = bbox[1];
  198.         float w = bbox[2];
  199.         float h = bbox[3];
  200.         // 计算
  201.         float x1 = x - w * 0.5f;
  202.         float y1 = y - h * 0.5f;
  203.         float x2 = x + w * 0.5f;
  204.         float y2 = y + h * 0.5f;
  205.         // 限制在图片区域内
  206.         return new float[]{
  207.                 x1 < 0 ? 0 : x1,
  208.                 y1 < 0 ? 0 : y1,
  209.                 x2 > maxWidth ? maxWidth:x2,
  210.                 y2 > maxHeight? maxHeight:y2};
  211.     }
  212.  
  213.     // 计算两个框的交并比
  214.     private static double calculateIoU(float[] box1, float[] box2) {
  215.         //  getXYXY() 返回 xmin-0 ymin-1 xmax-2 ymax-3
  216.         double x1 = Math.max(box1[0], box2[0]);
  217.         double y1 = Math.max(box1[1], box2[1]);
  218.         double x2 = Math.min(box1[2], box2[2]);
  219.         double y2 = Math.min(box1[3], box2[3]);
  220.         double intersectionArea = Math.max(0, x2 - x1 + 1) * Math.max(0, y2 - y1 + 1);
  221.         double box1Area = (box1[2] - box1[0] + 1) * (box1[3] - box1[1] + 1);
  222.         double box2Area = (box2[2] - box2[0] + 1) * (box2[3] - box2[1] + 1);
  223.         double unionArea = box1Area + box2Area - intersectionArea;
  224.         return intersectionArea / unionArea;
  225.     }
  226.  
  227. }
  228.  
  229.  
  230.  
  231.

项目详细代码：

https://github.com/TangYuFan/deeplearn-mobile

声明：本文内容由网友自发贡献，不代表【wpsshop博客】立场，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：https://www.wpsshop.cn/w/寸_铁/article/detail/747120
推荐阅读
相关标签

Copyright © 2003-2013 www.wpsshop.cn 版权所有，并保留所有权利。

  

闽ICP备14008679号