onnxruntime-gpu + windows + vs2019 cuda加速推理C++样例超详细_onnxruntime gpu推理

一、环境配置
全是windows 下的版本
cuda：11.1 11.4 11.7 三个版本都试过，都是ok的
cudnn：8.5.0
onnxruntime：1.12.1 relase版本

onnxruntime-gpu下载完后可以看到里面的头文件和静态库动态库，onnxruntime不需要安装，下载完之后需要把头文 件和库文件配置到工程中，下面有具体方法
PS D:\tools\onnxruntime-win-x64-gpu-1.12.1> tree /f
D:.
│ CodeSignSummary-e54fd8c5-34c1-462b-a8b2-0761efa3159d.md
│ GIT_COMMIT_ID
│ LICENSE
│ Privacy.md
│ README.md
│ ThirdPartyNotices.txt
│ VERSION_NUMBER
│
├─include
│ cpu_provider_factory.h
│ onnxruntime_cxx_api.h
│ onnxruntime_cxx_inline.h
│ onnxruntime_c_api.h
│ onnxruntime_run_options_config_keys.h
│ onnxruntime_session_options_config_keys.h
│ provider_options.h
│ tensorrt_provider_factory.h
│
└─lib
onnxruntime.dll
onnxruntime.lib
onnxruntime.pdb
onnxruntime_providers_cuda.dll
onnxruntime_providers_cuda.lib
onnxruntime_providers_cuda.pdb
onnxruntime_providers_shared.dll
onnxruntime_providers_shared.lib
onnxruntime_providers_shared.pdb
onnxruntime_providers_tensorrt.dll
onnxruntime_providers_tensorrt.lib
onnxruntime_providers_tensorrt.pdb

二、使用vs2019创建cmake工程或者解决方案工程，都行，这里我用的是cmake工程
1、创建vs2019创建cmake工程会自动创建build文件夹，就不需要单独创建了，创建完结果如下：

2、配置项目文件下的CMkaeLists.txt
将onnxruntime和opencv的静态库和头文件配置到CMakeLists.txt中，opencv的安装方式网上一大把，这里不在多说

此外将onnxruntime.dll、 onnxruntime_providers_cuda.dll、onnxruntime_providers_shared.dll、 onnxruntime_providers_tensorrt.dll放到C:\windows\system32中或者放到程序执行目录下，也就是.exe所在目录下

3、工程下的CMakeList.txt配置就很简单了

三、代码
模型文件，测试图片，标签获取请参考
https://blog.csdn.net/qq_44747572/article/details/120820964

头文件
CMaketest.h

// CMaketest.h: 标准系统包含文件的包含文件
// 或项目特定的包含文件。

#pragma once

#include <opencv2/core.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/opencv.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/core/core.hpp>
#include <opencv2/imgproc/imgproc_c.h>
#include <opencv2/dnn.hpp>
#include <iostream>  
#include <onnxruntime_cxx_api.h>
#include <assert.h>
#include <vector>
#include <fstream>
#include <numeric>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

源文件
CMaketest.cpp

#include "CMaketest.h"
bool CheckStatus(const OrtApi* g_ort, OrtStatus* status) {
	if (status != nullptr) {
		const char* msg = g_ort->GetErrorMessage(status);
		std::cerr << msg << std::endl;
		g_ort->ReleaseStatus(status);
		throw Ort::Exception(msg, OrtErrorCode::ORT_EP_FAIL);
	}
	return true;
}

using namespace cv;     
using namespace std;
using namespace Ort;
using namespace cv::dnn;

// 图像处理  标准化处理
void PreProcess(const Mat& image, Mat& image_blob)
{
	Mat input;
	image.copyTo(input);
	//数据处理 标准化
	std::vector<Mat> channels, channel_p;
	split(input, channels);
	Mat R, G, B;
	B = channels.at(0);
	G = channels.at(1);
	R = channels.at(2);

	B = (B / 255. - 0.406) / 0.225;
	G = (G / 255. - 0.456) / 0.224;
	R = (R / 255. - 0.485) / 0.229;

	channel_p.push_back(R);
	channel_p.push_back(G);
	channel_p.push_back(B);

	Mat outt;
	merge(channel_p, outt);
	image_blob = outt;
}


int main()         
{
	const OrtApi* g_ort = NULL;
	const OrtApiBase* ptr_api_base = OrtGetApiBase();
	g_ort = ptr_api_base->GetApi(ORT_API_VERSION);
	OrtEnv* env;
	CheckStatus(g_ort, g_ort->CreateEnv(ORT_LOGGING_LEVEL_WARNING, "test", &env));
	OrtSessionOptions* session_options;
	CheckStatus(g_ort, g_ort->CreateSessionOptions(&session_options));
	CheckStatus(g_ort, g_ort->SetIntraOpNumThreads(session_options, 1));
	CheckStatus(g_ort, g_ort->SetSessionGraphOptimizationLevel(session_options, ORT_ENABLE_BASIC));
	//CUDA option set
	OrtCUDAProviderOptions cuda_option;
	cuda_option.device_id = 0;
	cuda_option.arena_extend_strategy = 0;
	cuda_option.cudnn_conv_algo_search = OrtCudnnConvAlgoSearchExhaustive;
	cuda_option.gpu_mem_limit = SIZE_MAX;
	cuda_option.do_copy_in_default_stream = 1;
	//CUDA 加速
	CheckStatus(g_ort, g_ort->SessionOptionsAppendExecutionProvider_CUDA(session_options, &cuda_option));
	
	//load  model and creat session
	//模型文件路径
	const wchar_t* model_path = L"D:\\xx\\onnxruntimetest\\vgg16.onnx";
	printf("Using Onnxruntime C++ API\n");
	OrtSession* session;
	CheckStatus(g_ort, g_ort->CreateSession(env, model_path, session_options, &session));
	OrtAllocator* allocator;
	CheckStatus(g_ort, g_ort->GetAllocatorWithDefaultOptions(&allocator));

	
	//**********输入信息**********//
	size_t num_input_nodes; //输入节点的数量
	CheckStatus(g_ort, g_ort->SessionGetInputCount(session, &num_input_nodes));
	std::vector<const char*> input_node_names; //输入节点的名称
	std::vector<std::vector<int64_t>> input_node_dims; //输入节点的维度
	std::vector<ONNXTensorElementDataType> input_types; //输入节点的类型
	std::vector<OrtValue*> input_tensors; //输入节点的tensor

	input_node_names.resize(num_input_nodes);
	input_node_dims.resize(num_input_nodes);
	input_types.resize(num_input_nodes);
	input_tensors.resize(num_input_nodes);

	for (size_t i = 0; i < num_input_nodes; i++) {
		// Get input node names
		char* input_name;
		CheckStatus(g_ort, g_ort->SessionGetInputName(session, i, allocator, &input_name));
		input_node_names[i] = input_name;

		// Get input node types
		OrtTypeInfo* typeinfo;
		CheckStatus(g_ort, g_ort->SessionGetInputTypeInfo(session, i, &typeinfo));
		const OrtTensorTypeAndShapeInfo* tensor_info;
		CheckStatus(g_ort, g_ort->CastTypeInfoToTensorInfo(typeinfo, &tensor_info));
		ONNXTensorElementDataType type;
		CheckStatus(g_ort, g_ort->GetTensorElementType(tensor_info, &type));
		input_types[i] = type;

		// Get input shapes/dims
		size_t num_dims;
		CheckStatus(g_ort, g_ort->GetDimensionsCount(tensor_info, &num_dims));
		input_node_dims[i].resize(num_dims);
		CheckStatus(g_ort, g_ort->GetDimensions(tensor_info, input_node_dims[i].data(), num_dims));

		size_t tensor_size;
		CheckStatus(g_ort, g_ort->GetTensorShapeElementCount(tensor_info, &tensor_size));

		if (typeinfo) g_ort->ReleaseTypeInfo(typeinfo);
	}
	//---------------------------------------------------//

	//***********输出信息****************//

	size_t num_output_nodes;
	std::vector<const char*> output_node_names;
	std::vector<std::vector<int64_t>> output_node_dims;
	std::vector<OrtValue*> output_tensors;
	CheckStatus(g_ort, g_ort->SessionGetOutputCount(session, &num_output_nodes));
	output_node_names.resize(num_output_nodes);
	output_node_dims.resize(num_output_nodes);
	output_tensors.resize(num_output_nodes);

	for (size_t i = 0; i < num_output_nodes; i++) {
		// Get output node names
		char* output_name;
		CheckStatus(g_ort, g_ort->SessionGetOutputName(session, i, allocator, &output_name));
		output_node_names[i] = output_name;

		OrtTypeInfo* typeinfo;
		CheckStatus(g_ort, g_ort->SessionGetOutputTypeInfo(session, i, &typeinfo));
		const OrtTensorTypeAndShapeInfo* tensor_info;
		CheckStatus(g_ort, g_ort->CastTypeInfoToTensorInfo(typeinfo, &tensor_info));

		// Get output shapes/dims
		size_t num_dims;
		CheckStatus(g_ort, g_ort->GetDimensionsCount(tensor_info, &num_dims));
		output_node_dims[i].resize(num_dims);
		CheckStatus(g_ort, g_ort->GetDimensions(tensor_info, (int64_t*)output_node_dims[i].data(), num_dims));

		size_t tensor_size;
		CheckStatus(g_ort, g_ort->GetTensorShapeElementCount(tensor_info, &tensor_size));

		if (typeinfo) g_ort->ReleaseTypeInfo(typeinfo);
	}
	//---------------------------------------------------//

	printf("Number of inputs = %zu\n", num_input_nodes);
	printf("Number of output = %zu\n", num_output_nodes);
	std::cout << "input_name:" << input_node_names[0] << std::endl;
	std::cout << "output_name: " << output_node_names[0] << std::endl;

	//加载图片
	Mat img = imread("D:\\xx\\onnxruntimetest\\dog.jpg");
	Mat det1, det2, det3;
	resize(img, det1, Size(224, 224), INTER_AREA);
	det1.convertTo(det1, CV_32FC3);
	PreProcess(det1, det2);         //标准化处理
	//cvtColor(det2, det3, CV_BGR2RGB);
	//size_t input_data_length0 = det2.step * det2.rows;
	Mat blob = dnn::blobFromImage(det2, 1., Size(224, 224), Scalar(0, 0, 0), false, true);
	size_t input_data_length = blob.total() * blob.elemSize();
	printf("Load success!\n");

	clock_t startTime, endTime;
	//创建输入tensor
	OrtMemoryInfo* memory_info;

	CheckStatus(g_ort, g_ort->CreateCpuMemoryInfo(OrtArenaAllocator, OrtMemTypeDefault, &memory_info));
	CheckStatus(g_ort, g_ort->CreateTensorWithDataAsOrtValue(
						         memory_info, reinterpret_cast<void*>(blob.data), input_data_length,
						         input_node_dims[0].data(), input_node_dims[0].size(), input_types[0], &input_tensors[0]));

	g_ort->ReleaseMemoryInfo(memory_info);
    //循环推理看效果
	for (int i = 0; i < 10; i++)
	{
		startTime = clock();
		CheckStatus(g_ort, g_ort->Run(session, nullptr, input_node_names.data(), (const OrtValue* const*)input_tensors.data(),
					input_tensors.size(), output_node_names.data(), output_node_names.size(),
					output_tensors.data()));
		endTime = clock();
		cout << i << ": " << (double)(endTime - startTime) / CLOCKS_PER_SEC << "s" << endl;
	}

	void* output_buffer;
	CheckStatus(g_ort, g_ort->GetTensorMutableData(output_tensors[0], &output_buffer));
	float* float_buffer = reinterpret_cast<float*>(output_buffer);
	size_t output_data_size = output_node_dims[0][1];
	auto max = std::max_element(float_buffer, float_buffer + output_data_size);

	std::vector<float> optu(float_buffer, float_buffer + output_data_size);
	int max_index = static_cast<int>(std::distance(float_buffer, max));
	

	//标签文件加载
	std::fstream label_file("D:\\xx\\onnxruntimetest\\classification_classes_ILSVRC2012.txt", std::ios::in);
	std::unordered_map<int, std::string> label_table;
	label_table.reserve(output_data_size);
	int i = 0;
	std::string line;
	while (std::getline(label_file, line)) {
		label_table.emplace(i++, line);
	}

	printf("%d, %f, %s \n", max_index, *max, label_table[max_index].c_str());
	printf("Done!\n");
	return 0;
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213

四、结果
第一次推理时间很长，原因推测GPU唤醒需要时间，也跟模型有关

五、说一下遇到的坑
1、onnxruntime的动态库其实需要onnxruntime.dll、 onnxruntime_providers_cuda.dll、onnxruntime_providers_shared.dll这3个，而且一定要放对位置，否则会出各种内存异常的问题，可以看一下debug模式下，加载所需的dll，上文也已经说过也可以放在windows\system32下，优先从cmake可执行文件目录里面找dll。

、

2、提示找不到zlibwapi.dll文件，导致gpu推理失败
nvidia官网提供下载链接
https://docs.nvidia.com/deeplearning/cudnn/install-guide/index.html#prerequisites-windows