yolo的tensorrt加速，转化的trt/engine模型文件的加载和推理

0x00 经过yolo文件夹中的export.py文件实现onnx转化

0x01 onnx实现到trt文件的转化

参考博客：tensorRT加速遇到的若干问题

0x02 yolo文件夹中detect.py文件能实现trt和engine的加载和推理，但是新建单独的py文件如何加载和推理，直接给出代码。

import tensorrt as trt
import pycuda.driver as cuda
import pycuda.autoinit
import time
import cv2
import numpy as np
import torch
np.bool = np.bool_

f = open("best32.trt", "rb")
runtime = trt.Runtime(trt.Logger(trt.Logger.WARNING))
engine = runtime.deserialize_cuda_engine(f.read())
context = engine.create_execution_context()

# 加载图像
image_path = "left_2.jpg"
input_image = cv2.imread(image_path)  # 使用 OpenCV 加载图像
# 调整图像大小和通道顺序，以适应模型输入
input_image = cv2.resize(input_image, (640, 480))  # 调整大小为模型输入大小
input_image = cv2.cvtColor(input_image, cv2.COLOR_BGR2RGB)  # 调整通道顺序为 RGB
input_image = np.transpose(input_image, (2, 0, 1))  # 调整通道顺序为 CxHxW
input_image = input_image / 255.0
# 添加批处理维度
input_tensor = np.expand_dims(input_image, axis=0)
input_tensor = np.ascontiguousarray(input_tensor, dtype=np.float32)

output = np.empty([1, 18900, 6], dtype=np.float32)

# allocate device memory
d_input = cuda.mem_alloc(1 * input_tensor.nbytes)
d_output = cuda.mem_alloc(1 * output.nbytes)

bindings = [int(d_input), int(d_output)]
stream = cuda.Stream()

def predict(preprocessed_images):  # result gets copied into output
    # transfer input data to device
    cuda.memcpy_htod_async(d_input, preprocessed_images, stream)  # execute model
    context.execute_async_v2(bindings, stream.handle, None)  # transfer predictions back
    cuda.memcpy_dtoh_async(output, d_output, stream)  # syncronize threads
    stream.synchronize()
    d_input.free()
    d_output.free()
    return output

t0 = time.time()
pred = predict(input_tensor)
print(pred)
print(pred.shape)

t1 = time.time()
print(f'One frame spends time = ({t1 - t0:.3f}s)')


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54

运行此代码出现报错：
[04/11/2024-13:29:04] [TRT] [E] 3: [executionContext.cpp::enqueueInternal::622] Error Code 3: API Usage Error (Parameter check failed at: runtime/api/executionContext.cpp::enqueueInternal::622, condition: bindings[x] || nullBindingOK
)
或者
condition: binding[x] != nullptr
且输出打印出来全为0

[04/11/2024-13:29:04] [TRT] [E] 3: [executionContext.cpp::enqueueInternal::622] Error Code 3: API Usage Error (Parameter check failed at: runtime/api/executionContext.cpp::enqueueInternal::622, condition: bindings[x] || nullBindingOK
)
[[[0. 0. 0. 0. 0. 0.]
  [0. 0. 0. 0. 0. 0.]
  [0. 0. 0. 0. 0. 0.]
  ...
  [0. 0. 0. 0. 0. 0.]
  [0. 0. 0. 0. 0. 0.]
  [0. 0. 0. 0. 0. 0.]]]
(1, 18900, 6)
One frame spends time = (0.036s)

Process finished with exit code 0
1
2
3
4
5
6
7
8
9
10
11
12
13

参考博客：
TensorRT推理过程出现condition: binding[x] != nullptr，output全0

我们在代码：

f = open("best32.trt", "rb")
runtime = trt.Runtime(trt.Logger(trt.Logger.WARNING))
engine = runtime.deserialize_cuda_engine(f.read())
context = engine.create_execution_context()
1
2
3
4

后面加入代码：

for binding in engine:
    dims = engine.get_binding_shape(binding)
    size = trt.volume(dims)
    print("The size of binding is", size)
    print("The dimension of binding is", dims)
    print(binding)
    print("input = ", engine.binding_is_input(binding))
    print("dtype =", trt.nptype(engine.get_binding_dtype(binding)))
1
2
3
4
5
6
7
8

输出可以看到：

The size of binding is 921600
The dimension of binding is (1, 3, 480, 640)
images
input =  True
dtype = <class 'numpy.float32'>
The size of binding is 86400
The dimension of binding is (1, 3, 60, 80, 6)
onnx::Sigmoid_456
input =  False
dtype = <class 'numpy.float32'>
The size of binding is 21600
The dimension of binding is (1, 3, 30, 40, 6)
onnx::Sigmoid_509
input =  False
dtype = <class 'numpy.float32'>
The size of binding is 5400
The dimension of binding is (1, 3, 15, 20, 6)
onnx::Sigmoid_562
input =  False
dtype = <class 'numpy.float32'>
The size of binding is 113400
The dimension of binding is (1, 18900, 6)
output
input =  False
dtype = <class 'numpy.float32'>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

可以看出输入一个(1, 3, 480, 640)，输出不止一个(1, 18900, 6)
最开始的代码

image_path = "left_2.jpg"
input_image = cv2.imread(image_path)  # 使用 OpenCV 加载图像
# 调整图像大小和通道顺序，以适应模型输入
input_image = cv2.resize(input_image, (640, 480))  # 调整大小为模型输入大小
input_image = cv2.cvtColor(input_image, cv2.COLOR_BGR2RGB)  # 调整通道顺序为 RGB
input_image = np.transpose(input_image, (2, 0, 1))  # 调整通道顺序为 CxHxW
input_image = input_image / 255.0
# 添加批处理维度
input_tensor = np.expand_dims(input_image, axis=0)
input_tensor = np.ascontiguousarray(input_tensor, dtype=np.float32)
output = np.empty([1, 18900, 6], dtype=np.float32)

# allocate device memory
d_input = cuda.mem_alloc(1 * input_tensor.nbytes)
d_output = cuda.mem_alloc(1 * output.nbytes)

bindings = [int(d_input), int(d_output)]
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17

需要进行修改，不废话了，给出正确代码。也就是给另外的三个输出分配存储空间。给出完整代码。

import tensorrt as trt
import pycuda.driver as cuda
import pycuda.autoinit
import time
import cv2
import numpy as np
import torch
np.bool = np.bool_

f = open("best32.trt", "rb")
runtime = trt.Runtime(trt.Logger(trt.Logger.WARNING))
engine = runtime.deserialize_cuda_engine(f.read())
context = engine.create_execution_context()

# for binding in engine:
#     dims = engine.get_binding_shape(binding)
#     size = trt.volume(dims)
#     print("The size of binding is", size)
#     print("The dimension of binding is", dims)
#     print(binding)
#     print("input = ", engine.binding_is_input(binding))
#     print("dtype =", trt.nptype(engine.get_binding_dtype(binding)))

# 加载图像
image_path = "left_2.jpg"
input_image = cv2.imread(image_path)  # 使用 OpenCV 加载图像
# 调整图像大小和通道顺序，以适应模型输入
input_image = cv2.resize(input_image, (640, 480))  # 调整大小为模型输入大小
input_image = cv2.cvtColor(input_image, cv2.COLOR_BGR2RGB)  # 调整通道顺序为 RGB
input_image = np.transpose(input_image, (2, 0, 1))  # 调整通道顺序为 CxHxW
input_image = input_image / 255.0
# 添加批处理维度
input_tensor = np.expand_dims(input_image, axis=0)
input_tensor = np.ascontiguousarray(input_tensor, dtype=np.float32)
# print(input_tensor.shape)
# print(input_tensor)

output = np.empty([1, 18900, 6], dtype=np.float32)
output1 = np.empty([1, 3, 60, 80, 6], dtype=np.float32)
output2 = np.empty([1, 3, 30, 40, 6], dtype=np.float32)
output3 = np.empty([1, 3, 15, 20, 6], dtype=np.float32)


# allocate device memory
d_input = cuda.mem_alloc(1 * input_tensor.nbytes)
d_output = cuda.mem_alloc(1 * output.nbytes)
d1_output = cuda.mem_alloc(1 * output1.nbytes)
d2_output = cuda.mem_alloc(1 * output2.nbytes)
d3_output = cuda.mem_alloc(1 * output3.nbytes)
bindings = [int(d_input), int(d1_output), int(d2_output), int(d3_output), int(d_output)]
stream = cuda.Stream()

def predict(preprocessed_images):  # result gets copied into output
    # transfer input data to device
    cuda.memcpy_htod_async(d_input, preprocessed_images, stream)  # execute model
    context.execute_async_v2(bindings, stream.handle, None)  # transfer predictions back
    cuda.memcpy_dtoh_async(output, d_output, stream)  # syncronize threads 只需要d_output的结果。
    stream.synchronize()
    d_input.free()
    d_output.free()
    return output

t0 = time.time()

pred = predict(input_tensor)

print(pred)
print(pred.shape)

# 之后的pred 直接使用detect.py里面的non_max_suppression函数就好了  
# 显示输出图片之类的用detect.py里面的for i, det in enumerate(pred): 

t1 = time.time()
print(f'One frame spends time = ({t1 - t0:.3f}s)')


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76