DPDK初始化Mellanox失败（mlx5）无法获取VF_no legacy callbacks, legacy socket not created

记录一下，方便以后查阅。

错误如下

1、
EAL: VFIO support initialized
EAL: Probe PCI driver: mlx5_pci (15b3:101e) device: 0000:12:01.0 (socket 0)
mlx5_pci: no Verbs device matches PCI device 0000:12:01.0, are kernel drivers loaded?
common_mlx5: Failed to load driver = mlx5_pci.
EAL: Requested device 0000:b5:01.2 cannot be used
EAL: Bus (pci) probe failed.
EAL: No legacy callbacks, legacy socket not created

代码如下
code from：
https://github.com/zartbot/learn_dpdk

#include <stdint.h>
#include <unistd.h>
#include <inttypes.h>
#include <rte_eal.h>
#include <rte_ethdev.h>
#include <rte_cycles.h>
#include <rte_lcore.h>
#include <rte_mbuf.h>
#include <rte_ether.h>
#include <rte_ip.h>
#include <rte_udp.h>
#include <pthread.h>
#include <string.h>

#define MAX_PORTS 16
#define RX_RING_SIZE 1024
#define TX_RING_SIZE 1024

#define NUM_MBUFS 8191
#define MBUF_CACHE_SIZE 250
#define BURST_SIZE 32

static int hwts_dynfield_offset = -1;

static inline rte_mbuf_timestamp_t *
hwts_field(struct rte_mbuf *mbuf)
{
 return RTE_MBUF_DYNFIELD(mbuf,
   hwts_dynfield_offset, rte_mbuf_timestamp_t *);
}

typedef uint64_t tsc_t;
static int tsc_dynfield_offset = -1;

static inline tsc_t *
tsc_field(struct rte_mbuf *mbuf)
{
 return RTE_MBUF_DYNFIELD(mbuf, tsc_dynfield_offset, tsc_t *);
}

static uint16_t
add_timestamps(uint16_t port __rte_unused, uint16_t qidx __rte_unused,
  struct rte_mbuf **pkts, uint16_t nb_pkts,
  uint16_t max_pkts __rte_unused, void *_ __rte_unused)
{
 unsigned i;
 uint64_t now = rte_rdtsc();

 for (i = 0; i < nb_pkts; i++)
  printf("packet[%d] sendtime: %ld\n",i,now);
 return nb_pkts;
}

static const struct rte_eth_conf port_conf_default = {
    .rxmode = {
        .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
    },
};

static inline int
port_init(uint16_t port, struct rte_mempool *mbuf_pool)
{
    struct rte_eth_conf port_conf = port_conf_default;
    const uint16_t rx_rings = 1, tx_rings = 1;
    uint16_t nb_rxd = RX_RING_SIZE;
    uint16_t nb_txd = TX_RING_SIZE;
    int retval;
    uint16_t q;
    struct rte_eth_dev_info dev_info;
    struct rte_eth_txconf txconf;

    if (!rte_eth_dev_is_valid_port(port))
        return -1;

    retval = rte_eth_dev_info_get(port, &dev_info);
    if (retval != 0)
    {
        printf("Error during getting device (port %u) info: %s\n",
               port, strerror(-retval));
        return retval;
    }
    printf("\n\ninitializing port %d...\n", port);


    if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_CHECKSUM)
    {
        printf("port[%u] support RX cheksum offload.\n", port);
        port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_CHECKSUM;
    }


 if (!(dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TIMESTAMP)) {
   printf("\nERROR: Port %u does not support hardware timestamping\n"
     , port);
   return -1;
  }
  port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_TIMESTAMP;
  rte_mbuf_dyn_rx_timestamp_register(&hwts_dynfield_offset, NULL);
  if (hwts_dynfield_offset < 0) {
   printf("ERROR: Failed to register timestamp field\n");
   return -rte_errno;
 }

    if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
    {
        printf("port[%u] support TX mbuf fast free offload.\n", port);
        port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MBUF_FAST_FREE;
    }

    if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MT_LOCKFREE)
    {
        printf("port[%u] support TX MT lock free offload.\n", port);
        port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MT_LOCKFREE;
    }

    if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM)
    {
        printf("port[%u] support TX IPv4 checksum offload.\n", port);
        port_conf.txmode.offloads |= DEV_TX_OFFLOAD_IPV4_CKSUM;
    }

    if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM)
    {
        printf("port[%u] support TX UDP checksum offload.\n", port);
        port_conf.txmode.offloads |= DEV_TX_OFFLOAD_UDP_CKSUM;
    }

    if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM)
    {
        printf("port[%u] support TX TCP checksum offload.\n", port);
        port_conf.txmode.offloads |= DEV_TX_OFFLOAD_TCP_CKSUM;
    }

    if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_SCTP_CKSUM)
    {
        printf("port[%u] support TX SCTP checksum offload.\n", port);
        port_conf.txmode.offloads |= DEV_TX_OFFLOAD_SCTP_CKSUM;
    }

    /* Configure the Ethernet device. */
    retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf);
    if (retval != 0)
        return retval;

    retval = rte_eth_dev_adjust_nb_rx_tx_desc(port, &nb_rxd, &nb_txd);
    if (retval != 0)
        return retval;

    /* Allocate and set up 1 RX queue per Ethernet port. */
    for (q = 0; q < rx_rings; q++)
    {
        retval = rte_eth_rx_queue_setup(port, q, nb_rxd,
                                        rte_eth_dev_socket_id(port), NULL, mbuf_pool);
        if (retval < 0)
            return retval;
    }

    txconf = dev_info.default_txconf;
    txconf.offloads = port_conf.txmode.offloads;
    /* Allocate and set up 1 TX queue per Ethernet port. */
    for (q = 0; q < tx_rings; q++)
    {
        retval = rte_eth_tx_queue_setup(port, q, nb_txd,
                                        rte_eth_dev_socket_id(port), &txconf);
        if (retval < 0)
            return retval;
    }

    /* Start the Ethernet port. */
    retval = rte_eth_dev_start(port);
    if (retval < 0)
        return retval;

    struct rte_eth_link link;
    do
    {
        retval = rte_eth_link_get_nowait(port, &link);
        if (retval < 0)
        {
            printf("Failed link get (port %u): %s\n",
                   port, rte_strerror(-retval));
            return retval;
        }
        else if (link.link_status)
            break;

        printf("Waiting for Link up on port %" PRIu16 "\n", port);
        sleep(1);
    } while (!link.link_status);

    /* Display the port MAC address. */
    struct rte_ether_addr addr;
    retval = rte_eth_macaddr_get(port, &addr);
    if (retval != 0)
        return retval;

    printf("Port[%u] MAC: %02" PRIx8 ":%02" PRIx8 ":%02" PRIx8
           ":%02" PRIx8 ":%02" PRIx8 ":%02" PRIx8 "\n",
           port,
           addr.addr_bytes[0], addr.addr_bytes[1],
           addr.addr_bytes[2], addr.addr_bytes[3],
           addr.addr_bytes[4], addr.addr_bytes[5]);

    /* Enable RX in promiscuous mode for the Ethernet device. */
    retval = rte_eth_promiscuous_enable(port);
    if (retval != 0)
        return retval;

    //rte_eth_add_rx_callback(port, 0, add_timestamps, NULL);

    return 0;
}

rte_be32_t string_to_ip(char *s)
{
    unsigned char a[4];
    int rc = sscanf(s, "%hhd.%hhd.%hhd.%hhd", a + 0, a + 1, a + 2, a + 3);
    if (rc != 4)
    {
        fprintf(stderr, "bad source IP address format. Use like: 1.1.1.1\n");
        exit(1);
    }
    return (rte_be32_t)(a[3]) << 24 |
           (rte_be32_t)(a[2]) << 16 |
           (rte_be32_t)(a[1]) << 8 |
           (rte_be32_t)(a[0]);
}




static int
lcore_send(struct rte_mempool *mbuf_pool) {

    struct rte_ether_hdr *eth_hdr;
    struct rte_ipv4_hdr *ipv4_hdr;
    struct rte_udp_hdr *udp_hdr;

    //Defined header in UDP
    struct SRoU
    {
        uint8_t magic_num;
        uint8_t srou_length;
        uint8_t flags;
        uint8_t next_protcol;
        uint64_t pad;
    };

    //init mac
    struct rte_ether_addr s_addr = {{0x14, 0x02, 0xEC, 0x89, 0x8D, 0x24}};
    struct rte_ether_addr d_addr = {{0x3c, 0xfd, 0xfe, 0xa9, 0xa8, 0x89}};

    //init IP header
    rte_be32_t s_ip_addr = string_to_ip("1.0.0.253");
    rte_be32_t d_ip_addr = string_to_ip("1.0.0.1");
    uint16_t ether_type = rte_cpu_to_be_16(0x0800);

    //init udp payload
    struct SRoU obj = {
        .magic_num = 1,
        .srou_length = 4,
        .flags = 0xFF,
        .next_protcol = 0,
    };
    struct SRoU *msg;
    struct rte_mbuf *pkt[BURST_SIZE];

 

    for(;;) {
           for (int i = 0; i < BURST_SIZE; i++)
    {
        pkt[i] = rte_pktmbuf_alloc(mbuf_pool);
 }

 for (int i = 0; i < BURST_SIZE; i++)
    {
        eth_hdr = rte_pktmbuf_mtod(pkt[i], struct rte_ether_hdr *);
        eth_hdr->d_addr = d_addr;
         struct rte_ether_addr s_addr = {{0x14, 0x02, 0xEC, 0x89, 0x8D, i}};
        eth_hdr->s_addr = s_addr;
        eth_hdr->ether_type = ether_type;

        ipv4_hdr = rte_pktmbuf_mtod_offset(pkt[i], struct rte_ipv4_hdr *, sizeof(struct rte_ether_hdr));
        ipv4_hdr->version_ihl = 0x45;
        ipv4_hdr->next_proto_id = 0x11;
        ipv4_hdr->src_addr = s_ip_addr;
        ipv4_hdr->dst_addr = d_ip_addr;
        ipv4_hdr->time_to_live = 0x40;

        udp_hdr = rte_pktmbuf_mtod_offset(pkt[i], struct rte_udp_hdr *, sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr));
        udp_hdr->dgram_len = rte_cpu_to_be_16(sizeof(struct SRoU) + sizeof(struct rte_udp_hdr));
        udp_hdr->src_port = rte_cpu_to_be_16(1234);
        udp_hdr->dst_port = rte_cpu_to_be_16(6666);
        ipv4_hdr->total_length = rte_cpu_to_be_16(sizeof(struct SRoU) + sizeof(struct rte_udp_hdr) + sizeof(struct rte_ipv4_hdr));

        msg = (struct SRoU *)(rte_pktmbuf_mtod(pkt[i], char *) + sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) + sizeof(struct rte_udp_hdr));
        *msg = obj;
        int pkt_size = sizeof(struct SRoU) + sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) + sizeof(struct rte_udp_hdr);

        pkt[i]->l2_len = sizeof(struct rte_ether_hdr);
        pkt[i]->l3_len = sizeof(struct rte_ipv4_hdr);
        pkt[i]->l4_len = sizeof(struct rte_udp_hdr);
        pkt[i]->ol_flags |= PKT_TX_IPV4 | PKT_TX_IP_CKSUM | PKT_TX_UDP_CKSUM;
        ipv4_hdr->hdr_checksum = 0;
        udp_hdr->dgram_cksum = rte_ipv4_phdr_cksum(ipv4_hdr, pkt[i]->ol_flags);
        pkt[i]->data_len = pkt_size;
        pkt[i]->pkt_len = pkt_size;
    }

    uint16_t nb_tx = rte_eth_tx_burst(0, 0, pkt, BURST_SIZE);
    for (int i = 0; i < BURST_SIZE; i++)
    {
        rte_pktmbuf_free(pkt[i]);
    }

   }

}

int main(int argc, char *argv[])
{

    struct rte_mempool *mbuf_pool;
    unsigned nb_ports;
    uint16_t portid;
 	uint64_t now = rte_rdtsc();
 	uint64_t pkt_cnt = 0;

    int ret = rte_eal_init(argc, argv);
    if (ret < 0)
        rte_exit(EXIT_FAILURE, "initlize fail!");

    printf("\n\n\n*****************************************\n");

    nb_ports = rte_eth_dev_count_avail();
    printf("number of available port: %d\n", nb_ports);

    /* Creates a new mempool in memory to hold the mbufs. */
    mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL", NUM_MBUFS * nb_ports,
                                        MBUF_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());

    if (mbuf_pool == NULL)
        rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n");

    /* Initialize all ports. */
    RTE_ETH_FOREACH_DEV(portid)
    if (port_init(portid, mbuf_pool) != 0)
        rte_exit(EXIT_FAILURE, "Cannot init port %" PRIu16 "\n",
                 portid);

    /* start packet send function on lcore-1 */
    rte_eal_remote_launch((lcore_function_t *)lcore_send,mbuf_pool,1);

    struct rte_mbuf *rx_pkt[BURST_SIZE];
    for (int i = 0; i < BURST_SIZE; i++)
    {
        rx_pkt[i] = rte_pktmbuf_alloc(mbuf_pool);
    }


    uint64_t freq = rte_get_tsc_hz() ;
 
    for (;;)
    {
        uint16_t nb_rx = rte_eth_rx_burst(1, 0, rx_pkt, BURST_SIZE);
        if (unlikely(nb_rx == 0))
        {
            continue;
        }
  pkt_cnt += nb_rx;
  
  if (unlikely(rte_rdtsc() - now > freq)) {
   printf("PPS: %ld\n",pkt_cnt);
   pkt_cnt= 0;
   now = rte_rdtsc() ;
  }
        /*struct rte_ether_hdr *eth_hdr;  
        for (int i = 0; i < nb_rx; i++)
        {
            eth_hdr = rte_pktmbuf_mtod(rx_pkt[i], struct rte_ether_hdr *);
            printf("Recv Pkt[%d] from MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8
                   " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 " \n",i,
                   eth_hdr->s_addr.addr_bytes[0], eth_hdr->s_addr.addr_bytes[1],
                   eth_hdr->s_addr.addr_bytes[2], eth_hdr->s_addr.addr_bytes[3],
                   eth_hdr->s_addr.addr_bytes[4], eth_hdr->s_addr.addr_bytes[5]);
            rte_pktmbuf_free(rx_pkt[i]);
        }*/
    }
    return 0;
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391

使用的链接选项

-L/home/alen/dpdk-20.11.1/build/lib -L/home/alen/dpdk-20.11.1/build/drivers  -Wl,-Bdynamic -Wl,--whole-archive -Wl,-lrte_mempool_ring -Wl,-lrte_pci -Wl,-lrte_bus_pci -Wl,-lrte_bus_vdev -Wl,-lrte_net  -Wl,-lrte_distributor -Wl,-lrte_reorder -Wl,-lrte_kni -Wl,-lrte_pipeline -Wl,-lrte_table -Wl,-lrte_eventdev -Wl,-lrte_timer -Wl,-lrte_hash -Wl,-lrte_jobstats -Wl,-lrte_lpm -Wl,-lrte_power -Wl,-lrte_acl  -Wl,-lrte_meter -Wl,-lrte_sched -Wl,--start-group -Wl,-lrte_kvargs -Wl,-lrte_mbuf -Wl,-lrte_ip_frag -Wl,-lrte_ethdev -Wl,-lrte_cryptodev -Wl,-lrte_mempool -Wl,-lrte_ring -Wl,-lrte_eal -Wl,-lrte_cmdline  -Wl,-lrte_cfgfile  -Wl,-lrte_net_bond -Wl,-lrte_net_vmxnet3  -Wl,-lrte_net_virtio  -Wl,-lrte_net_cxgbe -Wl,-lrte_net_enic  -Wl,-lrte_net_i40e -Wl,-lrte_net_iavf -Wl,-lrte_net_ice -Wl,-lrte_net_fm10k -Wl,-lrte_net_ixgbe -Wl,-lrte_net_e1000  -Wl,-lrte_net_ring -Wl,-lrte_net_af_packet -Wl,-lrte_net_null -Wl,-lrte_telemetry -Wl,-lrte_security -Wl,-lrte_rcu -Wl,-lrte_common_iavf -Wl,-lrte_common_cpt -Wl,-lrte_common_dpaax -Wl,-lrte_common_octeontx -Wl,-lrte_common_octeontx2 -Wl,-lrte_common_sfc_efx -Wl,-lrte_bus_dpaa -Wl,-lrte_bus_fslmc -Wl,-lrte_bus_ifpga -Wl,-lrte_bus_vmbus -Wl,-lrte_mempool_bucket -Wl,-lrte_mempool_dpaa -Wl,-lrte_mempool_dpaa2 -Wl,-lrte_mempool_octeontx -Wl,-lrte_mempool_octeontx2 -Wl,-lrte_mempool_stack -Wl,-lrte_net_ark -Wl,-lrte_net_atlantic -Wl,-lrte_net_avp -Wl,-lrte_net_axgbe  -Wl,-lrte_net_bnx2x -Wl,-lrte_net_bnxt  -Wl,-lrte_net_dpaa -Wl,-lrte_net_dpaa2  -Wl,-lrte_net_ena -Wl,-lrte_net_enetc -Wl,-lrte_net_failsafe -Wl,-lrte_net_hinic -Wl,-lrte_net_hns3 -Wl,-lrte_net_igc  -Wl,-lrte_net_kni -Wl,-lrte_net_liquidio -Wl,-lrte_net_memif -Wl,-lrte_net_netvsc -Wl,-lrte_net_nfp -Wl,-lrte_net_octeontx -Wl,-lrte_net_octeontx2 -Wl,-lrte_net_pfe -Wl,-lrte_net_qede -Wl,-lrte_net_sfc -Wl,-lrte_net_tap -Wl,-lrte_net_thunderx -Wl,-lrte_net_txgbe -Wl,-lrte_net_vdev_netvsc -Wl,-lrte_net_vhost -Wl,-lrte_net_mlx5 -Wl,-lrte_common_mlx5 -Wl,-lrte_regex_mlx5 -Wl,-lrte_vdpa_mlx5 -Wl,-lrte_raw_dpaa2_cmdif -Wl,-lrte_raw_dpaa2_qdma -Wl,-lrte_raw_ioat -Wl,-lrte_raw_ntb -Wl,-lrte_raw_octeontx2_dma -Wl,-lrte_raw_octeontx2_ep -Wl,-lrte_raw_skeleton -Wl,-lrte_crypto_bcmfs -Wl,-lrte_crypto_caam_jr -Wl,-lrte_crypto_dpaa_sec -Wl,-lrte_crypto_dpaa2_sec -Wl,-lrte_crypto_nitrox -Wl,-lrte_crypto_null -Wl,-lrte_crypto_octeontx -Wl,-lrte_crypto_octeontx2 -Wl,-lrte_crypto_scheduler -Wl,-lrte_crypto_virtio -Wl,-lrte_compress_octeontx -Wl,-lrte_compress_zlib -Wl,-lrte_regex_octeontx2 -Wl,-lrte_vdpa_ifc -Wl,-lrte_event_dlb -Wl,-lrte_event_dlb2 -Wl,-lrte_event_dpaa -Wl,-lrte_event_dpaa2 -Wl,-lrte_event_octeontx2 -Wl,-lrte_event_opdl -Wl,-lrte_event_skeleton -Wl,-lrte_event_sw -Wl,-lrte_event_dsw -Wl,-lrte_event_octeontx -Wl,-lrte_node -Wl,-lrte_graph -Wl,-lrte_bpf -Wl,-lrte_flow_classify -Wl,-lrte_fib -Wl,-lrte_ipsec -Wl,-lrte_vhost -Wl,-lrte_stack -Wl,-lrte_rib -Wl,-lrte_regexdev -Wl,-lrte_rawdev -Wl,-lrte_pdump -Wl,-lrte_member -Wl,-lrte_latencystats -Wl,-lrte_gso -Wl,-lrte_gro -Wl,-lrte_efd -Wl,-lrte_compressdev -Wl,-lrte_bitratestats -Wl,-lrte_bbdev -Wl,-lrte_metrics -Wl,--end-group -Wl,--no-whole-archive -Wl,-Bdynamic -lhugetlbfs -lnuma -ldl -Wl,-lm -Wl,-lrt -Wl,-ldl -lz -lelf
1

解决方法
http://doc.dpdk.org/guides/nics/mlx5.html#how-to-configure-a-vf-as-trusted

1、 Create 2 VFs on the PF pf0 when in Legacy SR-IOV mode:
/sys/class/net/eth0/device/sriov_numvfs
用sriov_numvfs貌似不可以工作
移除VF

$ echo 0 > /sys/class/net/enp181s0f1/device/mlx5_num_vfs
$ echo 2 > /sys/class/net/enp181s0f1/device/mlx5_num_vfs
1
2

2、Verify the VFs are created:

$lspci |grep Mellanox
82:00.0 Ethernet controller: Mellanox Technologies MT27800 Family [ConnectX-5]
82:00.1 Ethernet controller: Mellanox Technologies MT27800 Family [ConnectX-5]
82:00.2 Ethernet controller: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function]
82:00.3 Ethernet controller: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function]
1
2
3
4
5

3、Unbind all VFs. For each VF PCIe, using the following command to unbind the driver:

$ echo "0000:b5:01.2" >> /sys/bus/pci/drivers/mlx5_core/unbind
1

4、Set the VFs to be trusted for the kernel by using one of the methods below:
• Using sysfs file:

$ echo ON | tee /sys/class/net/enp181s0f1/device/sriov/0/trust
$ echo ON | tee /sys/class/net/enp181s0f1/device/sriov/1/trust
1
2

5、Using “ip link” command:

$ ip link set enp181s0f1 vf 0 trust on
$ ip link set enp181s0f1 vf 1 trust on
1
2

6、Configure all VFs using mlxreg:
这一步我们可以忽略

$ mlxreg -d /dev/mst/mt4121_pciconf0 --reg_name VHCA_TRUST_LEVEL --yes --set "all_vhca=0x1,trust_level=0x1"
1

Note
Firmware version used must be >= xx.29.1016 and MFT >= 4.18
7. For each VF PCIe, using the following command to bind the driver:

$ echo "0000:b5:01.2" >> /sys/bus/pci/drivers/mlx5_core/bind
1

到这一步宿主机上是可以成功获取并初始化VF了，但是如果是docker里面运行，需要注意使用net --host参数，使得宿主机和容器用一个网络。同时，最为重要的是，如果是在宿主机中生成VF，则必须先生成VF，后启动docker，否则在docker中则不能访问的VF。如果已存在docker并启动了，那我们在echo 2生成VF时，就需要stop docker，然后echo 2生成VF，然后docker start。当然也有其他容器和主机网卡设置的技术，例如pipework等。
成功的结果如下

EAL: Detected 4 lcore(s)
EAL: Detected 1 NUMA nodes
EAL: Detected shared linkage of DPDK
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Selected IOVA mode 'PA'
EAL: No available 1048576 kB hugepages reported
EAL: VFIO support initialized
EAL:   Invalid NUMA socket, default to 0
EAL: Probe PCI driver: net_virtio (1af4:1041) device: 0000:01:00.0 (socket 0)
eth_virtio_pci_init(): Failed to init PCI device

EAL: Requested device 0000:01:00.0 cannot be used
EAL:   Invalid NUMA socket, default to 0
EAL: Probe PCI driver: mlx5_pci (15b3:101a) device: 0000:06:00.0 (socket 0)
EAL:   Invalid NUMA socket, default to 0
EAL: Probe PCI driver: mlx5_pci (15b3:101a) device: 0000:07:00.0 (socket 0)
TELEMETRY: No legacy callbacks, legacy socket not created



*****************************************
number of available port: 2


initializing port 0...
port[0] support RX cheksum offload.
port[0] support TX mbuf fast free offload.
port[0] support TX IPv4 checksum offload.
port[0] support TX UDP checksum offload.
port[0] support TX TCP checksum offload.
Port[0] MAC: 7a:99:ed:5f:e3:a6


initializing port 1...
port[1] support RX cheksum offload.
port[1] support TX mbuf fast free offload.
port[1] support TX IPv4 checksum offload.
port[1] support TX UDP checksum offload.
port[1] support TX TCP checksum offload.
Port[1] MAC: 5a:d8:51:db:17:2d
PPS: 2081708
PPS: 23912180
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

2、
EAL: Failed to open group 58
EAL: 0001:01:08.0 not managed by VFIO driver, skipping
EAL: Failed to open group 60
EAL: 0001:01:0e.0 not managed by VFIO driver, skipping

如果是这个错误，一般发生在container中。
1、重新生成vf，先检查是否是在container exit的状态下生成的VF。需要保证生成vf时候，容器是退出的，这点很关键。
2、如果第一种方式不行。在启动docker时加–device参数
ETH1_VFIO_0 = /sys/class/net/eth1/device/virtfn0/iommu_group
ETH2_VFIO_0 = /sys/class/net/eth2/device/virtfn0/iommu_group

docker run -dit --device /dev/vfio/$ETH1_VFIO_0 dockerimage bash

3、容器中使用vf
a、在启动容器时通过 --net host参数和主机共用网卡
b、使用pipework工具将host上的eth，pipework到容器。

./pipework --direct-phys eth1 -i eth1 container_name 0/0
1