learning：vppsb router插件编译_vppsb 编译

前面文章中介绍了一篇博客：Learning VPP: OSPF routing protocol，文章中介绍使用VPPsb的router插件，它实现了将控制报文推到Linux网络堆栈的逻辑，以及一个基于netlink的机制，它将Linux的路由表同步到VPP的FIB中。本文主要尝试vppsb router查询在最新vpp21.06版本编译。并配置简单的ping业务正常。

基本逻辑可以查询vpp官方指导文档:https://wiki.fd.io/view/VPP_Sandbox/router。源码解读后续有时间再研究。

基本环境及代码

基于Centos 7 Linux内核版本：3.10.0-957.21.3.el7.x86_64。
vpp ：最新master分支代码

learning_vpp# show version verbose
Version:                  v21.06-rc0~642-g87f6389 #运行后的版本号
Compiled by:              root
Compile host:             root
Compile date:             2021-05-11T04:44:10
Compile location:         /root/vpp
Compiler:                 GCC 9.3.1 20200408 (Red Hat 9.3.1-2)
Current PID:              17015
1
2
3
4
5
6
7
8

vppsb：https://github.com/garyachy/vppsb.git

代码编译

主要是参考了博客中提供的方案，在vpp的plugins目录下创建一个超链接方式，代码已经放在github上：

#原项目地址：https://github.com/garyachy/frr-vpp.git
#基于原项目地址fork：https://github.com/jin13417/vppsb.git ，创建了分支vpp_21.06
改动如下：
1、在vppsb目录下创建超链接
ln -sf ../netlink/librtnl router/librtnl
2、在vpp/src/plugins 创建超链接
 ln -sf ../../../vppsb/router vpp/src/plugins/router
3、在vppsb router目录下修改 CMakeLists.txt 
add_vpp_plugin(router
  SOURCES
  librtnl/mapper.c
  librtnl/netns.c
  librtnl/rtnl.c
  router/tap_inject.c
  router/tap_inject_netlink.c
  router/tap_inject_node.c
  router/tap_inject_tap.c

  INSTALL_HEADERS 
  librtnl/mapper.h
  librtnl/netns.h
  librtnl/rtnl.h
  router/tap_inject.h
)
4、在vpp目录下执行正常编译就OK了。
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

这种使用超链接方式将项目目录放在vpp plugins目录下，编译vpp时将本项目一起编译，和upg-vpp项目一致，感觉比flexiWAN中的编译方式要方便很多。

#upg-vpp 方式参考Makefile文件执行make vpp时 会调用脚本:hack/update-vpp.sh
#设置vpp相关的参数。
root@Server:~/workspace/upg-vpp# cat vpp.spec 
VPP_COMMIT=3d2d96e5547484290c9368bac0a420afa8c4c068
VPP_REPO=https://github.com/fdio/vpp.git
VPP_BRANCH=stable/2101
VPP_RELEASE=v21.01
#update-vpp.sh 脚本内容如下：
root@Server:~/workspace/upg-vpp# cat hack/update-vpp.sh 
#!/bin/bash
set -o errexit
set -o nounset
set -o pipefail

if [[ ${BASH:-} ]]; then
  # not compatible with alpine's sh
  set -o errtrace
  cd "$(dirname "${BASH_SOURCE}")/.."
fi

. vpp.spec
rm -rf vpp
#下载vpp代码，并切换到spec文件设置版本分支，打补丁，设置连接。
git clone -b "${VPP_BRANCH}" -n "${VPP_REPO}"
(
  cd vpp
  # FIXME: sometimes checkout on downstream branch fails
  # switch to VPP_BRANCH before creating downstream
  git checkout "${VPP_BRANCH}"
  git checkout -b downstream "${VPP_COMMIT}"
  git am ../vpp-patches/*
  cd src/plugins
  ln -s ../../../upf .
)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

flexiWAN项目中，是先编译vpp，再编译vppsb中的route插件.大家可以进入下面连接看一下。https://gitlab.com/flexiwangroup/flexirouter/-/blob/dev/scripts/vpp_build.sh

下面是在vpp-dev交流邮件中讨论的vppsb route插件的编译方法（没有实际测试，感兴趣的可以自己试试），交流邮件中还有很多配置和使用的相关讨论，感兴趣的可以去看一下。连接地址：https://lists.fd.io/g/vpp-dev/topic/32309215

In the home directory, git clone https://gerrit.fd.io/r/vppsb
cd vppsb
git checkout master

Then, in vpp directory,
ln -sf /home/xflow/vppsb/netlink
ln -sf /home/xflow/vppsb/router
ln -sf /home/xflow/netlink/netlink.mk build-data/packages/
ln -sf /home/xflow/router/router.mk build-data/packages/

cd build-root
sudo make V=0 PLATFORM=vpp TAG=vpp_debug install-deb netlink-install router-install
sudo dpkg -i *.deb
sudo ln -sf /root/xflow/install1/vpp/build-root/install-vpp_debug-native/router/lib64/router.so.0.0.0 /usr/lib/vpp_plugins/router.so
1
2
3
4
5
6
7
8
9
10
11
12
13
14

遇到问题

1、头文件找不到

#头文件引用方式 报错误，找不到。
-#include <librtnl/mapper.h>
-#include <librtnl/netns.h>
#修改双引号方式，也可以修改为 <router/librtnl/netns.h>方式。
+#include "mapper.h"
+#include "netns.h"
1
2
3
4
5
6

掰扯一下C语言头文件引用方式 用尖括号 #include <>：
一般用于包含标准的库头文件，编译器会去系统配置的库环境变量和者用户配置的路径去搜索，而不会在项目的当前目录去查找 用双引号 #include
“”： 一般用于包含用户自己编写的头文件，编译器会先在项目的当前目录查找，找不到后才会去系统配置的库环境变量和用户配置的路径去搜索

2、pool_foreach、pool_foreach_index参数不匹配
pool_foreach相关功能优化----与旧代码相比，减少了4次指令生成相关函数名称变更pool_foreach_old 、pool_foreach_index_old

commit b2c31b685fd2cf28436ca32bc93e23eb24c74878
commit 62c25abaa3e93be5815172d391295a6ab0390122
Author: Damjan Marion <damarion@cisco.com>
Date:   Sat Dec 12 23:32:12 2020 +0100
    vppinfra: simpler and faster pool_foreach
     - reduces number of instructions generated 4 times compared to old code
      #与旧代码相比，减少了4次指令生成
     - adds pool_foreach2 which is more friendly to clang-format #这个后面又删除了。
    Type: improvement

#define pool_foreach(VAR,POOL)                        \
  if (POOL)                                \
    for (VAR = POOL + pool_get_first_index (POOL);            \
     VAR < vec_end (POOL);                      \
     VAR = POOL + pool_get_next_index (POOL, VAR - POOL))

#define pool_foreach_old(VAR,POOL,BODY)                    \
  pool_foreach(VAR,POOL)                        \
   { BODY; }
#define pool_foreach_index(i,v)        \
  if (v)                    \
    for (i = pool_get_first_index (v);        \
     i < vec_len (v);           \
     i = pool_get_next_index (v, i))    \

/** Iterate pool by index. */
#define pool_foreach_index_old(i,v,body)        \
  pool_foreach_index (i,v)            \
    { body; }
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

3、Centos内核版本不支持RTNLGRP_MPLS_ROUTE宏定义
使用linux代码检索平台，基于3.10.0版本查询RTNLGRP_MPLS_ROUTE未找到此宏定义。

linux代码检索平台网址：https://elixir.bootlin.com/linux/v3.10.108/source，应该也是lxr(linux cross reference)搭建的

4、clib_memcpy参数via->rtvia_addr报错
这个原因和上面问题原因一样，因为内核版本不支持未查询到rtvia 结构，将相关mpls处理注释掉就ok了。

struct rtvia { 
      unsigned short rtvia_family; 
      unsigned char rtvia_addr[0]; 
}
 struct rtvia *via = (struct rtvia*) r->via;
clib_memcpy (&rpath.frp_addr.ip4, via->rtvia_addr, sizeof (rpath.frp_addr.ip4));
1
2
3
4
5
6

5、Missing separate debuginfos
在编译完执行make debug启动后报错却是下面安装包，直接按照提示:debuginfo-install xxxx 安装就可以了。

[Inferior 1 (process 14875) exited normally]
Missing separate debuginfos, use: debuginfo-install keyutils-libs-1.5.8-3.el7.x86_64 
libgcc-4.8.5-44.el7.x86_64 libselinux-2.5-15.el7.x86_64 libuuid-2.23.2-65.el7_9.1.x86_64 
mbedtls-2.7.17-1.el7.x86_64 numactl-libs-2.0.12-5.el7.x86_64 pcre-8.32-17.el7.x86_64
 pkcs11-helper-1.11-3.el7.x86_64
1
2
3
4
5

6、程序莫名的退出
make debug执行后程序正常退出。

[root@edge_auto_6 vpp]# make debug
Reading symbols from /root/vpp/build-root/install-vpp_debug-native/vpp/bin/vpp...done.
Signal        Stop    Print   Pass to program Description
SIGUSR1       No    No  Yes     User defined signal 1
(gdb) set args -c /etc/vpp/startup.conf
(gdb) r
Starting program: /root/vpp/build-root/install-vpp_debug-native/vpp/bin/vpp -c /etc/vpp/startup.conf
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
/root/vpp/build-root/install-vpp_debug-native/vpp/bin/vpp[16396]: perfmon: skipping source 'intel-uncore' - intel_uncore_init: no uncore units found
/root/vpp/build-root/install-vpp_debug-native/vpp/bin/vpp[16396]: tls_init_ca_chain:609: Could not initialize TLS CA certificates
/root/vpp/build-root/install-vpp_debug-native/vpp/bin/vpp[16396]: tls_mbedtls_init:644: failed to initialize TLS CA chain
[New Thread 0x7fffaa16d700 (LWP 16401)]
[New Thread 0x7fffa996c700 (LWP 16402)]
[Thread 0x7fffaa16d700 (LWP 16401) exited]
[Thread 0x7ffff3eea900 (LWP 16396) exited]
[Inferior 1 (process 16396) exited normally]  #程序正常退出。
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17

参考文章：gdb退出设置断点不起作用https://www.it1352.com/785461.html

尝试GDB使用设置断点参数，捕捉退出原因“catch syscall exit exit_group”，显示如下：

gdb) catch syscall exit exit_group
Catchpoint 2 (syscalls 'exit' [60] 'exit_group' [231])
(gdb) r
Starting program: /root/vpp/build-root/install-vpp_debug-native/vpp/bin/vpp -c /etc/vpp/startup.conf
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
/root/vpp/build-root/install-vpp_debug-native/vpp/bin/vpp[16423]: perfmon: skipping source 'intel-uncore' - intel_uncore_init: no uncore units found
/root/vpp/build-root/install-vpp_debug-native/vpp/bin/vpp[16423]: tls_init_ca_chain:609: Could not initialize TLS CA certificates
/root/vpp/build-root/install-vpp_debug-native/vpp/bin/vpp[16423]: tls_mbedtls_init:644: failed to initialize TLS CA chain
[New Thread 0x7fffaa16d700 (LWP 16427)]
[New Thread 0x7fffa996c700 (LWP 16428)]

Catchpoint 2 (call to syscall exit_group), 0x00007ffff55ebc99 in __GI__exit (status=status@entry=0) at ../sysdeps/unix/sysv/linux/_exit.c:31
31          INLINE_SYSCALL (exit_group, 1, status);
(gdb) nt
Undefined command: "nt".  Try "help".
(gdb) bt
#0  0x00007ffff55ebc99 in __GI__exit (status=status@entry=0) at ../sysdeps/unix/sysv/linux/_exit.c:31
#1  0x00007ffff555fcab in __run_exit_handlers (status=0, listp=<optimized out>, run_list_atexit=run_list_atexit@entry=true) at exit.c:92
#2  0x00007ffff555fd37 in __GI_exit (status=<optimized out>) at exit.c:99
#3  0x00007ffff554855c in __libc_start_main (main=0x407909 <main>, argc=3, argv=0x7fffffffe2d8, init=<optimized out>, fini=<optimized out>, rtld_fini=<optimized out>, stack_end=0x7fffffffe2c8)
    at ../csu/libc-start.c:300
#4  0x00000000004069a9 in _start ()
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

也没有定位到具体原因，最后发现通过看日志发现环境已经有一个vpp在运行中，bind 5002端口，导致异常退出，将其kill掉后，再启动就OK了。

journalctl -n 100 查询系统日志。  #程序异常时，日志的重要性。
vpp[17031]: clib_socket_init: bind (fd 26, '0.0.0.0:5002'): Address already in use。
1
2

基本配置及验证ping功能

基本环境配置如下：

    vpp1                  <-------->       vpp2
  GigabitEthernet13/0/0 (up): |  GigabitEthernet13/0/0 (up):
  L3 192.168.100.1/24         |   L3 192.168.100.2/24
1
2
3

vpp1上使能tap-inject功能。在内核上面配置vpp1的IP地址192.168.100.1
然后在内核上面执行ping 192.168.100.2 可以正常ping通。

#使能tap-inject
learning_vpp# enable tap-inject 
#查询tap-inject接口
learning_vpp# show tap-inject 
GigabitEthernet13/0/0 -> vpp1
GigabitEthernetb/0/0 -> vpp0
GigabitEthernet1b/0/0 -> vpp2
#
[root@edge_auto_6 ~]# ping -I vpp1 192.168.100.2
PING 192.168.100.2 (192.168.100.2) from 192.168.100.1 vpp1: 56(84) bytes of data.
64 bytes from 192.168.100.2: icmp_seq=1 ttl=64 time=5.80 ms
64 bytes from 192.168.100.2: icmp_seq=2 ttl=64 time=2.67 ms
#在vpp1上抓包
00:13:49:962220: dpdk-input
  GigabitEthernet13/0/0 rx queue 0
  buffer 0x9ff5f: current data 0, length 98, buffer-pool 0, ref-count 1, totlen-nifb 0, trace handle 0x1000002
                  ext-hdr-valid 
                  l4-cksum-computed l4-cksum-correct 
  PKT MBUF: port 1, nb_segs 1, pkt_len 98
    buf_len 2176, data_len 98, ol_flags 0x80, data_off 128, phys_addr 0x761fd840
    packet_type 0x91 l2_len 0 l3_len 0 outer_l2_len 0 outer_l3_len 0
    rss 0x0 fdir.hi 0x0 fdir.lo 0x0
    Packet Offload Flags
      PKT_RX_IP_CKSUM_GOOD (0x0080) IP cksum of RX pkt. is valid
    Packet Types
      RTE_PTYPE_L2_ETHER (0x0001) Ethernet packet
      RTE_PTYPE_L3_IPV4_EXT_UNKNOWN (0x0090) IPv4 packet with or without extension headers
  IP4: 00:0c:29:63:94:30 -> 00:0c:29:17:0a:44
  ICMP: 192.168.100.2 -> 192.168.100.1
    tos 0x00, ttl 64, length 84, checksum 0xaa46 dscp CS0 ecn NON_ECN
    fragment id 0x470e, flags DONT_FRAGMENT
  ICMP echo_reply checksum 0x65cd id 3116
00:13:49:962389: ethernet-input
  frame: flags 0x1, hw-if-index 2, sw-if-index 2
  IP4: 00:0c:29:63:94:30 -> 00:0c:29:17:0a:44
00:13:49:962410: ip4-input
  ICMP: 192.168.100.2 -> 192.168.100.1
    tos 0x00, ttl 64, length 84, checksum 0xaa46 dscp CS0 ecn NON_ECN
    fragment id 0x470e, flags DONT_FRAGMENT
  ICMP echo_reply checksum 0x65cd id 3116
00:13:49:962421: ip4-lookup
  fib 0 dpo-idx 7 flow hash: 0x00000000
  ICMP: 192.168.100.2 -> 192.168.100.1
    tos 0x00, ttl 64, length 84, checksum 0xaa46 dscp CS0 ecn NON_ECN
    fragment id 0x470e, flags DONT_FRAGMENT
  ICMP echo_reply checksum 0x65cd id 3116
00:13:49:962432: ip4-local
    ICMP: 192.168.100.2 -> 192.168.100.1
      tos 0x00, ttl 64, length 84, checksum 0xaa46 dscp CS0 ecn NON_ECN
      fragment id 0x470e, flags DONT_FRAGMENT
    ICMP echo_reply checksum 0x65cd id 3116
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

参考文章
1、VPP Sandbox/router 官方介绍
2、VPPSandBox中netlink和router编译
3、FRR与VPP路由和转发层面流程

4、骨干网PE中VPP与FRR的路由交互

如需要了解更多内容，微信公众号中搜索dpdk vpp 源码分析