技术交流

好好学习，天天向上。

0%

基于OVS的多虚机网络配置

发表于 更新于 分类于 ，

[toc]

背景

想打造一个NAS+软路由一体的机子，考虑在CentOS8上部署多个虚拟机实现。要实现的功能描述如下:

  • 软路由虚机给所有其他虚拟机、HOST以及其他通过网口接入的外部机器分配IP
  • NAS虚机负责管理机器上的所有磁盘

命令

在最终的设计中，Host我部署了一个OVS。软路由虚机用的OpenWrt，NAS虚机用的TrueNAS。OVS创立了两个网桥，一个lan，一个wan。OpenWrt虚机需要桥接两个虚拟网桥以进行网络管理，其他需要联网的机器都桥接lan桥即可。

OVS配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
yum install -y centos-release-openstack-train # 20220405更新这个源好像失效了，可以源码编译安装
yum install openvswitch
systemctl enable --now openvswitch
ovs-vsctl add-br ovsbr-lan # 如果需要删除，则执行 ovs-vsctl del-br ovsbr-lan
ovs-vsctl add-br ovsbr-wan 
ovs-vsctl add-port  ovsbr-wan enp0s31f6 # wan桥添加对外的网口
ovs-vsctl add-port  ovsbr-lan enp2s0f0  # lan桥添加对内的网口
ovs-vsctl add-port  ovsbr-lan enp2s0f1
ovs-vsctl add-port  ovsbr-lan enp2s0f2
ovs-vsctl add-port  ovsbr-lan enp2s0f3

# ifcfg文件,举例如下
[root@Host network-scripts]# cat ifcfg-enp0s31f6 # 这个是wan桥的物理口
TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=dhcp
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
NAME=enp0s31f6
UUID=9557329d-f610-4696-b8c1-a5c7e885e89b
DEVICE=enp0s31f6
ONBOOT=no
[root@Host network-scripts]# cat ifcfg-enp2s0f0 # 这个是lan桥的物理口之一
TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=dhcp
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
NAME=enp2s0f0
UUID=72321515-e762-4701-8db0-990041354705
DEVICE=enp2s0f0
ONBOOT=no

veth配置

如果Host需要联网，则需要用Veth将Host和Lan桥连接起来。Veth重启后会消失，所以需要设置开机的时候自动创建Veth。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
[root@Host ~]# cat /usr/lib/systemd/system/veth.service
[Unit]
Description=Create Veths Between OVS And Host
After=NetworkManager.service

[Service]
ExecStartPre=-/bin/sh -c "/usr/sbin/ip link del dev ovs2host type veth peer name host2ovs"
ExecStart=/usr/sbin/ip link add dev ovs2host type veth peer name host2ovs
ExecStartPost=-/bin/sh -c "/usr/bin/nmcli device set host2ovs managed yes"
ExecStartPost=-/bin/sh -c "/usr/bin/nmcli device set ovs2host managed yes"
ExecStartPost=-/bin/sh -c "/usr/sbin/ip link set up ovs2host; ip link set up host2ovs"
Restart=on-failure

[Install]
WantedBy=multi-user.target

[Install]
WantedBy=multi-user.target

[root@Host ~]# cat /etc/sysconfig/network-scripts/ifcfg-ovs2host 
TYPE=Ethernet
#PROXY_METHOD=none
#BROWSER_ONLY=no
BOOTPROTO=static
#DEFROUTE=yes
#IPV4_FAILURE_FATAL=no
#IPV6INIT=yes
#IPV6_AUTOCONF=yes
#IPV6_DEFROUTE=yes
#IPV6_FAILURE_FATAL=no
#IPV6_ADDR_GEN_MODE=stable-privacy
NAME=ovs2host
UUID=207797a5-553b-4100-9b5a-0571b79a0691
DEVICE=ovs2host
ONBOOT=yes
BRIDGE=ovsbr0

[root@Host ~]# cat /etc/sysconfig/network-scripts/ifcfg-host2ovs 
TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=dhcp
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=host2ovs
UUID=cb879745-0b86-4a19-ae89-10538fbfdd1d
MACADDR=6a:1c:57:6a:9f:d4
DEVICE=host2ovs
ONBOOT=yes

记得吧veth的ovs2host添加到ovsbr-lan中

1
ovs-vsctl add-port ovsbr-lan ovs2host

kvm配置

OpenWrt网络

1
2
3
4
5
6
7
8
9
10
11
12
13
14
<interface type='bridge'>
    <mac address='52:54:00:17:f3:f5'/>
    <source bridge='ovsbr-wan'/>
    <virtualport type='openvswitch'/>
    <model type='virtio'/>
    <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
</interface>
<interface type='bridge'>
    <mac address='52:54:00:94:2c:d5'/>
    <source bridge='ovsbr-lan'/>
    <virtualport type='openvswitch'/>
    <model type='virtio'/>
    <address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/>
</interface>

TrueNAS网络

1
2
3
4
5
6
7
<interface type='bridge'>
    <mac address='52:54:00:4d:84:f9'/>
    <source bridge='ovsbr-lan'/>
    <virtualport type='openvswitch'/>
    <model type='virtio'/>
    <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
</interface>

内存大页+绑核

1
2
3
4
5
6
7
8
9
<memoryBacking>
    <hugepages/>
</memoryBacking>
<vcpu placement='static' cpuset='1,3' current='1'>2</vcpu>
<cputune>
    <vcpupin vcpu='0' cpuset='1'/>
    <vcpupin vcpu='1' cpuset='3'/>
    <emulatorpin cpuset='1,3'/>
</cputune>
1
2
3
4
5
6
7
8
9
10
11
12
13
[root@Host ~]# cat /etc/default/grub 
GRUB_TIMEOUT=5
GRUB_DISTRIBUTOR="$(sed 's, release .*$,,g' /etc/system-release)"
GRUB_DEFAULT=saved
GRUB_DISABLE_SUBMENU=true
GRUB_TERMINAL_OUTPUT="console"
GRUB_CMDLINE_LINUX="crashkernel=auto resume=/dev/mapper/cl-swap rd.lvm.lv=cl/root rd.lvm.lv=cl/swap rhgb quiet intel_iommu=on iommu=pt default_hugepagesz=2M hugepagesz=2M hugepages=22528"
GRUB_DISABLE_RECOVERY="true"
GRUB_ENABLE_BLSCFG=true

[root@Host ~]# grub2-mkconfig -o /boot/efi/EFI/centos/grub.cfg # 如果执行报错，删除grub.cfg文件重新执行此命令（下面），还有注意uefi和legacy的区别,legacy启动的机器命令是grub2-mkconfig -o /boot/grub2/grub.cfg

[root@Host ~]# grub2-editenv /boot/grub2/grubenv create