openstack-openvswitch流表学习

发表于 | 分类于 ， |

前言

openstack版本是ocata，操作系统是ubuntu 1604.3。

架构图

逻辑架构图在这里

flow分析

这里只分析flow（neutron网络用的是openvswitch+vxlan+local vlan）

br-int

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
ovs-ofctl dump-flows br-int
 cookie=0x8035824320e90178, duration=62019.739s, table=0, n_packets=0, n_bytes=0, idle_age=65534, priority=10,icmp6,in_port=2,icmp_type=136 actions=resubmit(,24)
 cookie=0x8035824320e90178, duration=62019.721s, table=0, n_packets=0, n_bytes=0, idle_age=65534, priority=10,icmp6,in_port=3,icmp_type=136 actions=resubmit(,24)
 cookie=0x8035824320e90178, duration=62019.734s, table=0, n_packets=24, n_bytes=1008, idle_age=28257, priority=10,arp,in_port=2 actions=resubmit(,24)
 cookie=0x8035824320e90178, duration=62019.717s, table=0, n_packets=22, n_bytes=924, idle_age=2913, priority=10,arp,in_port=3 actions=resubmit(,24)
 cookie=0x8035824320e90178, duration=62019.744s, table=0, n_packets=99, n_bytes=10196, idle_age=28262, priority=9,in_port=2 actions=resubmit(,25)
 cookie=0x8035824320e90178, duration=62019.726s, table=0, n_packets=366, n_bytes=36340, idle_age=2918, priority=9,in_port=3 actions=resubmit(,25)
 cookie=0x8035824320e90178, duration=62040.987s, table=0, n_packets=0, n_bytes=0, idle_age=65534, priority=2,in_port=1,dl_src=fa:16:3f:7d:c0:dd actions=resubmit(,1)
 cookie=0x8035824320e90178, duration=62040.983s, table=0, n_packets=0, n_bytes=0, idle_age=65534, priority=2,in_port=1,dl_src=fa:16:3f:e9:2f:8a actions=resubmit(,1)
 cookie=0x8035824320e90178, duration=62040.979s, table=0, n_packets=0, n_bytes=0, idle_age=65534, priority=2,in_port=1,dl_src=fa:16:3f:f4:6e:4d actions=resubmit(,1)
 cookie=0x8035824320e90178, duration=62041.561s, table=0, n_packets=0, n_bytes=0, idle_age=65534, priority=0 actions=NORMAL
 cookie=0x8035824320e90178, duration=62041.009s, table=0, n_packets=450, n_bytes=43915, idle_age=2912, priority=1 actions=NORMAL
 cookie=0x8035824320e90178, duration=62041.013s, table=1, n_packets=0, n_bytes=0, idle_age=65534, priority=1 actions=drop
 cookie=0x8035824320e90178, duration=62041.012s, table=2, n_packets=0, n_bytes=0, idle_age=65534, priority=1 actions=drop
 cookie=0x8035824320e90178, duration=62041.015s, table=23, n_packets=0, n_bytes=0, idle_age=65534, priority=0 actions=drop
 cookie=0x8035824320e90178, duration=62019.741s, table=24, n_packets=0, n_bytes=0, idle_age=62019, priority=2,icmp6,in_port=2,icmp_type=136,nd_target=fe80::f816:3eff:fe13:fea1 actions=NORMAL
 cookie=0x8035824320e90178, duration=62019.723s, table=24, n_packets=0, n_bytes=0, idle_age=62019, priority=2,icmp6,in_port=3,icmp_type=136,nd_target=fe80::f816:3eff:fe1b:6be7 actions=NORMAL
 cookie=0x8035824320e90178, duration=62019.736s, table=24, n_packets=2, n_bytes=84, idle_age=28257, priority=2,arp,in_port=2,arp_spa=192.0.2.3 actions=resubmit(,25)
 cookie=0x8035824320e90178, duration=62019.719s, table=24, n_packets=20, n_bytes=840, idle_age=2913, priority=2,arp,in_port=3,arp_spa=192.0.2.10 actions=resubmit(,25)
 cookie=0x8035824320e90178, duration=62041.560s, table=24, n_packets=0, n_bytes=0, idle_age=65534, priority=0 actions=drop
 cookie=0x8035824320e90178, duration=62019.748s, table=25, n_packets=123, n_bytes=11204, idle_age=28257, priority=2,in_port=2,dl_src=fa:16:3e:13:fe:a1 actions=NORMAL
 cookie=0x8035824320e90178, duration=62019.730s, table=25, n_packets=388, n_bytes=37264, idle_age=2913, priority=2,in_port=3,dl_src=fa:16:3e:1b:6b:e7 actions=NORMAL

port=2和3是两个不同用户的虚机，port=1是patch-tun。
table=24 arp里的源IP和port验证。
table=25 源mac和port验证。
（<端口，源ip，源mac>的ipv4验证在iptables里）

br-tun

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
ovs-ofctl dump-flows br-tun
 cookie=0xa3c68df625ec7f59, duration=62929.875s, table=0, n_packets=525, n_bytes=49584, idle_age=3801, priority=1,in_port=1 actions=resubmit(,1)
 cookie=0xa3c68df625ec7f59, duration=62913.324s, table=0, n_packets=287, n_bytes=26502, idle_age=3801, priority=1,in_port=3 actions=resubmit(,4)
 cookie=0xa3c68df625ec7f59, duration=62907.929s, table=0, n_packets=168, n_bytes=18211, idle_age=29027, priority=1,in_port=2 actions=resubmit(,4)
 cookie=0xa3c68df625ec7f59, duration=62929.985s, table=0, n_packets=0, n_bytes=0, idle_age=65534, priority=0 actions=drop
 cookie=0xa3c68df625ec7f59, duration=62929.872s, table=1, n_packets=525, n_bytes=49584, idle_age=3801, priority=0 actions=resubmit(,2)
 cookie=0xa3c68df625ec7f59, duration=62929.983s, table=2, n_packets=27, n_bytes=1134, idle_age=4064, priority=1,arp,dl_dst=ff:ff:ff:ff:ff:ff actions=resubmit(,21)
 cookie=0xa3c68df625ec7f59, duration=62929.980s, table=2, n_packets=468, n_bytes=45046, idle_age=3801, priority=0,dl_dst=00:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,20)
 cookie=0xa3c68df625ec7f59, duration=62929.978s, table=2, n_packets=30, n_bytes=3404, idle_age=65534, priority=0,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,22)
 cookie=0xa3c68df625ec7f59, duration=62929.977s, table=3, n_packets=0, n_bytes=0, idle_age=65534, priority=0 actions=drop
 cookie=0xa3c68df625ec7f59, duration=62927.265s, table=4, n_packets=64, n_bytes=7237, idle_age=29146, priority=1,tun_id=0x3f actions=mod_vlan_vid:1,resubmit(,9)
 cookie=0xa3c68df625ec7f59, duration=62927.257s, table=4, n_packets=366, n_bytes=35310, idle_age=3801, priority=1,tun_id=0x57 actions=mod_vlan_vid:2,resubmit(,9)
 cookie=0xa3c68df625ec7f59, duration=62929.975s, table=4, n_packets=25, n_bytes=2166, idle_age=63148, priority=0 actions=drop
 cookie=0xa3c68df625ec7f59, duration=62929.974s, table=6, n_packets=0, n_bytes=0, idle_age=65534, priority=0 actions=drop
 cookie=0xa3c68df625ec7f59, duration=62929.853s, table=9, n_packets=0, n_bytes=0, idle_age=65534, priority=1,dl_src=fa:16:3f:7d:c0:dd actions=output:1
 cookie=0xa3c68df625ec7f59, duration=62929.849s, table=9, n_packets=0, n_bytes=0, idle_age=65534, priority=1,dl_src=fa:16:3f:e9:2f:8a actions=output:1
 cookie=0xa3c68df625ec7f59, duration=62929.845s, table=9, n_packets=0, n_bytes=0, idle_age=65534, priority=1,dl_src=fa:16:3f:f4:6e:4d actions=output:1
 cookie=0xa3c68df625ec7f59, duration=62929.873s, table=9, n_packets=430, n_bytes=42547, idle_age=3801, priority=0 actions=resubmit(,10)
 cookie=0xa3c68df625ec7f59, duration=62929.971s, table=10, n_packets=430, n_bytes=42547, idle_age=3801, priority=1 actions=learn(table=20,hard_timeout=300,priority=1,cookie=0xa3c68d
f625ec7f59,NXM_OF_VLAN_TCI[0..11],NXM_OF_ETH_DST[]=NXM_OF_ETH_SRC[],load:0->NXM_OF_VLAN_TCI[],load:NXM_NX_TUN_ID[]->NXM_NX_TUN_ID[],output:OXM_OF_IN_PORT[]),output:1
 cookie=0xa3c68df625ec7f59, duration=62907.916s, table=20, n_packets=92, n_bytes=9094, idle_age=29146, priority=2,dl_vlan=1,dl_dst=fa:16:3e:19:89:22 actions=strip_vlan,load:0x3f->NX
M_NX_TUN_ID[],output:2
 cookie=0xa3c68df625ec7f59, duration=62907.166s, table=20, n_packets=0, n_bytes=0, idle_age=63881, priority=2,dl_vlan=2,dl_dst=fa:16:3e:99:f0:a5 actions=strip_vlan,load:0x57->NXM_NX
_TUN_ID[],output:3
 cookie=0xa3c68df625ec7f59, duration=62907.161s, table=20, n_packets=275, n_bytes=25998, idle_age=3801, priority=2,dl_vlan=2,dl_dst=fa:16:3e:5c:4c:d8 actions=strip_vlan,load:0x57->N
XM_NX_TUN_ID[],output:3
 cookie=0xa3c68df625ec7f59, duration=62907.144s, table=20, n_packets=2, n_bytes=374, idle_age=29027, priority=2,dl_vlan=2,dl_dst=fa:16:3e:9a:b9:6f actions=strip_vlan,load:0x57->NXM_
NX_TUN_ID[],output:2
 cookie=0xa3c68df625ec7f59, duration=62907.139s, table=20, n_packets=0, n_bytes=0, idle_age=62907, priority=2,dl_vlan=2,dl_dst=fa:16:3e:0a:23:bd actions=strip_vlan,load:0x57->NXM_NX
_TUN_ID[],output:2
 cookie=0xa3c68df625ec7f59, duration=62929.970s, table=20, n_packets=0, n_bytes=0, idle_age=65534, priority=0 actions=resubmit(,22)
 cookie=0xa3c68df625ec7f59, duration=62907.918s, table=21, n_packets=2, n_bytes=84, idle_age=29151, priority=1,arp,dl_vlan=1,arp_tpa=192.0.2.2 actions=load:0x2->NXM_OF_ARP_OP[],move
:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[],load:0xfa163e198922->NXM_NX_ARP_SHA[],load:0xc0000202->NXM_OF_ARP_SPA[],move:NXM_OF_ETH_SRC[]->NXM_OF_ETH
_DST[],mod_dl_src:fa:16:3e:19:89:22,IN_PORT
 cookie=0xa3c68df625ec7f59, duration=62907.168s, table=21, n_packets=0, n_bytes=0, idle_age=63881, priority=1,arp,dl_vlan=2,arp_tpa=192.0.2.9 actions=load:0x2->NXM_OF_ARP_OP[],move:
NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[],load:0xfa163e99f0a5->NXM_NX_ARP_SHA[],load:0xc0000209->NXM_OF_ARP_SPA[],move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_
DST[],mod_dl_src:fa:16:3e:99:f0:a5,IN_PORT
 cookie=0xa3c68df625ec7f59, duration=62907.163s, table=21, n_packets=1, n_bytes=42, idle_age=4064, priority=1,arp,dl_vlan=2,arp_tpa=192.0.2.6 actions=load:0x2->NXM_OF_ARP_OP[],move:
NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[],load:0xfa163e5c4cd8->NXM_NX_ARP_SHA[],load:0xc0000206->NXM_OF_ARP_SPA[],move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_
DST[],mod_dl_src:fa:16:3e:5c:4c:d8,IN_PORT
 cookie=0xa3c68df625ec7f59, duration=62907.146s, table=21, n_packets=1, n_bytes=42, idle_age=29032, priority=1,arp,dl_vlan=2,arp_tpa=192.0.2.2 actions=load:0x2->NXM_OF_ARP_OP[],move
:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[],load:0xfa163e9ab96f->NXM_NX_ARP_SHA[],load:0xc0000202->NXM_OF_ARP_SPA[],move:NXM_OF_ETH_SRC[]->NXM_OF_ETH
_DST[],mod_dl_src:fa:16:3e:9a:b9:6f,IN_PORT
 cookie=0xa3c68df625ec7f59, duration=62907.141s, table=21, n_packets=0, n_bytes=0, idle_age=62907, priority=1,arp,dl_vlan=2,arp_tpa=192.0.2.1 actions=load:0x2->NXM_OF_ARP_OP[],move:
NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[],load:0xfa163e0a23bd->NXM_NX_ARP_SHA[],load:0xc0000201->NXM_OF_ARP_SPA[],move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_
DST[],mod_dl_src:fa:16:3e:0a:23:bd,IN_PORT
 cookie=0xa3c68df625ec7f59, duration=62929.968s, table=21, n_packets=21, n_bytes=882, idle_age=65534, priority=0 actions=resubmit(,22)
 cookie=0xa3c68df625ec7f59, duration=62907.926s, table=22, n_packets=29, n_bytes=2026, idle_age=65534, priority=1,dl_vlan=1 actions=strip_vlan,load:0x3f->NXM_NX_TUN_ID[],output:2
 cookie=0xa3c68df625ec7f59, duration=62907.158s, table=22, n_packets=8, n_bytes=1144, idle_age=65534, priority=1,dl_vlan=2 actions=strip_vlan,load:0x57->NXM_NX_TUN_ID[],output:2,out
put:3
 cookie=0xa3c68df625ec7f59, duration=62929.967s, table=22, n_packets=14, n_bytes=1116, idle_age=65534, priority=0 actions=drop

port=1是patch-int，port 2和3是vxlan接口，没有用通配tunnel。
table=0是根据in_port分出向和入向。
table=2是出向流表，按广播arp，单播，广播分发到table 21，20，22.
table=20是明细的单播flow，送到目的（匹配带vlan_id）
table=21是arp代答，然后默认转22
table=22是广播flow，送到目的（匹配带vlan_id）
table=4是入向，tun_id转成vlan_id，转table 9
table=9，转table 10
table=10入向流表，同时有学习，在table=20的后面添加一条priority=1的对应出向的流表，类似于cookie=0xa3c68df625ec7f59, duration=37.162s, table=20, n_packets=0, n_bytes=0, hard_timeout=300, idle_age=37, hard_age=0, priority=1,vlan_tci=0x0002/0x0fff,dl_dst=fa:16:3e:5c:4c:d8 actions=load:0->NXM_OF_VLAN_TCI[],load:0x57->NXM_NX_TUN_ID[],output:3，其中已经有了priority=2一样的推送下去的flow。
这条规则在配置了allowed-address-pair（单网卡多IP，多MAC支持）里用到了mac的情况下会用到。

tips,

  • vlan tag是在进入br-int时，设置在ovs port上的。
  • 本地vlan区分了不同的租户，同时在转发和arp应答时，减少了flow的条数。