ML2&&OVS&&VXLAN multi node setup requires correct version of plugin.ini -> /etc/neutron/plugins/ml2/ml2_conf.ini which appears to be generated with errors by packstack.
Two boxes have been setup , each one having 2 NICs (enp2s0,enp5s1) for
Controller && Compute Nodes setup. Before running
`packstack --answer-file=TwoNodeVXLAN.txt` SELINUX set to permissive on both nodes.Both enp5s1's assigned IPs and set support VXLAN tunnel (192.168.0.127, 192.168.0.137 ). Services firewalld and NetworkManager disabled, IPv4 firewall with iptables and service network are enabled and running. Packstack is bind to public IP of interface enp2s0 192.169.1.127, Compute Node is 192.169.1.137 ( view answer-file ).
Setup configuration
- Controller node: Nova, Keystone, Cinder, Glance, Neutron (using Open vSwitch plugin && VXLAN )
- Compute node: Nova (nova-compute), Neutron (openvswitch-agent)
icehouse1.localdomain - Controller (192.168.1.127)
icehouse2.localdomain - Compute (192.168.1.137)
[general]
CONFIG_SSH_KEY=/root/.ssh/id_rsa.pub
CONFIG_MYSQL_INSTALL=y
CONFIG_GLANCE_INSTALL=y
CONFIG_CINDER_INSTALL=y
CONFIG_NOVA_INSTALL=y
CONFIG_NEUTRON_INSTALL=y
CONFIG_HORIZON_INSTALL=y
CONFIG_SWIFT_INSTALL=n
CONFIG_CEILOMETER_INSTALL=y
CONFIG_HEAT_INSTALL=n
CONFIG_CLIENT_INSTALL=y
CONFIG_NTP_SERVERS=
CONFIG_NAGIOS_INSTALL=y
EXCLUDE_SERVERS=
CONFIG_DEBUG_MODE=n
CONFIG_VMWARE_BACKEND=n
CONFIG_MYSQL_HOST=192.168.1.127
CONFIG_MYSQL_USER=root
CONFIG_MYSQL_PW=a7f0349d1f7a4ab0
CONFIG_AMQP_SERVER=rabbitmq
CONFIG_AMQP_HOST=192.168.1.127
CONFIG_AMQP_ENABLE_SSL=n
CONFIG_AMQP_ENABLE_AUTH=n
CONFIG_AMQP_NSS_CERTDB_PW=0915db728b00409caf4b6e433b756308
CONFIG_AMQP_SSL_PORT=5671
CONFIG_AMQP_SSL_CERT_FILE=/etc/pki/tls/certs/amqp_selfcert.pem
CONFIG_AMQP_SSL_KEY_FILE=/etc/pki/tls/private/amqp_selfkey.pem
CONFIG_AMQP_SSL_SELF_SIGNED=y
CONFIG_AMQP_AUTH_USER=amqp_user
CONFIG_AMQP_AUTH_PASSWORD=f16d26ff54cd4033
CONFIG_KEYSTONE_HOST=192.168.1.127
CONFIG_KEYSTONE_DB_PW=32419736ee454c2c
CONFIG_KEYSTONE_ADMIN_TOKEN=836891519cb640458551556447a5a644
CONFIG_KEYSTONE_ADMIN_PW=4ebab181262d4224
CONFIG_KEYSTONE_DEMO_PW=56eb6360019e45bf
CONFIG_KEYSTONE_TOKEN_FORMAT=PKI
CONFIG_GLANCE_HOST=192.168.1.127
CONFIG_GLANCE_DB_PW=e51feef536104b49
CONFIG_GLANCE_KS_PW=2458775cd64848cb
CONFIG_CINDER_HOST=192.168.1.127
CONFIG_CINDER_DB_PW=bcf3b09c9c4144e2
CONFIG_CINDER_KS_PW=888c59cc113e4489
CONFIG_CINDER_BACKEND=lvm
CONFIG_CINDER_VOLUMES_CREATE=y
CONFIG_CINDER_VOLUMES_SIZE=15G
CONFIG_CINDER_GLUSTER_MOUNTS=
CONFIG_CINDER_NFS_MOUNTS=
CONFIG_VCENTER_HOST=192.168.1.127
CONFIG_VCENTER_USER=
CONFIG_VCENTER_PASSWORD=
CONFIG_NOVA_API_HOST=192.168.1.127
CONFIG_NOVA_CERT_HOST=192.168.1.127
CONFIG_NOVA_VNCPROXY_HOST=192.168.1.127
CONFIG_NOVA_COMPUTE_HOSTS=192.168.1.137
CONFIG_NOVA_CONDUCTOR_HOST=192.168.1.127
CONFIG_NOVA_DB_PW=8cc18e22eaeb4c4d
CONFIG_NOVA_KS_PW=aaf8cf4c60224150
CONFIG_NOVA_SCHED_HOST=192.168.1.127
CONFIG_NOVA_SCHED_CPU_ALLOC_RATIO=16.0
CONFIG_NOVA_SCHED_RAM_ALLOC_RATIO=1.5
CONFIG_NOVA_COMPUTE_PRIVIF=enp5s1
CONFIG_NOVA_NETWORK_HOSTS=192.168.1.127
CONFIG_NOVA_NETWORK_MANAGER=nova.network.manager.FlatDHCPManager
CONFIG_NOVA_NETWORK_PUBIF=enp2s0
CONFIG_NOVA_NETWORK_PRIVIF=enp5s1
CONFIG_NOVA_NETWORK_FIXEDRANGE=192.168.32.0/22
CONFIG_NOVA_NETWORK_FLOATRANGE=10.3.4.0/22
CONFIG_NOVA_NETWORK_DEFAULTFLOATINGPOOL=nova
CONFIG_NOVA_NETWORK_AUTOASSIGNFLOATINGIP=n
CONFIG_NOVA_NETWORK_VLAN_START=100
CONFIG_NOVA_NETWORK_NUMBER=1
CONFIG_NOVA_NETWORK_SIZE=255
CONFIG_VCENTER_HOST=192.168.1.127
CONFIG_VCENTER_USER=
CONFIG_VCENTER_PASSWORD=
CONFIG_VCENTER_CLUSTER_NAME=
CONFIG_NEUTRON_SERVER_HOST=192.168.1.127
CONFIG_NEUTRON_KS_PW=5f11f559abc94440
CONFIG_NEUTRON_DB_PW=0302dcfeb69e439f
CONFIG_NEUTRON_L3_HOSTS=192.168.1.127
CONFIG_NEUTRON_L3_EXT_BRIDGE=br-ex
CONFIG_NEUTRON_DHCP_HOSTS=192.168.1.127
CONFIG_NEUTRON_LBAAS_HOSTS=
CONFIG_NEUTRON_L2_PLUGIN=ml2
CONFIG_NEUTRON_METADATA_HOSTS=192.168.1.127
CONFIG_NEUTRON_METADATA_PW=227f7bbc8b6f4f74
############################################
CONFIG_NEUTRON_ML2_TYPE_DRIVERS=vxlan
CONFIG_NEUTRON_ML2_TENANT_NETWORK_TYPES=vxlan
############################################
CONFIG_NEUTRON_ML2_MECHANISM_DRIVERS=openvswitch
CONFIG_NEUTRON_ML2_FLAT_NETWORKS=*
CONFIG_NEUTRON_ML2_VLAN_RANGES=
CONFIG_NEUTRON_ML2_TUNNEL_ID_RANGES=1001:2000
CONFIG_NEUTRON_ML2_VXLAN_GROUP=239.1.1.2
CONFIG_NEUTRON_ML2_VNI_RANGES=1001:2000
CONFIG_NEUTRON_L2_AGENT=openvswitch
CONFIG_NEUTRON_LB_TENANT_NETWORK_TYPE=local
CONFIG_NEUTRON_LB_VLAN_RANGES=
CONFIG_NEUTRON_LB_INTERFACE_MAPPINGS=
#########################################
CONFIG_NEUTRON_OVS_TENANT_NETWORK_TYPE=vxlan
CONFIG_NEUTRON_OVS_VLAN_RANGES=
CONFIG_NEUTRON_OVS_BRIDGE_MAPPINGS=physnet1:br-ex
CONFIG_NEUTRON_OVS_BRIDGE_IFACES=
CONFIG_NEUTRON_OVS_TUNNEL_RANGES=1001:2000
CONFIG_NEUTRON_OVS_TUNNEL_IF=enp5s1
########################################
CONFIG_NEUTRON_OVS_VXLAN_UDP_PORT=4789
CONFIG_OSCLIENT_HOST=192.168.1.127
CONFIG_HORIZON_HOST=192.168.1.127
CONFIG_HORIZON_SSL=n
CONFIG_SSL_CERT=
CONFIG_SSL_KEY=
CONFIG_SWIFT_PROXY_HOSTS=192.168.1.127
CONFIG_SWIFT_KS_PW=63d3108083ac495b
CONFIG_SWIFT_STORAGE_HOSTS=192.168.1.127
CONFIG_SWIFT_STORAGE_ZONES=1
CONFIG_SWIFT_STORAGE_REPLICAS=1
CONFIG_SWIFT_STORAGE_FSTYPE=ext4
CONFIG_SWIFT_HASH=ebf91dbf930c49ca
CONFIG_SWIFT_STORAGE_SIZE=2G
CONFIG_PROVISION_DEMO=y
CONFIG_PROVISION_TEMPEST=n
CONFIG_PROVISION_DEMO_FLOATRANGE=172.24.4.224/28
CONFIG_PROVISION_TEMPEST_REPO_URI=https://github.com/openstack/tempest.git
CONFIG_PROVISION_TEMPEST_REPO_REVISION=master
CONFIG_PROVISION_ALL_IN_ONE_OVS_BRIDGE=n
CONFIG_HEAT_HOST=192.168.1.127
CONFIG_HEAT_DB_PW=f0be2b0fa2044183
CONFIG_HEAT_AUTH_ENC_KEY=29419b1f4e574e5e
CONFIG_HEAT_KS_PW=d5c39c630c364c5b
CONFIG_HEAT_CLOUDWATCH_INSTALL=n
CONFIG_HEAT_CFN_INSTALL=n
CONFIG_HEAT_CLOUDWATCH_HOST=192.168.1.127
CONFIG_HEAT_CFN_HOST=192.168.1.127
CONFIG_CEILOMETER_HOST=192.168.1.127
CONFIG_CEILOMETER_SECRET=d1ed1459830e4288
CONFIG_CEILOMETER_KS_PW=84f18f2e478f4230
CONFIG_MONGODB_HOST=192.168.1.127
CONFIG_NAGIOS_HOST=192.168.1.127
CONFIG_NAGIOS_PW=e2d02c03b5664ffe
CONFIG_USE_EPEL=y
CONFIG_REPO=
CONFIG_RH_USER=
CONFIG_RH_PW=
CONFIG_RH_BETA_REPO=n
CONFIG_SATELLITE_URL=
CONFIG_SATELLITE_USER=
CONFIG_SATELLITE_PW=
CONFIG_SATELLITE_AKEY=
CONFIG_SATELLITE_CACERT=
CONFIG_SATELLITE_PROFILE=
CONFIG_SATELLITE_FLAGS=
CONFIG_SATELLITE_PROXY=
CONFIG_SATELLITE_PROXY_USER=
CONFIG_SATELLITE_PROXY_PW=
On Controller :-
[root@icehouse1 ~(keystone_admin)]# cat /etc/neutron/plugin.ini
[ml2]
type_drivers = vxlan
tenant_network_types = vxlan
mechanism_drivers =openvswitch
[ml2_type_flat]
[ml2_type_vlan]
[ml2_type_gre]
[ml2_type_vxlan]
vni_ranges =1001:2000
vxlan_group =239.1.1.2
[OVS]
local_ip=192.168.0.127
enable_tunneling=True
integration_bridge=br-int
tunnel_bridge=br-tun
[securitygroup]
enable_security_group = True
firewall_driver=neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
[agent]
polling_interval=2
[root@icehouse1 ~(keystone_admin)]# ls -l /etc/neutron
total 64
-rw-r--r--. 1 root root 193 Jul 29 16:15 api-paste.ini
-rw-r-----. 1 root neutron 3853 Jul 29 16:14 dhcp_agent.ini
-rw-r-----. 1 root neutron 208 Jul 29 16:15 fwaas_driver.ini
-rw-r-----. 1 root neutron 3431 Jul 29 16:14 l3_agent.ini
-rw-r-----. 1 root neutron 1400 Jun 8 01:38 lbaas_agent.ini
-rw-r-----. 1 root neutron 1481 Jul 29 16:15 metadata_agent.ini
-rw-r-----. 1 root neutron 19150 Jul 29 16:15 neutron.conf
lrwxrwxrwx. 1 root root 37 Jul 29 16:14 plugin.ini -> /etc/neutron/plugins/ml2/ml2_conf.ini
-rw-r--r--. 1 root root 452 Jul 29 17:11 plugin.out
drwxr-xr-x. 4 root root 34 Jul 29 16:14 plugins
-rw-r-----. 1 root neutron 6148 Jun 8 01:38 policy.json
-rw-r--r--. 1 root root 78 Jul 2 15:11 release
-rw-r--r--. 1 root root 1216 Jun 8 01:38 rootwrap.conf
On Compute (automatically will be created):-
[root@icehouse1 openvswitch]# pwd
/etc/neutron/plugins/openvswitch
[root@icehouse2 openvswitch]# cat ovs_neutron_plugin.ini | grep -v ^# | grep -v ^$
[ovs]
enable_tunneling = True
integration_bridge = br-int
tunnel_bridge = br-tun
local_ip =192.168.0.137
bridge_mappings =physnet1:br-ex
[agent]
polling_interval = 2
tunnel_types =vxlan
vxlan_udp_port =4789
l2_population = False
arp_responder=False
[securitygroup]
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
On Controller
[root@icehouse1 ~(keystone_admin)]# ovs-vsctl show
2742fa6e-78bf-440e-a2c1-cb48242ea565
Bridge br-ex
Port phy-br-ex
Interface phy-br-ex
Port "qg-76f29fee-9c"
Interface "qg-76f29fee-9c"
type: internal
Port br-ex
Interface br-ex
type: internal
Port "enp2s0"
Interface "enp2s0"
Bridge br-tun
Port "vxlan-c0a80089"
Interface "vxlan-c0a80089"
type: vxlan
options: {in_key=flow, local_ip="192.168.0.127", out_key=flow, remote_ip="192.168.0.137"}
Port patch-int
Interface patch-int
type: patch
options: {peer=patch-tun}
Port br-tun
Interface br-tun
type: internal
Bridge br-int
Port "qr-8cad61e3-ce"
tag: 1
Interface "qr-8cad61e3-ce"
type: internal
Port patch-tun
Interface patch-tun
type: patch
options: {peer=patch-int}
Port "tapff8659ee-8d"
tag: 1
Interface "tapff8659ee-8d"
type: internal
Port br-int
Interface br-int
type: internal
Port int-br-ex
Interface int-br-ex
ovs_version: "2.0.0"
On Compute
[root@icehouse2 ~]# ovs-vsctl show
642d8c9f-116e-4b44-842a-e975e506fe24
Bridge br-ex
Port phy-br-ex
Interface phy-br-ex
Port br-ex
Interface br-ex
type: internal
Bridge br-tun
Port br-tun
Interface br-tun
type: internal
Port patch-int
Interface patch-int
type: patch
options: {peer=patch-tun}
Port "vxlan-c0a8007f"
Interface "vxlan-c0a8007f"
type: vxlan
options: {in_key=flow, local_ip="192.168.0.137", out_key=flow, remote_ip="192.168.0.127"}
Bridge br-int
Port patch-tun
Interface patch-tun
type: patch
options: {peer=patch-int}
Port int-br-ex
Interface int-br-ex
Port "qvodc2c598a-b3"
tag: 1
Interface "qvodc2c598a-b3"
Port br-int
Interface br-int
type: internal
Port "qvo25cbd1fa-96"
tag: 1
Interface "qvo25cbd1fa-96"
ovs_version: "2.0.0"
[root@icehouse2 ~]# ovs-ofctl show br-tun
OFPT_FEATURES_REPLY (xid=0x2): dpid:00000630614cd749
n_tables:254, n_buffers:256
capabilities: FLOW_STATS TABLE_STATS PORT_STATS QUEUE_STATS ARP_MATCH_IP
actions: OUTPUT SET_VLAN_VID SET_VLAN_PCP STRIP_VLAN SET_DL_SRC SET_DL_DST SET_NW_SRC SET_NW_DST SET_NW_TOS SET_TP_SRC SET_TP_DST ENQUEUE
1(patch-int): addr:ce:29:c5:27:d3:de
config: 0
state: 0
speed: 0 Mbps now, 0 Mbps max
2(vxlan-c0a8007f): addr:12:3b:55:68:eb:cd
config: 0
state: 0
speed: 0 Mbps now, 0 Mbps max
LOCAL(br-tun): addr:06:30:61:4c:d7:49
config: 0
state: 0
speed: 0 Mbps now, 0 Mbps max
OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0
[root@icehouse2 ~]# ovs-ofctl dump-flows br-tun
NXST_FLOW reply (xid=0x4):
cookie=0x0, duration=10772.245s, table=0, n_packets=286236, n_bytes=22822087, idle_age=105, priority=1,in_port=1 actions=resubmit(,1)
cookie=0x0, duration=10722.259s, table=0, n_packets=564293, n_bytes=817326613, idle_age=105, priority=1,in_port=2 actions=resubmit(,3)
cookie=0x0, duration=10772.194s, table=0, n_packets=6, n_bytes=468, idle_age=10762, priority=0 actions=drop
cookie=0x0, duration=10772.142s, table=1, n_packets=286129, n_bytes=22808125, idle_age=105, priority=1,dl_dst=00:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,20)
cookie=0x0, duration=10772.091s, table=1, n_packets=107, n_bytes=13962, idle_age=948, priority=1,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,21)
cookie=0x0, duration=10772.04s, table=2, n_packets=0, n_bytes=0, idle_age=10772, priority=0 actions=drop
cookie=0x0, duration=10770.529s, table=3, n_packets=1, n_bytes=70, idle_age=10719, priority=1,tun_id=0x3eb actions=mod_vlan_vid:1,resubmit(,10)
cookie=0x0, duration=10770.119s, table=3, n_packets=564291, n_bytes=817326473, idle_age=105, priority=1,tun_id=0x3ec actions=mod_vlan_vid:2,resubmit(,10)
cookie=0x0, duration=10771.989s, table=3, n_packets=4, n_bytes=280, idle_age=10719, priority=0 actions=drop
cookie=0x0, duration=10771.938s, table=10, n_packets=564292, n_bytes=817326543, idle_age=105, priority=1
actions=learn(table=20,hard_timeout=300,priority=1,NXM_OF_VLAN_TCI[0..11],NXM_OF_ETH_DST[]=NXM_OF_ETH_SRC[],load:0->NXM_OF_VLAN_TCI[],load:NXM_NX_TUN_ID[]->NXM_NX_TUN_ID[],output:NXM_OF_IN_PORT[]),output:1
cookie=0x0, duration=5378.415s, table=20, n_packets=283630, n_bytes=22567138, hard_timeout=300, idle_age=105, hard_age=105, priority=1,vlan_tci=0x0002/0x0fff,dl_dst=fa:16:3e:7b:d4:9d
actions=load:0->NXM_OF_VLAN_TCI[],load:0x3ec->NXM_NX_TUN_ID[],output:2
cookie=0x0, duration=10771.887s, table=20, n_packets=4, n_bytes=324, idle_age=7112, priority=0 actions=resubmit(,21)
cookie=0x0, duration=10770.17s, table=21, n_packets=105, n_bytes=13778, idle_age=948, hard_age=10722, dl_vlan=2 actions=strip_vlan,set_tunnel:0x3ec,output:2
cookie=0x0, duration=10770.58s, table=21, n_packets=0, n_bytes=0, idle_age=10770, hard_age=10722, dl_vlan=1 actions=strip_vlan,set_tunnel:0x3eb,output:2
cookie=0x0, duration=10771.835s, table=21, n_packets=6, n_bytes=508, idle_age=5396, priority=0 actions=drop
