DVR_SNAT && DVR on RDO Mitaka M1 (CentOS 7.1)

********************************************************
Setup Delorean Repos for Mitaka M1 on CentOS 7.1
********************************************************

yum -y install yum-plugin-priorities
cd /etc/yum.repos.d/
# for Centos 7 and RHEL 7
wget http://trunk.rdoproject.org/centos7/delorean-deps.repo
wget http://trunk.rdoproject.org/centos7/current-passed-ci/delorean.repo

yum -y install openstack-packstack 

********************************************************************
Before running packstack on RDO Mitaka M1 be aware of
********************************************************************

1.  https://bugzilla.redhat.com/show_bug.cgi?id=1288179
2.  https://bugzilla.redhat.com/show_bug.cgi?id=1285314

[root@ip-192-169-142-127 ~(keystone_admin)]# cat answer3Node.txt
[general]
CONFIG_SSH_KEY=/root/.ssh/id_rsa.pub
CONFIG_DEFAULT_PASSWORD=
CONFIG_MARIADB_INSTALL=y
CONFIG_GLANCE_INSTALL=y
CONFIG_CINDER_INSTALL=y
CONFIG_NOVA_INSTALL=y
CONFIG_NEUTRON_INSTALL=y
CONFIG_HORIZON_INSTALL=y
CONFIG_SWIFT_INSTALL=y
# For now, just to avoid headaches
CONFIG_CEILOMETER_INSTALL=n
CONFIG_HEAT_INSTALL=n
CONFIG_CLIENT_INSTALL=y
CONFIG_NTP_SERVERS=
CONFIG_NAGIOS_INSTALL=y
EXCLUDE_SERVERS=
CONFIG_DEBUG_MODE=n
CONFIG_CONTROLLER_HOST=192.169.142.127
CONFIG_COMPUTE_HOSTS=192.169.142.137,192.169.142.147
CONFIG_NETWORK_HOSTS=192.169.142.127
CONFIG_VMWARE_BACKEND=n
CONFIG_UNSUPPORTED=n
CONFIG_VCENTER_HOST=
CONFIG_VCENTER_USER=
CONFIG_VCENTER_PASSWORD=
CONFIG_VCENTER_CLUSTER_NAME=
CONFIG_STORAGE_HOST=192.169.142.127
CONFIG_USE_EPEL=y
CONFIG_REPO=
CONFIG_RH_USER=
CONFIG_SATELLITE_URL=
CONFIG_RH_PW=
CONFIG_RH_OPTIONAL=y
CONFIG_RH_PROXY=
CONFIG_RH_PROXY_PORT=
CONFIG_RH_PROXY_USER=
CONFIG_RH_PROXY_PW=
CONFIG_SATELLITE_USER=
CONFIG_SATELLITE_PW=
CONFIG_SATELLITE_AKEY=
CONFIG_SATELLITE_CACERT=
CONFIG_SATELLITE_PROFILE=
CONFIG_SATELLITE_FLAGS=
CONFIG_SATELLITE_PROXY=
CONFIG_SATELLITE_PROXY_USER=
CONFIG_SATELLITE_PROXY_PW=
CONFIG_AMQP_BACKEND=rabbitmq
CONFIG_AMQP_HOST=192.169.142.127
CONFIG_AMQP_ENABLE_SSL=n
CONFIG_AMQP_ENABLE_AUTH=n
CONFIG_AMQP_NSS_CERTDB_PW=PW_PLACEHOLDER
CONFIG_AMQP_SSL_PORT=5671
CONFIG_AMQP_SSL_CERT_FILE=/etc/pki/tls/certs/amqp_selfcert.pem
CONFIG_AMQP_SSL_KEY_FILE=/etc/pki/tls/private/amqp_selfkey.pem
CONFIG_AMQP_SSL_SELF_SIGNED=y
CONFIG_AMQP_AUTH_USER=amqp_user
CONFIG_AMQP_AUTH_PASSWORD=PW_PLACEHOLDER
CONFIG_MARIADB_HOST=192.169.142.127
CONFIG_MARIADB_USER=root
CONFIG_MARIADB_PW=7207ae344ed04957
CONFIG_KEYSTONE_DB_PW=abcae16b785245c3
CONFIG_KEYSTONE_REGION=RegionOne
CONFIG_KEYSTONE_ADMIN_TOKEN=3ad2de159f9649afb0c342ba57e637d9
CONFIG_KEYSTONE_ADMIN_PW=7049f834927e4468
CONFIG_KEYSTONE_DEMO_PW=bf737b785cfa4398
CONFIG_KEYSTONE_TOKEN_FORMAT=UUID
CONFIG_KEYSTONE_SERVICE_NAME=httpd
CONFIG_GLANCE_DB_PW=41264fc52ffd4fe8
CONFIG_GLANCE_KS_PW=f6a9398960534797
CONFIG_GLANCE_BACKEND=file
CONFIG_CINDER_DB_PW=5ac08c6d09ba4b69
CONFIG_CINDER_KS_PW=c8cb1ecb8c2b4f6f
CONFIG_CINDER_BACKEND=lvm
CONFIG_CINDER_VOLUMES_CREATE=y
CONFIG_CINDER_VOLUMES_SIZE=5G
CONFIG_CINDER_GLUSTER_MOUNTS=
CONFIG_CINDER_NFS_MOUNTS=
CONFIG_CINDER_NETAPP_LOGIN=
CONFIG_CINDER_NETAPP_PASSWORD=
CONFIG_CINDER_NETAPP_HOSTNAME=
CONFIG_CINDER_NETAPP_SERVER_PORT=80
CONFIG_CINDER_NETAPP_STORAGE_FAMILY=ontap_cluster
CONFIG_CINDER_NETAPP_TRANSPORT_TYPE=http
CONFIG_CINDER_NETAPP_STORAGE_PROTOCOL=nfs
CONFIG_CINDER_NETAPP_SIZE_MULTIPLIER=1.0
CONFIG_CINDER_NETAPP_EXPIRY_THRES_MINUTES=720
CONFIG_CINDER_NETAPP_THRES_AVL_SIZE_PERC_START=20
CONFIG_CINDER_NETAPP_THRES_AVL_SIZE_PERC_STOP=60
CONFIG_CINDER_NETAPP_NFS_SHARES_CONFIG=
CONFIG_CINDER_NETAPP_VOLUME_LIST=
CONFIG_CINDER_NETAPP_VFILER=
CONFIG_CINDER_NETAPP_VSERVER=
CONFIG_CINDER_NETAPP_CONTROLLER_IPS=
CONFIG_CINDER_NETAPP_SA_PASSWORD=
CONFIG_CINDER_NETAPP_WEBSERVICE_PATH=/devmgr/v2
CONFIG_CINDER_NETAPP_STORAGE_POOLS=
CONFIG_NOVA_DB_PW=1e1b5aeeeaf342a8
CONFIG_NOVA_KS_PW=d9583177a2444f06
CONFIG_NOVA_SCHED_CPU_ALLOC_RATIO=16.0
CONFIG_NOVA_SCHED_RAM_ALLOC_RATIO=1.5
CONFIG_NOVA_COMPUTE_MIGRATE_PROTOCOL=tcp
CONFIG_NOVA_COMPUTE_PRIVIF=eth1
CONFIG_NOVA_NETWORK_MANAGER=nova.network.manager.FlatDHCPManager
CONFIG_NOVA_NETWORK_PUBIF=eth0
CONFIG_NOVA_NETWORK_PRIVIF=eth1
CONFIG_NOVA_NETWORK_FIXEDRANGE=192.168.32.0/22
CONFIG_NOVA_NETWORK_FLOATRANGE=10.3.4.0/22
CONFIG_NOVA_NETWORK_DEFAULTFLOATINGPOOL=nova
CONFIG_NOVA_NETWORK_AUTOASSIGNFLOATINGIP=n
CONFIG_NOVA_NETWORK_VLAN_START=100
CONFIG_NOVA_NETWORK_NUMBER=1
CONFIG_NOVA_NETWORK_SIZE=255
CONFIG_NEUTRON_KS_PW=808e36e154bd4cee
CONFIG_NEUTRON_DB_PW=0e2b927a21b44737
CONFIG_NEUTRON_L3_EXT_BRIDGE=br-ex
CONFIG_NEUTRON_L2_PLUGIN=ml2
CONFIG_NEUTRON_METADATA_PW=a965cd23ed2f4502
CONFIG_LBAAS_INSTALL=n
CONFIG_NEUTRON_METERING_AGENT_INSTALL=n
CONFIG_NEUTRON_FWAAS=n
CONFIG_NEUTRON_ML2_TYPE_DRIVERS=vxlan
CONFIG_NEUTRON_ML2_TENANT_NETWORK_TYPES=vxlan
CONFIG_NEUTRON_ML2_MECHANISM_DRIVERS=openvswitch
CONFIG_NEUTRON_ML2_FLAT_NETWORKS=*
CONFIG_NEUTRON_ML2_VLAN_RANGES=
CONFIG_NEUTRON_ML2_TUNNEL_ID_RANGES=1001:2000
CONFIG_NEUTRON_ML2_VXLAN_GROUP=239.1.1.2
CONFIG_NEUTRON_ML2_VNI_RANGES=1001:2000
CONFIG_NEUTRON_L2_AGENT=openvswitch
CONFIG_NEUTRON_LB_TENANT_NETWORK_TYPE=local
CONFIG_NEUTRON_LB_VLAN_RANGES=
CONFIG_NEUTRON_LB_INTERFACE_MAPPINGS=
CONFIG_NEUTRON_OVS_TENANT_NETWORK_TYPE=vxlan
CONFIG_NEUTRON_OVS_VLAN_RANGES=
CONFIG_NEUTRON_OVS_BRIDGE_MAPPINGS=physnet1:br-ex
CONFIG_NEUTRON_OVS_BRIDGE_IFACES=
CONFIG_NEUTRON_OVS_TUNNEL_RANGES=1001:2000
CONFIG_NEUTRON_OVS_TUNNEL_IF=eth1
CONFIG_NEUTRON_OVS_VXLAN_UDP_PORT=4789
CONFIG_HORIZON_SSL=n
CONFIG_SSL_CERT=
CONFIG_SSL_KEY=
CONFIG_SSL_CACHAIN=
CONFIG_SWIFT_KS_PW=8f75bfd461234c30
CONFIG_SWIFT_STORAGES=
CONFIG_SWIFT_STORAGE_ZONES=1
CONFIG_SWIFT_STORAGE_REPLICAS=1
CONFIG_SWIFT_STORAGE_FSTYPE=ext4
CONFIG_SWIFT_HASH=a60aacbedde7429a
CONFIG_SWIFT_STORAGE_SIZE=2G
CONFIG_PROVISION_DEMO=y
CONFIG_PROVISION_TEMPEST=n
CONFIG_PROVISION_TEMPEST_USER=
CONFIG_PROVISION_TEMPEST_USER_PW=44faa4ebc3da4459
CONFIG_PROVISION_DEMO_FLOATRANGE=172.24.4.224/28
CONFIG_PROVISION_TEMPEST_REPO_URI=https://github.com/openstack/tempest.git
CONFIG_PROVISION_TEMPEST_REPO_REVISION=master
CONFIG_PROVISION_ALL_IN_ONE_OVS_BRIDGE=n
CONFIG_HEAT_DB_PW=PW_PLACEHOLDER
CONFIG_HEAT_AUTH_ENC_KEY=fc3fb7fee61e46b0
CONFIG_HEAT_KS_PW=PW_PLACEHOLDER
CONFIG_HEAT_CLOUDWATCH_INSTALL=n
CONFIG_HEAT_USING_TRUSTS=y
CONFIG_HEAT_CFN_INSTALL=n
CONFIG_HEAT_DOMAIN=heat
CONFIG_HEAT_DOMAIN_ADMIN=heat_admin

CONFIG_HEAT_DOMAIN_PASSWORD=PW_PLACEHOLDER
CONFIG_CEILOMETER_SECRET=19ae0e7430174349
CONFIG_CEILOMETER_KS_PW=337b08d4b3a44753
CONFIG_MONGODB_HOST=192.169.142.127
CONFIG_NAGIOS_PW=02f168ee8edd44e4

********************
Then follow :-
********************
RDO Liberty DVR Neutron workflow on CentOS 7.1
http://dbaxps.blogspot.ru/2015/10/rdo-liberty-rc-dvr-deployment.html

[root@ip-192-169-142-127 ~(keystone_admin)]# nova-manage --version
No handlers could be found for logger "oslo_config.cfg"
13.0.0

[root@ip-192-169-142-127 ~(keystone_admin)]# neutron l3-agent-list-hosting-router RouterDMS
+--------------------------------------+----------------------------------------+----------------+-------+----------+
| id                                   | host                                   | admin_state_up | alive | ha_state |
+--------------------------------------+----------------------------------------+----------------+-------+----------+
| 0e5f8de4-bae4-4b92-872c-b4a692ffca2b | ip-192-169-142-147.ip.secureserver.net | True           | :-)   |          |
| a08b319b-ce27-4b7c-8f72-e530c148ab70 | ip-192-169-142-127.ip.secureserver.net | True           | :-)   |          |
| ebab6768-cd2f-4d09-a1ba-a6e3aa6b4751 | ip-192-169-142-137.ip.secureserver.net | True           | :-)   |          |
+--------------------------------------+----------------------------------------+----------------+-------+----------+

[root@ip-192-169-142-127 ~(keystone_admin)]# nova service-list
+----+------------------+----------------------------------------+----------+---------+-------+----------------------------+-----------------+
| Id | Binary           | Host                                   | Zone     | Status  | State | Updated_at                 | Disabled Reason |
+----+------------------+----------------------------------------+----------+---------+-------+----------------------------+-----------------+
| 1  | nova-consoleauth | ip-192-169-142-127.ip.secureserver.net | internal | enabled | up    | 2015-12-08T15:00:31.000000 | -               |
| 2  | nova-scheduler   | ip-192-169-142-127.ip.secureserver.net | internal | enabled | up    | 2015-12-08T15:00:31.000000 | -               |
| 3  | nova-conductor   | ip-192-169-142-127.ip.secureserver.net | internal | enabled | up    | 2015-12-08T15:00:30.000000 | -               |
| 4  | nova-cert        | ip-192-169-142-127.ip.secureserver.net | internal | enabled | up    | 2015-12-08T15:00:30.000000 | -               |
| 5  | nova-compute     | ip-192-169-142-147.ip.secureserver.net | nova     | enabled | up    | 2015-12-08T15:00:26.000000 | -               |
| 6  | nova-compute     | ip-192-169-142-137.ip.secureserver.net | nova     | enabled | up    | 2015-12-08T15:00:32.000000 | -               |
+----+------------------+----------------------------------------+----------+---------+-------+----------------------------+-----------------+

[root@ip-192-169-142-127 ~(keystone_admin)]# neutron agent-list
+--------------------------------------+--------------------+----------------------------------------+-------+----------------+---------------------------+
| id                                   | agent_type         | host                                   | alive | admin_state_up | binary                    |
+--------------------------------------+--------------------+----------------------------------------+-------+----------------+---------------------------+
| 0e5f8de4-bae4-4b92-872c-b4a692ffca2b | L3 agent           | ip-192-169-142-147.ip.secureserver.net | :-)   | True           | neutron-l3-agent          |
| 18f3207c-9cde-441c-bfda-26e35b393b5b | Open vSwitch agent | ip-192-169-142-147.ip.secureserver.net | :-)   | True           | neutron-openvswitch-agent |
| 1d22e3ae-9f27-4cf2-aff7-9febb6745a28 | Open vSwitch agent | ip-192-169-142-127.ip.secureserver.net | :-)   | True           | neutron-openvswitch-agent |
| 39dd0e7b-9359-4ea8-890d-bfe2ebda95b8 | Open vSwitch agent | ip-192-169-142-137.ip.secureserver.net | :-)   | True           | neutron-openvswitch-agent |
| 3fd7c6d3-f2bd-4612-9f88-4e7c01ceb90d | DHCP agent         | ip-192-169-142-127.ip.secureserver.net | :-)   | True           | neutron-dhcp-agent        |
| 55f597f5-0b7a-414a-aae2-1098b79fdec4 | Metadata agent     | ip-192-169-142-137.ip.secureserver.net | :-)   | True           | neutron-metadata-agent    |
| a08b319b-ce27-4b7c-8f72-e530c148ab70 | L3 agent           | ip-192-169-142-127.ip.secureserver.net | :-)   | True           | neutron-l3-agent          |
| d8bed085-b87c-4d7d-be8d-503b9ce60eda | Metadata agent     | ip-192-169-142-127.ip.secureserver.net | :-)   | True           | neutron-metadata-agent    |
| ddf1906c-09bd-4415-ab62-4a8d290f7fe9 | Metadata agent     | ip-192-169-142-147.ip.secureserver.net | :-)   | True           | neutron-metadata-agent    |
| ebab6768-cd2f-4d09-a1ba-a6e3aa6b4751 | L3 agent           | ip-192-169-142-137.ip.secureserver.net | :-)   | True           | neutron-l3-agent          |
+--------------------------------------+-------------------+----------------------------------------+-------+----------------+---------------------------+

  

*******************************
DVR_SNAT Section 
*******************************
Controller :-

  

OVS Flows

[root@ip-192-169-142-127 ~(keystone_admin)]# ovs-ofctl show br-int | grep "sg-"
 8(sg-48ec616f-1c): addr:00:00:00:00:00:00

[root@ip-192-169-142-127 ~(keystone_admin)]# ovs-ofctl dump-flows  br-int| grep "output:8"
 cookie=0x976c449145b02ae6, duration=4239.096s, table=1, n_packets=1427417, n_bytes=94772642, idle_age=0, priority=4,dl_vlan=1,dl_dst=fa:16:3e:4b:05:09 actions=strip_vlan,mod_dl_src:fa:16:3e:28:0d:19,output:8
[root@ip-192-169-142-127 ~(keystone_admin)]# ovs-ofctl dump-flows  br-int| grep "output:8"
 cookie=0x976c449145b02ae6, duration=4241.129s, table=1, n_packets=1429159, n_bytes=94888334, idle_age=0, priority=4,dl_vlan=1,dl_dst=fa:16:3e:4b:05:09 actions=strip_vlan,mod_dl_src:fa:16:3e:28:0d:19,output:8
[root@ip-192-169-142-127 ~(keystone_admin)]# ovs-ofctl dump-flows  br-int| grep "output:8"
 cookie=0x976c449145b02ae6, duration=4245.441s, table=1, n_packets=1432026, n_bytes=95078792, idle_age=0, priority=4,dl_vlan=1,dl_dst=fa:16:3e:4b:05:09 actions=strip_vlan,mod_dl_src:fa:16:3e:28:0d:19,output:8
[root@ip-192-169-142-127 ~(keystone_admin)]# ovs-ofctl dump-flows  br-int| grep "output:8"
 cookie=0x976c449145b02ae6, duration=4249.608s, table=1, n_packets=1434642, n_bytes=95252072, idle_age=0, priority=4,dl_vlan=1,dl_dst=fa:16:3e:4b:05:09 actions=strip_vlan,mod_dl_src:fa:16:3e:28:0d:19,output:8
[root@ip-192-169-142-127 ~(keystone_admin)]# ovs-ofctl dump-flows  br-int| grep "output:8"
 cookie=0x976c449145b02ae6, duration=4256.953s, table=1, n_packets=1439185, n_bytes=95553634, idle_age=0, priority=4,dl_vlan=1,dl_dst=fa:16:3e:4b:05:09 actions=strip_vlan,mod_dl_src:fa:16:3e:28:0d:19,output:8
[root@ip-192-169-142-127 ~(keystone_admin)]# ovs-ofctl dump-flows  br-int| grep "output:8"
 cookie=0x976c449145b02ae6, duration=4262.048s, table=1, n_packets=1442924, n_bytes=95800408, idle_age=0, priority=4,dl_vlan=1,dl_dst=fa:16:3e:4b:05:09 actions=strip_vlan,mod_dl_src:fa:16:3e:28:0d:19,

Nova and Neutron work-flow && CLI for HAProxy/Keepalived 3 Node Controller RDO Liberty

The correct name of this post is supposed to be "Nova and Neutron workflow && CLI for HAProxy/Keepalived 3 Node Controller RDO Liberty in an appropriate amount of detail". It follows up http://lxer.com/module/newswire/view/222164/index.html . All environment has been built via Nova and Neutron CLI ( no Horizon involvement ).
Neutron work-flow on Controller is described including OVS flow rules on external bridge created by flat external network provider , eth0 external interface as VLAN OVS port of bridge br-eth0 and br-int

First create keystonerc_admin to provide admin ability manage via CLI

[root@hacontroller1 ~(keystone_admin)]# cat keystonerc_admin
export OS_USERNAME=admin
export OS_TENANT_NAME=admin
export OS_PROJECT_NAME=admin
export OS_REGION_NAME=regionOne
export OS_PASSWORD=keystonetest
export OS_AUTH_URL=http://controller-vip.example.com:35357/v2.0/
export OS_SERVICE_ENDPOINT=http://controller-vip.example.com:35357/v2.0 export OS_SERVICE_TOKEN=$(cat /root/keystone_service_token)
export PS1='[\u@\h \W(keystone_admin)]\$ '

[root@hacontroller1 ~(keystone_admin)]# cat keystonerc_demo
export OS_USERNAME=demo
export OS_TENANT_NAME=demo
export OS_PROJECT_NAME=demo
export OS_REGION_NAME=regionOne
export OS_PASSWORD=redhat
export OS_AUTH_URL=http://controller-vip.example.com:5000/v2.0/
export PS1='[\u@\h \W(keystone_demo)]\$ '

 

[root@hacontroller1 ~(keystone_admin)]#  keystone tenant-list
+----------------------------------+----------+---------+
|                id                |   name   | enabled |
+----------------------------------+----------+---------+
| b2be742697534c3188bdc5ec56038853 |  admin   |   True  |
| efe017b919c1487bab8c58281fcaceeb |   demo   |   True  |
| 4cd322b30ca947eeb86c0a883e549a27 | services |   True  |
+----------------------------------+----------+---------+

****************************************************
Creating HA Neutron router belongs tenant demo
****************************************************

[root@hacontroller1 ~(keystone_admin)]# neutron router-create --ha True \
--tenant-id efe017b919c1487bab8c58281fcaceeb RouterDMS

[root@hacontroller1 ~(keystone_admin)]# neutron l3-agent-list-hosting-router RouterDMS
+--------------------------------------+---------------------------+----------------+-------+----------+
| id                                   | host                      | admin_state_up | alive | ha_state |
+--------------------------------------+---------------------------+----------------+-------+----------+
| 9c83e688-e7b4-4101-97df-844510d0ee52 | hacontroller1.example.com | True           | :-)   | active   |
| a7bdf03e-4550-4f1b-ae6f-25744894086d | hacontroller2.example.com | True           | :-)   | standby  |
+--------------------------------------+---------------------------+----------------+-------+------- 

[root@hacontroller1 ~(keystone_admin)]# neutron router-port-list RouterDMS
+--------------------------------------+-------------------------------------------------+-------------------+--------------------------------------------------------------------------------------+
| id                                   | name                                            | mac_address       | fixed_ips                                                                            |
+--------------------------------------+-------------------------------------------------+-------------------+--------------------------------------------------------------------------------------+
| 013404f6-0bb8-4885-ab64-19e3970d7046 | HA port tenant efe017b919c1487bab8c58281fcaceeb | fa:16:3e:d5:7e:6f | {"subnet_id": "6886d46c-4947-455d-8656-ff0f2a649632", "ip_address": "169.254.192.2"} |
| 4a6bcdf6-7895-45b7-b32b-c63d6e1c6bc5 | HA port tenant efe017b919c1487bab8c58281fcaceeb | fa:16:3e:35:f7:73 | {"subnet_id": "6886d46c-4947-455d-8656-ff0f2a649632", "ip_address": "169.254.192.1"} |
| a37e106e-70a1-47bc-b8de-1153c8cacd2a |                                                 | fa:16:3e:0e:4b:eb | {"subnet_id": "a2c617b1-17cc-4768-b213-9f0795d07b40", "ip_address": "10.10.10.100"}  |
| dd6507fd-73e3-45f6-a935-8bbf29dacbb9 |                                                 | fa:16:3e:26:55:06 | {"subnet_id": "1c47d964-d7ec-4a72-a5a7-bc390c96359d", "ip_address": "30.0.0.1"}      |
+--------------------------------------+-------------------------------------------------+-------------------+------------------------------------------------------------------------------------ 

**************************************
Creating private network as demo
**************************************

[root@hacontroller2 ~(keystone_demo)]#  neutron net-create private
[root@hacontroller2 ~(keystone_demo)]#  neutron subnet-create private \
30.0.0.0/24 --dns_nameservers list=true 83.221.202.254

**************************************
Creating public  network as admin
**************************************

[root@hacontroller1 ~(keystone_admin)]# neutron net-create public --shared \
--provider:network_type flat --provider:physical_network physnet1 --router:external

[root@hacontroller1 ~(keystone_admin)]# neutron subnet-create --gateway 10.10.10.1 \
 --allocation-pool start=10.10.10.100,end=10.10.10.150 --disable-dhcp \
--name public_subnet public 10.10.10.0/24 

[root@hacontroller1 neutron(keystone_demo)]# cat l3_agent.ini | grep -v ^# | grep -v ^$
[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
handle_internal_only_routers = True
send_arp_for_ha = 3
metadata_ip = controller-vip.example.com
external_network_bridge = 
[AGENT] 

[root@hacontroller1 ml2(keystone_admin)]# cat ml2_conf.ini|grep -v ^#|grep -v ^$
[ml2]
type_drivers = local,gre,flat,vxlan,vlan
tenant_network_types = vxlan
mechanism_drivers = openvswitch
[ml2_type_flat]
flat_networks = *
[ml2_type_vlan]
[ml2_type_gre]
tunnel_id_ranges = 10:10000
[ml2_type_vxlan]
vni_ranges = 10:10000
vxlan_group = 224.0.0.1
[ml2_type_geneve]
[securitygroup]
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver

[root@hacontroller1 ml2(keystone_admin)]# cat openvswitch_agent.ini | grep -v ^#|grep -v ^$
[ovs]
local_ip = 192.169.142.221
enable_tunneling = True
integration_bridge = br-int
tunnel_bridge = br-tun
bridge_mappings = physnet1:br-eth0
network_vlan_ranges = physnet1
[agent]
tunnel_types = vxlan
vxlan_udp_port = 4789
l2_population = False
[securitygroup]
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver

When "external_network_bridge = " , Neutron places the external 
interface of the router into the OVS bridge specified by the 
"provider_network" provider attribute in the Neutron network. Traffic is
processed by Open vSwitch flow rules. In this configuration it is 
possible to utilize flat and VLAN provider networks.  

[root@hacontroller1 ~(keystone_admin)]# ovs-ofctl show br-eth0
OFPT_FEATURES_REPLY (xid=0x2): dpid:00003e31a75b624a
n_tables:254, n_buffers:256
capabilities: FLOW_STATS TABLE_STATS PORT_STATS QUEUE_STATS ARP_MATCH_IP
actions: output enqueue set_vlan_vid set_vlan_pcp strip_vlan mod_dl_src mod_dl_dst mod_nw_src mod_nw_dst mod_nw_tos mod_tp_src mod_tp_dst
 1(eth0): addr:52:54:00:41:74:39
     config:     0
     state:      0
     speed: 0 Mbps now, 0 Mbps max
 2(phy-br-eth0): addr:de:0e:37:e4:28:49
     config:     0
     state:      0
     speed: 0 Mbps now, 0 Mbps max
 LOCAL(br-eth0): addr:3e:31:a7:5b:62:4a
     config:     0
     state:      0
     speed: 0 Mbps now, 0 Mbps max
OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0

[root@hacontroller1 ~(keystone_admin)]# ovs-ofctl dump-flows  br-eth0
NXST_FLOW reply (xid=0x4):
 cookie=0x0, duration=6785.707s, table=0, n_packets=18476, n_bytes=1202867, idle_age=3, priority=4,
in_port=2,dl_vlan=3 actions=strip_vlan,NORMAL <==== VLAN tag is striped
 cookie=0x0, duration=6977.001s, table=0, n_packets=13639, n_bytes=766402, idle_age=1, priority=2,in_port=2 actions=drop
 cookie=0x0, duration=6977.041s, table=0, n_packets=11557, n_bytes=10607506, idle_age=1, priority=0 actions=NORMAL

[root@hacontroller1 ~(keystone_admin)]# ovs-vsctl show
eae701a9-447e-4b75-98b5-4f7ce026ddbb
    Bridge br-tun
        fail_mode: secure
        Port "vxlan-c0a98ee0"
            Interface "vxlan-c0a98ee0"
                type: vxlan
                options: {df_default="true", in_key=flow, local_ip="192.169.142.221", out_key=flow, remote_ip="192.169.142.224"}
        Port br-tun
            Interface br-tun
                type: internal
        Port "vxlan-c0a98ede"
            Interface "vxlan-c0a98ede"
                type: vxlan
                options: {df_default="true", in_key=flow, local_ip="192.169.142.221", out_key=flow, remote_ip="192.169.142.222"}
        Port "vxlan-c0a98edf"
            Interface "vxlan-c0a98edf"
                type: vxlan
                options: {df_default="true", in_key=flow, local_ip="192.169.142.221", out_key=flow, remote_ip="192.169.142.223"}
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
    Bridge "br-eth0"
        Port "br-eth0"
            Interface "br-eth0"
                type: internal
        Port "eth0"
            Interface "eth0"               <=============
        Port "phy-br-eth0"
            Interface "phy-br-eth0"
                type: patch
                options: {peer="int-br-eth0"}
    Bridge br-int
        fail_mode: secure
        Port br-int
            Interface br-int
                type: internal
        Port "ha-013404f6-0b"
            tag: 2
            Interface "ha-013404f6-0b"
                type: internal
        Port "int-br-eth0"
            Interface "int-br-eth0"
                type: patch
                options: {peer="phy-br-eth0"}
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "qr-dd6507fd-73"
            tag: 1
            Interface "qr-dd6507fd-73"
                type: internal
        Port "qg-a37e106e-70"             ===============>
            tag: 3
            Interface "qg-a37e106e-70"
                type: internal
        Port "tap7e8e240c-aa"
            tag: 1
            Interface "tap7e8e240c-aa"
                type: internal
    ovs_version: "2.4.0"

  The packet exits the qg-a37e106e-70 (which is outgoing interface of 
corresponding qrouter-namespace attached to br-int due to external network provider involvment)
interface, where it is assigned the VLAN tag associated with the external network 3.
The packet is delivered to the external bridge, where a flow rule strip the VLAN tag 3.
The packet is sent out the physical interface associated with the bridge.
 

**************************************
Now check OVS flow at br-int
**************************************

[root@hacontroller1 ~(keystone_admin)]# ovs-ofctl show  br-int

OFPT_FEATURES_REPLY (xid=0x2): dpid:0000e6b4e9fe1044
n_tables:254, n_buffers:256
capabilities: FLOW_STATS TABLE_STATS PORT_STATS QUEUE_STATS ARP_MATCH_IP
actions: output enqueue set_vlan_vid set_vlan_pcp strip_vlan mod_dl_src mod_dl_dst mod_nw_src mod_nw_dst mod_nw_tos mod_tp_src mod_tp_dst

 1(int-br-eth0): addr:d2:35:eb:fc:ba:27
     config:     0
     state:      0
     speed: 0 Mbps now, 0 Mbps max
 2(patch-tun): addr:1e:95:23:46:9e:a4
     config:     0
     state:      0
     speed: 0 Mbps now, 0 Mbps max
 4(tap73cbf393-29): addr:00:00:00:00:00:00
     config:     PORT_DOWN
     state:      LINK_DOWN
     speed: 0 Mbps now, 0 Mbps max
 5(ha-013404f6-0b): addr:00:00:00:00:00:00
     config:     PORT_DOWN
     state:      LINK_DOWN
     speed: 0 Mbps now, 0 Mbps max
 6(qr-dd6507fd-73): addr:00:00:00:00:00:00
     config:     PORT_DOWN
     state:      LINK_DOWN
     speed: 0 Mbps now, 0 Mbps max
 7(qg-a37e106e-70): addr:00:00:00:00:00:00
     config:     PORT_DOWN
     state:      LINK_DOWN
     speed: 0 Mbps now, 0 Mbps max
 LOCAL(br-int): addr:e6:b4:e9:fe:10:44
     config:     PORT_DOWN
     state:      LINK_DOWN
     speed: 0 Mbps now, 0 Mbps max
OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0

[root@hacontroller1 ~(keystone_admin)]# ovs-ofctl dump-flows  br-int
NXST_FLOW reply (xid=0x4):
 cookie=0x87b766cda5a1f568, duration=5899.822s, table=0, n_packets=14899, n_bytes=18740788, idle_age=1334, priority=3,in_port=1,vlan_tci=0x0000 actions=mod_vlan_vid:3,NORMAL

 cookie=0x87b766cda5a1f568, duration=6141.616s, table=0, n_packets=0, n_bytes=0, idle_age=6141, priority=2,in_port=1 actions=drop
 cookie=0x87b766cda5a1f568, duration=5900.707s, table=0, n_packets=5, n_bytes=390, idle_age=5892, priority=2,in_port=4 actions=drop
 cookie=0x87b766cda5a1f568, duration=6141.675s, table=0, n_packets=59875, n_bytes=21916362, idle_age=0, priority=0 actions=NORMAL
 cookie=0x87b766cda5a1f568, duration=6141.671s, table=23, n_packets=0, n_bytes=0, idle_age=6141, priority=0 actions=drop
 cookie=0x87b766cda5a1f568, duration=6141.667s, table=24, n_packets=0, n_bytes=0, idle_age=6141, priority=0 actions=drop

************************************************************************
Notice that int-br-eth0 and phy-br-eth0 is a veth pair connecting
br-int and br-eth0
************************************************************************

[root@hacontroller1 ~(keystone_admin)]# ip netns
qrouter-afe13460-e106-4a0a-abf5-a618f97de6b9
qdhcp-847e5c9c-ce9f-4b2c-86fb-d7597017e8e3

[root@hacontroller1 ~(keystone_admin)]# ip netns exec qrouter-afe13460-e106-4a0a-abf5-a618f97de6b9  ip route
default via 10.10.10.1  dev  qg-a37e106e-70
10.10.10.0/24 qg-a37e106e-70 proto kernel scope link src  10.10.10.100
30.0.0.0/24 qr-dd6507fd-73 proto kernel scope link src 30.0.0.1
169.254.0.0/24 ha-013404f6-0b proto kernel scope link src 169.254.0.1
169.254.192.0/18 dev ha-013404f6-0b proto kernel scope link src 169.254.192.2

 

Per https://github.com/beekhof/osp-ha-deploy/commit/b2e01e86ca93cfad9ad01d533b386b4c9607c60d#diff-ee239d1187adb09f970dc4ddcf0df1c2 

Assuming eth0 is your interface attached to the external network, create two files in /etc/sysconfig/network-scripts/ as follows (change MTU if you need):

cat < /etc/sysconfig/network-scripts/ifcfg-eth0 
DEVICE=eth0 
ONBOOT=yes 
DEVICETYPE=ovs 
TYPE=OVSPort 
OVS_BRIDGE=br-eth0 
ONBOOT=yes 
BOOTPROTO=none 
VLAN=yes 
MTU="9000" 
NM_CONTROLLED=no 
EOF 

cat < /etc/sysconfig/network-scripts/ifcfg-br-eth0 
DEVICE=br-eth0 
DEVICETYPE=ovs 
OVSBOOTPROTO=none 
TYPE=OVSBridge 
ONBOOT=yes BOOTPROTO=static 
MTU="9000" 
NM_CONTROLLED=no 
EOF

Restart the network for the changes to take effect

[root@hacontroller1 ~(keystone_admin)]# neutron net-list
+--------------------------------------+----------------------------------------------------+-------------------------------------------------------+
| id                                   | name                                               | subnets                                               |
+--------------------------------------+----------------------------------------------------+-------------------------------------------------------+
| b4580386-bc02-4aa7-8792-ea4c40c41573 | public                                             | a2c617b1-17cc-4768-b213-9f0795d07b40 10.10.10.0/24    |
| ab421dc7-27fa-4984-ae21-ba9518887293 | HA network tenant efe017b919c1487bab8c58281fcaceeb | 6886d46c-4947-455d-8656-ff0f2a649632 169.254.192.0/18 |
| 847e5c9c-ce9f-4b2c-86fb-d7597017e8e3 | private                                            | 1c47d964-d7ec-4a72-a5a7-bc390c96359d 30.0.0.0/24      |
+--------------------------------------+----------------------------------------------------+-------------------------------------------------------+ 

[root@hacontroller1 ~(keystone_admin)]# neutron subnet-list
+--------------------------------------+---------------------------------------------------+------------------+------------------------------------------------------+
| id                                   | name                                              | cidr             | allocation_pools                                     |
+--------------------------------------+---------------------------------------------------+------------------+------------------------------------------------------+
| a2c617b1-17cc-4768-b213-9f0795d07b40 | public_subnet                                     | 10.10.10.0/24    | {"start": "10.10.10.100", "end": "10.10.10.150"}     |
| 6886d46c-4947-455d-8656-ff0f2a649632 | HA subnet tenant efe017b919c1487bab8c58281fcaceeb | 169.254.192.0/18 | {"start": "169.254.192.1", "end": "169.254.255.254"} |
| 1c47d964-d7ec-4a72-a5a7-bc390c96359d |                                                   | 30.0.0.0/24      | {"start": "30.0.0.2", "end": "30.0.0.254"}           |
+--------------------------------------+---------------------------------------------------+------------------+------------------------------------------------------+
  

[root@hacontroller2 ~(keystone_demo)]#  neutron router-gateway-set RouterDMS public

[root@hacontroller2 ~(keystone_demo)]#  neutron router-interface-add RouterDMS \
1c47d964-d7ec-4a72-a5a7-bc390c96359d 

[root@hacontroller2 ~(keystone_demo)]# neutron router-port-list RouterDMS
+--------------------------------------+------+-------------------+---------------------------------------------------------------------------------+
| id                                   | name | mac_address       | fixed_ips                                                                       |
+--------------------------------------+------+-------------------+---------------------------------------------------------------------------------+
| dd6507fd-73e3-45f6-a935-8bbf29dacbb9 |      | fa:16:3e:26:55:06 | {"subnet_id": "1c47d964-d7ec-4a72-a5a7-bc390c96359d", "ip_address": "30.0.0.1"} |
+--------------------------------------+------+-------------------+---------------------------------------------------------------------------------+
[root@hacontroller2 ~(keystone_demo)]# neutron port-show dd6507fd-73e3-45f6-a935-8bbf29dacbb9
+-----------------------+--------------------------------------------------------------------------------------------------+
| Field                 | Value                                                                                            |
+-----------------------+--------------------------------------------------------------------------------------------------+
| admin_state_up        | True                                                                                             |
| allowed_address_pairs |                                                                                                  |
| binding:vnic_type     | normal                                                                                           |
| device_id             | afe13460-e106-4a0a-abf5-a618f97de6b9                                                             |
| device_owner          | network:router_interface                                                                         |
| dns_assignment        | {"hostname": "host-30-0-0-1", "ip_address": "30.0.0.1", "fqdn": "host-30-0-0-1.openstacklocal."} |
| dns_name              |                                                                                                  |
| extra_dhcp_opts       |                                                                                                  |
| fixed_ips             | {"subnet_id": "1c47d964-d7ec-4a72-a5a7-bc390c96359d", "ip_address": "30.0.0.1"}                  |
| id                    | dd6507fd-73e3-45f6-a935-8bbf29dacbb9                                                             |
| mac_address           | fa:16:3e:26:55:06                                                                                |
| name                  |                                                                                                  |
| network_id            | 847e5c9c-ce9f-4b2c-86fb-d7597017e8e3                                                             |
| security_groups       |                                                                                                  |
| status                | ACTIVE                                                                                           |
| tenant_id             | efe017b919c1487bab8c58281fcaceeb                                                                 |
+-----------------------+--------------------------------------------------------------------------------------------------+

********************************************
Creating security rules for tenant demo
********************************************

[root@hacontroller2 ~(keystone_demo)]#  neutron security-group-rule-create --protocol icmp \
--direction ingress --remote-ip-prefix 0.0.0.0/0 default

[root@hacontroller2 ~(keystone_demo)]#  neutron security-group-rule-create --protocol tcp \
--port-range-min 22 --port-range-max 22   --direction ingress --remote-ip-prefix 0.0.0.0/0 default 

********************************************
Creating ssh keypair for tenant demo
********************************************

[root@hacontroller2 ~(keystone_demo)]#  nova keypair-add oskey1 > oskey1.priv
[root@hacontroller2 ~(keystone_demo)]#  chmod 600  oskey1.priv

[root@hacontroller2 ~(keystone_demo)]# neutron net-list
+--------------------------------------+---------+----------------------------------------------------+
| id                                   | name    | subnets                                            |
+--------------------------------------+---------+----------------------------------------------------+
| b4580386-bc02-4aa7-8792-ea4c40c41573 | public  | a2c617b1-17cc-4768-b213-9f0795d07b40 10.10.10.0/24 |
| 847e5c9c-ce9f-4b2c-86fb-d7597017e8e3 | private | 1c47d964-d7ec-4a72-a5a7-bc390c96359d 30.0.0.0/24   |
+--------------------------------------+---------+--------------------------------------------------
[root@hacontroller2 ~(keystone_demo)]# glance image-list
+--------------------------------------+-----------+
| ID                                   | Name      |
+--------------------------------------+-----------+
| 6b4ee270-41ca-4a14-b584-d21f6ff5d6be | cirros    |
| e6945bf1-0a0d-4e99-a1fc-64ca45479095 | VF23Cloud |
+--------------------------------------+-----------+

[root@hacontroller2 ~(keystone_demo)]#  nova boot --flavor 2 --key_name oskey1 --image \
e6945bf1-0a0d-4e99-a1fc-64ca45479095 --nic net-id=847e5c9c-ce9f-4b2c-86fb-d7597017e8e3 VF23Devs05

 +--------------------------------------+--------------------------------------------------+
| Property                             | Value                                            |
+--------------------------------------+--------------------------------------------------+
| OS-DCF:diskConfig                    | MANUAL                                           |
| OS-EXT-AZ:availability_zone          |                                                  |
| OS-EXT-STS:power_state               | 0                                                |
| OS-EXT-STS:task_state                | scheduling                                       |
| OS-EXT-STS:vm_state                  | building                                         |
| OS-SRV-USG:launched_at               | -                                                |
| OS-SRV-USG:terminated_at             | -                                                |
| accessIPv4                           |                                                  |
| accessIPv6                           |                                                  |
| adminPass                            | 8c3HZUTS3jZ3                                     |
| config_drive                         |                                                  |
| created                              | 2015-11-28T17:44:02Z                             |
| flavor                               | m1.small (2)                                     |
| hostId                               |                                                  |
| id                                   | 68db2410-5d7d-42ca-82ab-6000123ab8d2             |
| image                                | VF23Cloud (e6945bf1-0a0d-4e99-a1fc-64ca45479095) |
| key_name                             | oskey1                                           |
| metadata                             | {}                                               |
| name                                 | VF23Devs05                                       |
| os-extended-volumes:volumes_attached | []                                               |
| progress                             | 0                                                |
| security_groups                      | default                                          |
| status                               | BUILD                                            |
| tenant_id                            | efe017b919c1487bab8c58281fcaceeb                 |
| updated                              | 2015-11-28T17:44:03Z                             |
| user_id                              | 426a9a98019f4055a2edb3d145355646                 |
+--------------------------------------+--------------------------------------------------+
[root@hacontroller2 ~(keystone_demo)]# nova list

+--------------------------------------+------------+---------+------------+-------------+--------------------------------+
| ID                                   | Name       | Status  | Task State | Power State | Networks                       |
+--------------------------------------+------------+---------+------------+-------------+--------------------------------+
| 2b0f822f-be17-43c1-b127-f626d5a62823 | CirrOSDevs | SHUTOFF | -          | Shutdown    | private=30.0.0.4, 10.10.10.101 |
| 68db2410-5d7d-42ca-82ab-6000123ab8d2 | VF23Devs05 | BUILD   | spawning   | NOSTATE     |                                |
+--------------------------------------+------------+---------+------------+-------------+--------------------------------+
[root@hacontroller2 ~(keystone_demo)]# nova list
+--------------------------------------+------------+---------+------------+-------------+--------------------------------+
| ID                                   | Name       | Status  | Task State | Power State | Networks                       |
+--------------------------------------+------------+---------+------------+-------------+--------------------------------+
| 2b0f822f-be17-43c1-b127-f626d5a62823 | CirrOSDevs | SHUTOFF | -          | Shutdown    | private=30.0.0.4, 10.10.10.101 |
| 68db2410-5d7d-42ca-82ab-6000123ab8d2 | VF23Devs05 | ACTIVE  | -          | Running     | private=30.0.0.10              |
+--------------------------------------+------------+---------+------------+-------------+--------------------------------+

[root@hacontroller2 ~(keystone_demo)]# neutron port-list --device-id \
68db2410-5d7d-42ca-82ab-6000123ab8d2
+--------------------------------------+------+-------------------+----------------------------------------------------------------------------------+
| id                                   | name | mac_address       | fixed_ips                                                                        |
+--------------------------------------+------+-------------------+----------------------------------------------------------------------------------+
| 879c8ca8-fe8e-42d7-8b6b-34be981d03d0 |      | fa:16:3e:32:47:49 | {"subnet_id": "1c47d964-d7ec-4a72-a5a7-bc390c96359d", "ip_address": "30.0.0.10"} |
+--------------------------------------+------+-------------------+----------------------------------------------------------------------------------+

[root@hacontroller2 ~(keystone_demo)]# neutron floatingip-create public
Created a new floatingip:
+---------------------+--------------------------------------+
| Field               | Value                                |
+---------------------+--------------------------------------+
| fixed_ip_address    |                                      |
| floating_ip_address | 10.10.10.102                         |
| floating_network_id | b4580386-bc02-4aa7-8792-ea4c40c41573 |
| id                  | aa48fd10-bb25-46ae-8f76-eb90e343b3f1 |
| port_id             |                                      |
| router_id           |                                      |
| status              | DOWN                                 |
| tenant_id           | efe017b919c1487bab8c58281fcaceeb     |
+---------------------+--------------------------------------+

[root@hacontroller2 ~(keystone_demo)]# neutron floatingip-associate \
aa48fd10-bb25-46ae-8f76-eb90e343b3f1 879c8ca8-fe8e-42d7-8b6b-34be981d03d0
Associated floating IP aa48fd10-bb25-46ae-8f76-eb90e343b3f1

[root@hacontroller2 ~(keystone_demo)]# nova list
+--------------------------------------+------------+---------+------------+-------------+---------------------------------+
| ID                                   | Name       | Status  | Task State | Power State | Networks                        |
+--------------------------------------+------------+---------+------------+-------------+---------------------------------+
| 2b0f822f-be17-43c1-b127-f626d5a62823 | CirrOSDevs | SHUTOFF | -          | Shutdown    | private=30.0.0.4, 10.10.10.101  |
| 68db2410-5d7d-42ca-82ab-6000123ab8d2 | VF23Devs05 | ACTIVE  | -          | Running     | private=30.0.0.10, 10.10.10.102 |
+--------------------------------------+------------+---------+------------+-------------+---------------------------------+

[root@hacontroller1 ~(keystone_admin)]# ip netns exec qrouter-afe13460-e106-4a0a-abf5-a618f97de6b9   ip a
1: lo:  mtu 65536 qdisc noqueue state UNKNOWN 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
8: ha-013404f6-0b:  mtu 1500 qdisc noqueue state UNKNOWN 
    link/ether fa:16:3e:d5:7e:6f brd ff:ff:ff:ff:ff:ff
    inet 169.254.192.2/18 brd 169.254.255.255 scope global ha-013404f6-0b
       valid_lft forever preferred_lft forever
    inet 169.254.0.1/24 scope global ha-013404f6-0b
       valid_lft forever preferred_lft forever
    inet6 fe80::f816:3eff:fed5:7e6f/64 scope link 
       valid_lft forever preferred_lft forever
9: qr-dd6507fd-73:  mtu 1500 qdisc noqueue state UNKNOWN 
    link/ether fa:16:3e:26:55:06 brd ff:ff:ff:ff:ff:ff
    inet 30.0.0.1/24 scope global qr-dd6507fd-73
       valid_lft forever preferred_lft forever
    inet6 fe80::f816:3eff:fe26:5506/64 scope link nodad 
       valid_lft forever preferred_lft forever
10: qg-a37e106e-70:  mtu 1500 qdisc noqueue state UNKNOWN 
    link/ether fa:16:3e:0e:4b:eb brd ff:ff:ff:ff:ff:ff
    inet 10.10.10.100/24 scope global qg-a37e106e-70
       valid_lft forever preferred_lft forever
    inet 10.10.10.101/32 scope global qg-a37e106e-70
       valid_lft forever preferred_lft forever
    inet 10.10.10.102/32 scope global qg-a37e106e-70
       valid_lft forever preferred_lft forever
    inet6 fe80::f816:3eff:fe0e:4beb/64 scope link nodad 
       valid_lft forever preferred_lft forever

  
References
1.http://blog.oddbit.com/2015/08/13/provider-external-networks-details/
2.https://github.com/beekhof/osp-ha-deploy/blob/master/keepalived/neutron-config.md