Tuesday, July 29, 2014

RDO Setup Two Real Node (Controller+Compute) IceHouse Neutron ML2&OVS&VXLAN Cluster on CentOS 7


ML2&&OVS&&VXLAN multi node setup requires correct version of plugin.ini -> /etc/neutron/plugins/ml2/ml2_conf.ini which appears to be generated with errors by packstack.
Two boxes  have been setup , each one having 2  NICs (enp2s0,enp5s1) for
Controller && Compute Nodes setup. Before running
`packstack --answer-file=TwoNodeVXLAN.txt` SELINUX set to permissive on both nodes.Both enp5s1's assigned IPs and set support VXLAN tunnel  (192.168.0.127, 192.168.0.137 ). Services firewalld and NetworkManager disabled, IPv4 firewall with iptables and service network are enabled and running. Packstack is bind to public IP of interface enp2s0 192.169.1.127, Compute Node is 192.169.1.137 ( view answer-file ).

Setup configuration

- Controller node: Nova, Keystone, Cinder, Glance, Neutron (using Open vSwitch plugin && VXLAN )
- Compute node: Nova (nova-compute), Neutron (openvswitch-agent)


icehouse1.localdomain   -  Controller (192.168.1.127)
icehouse2.localdomain   -  Compute   (192.168.1.137)


[root@icehouse1 ~(keystone_admin)]# cat TwoNodeVXLAN.txt
[general]
CONFIG_SSH_KEY=/root/.ssh/id_rsa.pub
CONFIG_MYSQL_INSTALL=y
CONFIG_GLANCE_INSTALL=y
CONFIG_CINDER_INSTALL=y
CONFIG_NOVA_INSTALL=y
CONFIG_NEUTRON_INSTALL=y
CONFIG_HORIZON_INSTALL=y
CONFIG_SWIFT_INSTALL=n
CONFIG_CEILOMETER_INSTALL=y
CONFIG_HEAT_INSTALL=n
CONFIG_CLIENT_INSTALL=y
CONFIG_NTP_SERVERS=
CONFIG_NAGIOS_INSTALL=y
EXCLUDE_SERVERS=
CONFIG_DEBUG_MODE=n
CONFIG_VMWARE_BACKEND=n
CONFIG_MYSQL_HOST=192.168.1.127
CONFIG_MYSQL_USER=root
CONFIG_MYSQL_PW=a7f0349d1f7a4ab0
CONFIG_AMQP_SERVER=rabbitmq
CONFIG_AMQP_HOST=192.168.1.127
CONFIG_AMQP_ENABLE_SSL=n
CONFIG_AMQP_ENABLE_AUTH=n
CONFIG_AMQP_NSS_CERTDB_PW=0915db728b00409caf4b6e433b756308
CONFIG_AMQP_SSL_PORT=5671
CONFIG_AMQP_SSL_CERT_FILE=/etc/pki/tls/certs/amqp_selfcert.pem
CONFIG_AMQP_SSL_KEY_FILE=/etc/pki/tls/private/amqp_selfkey.pem
CONFIG_AMQP_SSL_SELF_SIGNED=y
CONFIG_AMQP_AUTH_USER=amqp_user
CONFIG_AMQP_AUTH_PASSWORD=f16d26ff54cd4033
CONFIG_KEYSTONE_HOST=192.168.1.127
CONFIG_KEYSTONE_DB_PW=32419736ee454c2c
CONFIG_KEYSTONE_ADMIN_TOKEN=836891519cb640458551556447a5a644
CONFIG_KEYSTONE_ADMIN_PW=4ebab181262d4224
CONFIG_KEYSTONE_DEMO_PW=56eb6360019e45bf
CONFIG_KEYSTONE_TOKEN_FORMAT=PKI
CONFIG_GLANCE_HOST=192.168.1.127
CONFIG_GLANCE_DB_PW=e51feef536104b49
CONFIG_GLANCE_KS_PW=2458775cd64848cb
CONFIG_CINDER_HOST=192.168.1.127
CONFIG_CINDER_DB_PW=bcf3b09c9c4144e2
CONFIG_CINDER_KS_PW=888c59cc113e4489
CONFIG_CINDER_BACKEND=lvm
CONFIG_CINDER_VOLUMES_CREATE=y
CONFIG_CINDER_VOLUMES_SIZE=15G
CONFIG_CINDER_GLUSTER_MOUNTS=
CONFIG_CINDER_NFS_MOUNTS=
CONFIG_VCENTER_HOST=192.168.1.127
CONFIG_VCENTER_USER=
CONFIG_VCENTER_PASSWORD=
CONFIG_NOVA_API_HOST=192.168.1.127
CONFIG_NOVA_CERT_HOST=192.168.1.127
CONFIG_NOVA_VNCPROXY_HOST=192.168.1.127
CONFIG_NOVA_COMPUTE_HOSTS=192.168.1.137
CONFIG_NOVA_CONDUCTOR_HOST=192.168.1.127
CONFIG_NOVA_DB_PW=8cc18e22eaeb4c4d
CONFIG_NOVA_KS_PW=aaf8cf4c60224150
CONFIG_NOVA_SCHED_HOST=192.168.1.127
CONFIG_NOVA_SCHED_CPU_ALLOC_RATIO=16.0
CONFIG_NOVA_SCHED_RAM_ALLOC_RATIO=1.5
CONFIG_NOVA_COMPUTE_PRIVIF=enp5s1
CONFIG_NOVA_NETWORK_HOSTS=192.168.1.127
CONFIG_NOVA_NETWORK_MANAGER=nova.network.manager.FlatDHCPManager
CONFIG_NOVA_NETWORK_PUBIF=enp2s0
CONFIG_NOVA_NETWORK_PRIVIF=enp5s1
CONFIG_NOVA_NETWORK_FIXEDRANGE=192.168.32.0/22
CONFIG_NOVA_NETWORK_FLOATRANGE=10.3.4.0/22
CONFIG_NOVA_NETWORK_DEFAULTFLOATINGPOOL=nova
CONFIG_NOVA_NETWORK_AUTOASSIGNFLOATINGIP=n
CONFIG_NOVA_NETWORK_VLAN_START=100
CONFIG_NOVA_NETWORK_NUMBER=1
CONFIG_NOVA_NETWORK_SIZE=255
CONFIG_VCENTER_HOST=192.168.1.127
CONFIG_VCENTER_USER=
CONFIG_VCENTER_PASSWORD=
CONFIG_VCENTER_CLUSTER_NAME=
CONFIG_NEUTRON_SERVER_HOST=192.168.1.127
CONFIG_NEUTRON_KS_PW=5f11f559abc94440
CONFIG_NEUTRON_DB_PW=0302dcfeb69e439f
CONFIG_NEUTRON_L3_HOSTS=192.168.1.127
CONFIG_NEUTRON_L3_EXT_BRIDGE=br-ex
CONFIG_NEUTRON_DHCP_HOSTS=192.168.1.127
CONFIG_NEUTRON_LBAAS_HOSTS=
CONFIG_NEUTRON_L2_PLUGIN=ml2
CONFIG_NEUTRON_METADATA_HOSTS=192.168.1.127
CONFIG_NEUTRON_METADATA_PW=227f7bbc8b6f4f74
############################################
CONFIG_NEUTRON_ML2_TYPE_DRIVERS=vxlan
CONFIG_NEUTRON_ML2_TENANT_NETWORK_TYPES=vxlan
############################################
CONFIG_NEUTRON_ML2_MECHANISM_DRIVERS=openvswitch
CONFIG_NEUTRON_ML2_FLAT_NETWORKS=*
CONFIG_NEUTRON_ML2_VLAN_RANGES=
CONFIG_NEUTRON_ML2_TUNNEL_ID_RANGES=1001:2000
CONFIG_NEUTRON_ML2_VXLAN_GROUP=239.1.1.2
CONFIG_NEUTRON_ML2_VNI_RANGES=1001:2000
CONFIG_NEUTRON_L2_AGENT=openvswitch
CONFIG_NEUTRON_LB_TENANT_NETWORK_TYPE=local
CONFIG_NEUTRON_LB_VLAN_RANGES=
CONFIG_NEUTRON_LB_INTERFACE_MAPPINGS=
#########################################
CONFIG_NEUTRON_OVS_TENANT_NETWORK_TYPE=vxlan
CONFIG_NEUTRON_OVS_VLAN_RANGES=
CONFIG_NEUTRON_OVS_BRIDGE_MAPPINGS=physnet1:br-ex
CONFIG_NEUTRON_OVS_BRIDGE_IFACES=
CONFIG_NEUTRON_OVS_TUNNEL_RANGES=1001:2000
CONFIG_NEUTRON_OVS_TUNNEL_IF=enp5s1
########################################
CONFIG_NEUTRON_OVS_VXLAN_UDP_PORT=4789
CONFIG_OSCLIENT_HOST=192.168.1.127
CONFIG_HORIZON_HOST=192.168.1.127
CONFIG_HORIZON_SSL=n
CONFIG_SSL_CERT=
CONFIG_SSL_KEY=
CONFIG_SWIFT_PROXY_HOSTS=192.168.1.127
CONFIG_SWIFT_KS_PW=63d3108083ac495b
CONFIG_SWIFT_STORAGE_HOSTS=192.168.1.127
CONFIG_SWIFT_STORAGE_ZONES=1
CONFIG_SWIFT_STORAGE_REPLICAS=1
CONFIG_SWIFT_STORAGE_FSTYPE=ext4
CONFIG_SWIFT_HASH=ebf91dbf930c49ca
CONFIG_SWIFT_STORAGE_SIZE=2G
CONFIG_PROVISION_DEMO=y
CONFIG_PROVISION_TEMPEST=n
CONFIG_PROVISION_DEMO_FLOATRANGE=172.24.4.224/28
CONFIG_PROVISION_TEMPEST_REPO_URI=https://github.com/openstack/tempest.git
CONFIG_PROVISION_TEMPEST_REPO_REVISION=master
CONFIG_PROVISION_ALL_IN_ONE_OVS_BRIDGE=n
CONFIG_HEAT_HOST=192.168.1.127
CONFIG_HEAT_DB_PW=f0be2b0fa2044183
CONFIG_HEAT_AUTH_ENC_KEY=29419b1f4e574e5e
CONFIG_HEAT_KS_PW=d5c39c630c364c5b
CONFIG_HEAT_CLOUDWATCH_INSTALL=n
CONFIG_HEAT_CFN_INSTALL=n
CONFIG_HEAT_CLOUDWATCH_HOST=192.168.1.127
CONFIG_HEAT_CFN_HOST=192.168.1.127
CONFIG_CEILOMETER_HOST=192.168.1.127
CONFIG_CEILOMETER_SECRET=d1ed1459830e4288
CONFIG_CEILOMETER_KS_PW=84f18f2e478f4230
CONFIG_MONGODB_HOST=192.168.1.127
CONFIG_NAGIOS_HOST=192.168.1.127
CONFIG_NAGIOS_PW=e2d02c03b5664ffe
CONFIG_USE_EPEL=y
CONFIG_REPO=
CONFIG_RH_USER=
CONFIG_RH_PW=
CONFIG_RH_BETA_REPO=n
CONFIG_SATELLITE_URL=
CONFIG_SATELLITE_USER=
CONFIG_SATELLITE_PW=
CONFIG_SATELLITE_AKEY=
CONFIG_SATELLITE_CACERT=
CONFIG_SATELLITE_PROFILE=
CONFIG_SATELLITE_FLAGS=
CONFIG_SATELLITE_PROXY=
CONFIG_SATELLITE_PROXY_USER=
CONFIG_SATELLITE_PROXY_PW=

On Controller :-

[root@icehouse1 ~(keystone_admin)]# cat /etc/neutron/plugin.ini
[ml2]
type_drivers = vxlan
tenant_network_types = vxlan
mechanism_drivers =openvswitch
[ml2_type_flat]
[ml2_type_vlan]
[ml2_type_gre]
[ml2_type_vxlan]
vni_ranges =1001:2000
vxlan_group =239.1.1.2
[OVS]
local_ip=192.168.0.127
enable_tunneling=True
integration_bridge=br-int
tunnel_bridge=br-tun
[securitygroup]
enable_security_group = True
firewall_driver=neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
[agent]
polling_interval=2


[root@icehouse1 ~(keystone_admin)]# ls -l /etc/neutron
total 64
-rw-r--r--. 1 root root      193 Jul 29 16:15 api-paste.ini
-rw-r-----. 1 root neutron  3853 Jul 29 16:14 dhcp_agent.ini
-rw-r-----. 1 root neutron   208 Jul 29 16:15 fwaas_driver.ini
-rw-r-----. 1 root neutron  3431 Jul 29 16:14 l3_agent.ini
-rw-r-----. 1 root neutron  1400 Jun  8 01:38 lbaas_agent.ini
-rw-r-----. 1 root neutron  1481 Jul 29 16:15 metadata_agent.ini
-rw-r-----. 1 root neutron 19150 Jul 29 16:15 neutron.conf
lrwxrwxrwx. 1 root root       37 Jul 29 16:14 plugin.ini -> /etc/neutron/plugins/ml2/ml2_conf.ini
-rw-r--r--. 1 root root      452 Jul 29 17:11 plugin.out
drwxr-xr-x. 4 root root       34 Jul 29 16:14 plugins
-rw-r-----. 1 root neutron  6148 Jun  8 01:38 policy.json
-rw-r--r--. 1 root root       78 Jul  2 15:11 release
-rw-r--r--. 1 root root     1216 Jun  8 01:38 rootwrap.conf


On Compute (automatically will be created):-

[root@icehouse1 openvswitch]# pwd
/etc/neutron/plugins/openvswitch
 

[root@icehouse2 openvswitch]# cat ovs_neutron_plugin.ini | grep -v ^# | grep -v ^$
[ovs]
enable_tunneling = True
integration_bridge = br-int
tunnel_bridge = br-tun
local_ip =192.168.0.137
bridge_mappings =physnet1:br-ex
[agent]
polling_interval = 2
tunnel_types =vxlan
vxlan_udp_port =4789
l2_population = False
arp_responder=False
[securitygroup]
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver


On Controller

[root@icehouse1 ~(keystone_admin)]# ovs-vsctl show
2742fa6e-78bf-440e-a2c1-cb48242ea565
    Bridge br-ex
        Port phy-br-ex
            Interface phy-br-ex
        Port "qg-76f29fee-9c"
            Interface "qg-76f29fee-9c"
                type: internal
        Port br-ex
            Interface br-ex
                type: internal
        Port "enp2s0"
            Interface "enp2s0"
    Bridge br-tun
        Port "vxlan-c0a80089"
            Interface "vxlan-c0a80089"
                type: vxlan
                options: {in_key=flow, local_ip="192.168.0.127", out_key=flow, remote_ip="192.168.0.137"}
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
        Port br-tun
            Interface br-tun
                type: internal
    Bridge br-int
        Port "qr-8cad61e3-ce"
            tag: 1
            Interface "qr-8cad61e3-ce"
                type: internal
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "tapff8659ee-8d"
            tag: 1
            Interface "tapff8659ee-8d"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port int-br-ex
            Interface int-br-ex
    ovs_version: "2.0.0"

On Compute

[root@icehouse2 ~]# ovs-vsctl show
642d8c9f-116e-4b44-842a-e975e506fe24
    Bridge br-ex
        Port phy-br-ex
            Interface phy-br-ex
        Port br-ex
            Interface br-ex
                type: internal
    Bridge br-tun
        Port br-tun
            Interface br-tun
                type: internal
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
        Port "vxlan-c0a8007f"
            Interface "vxlan-c0a8007f"
                type: vxlan
                options: {in_key=flow, local_ip="192.168.0.137", out_key=flow, remote_ip="192.168.0.127"}
    Bridge br-int
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port int-br-ex
            Interface int-br-ex
        Port "qvodc2c598a-b3"
            tag: 1
            Interface "qvodc2c598a-b3"
        Port br-int
            Interface br-int
                type: internal
        Port "qvo25cbd1fa-96"
            tag: 1
            Interface "qvo25cbd1fa-96"
    ovs_version: "2.0.0"

[root@icehouse2 ~]# ovs-ofctl show br-tun
OFPT_FEATURES_REPLY (xid=0x2): dpid:00000630614cd749
n_tables:254, n_buffers:256
capabilities: FLOW_STATS TABLE_STATS PORT_STATS QUEUE_STATS ARP_MATCH_IP
actions: OUTPUT SET_VLAN_VID SET_VLAN_PCP STRIP_VLAN SET_DL_SRC SET_DL_DST SET_NW_SRC SET_NW_DST SET_NW_TOS SET_TP_SRC SET_TP_DST ENQUEUE
 1(patch-int): addr:ce:29:c5:27:d3:de
     config:     0
     state:      0
     speed: 0 Mbps now, 0 Mbps max
 2(vxlan-c0a8007f): addr:12:3b:55:68:eb:cd
     config:     0
     state:      0
     speed: 0 Mbps now, 0 Mbps max
 LOCAL(br-tun): addr:06:30:61:4c:d7:49
     config:     0
     state:      0
     speed: 0 Mbps now, 0 Mbps max
OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0
[root@icehouse2 ~]# ovs-ofctl dump-flows  br-tun
NXST_FLOW reply (xid=0x4):
 cookie=0x0, duration=10772.245s, table=0, n_packets=286236, n_bytes=22822087, idle_age=105, priority=1,in_port=1 actions=resubmit(,1)
 cookie=0x0, duration=10722.259s, table=0, n_packets=564293, n_bytes=817326613, idle_age=105, priority=1,in_port=2 actions=resubmit(,3)
 cookie=0x0, duration=10772.194s, table=0, n_packets=6, n_bytes=468, idle_age=10762, priority=0 actions=drop
 cookie=0x0, duration=10772.142s, table=1, n_packets=286129, n_bytes=22808125, idle_age=105, priority=1,dl_dst=00:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,20)
 cookie=0x0, duration=10772.091s, table=1, n_packets=107, n_bytes=13962, idle_age=948, priority=1,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,21)
 cookie=0x0, duration=10772.04s, table=2, n_packets=0, n_bytes=0, idle_age=10772, priority=0 actions=drop
 cookie=0x0, duration=10770.529s, table=3, n_packets=1, n_bytes=70, idle_age=10719, priority=1,tun_id=0x3eb actions=mod_vlan_vid:1,resubmit(,10)
 cookie=0x0, duration=10770.119s, table=3, n_packets=564291, n_bytes=817326473, idle_age=105, priority=1,tun_id=0x3ec actions=mod_vlan_vid:2,resubmit(,10)
 cookie=0x0, duration=10771.989s, table=3, n_packets=4, n_bytes=280, idle_age=10719, priority=0 actions=drop
 cookie=0x0, duration=10771.938s, table=10, n_packets=564292, n_bytes=817326543, idle_age=105, priority=1

actions=learn(table=20,hard_timeout=300,priority=1,NXM_OF_VLAN_TCI[0..11],NXM_OF_ETH_DST[]=NXM_OF_ETH_SRC[],load:0->NXM_OF_VLAN_TCI[],load:NXM_NX_TUN_ID[]->NXM_NX_TUN_ID[],output:NXM_OF_IN_PORT[]),output:1

cookie=0x0, duration=5378.415s, table=20, n_packets=283630, n_bytes=22567138, hard_timeout=300, idle_age=105, hard_age=105, priority=1,vlan_tci=0x0002/0x0fff,dl_dst=fa:16:3e:7b:d4:9d 
actions=load:0->NXM_OF_VLAN_TCI[],load:0x3ec->NXM_NX_TUN_ID[],output:2

 cookie=0x0, duration=10771.887s, table=20, n_packets=4, n_bytes=324, idle_age=7112, priority=0 actions=resubmit(,21)
 cookie=0x0, duration=10770.17s, table=21, n_packets=105, n_bytes=13778, idle_age=948, hard_age=10722, dl_vlan=2 actions=strip_vlan,set_tunnel:0x3ec,output:2
 cookie=0x0, duration=10770.58s, table=21, n_packets=0, n_bytes=0, idle_age=10770, hard_age=10722, dl_vlan=1 actions=strip_vlan,set_tunnel:0x3eb,output:2
 cookie=0x0, duration=10771.835s, table=21, n_packets=6, n_bytes=508, idle_age=5396, priority=0 actions=drop

Monday, July 14, 2014

RDO Setup Two Real Node (Controller+Compute) IceHouse Neutron ML2&OVS&VLAN Cluster on CentOS 7

As of 07/14/2014 Bug https://ask.openstack.org/en/question/35705/attempt-of-rdo-aio-install-icehouse-on-centos-7/ is still pending and workaround suggested above should be applied during two node RDO packstack installation.
  Successful implementation of Neutron ML2&&OVS&&VLAN multi node setup requires correct version of plugin.ini -> /etc/neutron/plugins/ml2/ml2_conf.ini which appears to be generated with errors by packstack.

Two boxes  have been setup , each one having 2  NICs (enp2s0,enp5s1) for
Controller && Compute Nodes setup. Before running
`packstack --answer-file=TwoRealNode Neutron ML2&OVS&VLAN.txt` SELINUX set to permissive on both nodes.Both enp5s1's assigned IPs and set to promiscuous mode (192.168.0.127, 192.168.0.137 ). Services firewalld and NetworkManager disabled, IPv4 firewall with iptables and service network are enabled and running. Packstack is bind to public IP of interface enp2s0 192.169.1.127, Compute Node is 192.169.1.137 ( view answer-file ).

Setup configuration

- Controller node: Nova, Keystone, Cinder, Glance, Neutron (using Open vSwitch plugin && VLAN )
- Compute node: Nova (nova-compute), Neutron (openvswitch-agent)


icehouse1.localdomain   -  Controller (192.168.1.127)
icehouse2.localdomain   -  Compute   (192.168.1.137)

Status after packstack install and updating /etc/neutron/plugin.ini as shown bellow

[root@icehouse1 neutron]# cat plugin.ini
[ml2]
type_drivers = vlan
tenant_network_types = vlan
mechanism_drivers = openvswitch
[ml2_type_vlan]
network_vlan_ranges = physnet1:100:200
[ovs]
network_vlan_ranges = physnet1:100:200
tenant_network_type = vlan
enable_tunneling = False
integration_bridge = br-int
bridge_mappings = physnet1:br-enp5s1
local_ip = 192.168.1.127
[AGENT]
polling_interval = 2
[SECURITYGROUP]
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver


[root@icehouse1 ~(keystone_admin)]# ls -l /etc/neutron
total 60
-rw-r--r--. 1 root root      193 Jul 13 09:06 api-paste.ini
-rw-r-----. 1 root neutron  3853 Jul 13 09:05 dhcp_agent.ini
-rw-r-----. 1 root neutron   208 Jul 13 09:06 fwaas_driver.ini
-rw-r-----. 1 root neutron  3431 Jul 13 09:06 l3_agent.ini
-rw-r-----. 1 root neutron  1400 Jun  8 01:38 lbaas_agent.ini
-rw-r-----. 1 root neutron  1481 Jul 13 09:06 metadata_agent.ini
-rw-r-----. 1 root neutron 19148 Jul 13 09:06 neutron.conf
lrwxrwxrwx. 1 root root       37 Jul 13 09:06 plugin.ini -> /etc/neutron/plugins/ml2/ml2_conf.ini
drwxr-xr-x. 4 root root       34 Jul 13 09:06 plugins
-rw-r-----. 1 root neutron  6148 Jun  8 01:38 policy.json
-rw-r--r--. 1 root root       78 Jun 24 15:14 release
-rw-r--r--. 1 root root     1216 Jun  8 01:38 rootwrap.conf

# systemctl disable NetworkManager
# systemctl stop  NetworkManager
# chkconfig network on 

Assigning IP to OVS bridge br-ex and making enp2s0 OVS port of br-ex :-

[root@icehouse1 network-scripts(keystone_admin)]# cat ifcfg-br-ex
DEVICE="br-ex"
BOOTPROTO="static"
IPADDR="192.168.1.127"
NETMASK="255.255.255.0"
DNS1="83.221.202.254"
BROADCAST="192.168.1.255"
GATEWAY="192.168.1.1"
NM_CONTROLLED="no"
DEFROUTE="yes"
IPV4_FAILURE_FATAL="yes"
IPV6INIT=no
ONBOOT="yes"
TYPE="OVSBridge"
DEVICETYPE="ovs"

[root@icehouse1 network-scripts(keystone_admin)]# cat ifcfg-enp2s0
DEVICE=enp2s0
ONBOOT="yes"
TYPE="OVSPort"
DEVICETYPE="ovs"
OVS_BRIDGE=br-ex
NM_CONTROLLED=no
IPV6INIT=no

# service network restart

[root@icehouse1 ~(keystone_admin)]# openstack-status
== Nova services ==
openstack-nova-api:                     active
openstack-nova-cert:                    active
openstack-nova-compute:                 inactive  (disabled on boot)
openstack-nova-network:                 inactive  (disabled on boot)
openstack-nova-scheduler:               active
openstack-nova-volume:                  inactive  (disabled on boot)
openstack-nova-conductor:               active
== Glance services ==
openstack-glance-api:                   active
openstack-glance-registry:              active
== Keystone service ==
openstack-keystone:                     active
== Horizon service ==
openstack-dashboard:                    active
== neutron services ==
neutron-server:                         active
neutron-dhcp-agent:                     active
neutron-l3-agent:                       active
neutron-metadata-agent:                 active
neutron-lbaas-agent:                    inactive  (disabled on boot)
neutron-openvswitch-agent:              active
neutron-linuxbridge-agent:              inactive  (disabled on boot)
neutron-ryu-agent:                      inactive  (disabled on boot)
neutron-nec-agent:                      inactive  (disabled on boot)
neutron-mlnx-agent:                     inactive  (disabled on boot)
== Cinder services ==
openstack-cinder-api:                   active
openstack-cinder-scheduler:             active
openstack-cinder-volume:                active
openstack-cinder-backup:                inactive  (disabled on boot)
== Support services ==
libvirtd:                               active
openvswitch:                            active
dbus:                                   active
tgtd:                                   inactive  (disabled on boot)
rabbitmq-server:                        active
memcached:                              active
== Keystone users ==
+----------------------------------+------------+---------+----------------------+
|                id                |    name    | enabled |        email         |
+----------------------------------+------------+---------+----------------------+
| d8a581fed7004a73b38ccdccfc3bc7ab |   admin    |   True  |    test@test.com     |
| d4c7dd1c19a54a89a084680b0a301cba |   boris    |   True  |                      |
| 1cf40879e9e74e8b893fb617ecd8a6fc | ceilometer |   True  | ceilometer@localhost |
| 84d41c99fa944eaea81ab5107d360d12 |   cinder   |   True  |   cinder@localhost   |
| 928a6cceed90448fbb93e78ddad4c3f8 |    demo    |   True  |                      |
| b9ff3b2044d14d70944d0af717aff468 |   glance   |   True  |   glance@localhost   |
| fd12b31ef7794fa2a57498e5195747eb |  neutron   |   True  |  neutron@localhost   |
| 4b3859e66b2f4fc9b924bc8f718b9b2f |    nova    |   True  |    nova@localhost    |
| c9921012c6d543c8ba109a2726599826 |   undef    |   True  |                      |
+----------------------------------+------------+---------+----------------------+
== Glance images ==
+--------------------------------------+-----------------+-------------+------------------+-----------+--------+
| ID                                   | Name            | Disk Format | Container Format | Size      | Status |
+--------------------------------------+-----------------+-------------+------------------+-----------+--------+
| 72dbae96-62b5-4011-ab3a-bb45872107c4 | cirros          | qcow2       | bare             | 13147648  | active |
| 94a45bd6-1872-4f1a-8276-6679329b063e | Ubuntu 07/12/14 | qcow2       | bare             | 254738944 | active |
| 70fc1c75-c8f9-4a3b-834c-302f8430aaa0 | VF20image       | qcow2       | bare             | 210829312 | active |
+--------------------------------------+-----------------+-------------+------------------+-----------+--------+
== Nova managed services ==
+------------------+-----------------------+----------+---------+-------+----------------------------+-----------------+
| Binary           | Host                  | Zone     | Status  | State | Updated_at                 | Disabled Reason |
+------------------+-----------------------+----------+---------+-------+----------------------------+-----------------+
| nova-consoleauth | icehouse1.localdomain | internal | enabled | up    | 2014-07-14T08:29:55.000000 | -               |
| nova-scheduler   | icehouse1.localdomain | internal | enabled | up    | 2014-07-14T08:29:55.000000 | -               |
| nova-conductor   | icehouse1.localdomain | internal | enabled | up    | 2014-07-14T08:29:52.000000 | -               |
| nova-cert        | icehouse1.localdomain | internal | enabled | up    | 2014-07-14T08:29:55.000000 | -               |
| nova-compute     | icehouse2.localdomain | nova     | enabled | up    | 2014-07-14T08:29:51.000000 | -               |
+------------------+-----------------------+----------+---------+-------+----------------------------+-----------------+
== Nova networks ==
+--------------------------------------+----------+------+
| ID                                   | Label    | Cidr |
+--------------------------------------+----------+------+
| 702e1fe7-62b5-47c8-95fc-a169231a5dde | public   | -    |
| 430a0ef1-7fc8-40d4-9d25-d69e068b1c98 | private  | -    |
| 1722093c-1852-4eab-a76f-1f9e135880a8 | internal | -    |
+--------------------------------------+----------+------+
== Nova instance flavors ==
+----+-----------+-----------+------+-----------+------+-------+-------------+-----------+
| ID | Name      | Memory_MB | Disk | Ephemeral | Swap | VCPUs | RXTX_Factor | Is_Public |
+----+-----------+-----------+------+-----------+------+-------+-------------+-----------+
| 1  | m1.tiny   | 512       | 1    | 0         |      | 1     | 1.0         | True      |
| 2  | m1.small  | 2048      | 20   | 0         |      | 1     | 1.0         | True      |
| 3  | m1.medium | 4096      | 40   | 0         |      | 2     | 1.0         | True      |
| 4  | m1.large  | 8192      | 80   | 0         |      | 4     | 1.0         | True      |
| 5  | m1.xlarge | 16384     | 160  | 0         |      | 8     | 1.0         | True      |
+----+-----------+-----------+------+-----------+------+-------+-------------+-----------+
== Nova instances ==
+----+------+--------+------------+-------------+----------+
| ID | Name | Status | Task State | Power State | Networks |
+----+------+--------+------------+-------------+----------+
+----+------+--------+------------+-------------+----------+

[root@icehouse1 ~(keystone_admin)]# ovs-vsctl show
bf332dbe-de2a-4485-8d76-21e15d44fbcb
    Bridge "br-enp5s1"
        Port "phy-br-enp5s1"
            Interface "phy-br-enp5s1"
        Port "enp5s1"
            Interface "enp5s1"
        Port "br-enp5s1"
            Interface "br-enp5s1"
                type: internal
    Bridge br-int
        Port br-int
            Interface br-int
                type: internal
        Port "qr-61cb5e96-a8"
            tag: 1
            Interface "qr-61cb5e96-a8"
                type: internal
        Port "tap50c5ff29-c0"
            tag: 3
            Interface "tap50c5ff29-c0"
                type: internal
        Port "qr-b4379334-1b"
            tag: 2
            Interface "qr-b4379334-1b"
                type: internal
        Port "qr-6cc14b98-7b"
            tag: 3
            Interface "qr-6cc14b98-7b"
                type: internal
        Port "tapc5a05b89-ae"
            tag: 2
            Interface "tapc5a05b89-ae"
                type: internal
        Port "tapb1808221-e9"
            tag: 1
            Interface "tapb1808221-e9"
                type: internal
        Port "int-br-enp5s1"
            Interface "int-br-enp5s1"
    Bridge br-ex
        Port "qg-877c3024-c7"
            Interface "qg-877c3024-c7"
                type: internal
        Port "qg-c5751b1f-b9"
            Interface "qg-c5751b1f-b9"
                type: internal
        Port br-ex
            Interface br-ex
                type: internal
        Port "enp2s0"
            Interface "enp2s0"
        Port "qg-1ede8ba6-a0"
            Interface "qg-1ede8ba6-a0"
                type: internal
    ovs_version: "2.0.0"

************************
Metadata Trobleshooting
************************

[root@icehouse1 ~(keystone_admin)]# netstat -antp| grep 8775
tcp        0      0 0.0.0.0:8775            0.0.0.0:*               LISTEN      1580/python      
[root@icehouse1 ~(keystone_admin)]# ps -ef | grep 1580
nova      1580     1  0 07:50 ?        00:01:28 /usr/bin/python /usr/bin/nova-api
nova      5192  1580  0 07:51 ?        00:00:00 /usr/bin/python /usr/bin/nova-api
nova      5193  1580  0 07:51 ?        00:00:00 /usr/bin/python /usr/bin/nova-api
nova      5218  1580  0 07:51 ?        00:00:04 /usr/bin/python /usr/bin/nova-api
nova      5219  1580  0 07:51 ?        00:00:05 /usr/bin/python /usr/bin/nova-api
nova      5226  1580  0 07:51 ?        00:00:00 /usr/bin/python /usr/bin/nova-api
nova      5228  1580  0 07:51 ?        00:00:00 /usr/bin/python /usr/bin/nova-api
root     19054 15837  0 12:16 pts/0    00:00:00 grep --color=auto 1580

[root@icehouse1 ~(keystone_admin)]# ip netns
qrouter-9bfc940b-ca64-4e36-9597-188f67ba44a4
qrouter-a3fad1f2-7c02-4952-ba35-fc1e836a1805
qrouter-9d4283e9-8d53-415e-aa5f-427e3ca69da1
qdhcp-12e1088e-1edf-488a-b8c9-0a4085ea2803
qdhcp-1722093c-1852-4eab-a76f-1f9e135880a8
qdhcp-430a0ef1-7fc8-40d4-9d25-d69e068b1c98

[root@icehouse1 ~(keystone_admin)]# ip netns exec qrouter-9bfc940b-ca64-4e36-9597-188f67ba44a4 iptables -S -t nat | grep 169.254
-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 9697

[root@icehouse1 ~(keystone_admin)]# ip netns exec qrouter-9bfc940b-ca64-4e36-9597-188f67ba44a4 netstat -antp
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name  
tcp        0      0 0.0.0.0:9697            0.0.0.0:*               LISTEN      6954/python      

[root@icehouse1 ~(keystone_admin)]# ps -ef | grep 6954
root      6954     1  0 07:52 ?        00:00:00 /usr/bin/python /bin/neutron-ns-metadata-proxy --pid_file=/var/lib/neutron/external/pids/9bfc940b-ca64-4e36-9597-188f67ba44a4.pid --metadata_proxy_socket=/var/lib/neutron/metadata_proxy --router_id=9bfc940b-ca64-4e36-9597-188f67ba44a4 --state_path=/var/lib/neutron --metadata_port=9697 --verbose --log-file=neutron-ns-metadata-proxy-9bfc940b-ca64-4e36-9597-188f67ba44a4.log --log-dir=/var/log/neutron
root     19599 15837  0 12:20 pts/0    00:00:00 grep --color=auto 6954