Basic troubleshooting AIO Havana Instance on CentOS 6.5

 This Howto is supposed to show up some internals which usually are not known by  people just starting with AIO ( allinone) Havana setup on CentOS 6.5
View also RDO Havana Neutron Namespaces Troubleshooting for OVS&VLAN (GRE) Config  http://bderzhavets.blogspot.com/2014/04/rdo-havana-neutron-namespaces.html



Getting routers and network ids for corresponding namespaces


[root@havana ~(keystone_admin)]# neutron router-list
+--------------------------------------+---------+-----------------------------------------------------------------------------+
| id                                   | name    | external_gateway_info                                                       |
+--------------------------------------+---------+-----------------------------------------------------------------------------+
| 6c0825b7-7cb1-4327-a97b-04a7837ceba4 | router2 | {"network_id": "7acdb5bb-f366-4a77-9eab-b232fcfc5b7e", "enable_snat": true} |
+--------------------------------------+---------+-----------------------------------------------------------------------------+

[root@havana ~(keystone_admin)]# ip netns | grep 6c0825b7-7cb1-4327-a97b-04a7837ceba4
qrouter-6c0825b7-7cb1-4327-a97b-04a7837ceba4

[root@havana ~(keystone_admin)]# neutron net-list
+--------------------------------------+----------+-------------------------------------------------------+
| id                                   | name     | subnets                                               |
+--------------------------------------+----------+-------------------------------------------------------+
| 30f38c89-4d4f-4d1d-bdea-e1f252221a57 | private1 | 2e143cbd-d965-417b-b366-d58b75952e46 40.0.0.0/24      |
| 57b12ecb-bc00-4482-9653-d84c67c79e04 | private  | a42fb194-3751-4f64-afcf-cddcfc5d6432 10.0.0.0/24      |
| 7acdb5bb-f366-4a77-9eab-b232fcfc5b7e | public   | 9f2e36cb-47b7-427a-8948-1184f0c6507e 192.168.122.0/24 |
+--------------------------------------+----------+-------------------------------------------------------+


Neutron agents status verification


[root@havana ~(keystone_admin)]# neutron agent-list
+--------------------------------------+--------------------+--------------------+-------+----------------+
| id                                   | agent_type         | host               | alive | admin_state_up |
+--------------------------------------+--------------------+--------------------+-------+----------------+
| 8a198929-b5d2-4e30-8f9c-ad6419e00349 | Open vSwitch agent | havana.localdomain | :-)   | True           |
| a82e3f96-2849-47d3-bf39-bf83f907cf92 | DHCP agent         | havana.localdomain | :-)   | True           |
| f164778e-de98-4e51-9aeb-59568b30bca8 | L3 agent           | havana.localdomain | :-)   | True           |
+--------------------------------------+--------------------+--------------------+-------+----------

[root@havana ~(keystone_admin)]# ip netns | grep 30f38c89-4d4f-4d1d-bdea-e1f252221a57
qdhcp-30f38c89-4d4f-4d1d-bdea-e1f252221a57

[root@havana ~(keystone_admin)]# ip netns | grep 57b12ecb-bc00-4482-9653-d84c67c79e04
qdhcp-57b12ecb-bc00-4482-9653-d84c67c79e04

Getting internal and external qrouter namespace interfaces names

[root@havana ~(keystone_admin)]# ip netns exec qrouter-6c0825b7-7cb1-4327-a97b-04a7837ceba4 ifconfig
lo        Link encap:Local Loopback 
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:16436  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 b)  TX bytes:0 (0.0 b)

qg-876fd79e-d1 Link encap:Ethernet  HWaddr FA:16:3E:27:DC:DF 
          inet addr:192.168.122.102  Bcast:192.168.122.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe27:dcdf/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:14556 errors:0 dropped:0 overruns:0 frame:0
          TX packets:3054 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:20754615 (19.7 MiB)  TX bytes:290113 (283.3 KiB)

qr-b9f5e2fa-cb Link encap:Ethernet  HWaddr FA:16:3E:CC:6A:11 
          inet addr:40.0.0.1  Bcast:40.0.0.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fecc:6a11/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:3033 errors:0 dropped:0 overruns:0 frame:0
          TX packets:14550 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:290129 (283.3 KiB)  TX bytes:20754691 (19.7 MiB)

Getting internal interfaces of qdhcp namespaces


[root@havana ~(keystone_admin)]# ip netns exec qdhcp-30f38c89-4d4f-4d1d-bdea-e1f252221a57 ifconfig
lo        Link encap:Local Loopback 
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:16436  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 b)  TX bytes:0 (0.0 b)

tapd265d281-00 Link encap:Ethernet  HWaddr FA:16:3E:57:C6:94 
          inet addr:40.0.0.11  Bcast:40.0.0.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe57:c694/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:59 errors:0 dropped:0 overruns:0 frame:0
          TX packets:53 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:8640 (8.4 KiB)  TX bytes:8746 (8.5 KiB)

[root@havana ~(keystone_admin)]# ip netns exec qdhcp-57b12ecb-bc00-4482-9653-d84c67c79e04 ifconfig
lo        Link encap:Local Loopback 
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:16436  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 b)  TX bytes:0 (0.0 b)

tap7e72dcf0-96 Link encap:Ethernet  HWaddr FA:16:3E:6E:BF:64 
          inet addr:10.0.0.2  Bcast:10.0.0.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe6e:bf64/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:9 errors:0 dropped:0 overruns:0 frame:0
          TX packets:10 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:726 (726.0 b)  TX bytes:796 (796.0 b)

[root@havana ~(keystone_admin)]# ovs-vsctl show
20611b75-2436-46a8-8016-21588b858a0c
    Bridge br-ex
        Port "eth0"
            Interface "eth0"
        Port "qg-876fd79e-d1"
            Interface "qg-876fd79e-d1"
                type: internal
        Port br-ex
            Interface br-ex
                type: internal
    Bridge br-int
        Port "tap7e72dcf0-96"
            tag: 2
            Interface "tap7e72dcf0-96"
                type: internal
        Port "tapd265d281-00"
            tag: 1
            Interface "tapd265d281-00"
                type: internal
        Port "qr-b9f5e2fa-cb"
            tag: 1
            Interface "qr-b9f5e2fa-cb"
                type: internal
        Port "qvo266c9054-1a"
            tag: 1
            Interface "qvo266c9054-1a"
        Port "qvob710d1a4-90"
            tag: 1
            Interface "qvob710d1a4-90"
        Port br-int
            Interface br-int
                type: internal
    ovs_version: "1.11.0


****************************
Veth pairs verification
****************************

[root@havana ~(keystone_admin)]# brctl show

bridge name    bridge id        STP enabled    interfaces
qbr266c9054-1a        8000.cac48b4d0dcd    no        qvb266c9054-1a
qbrb710d1a4-90        8000.666011280442    no        qvbb710d1a4-90

[root@havana ~(keystone_admin)]# ovs-vsctl show | grep 266c9054-1a
        Port "qvo266c9054-1a"
            Interface "qvo266c9054-1a"

[root@havana ~(keystone_admin)]# ovs-vsctl show | grep 710d1a4-90
        Port "qvob710d1a4-90"
            Interface "qvob710d1a4-90"

The qvo prefix stands for neutron veth Open vSwitch side and the qvb prefix stands for neutron veth linux bridge side . Details here :- http://openstack.redhat.com/PackStack_All-in-One_DIY_Configuration

*************************************************
Getting routers DNAT&SNAT forwarding rools
*************************************************

[root@havana ~(keystone_admin)]# ip netns exec qrouter-6c0825b7-7cb1-4327-a97b-04a7837ceba4 iptables -S -t nat
-P PREROUTING ACCEPT
-P POSTROUTING ACCEPT
-P OUTPUT ACCEPT
-N neutron-l3-agent-OUTPUT
-N neutron-l3-agent-POSTROUTING
-N neutron-l3-agent-PREROUTING
-N neutron-l3-agent-float-snat
-N neutron-l3-agent-snat
-N neutron-postrouting-bottom
-A PREROUTING -j neutron-l3-agent-PREROUTING
-A POSTROUTING -j neutron-l3-agent-POSTROUTING
-A POSTROUTING -j neutron-postrouting-bottom
-A OUTPUT -j neutron-l3-agent-OUTPUT
-A neutron-l3-agent-OUTPUT -d 192.168.122.105/32 -j DNAT --to-destination 40.0.0.12
-A neutron-l3-agent-OUTPUT -d 192.168.122.103/32 -j DNAT --to-destination 40.0.0.10
-A neutron-l3-agent-POSTROUTING ! -i qg-876fd79e-d1 ! -o qg-876fd79e-d1 -m conntrack ! --ctstate DNAT -j ACCEPT
-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 9697
-A neutron-l3-agent-PREROUTING -d 192.168.122.105/32 -j DNAT --to-destination 40.0.0.12
-A neutron-l3-agent-PREROUTING -d 192.168.122.103/32 -j DNAT --to-destination 40.0.0.10
-A neutron-l3-agent-float-snat -s 40.0.0.12/32 -j SNAT --to-source 192.168.122.105
-A neutron-l3-agent-float-snat -s 40.0.0.10/32 -j SNAT --to-source 192.168.122.103
-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat
-A neutron-l3-agent-snat -s 40.0.0.0/24 -j SNAT --to-source 192.168.122.102
-A neutron-postrouting-bottom -j neutron-l3-agent-snat


********************************************************************************
Pinging  running VM from host and capturing packages via tcpdump on internal external interfaces of qrouter  namespace
********************************************************************************


[root@havana ~]# ip netns exec qrouter-6c0825b7-7cb1-4327-a97b-04a7837ceba4 tcpdump -ln -i qg-876fd79e-d1
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on qg-876fd79e-d1, link-type EN10MB (Ethernet), capture size 65535 bytes
17:47:27.402753 IP 192.168.122.100 > 192.168.122.105: ICMP echo request, id 33844, seq 12, length 64
17:47:27.403643 IP 192.168.122.105 > 192.168.122.100: ICMP echo reply, id 33844, seq 12, length 64
17:47:28.404095 IP 192.168.122.100 > 192.168.122.105: ICMP echo request, id 33844, seq 13, length 64
17:47:28.405191 IP 192.168.122.105 > 192.168.122.100: ICMP echo reply, id 33844, seq 13, length 64
17:47:29.405633 IP 192.168.122.100 > 192.168.122.105: ICMP echo request, id 33844, seq 14, length 64
17:47:29.406348 IP 192.168.122.105 > 192.168.122.100: ICMP echo reply, id 33844, seq 14, length 64
17:47:30.406741 IP 192.168.122.100 > 192.168.122.105: ICMP echo request, id 33844, seq 15, length 64
17:47:30.407517 IP 192.168.122.105 > 192.168.122.100: ICMP echo reply, id 33844, seq 15, length 64
17:47:31.407896 IP 192.168.122.100 > 192.168.122.105: ICMP echo request, id 33844, seq 16, length 64
17:47:31.408567 IP 192.168.122.105 > 192.168.122.100: ICMP echo reply, id 33844, seq 16, length 64
17:47:32.408970 IP 192.168.122.100 > 192.168.122.105: ICMP echo request, id 33844, seq 17, length 64
17:47:32.409735 IP 192.168.122.105 > 192.168.122.100: ICMP echo reply, id 33844, seq 17, length 64
17:47:33.410087 IP 192.168.122.100 > 192.168.122.105: ICMP echo request, id 33844, seq 18, length 64
17:47:33.410886 IP 192.168.122.105 > 192.168.122.100: ICMP echo reply, id 33844, seq 18, length 64
17:47:34.411608 IP 192.168.122.100 > 192.168.122.105: ICMP echo request, id 33844, seq 19, length 64
17:47:34.412400 IP 192.168.122.105 > 192.168.122.100: ICMP echo reply, id 33844, seq 19, length 64


[root@havana ~]# ip netns exec qrouter-6c0825b7-7cb1-4327-a97b-04a7837ceba4 tcpdump -ln -i qr-b9f5e2fa-cb

tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on qr-b9f5e2fa-cb, link-type EN10MB (Ethernet), capture size 65535 bytes
17:47:34.411634 IP 192.168.122.100 > 40.0.0.12: ICMP echo request, id 33844, seq 19, length 64
17:47:34.412344 IP 40.0.0.12 > 192.168.122.100: ICMP echo reply, id 33844, seq 19, length 64
17:47:35.412857 IP 192.168.122.100 > 40.0.0.12: ICMP echo request, id 33844, seq 20, length 64
17:47:35.413579 IP 40.0.0.12 > 192.168.122.100: ICMP echo reply, id 33844, seq 20, length 64
17:47:36.414018 IP 192.168.122.100 > 40.0.0.12: ICMP echo request, id 33844, seq 21, length 64
17:47:36.414782 IP 40.0.0.12 > 192.168.122.100: ICMP echo reply, id 33844, seq 21, length 64
17:47:37.415336 IP 192.168.122.100 > 40.0.0.12: ICMP echo request, id 33844, seq 22, length 64
17:47:37.421958 IP 40.0.0.12 > 192.168.122.100: ICMP echo reply, id 33844, seq 22, length 64
17:47:38.417196 IP 192.168.122.100 > 40.0.0.12: ICMP echo request, id 33844, seq 23, length 64
17:47:38.417931 IP 40.0.0.12 > 192.168.122.100: ICMP echo reply, id 33844, seq 23, length 64
17:47:39.418426 IP 192.168.122.100 > 40.0.0.12: ICMP echo request, id 33844, seq 24, length 64
17:47:39.419180 IP 40.0.0.12 > 192.168.122.100: ICMP echo reply, id 33844, seq 24, length 64
17:47:40.418793 IP 192.168.122.100 > 40.0.0.12: ICMP echo request, id 33844, seq 25, length 64
17:47:40.419562 IP 40.0.0.12 > 192.168.122.100: ICMP echo reply, id 33844, seq 25, length 64

Troubleshooting DHCP inside private network (VM is running and is requesting
dhcp-lease on regular basis)

[root@havana ~(keystone_admin)]# neutron net-list
+--------------------------------------+----------+-------------------------------------------------------+
| id                                   | name     | subnets                                               |
+--------------------------------------+----------+-------------------------------------------------------+
| 30f38c89-4d4f-4d1d-bdea-e1f252221a57 | private1 | 2e143cbd-d965-417b-b366-d58b75952e46 40.0.0.0/24      |
| 57b12ecb-bc00-4482-9653-d84c67c79e04 | private  | a42fb194-3751-4f64-afcf-cddcfc5d6432 10.0.0.0/24      |
| 7acdb5bb-f366-4a77-9eab-b232fcfc5b7e | public   | 9f2e36cb-47b7-427a-8948-1184f0c6507e 192.168.122.0/24 |
+--------------------------------------+----------+-------------------------------------------------------+
[root@havana ~(keystone_admin)]# ip netns | grep 30f38c89-4d4f-4d1d-bdea-e1f252221a57
qdhcp-30f38c89-4d4f-4d1d-bdea-e1f252221a57

[root@havana ~(keystone_admin)]# ip netns exec qdhcp-30f38c89-4d4f-4d1d-bdea-e1f252221a57 ifconfig
lo        Link encap:Local Loopback 
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:16436  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 b)  TX bytes:0 (0.0 b)

tapd265d281-00 Link encap:Ethernet  HWaddr FA:16:3E:57:C6:94 
          inet addr:40.0.0.11  Bcast:40.0.0.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe57:c694/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:22 errors:0 dropped:0 overruns:0 frame:0
          TX packets:15 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:2486 (2.4 KiB)  TX bytes:2214 (2.1 KiB)

[root@havana ~(keystone_admin)]# ip netns exec qdhcp-30f38c89-4d4f-4d1d-bdea-e1f252221a57 tcpdump -ln -i tapd265d281-00

tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on tapd265d281-00, link-type EN10MB (Ethernet), capture size 65535 bytes
10:12:56.093405 IP 40.0.0.12.bootpc > 40.0.0.11.bootps: BOOTP/DHCP, Request from fa:16:3e:de:f7:ec, length 300
10:12:56.093614 IP 40.0.0.11.bootps > 40.0.0.12.bootpc: BOOTP/DHCP, Reply, length 324
10:13:01.092868 ARP, Request who-has 40.0.0.12 tell 40.0.0.11, length 28
10:13:01.093960 ARP, Reply 40.0.0.12 is-at fa:16:3e:de:f7:ec, length 28
10:13:49.834522 IP 40.0.0.12.bootpc > 40.0.0.11.bootps: BOOTP/DHCP, Request from fa:16:3e:de:f7:ec, length 300
10:13:49.837596 IP 40.0.0.11.bootps > 40.0.0.12.bootpc: BOOTP/DHCP, Reply, length 324
10:13:54.836873 ARP, Request who-has 40.0.0.12 tell 40.0.0.11, length 28
10:13:54.838265 ARP, Reply 40.0.0.12 is-at fa:16:3e:de:f7:ec, length 28
10:14:43.800255 IP 40.0.0.12.bootpc > 40.0.0.11.bootps: BOOTP/DHCP, Request from fa:16:3e:de:f7:ec, length 300
10:14:43.809635 IP 40.0.0.11.bootps > 40.0.0.12.bootpc: BOOTP/DHCP, Reply, length 324
10:14:48.807599 ARP, Request who-has 40.0.0.11 tell 40.0.0.12, length 28
10:14:48.807631 ARP, Reply 40.0.0.11 is-at fa:16:3e:57:c6:94, length 28
10:15:26.138172 IP 40.0.0.12.bootpc > 40.0.0.11.bootps: BOOTP/DHCP, Request from fa:16:3e:de:f7:ec, length 300
10:15:26.140720 IP 40.0.0.11.bootps > 40.0.0.12.bootpc: BOOTP/DHCP, Reply, length 324
10:15:31.140869 ARP, Request who-has 40.0.0.12 tell 40.0.0.11, length 28
10:15:31.142128 ARP, Reply 40.0.0.12 is-at fa:16:3e:de:f7:ec, length 28


********************************************************************
List of /etc/libvirt/qemu XML definitions of VMs running in the cloud
to compare with `brctl show` output
********************************************************************

[root@havana ~(keystone_admin)]# cd /etc/libvirt/qemu
[root@havana qemu(keystone_admin)]# ls -l
total 12
-rw-------. 1 root root 2889 Apr 12 17:58 instance-00000001.xml
-rw-------. 1 root root 3007 Apr 28 17:23 instance-00000003.xml
drwx------. 3 root root 4096 Apr 12 15:26 networks
[root@havana qemu(keystone_admin)]# cat instance-00000001.xml
<!--
WARNING: THIS IS AN AUTO-GENERATED FILE. CHANGES TO IT ARE LIKELY TO BE
OVERWRITTEN AND LOST. Changes to this xml configuration should be made using:
  virsh edit instance-00000001
or other application using the libvirt API.
-->

<domain type='qemu'>
  <name>instance-00000001</name>
  <uuid>00a43ef7-a3a7-47fe-942a-fcbef48d5fe2</uuid>
  <memory unit='KiB'>524288</memory>
  <currentMemory unit='KiB'>524288</currentMemory>
  <vcpu placement='static'>1</vcpu>
  <sysinfo type='smbios'>
    <system>
      <entry name='manufacturer'>Red Hat Inc.</entry>
      <entry name='product'>OpenStack Nova</entry>
      <entry name='version'>2013.2.2-1.el6</entry>
      <entry name='serial'>d9e7bce0-ee66-4c7f-aa96-1950d86b5bb6</entry>
      <entry name='uuid'>00a43ef7-a3a7-47fe-942a-fcbef48d5fe2</entry>
    </system>
  </sysinfo>
  <os>
    <type arch='x86_64' machine='rhel6.5.0'>hvm</type>
    <boot dev='hd'/>
    <smbios mode='sysinfo'/>
  </os>
  <features>
    <acpi/>
    <apic/>
  </features>
  <clock offset='utc'/>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>destroy</on_crash>
  <devices>
    <emulator>/usr/libexec/qemu-kvm</emulator>
    <disk type='file' device='disk'>
      <driver name='qemu' type='qcow2' cache='none'/>
      <source file='/var/lib/nova/instances/00a43ef7-a3a7-47fe-942a-fcbef48d5fe2/disk'/>
      <target dev='vda' bus='virtio'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
    </disk>
    <controller type='usb' index='0'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
    </controller>
    <interface type='bridge'>
      <mac address='fa:16:3e:1f:2e:ab'/>
      <source bridge='qbrb710d1a4-90'/>
      <target dev='tapb710d1a4-90'/>
      <model type='virtio'/>
      <driver name='qemu'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
    </interface>
    <serial type='file'>
      <source path='/var/lib/nova/instances/00a43ef7-a3a7-47fe-942a-fcbef48d5fe2/console.log'/>
      <target port='0'/>
    </serial>
    <serial type='pty'>
      <target port='1'/>
    </serial>
    <console type='file'>
      <source path='/var/lib/nova/instances/00a43ef7-a3a7-47fe-942a-fcbef48d5fe2/console.log'/>
      <target type='serial' port='0'/>
    </console>
    <input type='tablet' bus='usb'/>
    <input type='mouse' bus='ps2'/>
    <graphics type='vnc' port='-1' autoport='yes' listen='192.168.122.100' keymap='en-us'>
      <listen type='address' address='192.168.122.100'/>
    </graphics>
    <video>
      <model type='cirrus' vram='9216' heads='1'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
    </video>
    <memballoon model='virtio'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
    </memballoon>
  </devices>
</domain>
[root@havana qemu(keystone_admin)]# cat instance-00000003.xml
<!--
WARNING: THIS IS AN AUTO-GENERATED FILE. CHANGES TO IT ARE LIKELY TO BE
OVERWRITTEN AND LOST. Changes to this xml configuration should be made using:
  virsh edit instance-00000003
or other application using the libvirt API.
-->

<domain type='qemu'>
  <name>instance-00000003</name>
  <uuid>7306a5a1-4148-4744-bd55-edb4fae0e98d</uuid>
  <memory unit='KiB'>2097152</memory>
  <currentMemory unit='KiB'>2097152</currentMemory>
  <vcpu placement='static'>1</vcpu>
  <sysinfo type='smbios'>
    <system>
      <entry name='manufacturer'>Red Hat Inc.</entry>
      <entry name='product'>OpenStack Nova</entry>
      <entry name='version'>2013.2.2-1.el6</entry>
      <entry name='serial'>d9e7bce0-ee66-4c7f-aa96-1950d86b5bb6</entry>
      <entry name='uuid'>7306a5a1-4148-4744-bd55-edb4fae0e98d</entry>
    </system>
  </sysinfo>
  <os>
    <type arch='x86_64' machine='rhel6.5.0'>hvm</type>
    <boot dev='hd'/>
    <smbios mode='sysinfo'/>
  </os>
  <features>
    <acpi/>
    <apic/>
  </features>
  <clock offset='utc'/>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>destroy</on_crash>
  <devices>
    <emulator>/usr/libexec/qemu-kvm</emulator>
    <disk type='block' device='disk'>
      <driver name='qemu' type='raw' cache='none'/>
      <source dev='/dev/disk/by-path/ip-192.168.122.100:3260-iscsi-iqn.2010-10.org.openstack:volume-a14f3b05-a476-46f6-a763-bb3cad621277-lun-1'/>
      <target dev='vda' bus='virtio'/>
      <serial>a14f3b05-a476-46f6-a763-bb3cad621277</serial>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
    </disk>
    <controller type='usb' index='0'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
    </controller>
    <interface type='bridge'>
      <mac address='fa:16:3e:de:f7:ec'/>
      <source bridge='qbr266c9054-1a'/>
      <target dev='tap266c9054-1a'/>
      <model type='virtio'/>
      <driver name='qemu'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
    </interface>
    <serial type='file'>
      <source path='/var/lib/nova/instances/7306a5a1-4148-4744-bd55-edb4fae0e98d/console.log'/>
      <target port='0'/>
    </serial>
    <serial type='pty'>
      <target port='1'/>
    </serial>
    <console type='file'>
      <source path='/var/lib/nova/instances/7306a5a1-4148-4744-bd55-edb4fae0e98d/console.log'/>
      <target type='serial' port='0'/>
    </console>
    <input type='tablet' bus='usb'/>
    <input type='mouse' bus='ps2'/>
    <graphics type='vnc' port='-1' autoport='yes' listen='192.168.122.100' keymap='en-us'>
      <listen type='address' address='192.168.122.100'/>
    </graphics>
    <video>
      <model type='cirrus' vram='9216' heads='1'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
    </video>
    <memballoon model='virtio'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
    </memballoon>
  </devices>
</domain>

Overlapping private networks in Havana

Neutron.conf on Neutron Server && Compute nodes should have an entry :
allow_overlapping_ips = True  

New user "andrew" under ostenant2 created router5 and private network int2 overlapping with int
New VM VF20RS025 (10.0.0.13,192.168.1.108) has been launched with Internet access (F20 OS)


[root@dfw02 ~(keystone_admin)]$ neutron router-list
-+-----------------------------------------------------------------------------++--------------------------------------+--------
| id                                   | name    | external_gateway_info                                                       |
+--------------------------------------+---------+-----------------------------------------------------------------------------+
| 0686e526-0584-4004-a8b0-7d64b376f0ce | router5 | {"network_id": "780ce2f3-2e6e-4881-bbac-857813f9a8e0", "enable_snat": true} |
| 86b3008c-297f-4301-9bdc-766b839785f1 | router2 | {"network_id": "780ce2f3-2e6e-4881-bbac-857813f9a8e0", "enable_snat": true} |
| bf360d81-79fb-4636-8241-0a843f228fc8 | router1 | {"network_id": "780ce2f3-2e6e-4881-bbac-857813f9a8e0", "enable_snat": true} |
+--------------------------------------+---------+-----------------------------------------------------------------------------+

Qrouter namespace created

[root@dfw02 ~(keystone_admin)]$ ip netns list | grep 0686e526-0584-4004-a8b0-7d64b376f0ce
qrouter-0686e526-0584-4004-a8b0-7d64b376f0ce

[root@dfw02 ~(keystone_admin)]$ neutron net-list
+--------------------------------------+------+-----------------------------------------------------+
| id                                   | name | subnets                                             |
+--------------------------------------+------+-----------------------------------------------------+
| 1eea88bb-4952-4aa4-9148-18b61c22d5b7 | int  | fa930cea-3d51-4cbe-a305-579f12aa53c0 10.0.0.0/24    |
| 426bb226-0ab9-440d-ba14-05634a17fb2b | int1 | 9e0d457b-c4c4-45cf-84e2-4ac7550f3b06 40.0.0.0/24    |
| 780ce2f3-2e6e-4881-bbac-857813f9a8e0 | ext  | f30e5a16-a055-4388-a6ea-91ee142efc3d 192.168.1.0/24 |
| 971fd74b-66e4-4016-b997-433e124a22ad | int2 | caca1f54-3211-4395-ba3c-a0c26d9605c0 10.0.0.0/24    |
+--------------------------------------+------+-----------------------------------------------------+ 

[root@dfw02 ~(keystone_admin)]$ neutron subnet-list
+--------------------------------------+------+----------------+----------------------------------------------------+
| id                                   | name | cidr           | allocation_pools                                   |
+--------------------------------------+------+----------------+----------------------------------------------------+
| 9e0d457b-c4c4-45cf-84e2-4ac7550f3b06 |      | 40.0.0.0/24    | {"start": "40.0.0.2", "end": "40.0.0.254"}         |
| fa930cea-3d51-4cbe-a305-579f12aa53c0 |      | 10.0.0.0/24    | {"start": "10.0.0.2", "end": "10.0.0.254"}         |
| caca1f54-3211-4395-ba3c-a0c26d9605c0 |      | 10.0.0.0/24    | {"start": "10.0.0.11", "end": "10.0.0.254"}        |
| f30e5a16-a055-4388-a6ea-91ee142efc3d |      | 192.168.1.0/24 | {"start": "192.168.1.100", "end": "192.168.1.200"} |
+--------------------------------------+------+----------------+----------------------------------------------------+
 

 Qdhcp namespace created

[root@dfw02 ~(keystone_admin)]$ ip netns list | grep 971fd74b-66e4-4016-b997-433e124a22ad 
qdhcp-971fd74b-66e4-4016-b997-433e124a22ad

[root@dfw02 ~(keystone_admin)]$ ip netns exec qrouter-0686e526-0584-4004-a8b0-7d64b376f0ce iptables -S -tnat
-P PREROUTING ACCEPT
-P INPUT ACCEPT
-P OUTPUT ACCEPT
-P POSTROUTING ACCEPT
-N neutron-l3-agent-OUTPUT
-N neutron-l3-agent-POSTROUTING
-N neutron-l3-agent-PREROUTING
-N neutron-l3-agent-float-snat
-N neutron-l3-agent-snat
-N neutron-postrouting-bottom
-A PREROUTING -j neutron-l3-agent-PREROUTING
-A OUTPUT -j neutron-l3-agent-OUTPUT
-A POSTROUTING -j neutron-l3-agent-POSTROUTING
-A POSTROUTING -j neutron-postrouting-bottom
-A neutron-l3-agent-OUTPUT -d 192.168.1.112/32 -j DNAT --to-destination 10.0.0.11
-A neutron-l3-agent-OUTPUT -d 192.168.1.108/32 -j DNAT --to-destination 10.0.0.13
-A neutron-l3-agent-POSTROUTING ! -i qg-edd58124-43 ! -o qg-edd58124-43 -m conntrack ! --ctstate DNAT -j ACCEPT
-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 8700
-A neutron-l3-agent-PREROUTING -d 192.168.1.112/32 -j DNAT --to-destination 10.0.0.11
-A neutron-l3-agent-PREROUTING -d 192.168.1.108/32 -j DNAT --to-destination 10.0.0.13
-A neutron-l3-agent-float-snat -s 10.0.0.11/32 -j SNAT --to-source 192.168.1.112
-A neutron-l3-agent-float-snat -s 10.0.0.13/32 -j SNAT --to-source 192.168.1.108
-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat
-A neutron-l3-agent-snat -s 10.0.0.0/24 -j SNAT --to-source 192.168.1.107
-A neutron-postrouting-bottom -j neutron-l3-agent-snat


[root@dfw02 ~]# ssh -l fedora -i oskey35.pem 192.168.1.108
The authenticity of host '192.168.1.108 (192.168.1.108)' can't be established.
RSA key fingerprint is 9b:b3:07:5e:61:71:67:01:c5:9f:7a:53:e0:ca:60:6a.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '192.168.1.108' (RSA) to the list of known hosts.
[fedora@vf20rs025 ~]$ sudo su -
[root@vf20rs025 ~]# ifconfig
eth0: flags=4163  mtu 1454
        inet 10.0.0.13  netmask 255.255.255.0  broadcast 10.0.0.255
        inet6 fe80::f816:3eff:fe74:68c  prefixlen 64  scopeid 0x20
        ether fa:16:3e:74:06:8c  txqueuelen 1000  (Ethernet)
        RX packets 279  bytes 29979 (29.2 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 247  bytes 26077 (25.4 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

lo: flags=73  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10
        loop  txqueuelen 0  (Local Loopback)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

[root@vf20rs025 ~]# yum -y install links
fedora/20/x86_64/metalink                                                         |  35 kB  00:00:00     
fedora                                                                            | 3.8 kB  00:00:00     
updates/20/x86_64/metalink                                                        |  22 kB  00:00:00     
updates                                                                           | 4.9 kB  00:00:00     
(1/4): fedora/20/x86_64/group_gz                                                  | 394 kB  00:00:07     
(2/4): updates/20/x86_64/group_gz                                                 | 394 kB  00:00:06     
(3/4): updates/20/x86_64/primary_db                                               | 9.6 MB  00:00:11     
(4/4): fedora/20/x86_64/primary_db                                                |  18 MB  00:00:18     
(1/2): updates/20/x86_64/updateinfo                                               | 1.0 MB  00:00:04     
(2/2): updates/20/x86_64/pkgtags                                                  | 1.0 MB  00:00:08     
Resolving Dependencies
--> Running transaction check
---> Package links.x86_64 1:2.6-6.fc20 will be installed
--> Processing Dependency: libgpm.so.2()(64bit) for package: 1:links-2.6-6.fc20.x86_64
--> Processing Dependency: libX11.so.6()(64bit) for package: 1:links-2.6-6.fc20.x86_64
--> Running transaction check
---> Package gpm-libs.x86_64 0:1.20.7-3.fc20 will be installed
---> Package libX11.x86_64 0:1.6.1-1.fc20 will be installed
--> Processing Dependency: libX11-common = 1.6.1-1.fc20 for package: libX11-1.6.1-1.fc20.x86_64
--> Processing Dependency: libxcb.so.1()(64bit) for package: libX11-1.6.1-1.fc20.x86_64
--> Running transaction check
---> Package libX11-common.noarch 0:1.6.1-1.fc20 will be installed
---> Package libxcb.x86_64 0:1.9.1-3.fc20 will be installed
--> Processing Dependency: libXau.so.6()(64bit) for package: libxcb-1.9.1-3.fc20.x86_64
--> Running transaction check
---> Package libXau.x86_64 0:1.0.8-2.fc20 will be installed
--> Finished Dependency Resolution

Dependencies Resolved

=========================================================================================================
 Package                     Arch                 Version                     Repository            Size
=========================================================================================================
Installing:
 links                       x86_64               1:2.6-6.fc20                fedora               1.8 M
Installing for dependencies:
 gpm-libs                    x86_64               1.20.7-3.fc20               fedora                32 k
 libX11                      x86_64               1.6.1-1.fc20                fedora               597 k
 libX11-common               noarch               1.6.1-1.fc20                fedora               181 k
 libXau                      x86_64               1.0.8-2.fc20                fedora                29 k
 libxcb                      x86_64               1.9.1-3.fc20                fedora               167 k

Transaction Summary
=========================================================================================================
Install  1 Package (+5 Dependent packages)

Total download size: 2.8 M
Installed size: 6.6 M
Downloading packages:
(1/6): gpm-libs-1.20.7-3.fc20.x86_64.rpm                                          |  32 kB  00:00:10     
(2/6): libX11-1.6.1-1.fc20.x86_64.rpm                                             | 597 kB  00:00:08     
(3/6): links-2.6-6.fc20.x86_64.rpm                                                | 1.8 MB  00:00:01     
(4/6): libXau-1.0.8-2.fc20.x86_64.rpm                                             |  29 kB  00:00:05     
(5/6): libxcb-1.9.1-3.fc20.x86_64.rpm                                             | 167 kB  00:00:05     
(6/6): libX11-common-1.6.1-1.fc20.noarch.rpm                                      | 181 kB  00:00:11     
---------------------------------------------------------------------------------------------------------
Total                                                                    158 kB/s | 2.8 MB     00:17     
Running transaction check
Running transaction test
Transaction test succeeded
Running transaction
  Installing : gpm-libs-1.20.7-3.fc20.x86_64                                                         1/6 
  Installing : libXau-1.0.8-2.fc20.x86_64                                                            2/6 
  Installing : libxcb-1.9.1-3.fc20.x86_64                                                            3/6 
  Installing : libX11-common-1.6.1-1.fc20.noarch                                                     4/6 
  Installing : libX11-1.6.1-1.fc20.x86_64                                                            5/6 
  Installing : 1:links-2.6-6.fc20.x86_64                                                             6/6 
  Verifying  : libX11-common-1.6.1-1.fc20.noarch                                                     1/6 
  Verifying  : 1:links-2.6-6.fc20.x86_64                                                             2/6 
  Verifying  : libXau-1.0.8-2.fc20.x86_64                                                            3/6 
  Verifying  : gpm-libs-1.20.7-3.fc20.x86_64                                                         4/6 
  Verifying  : libxcb-1.9.1-3.fc20.x86_64                                                            5/6 
  Verifying  : libX11-1.6.1-1.fc20.x86_64                                                            6/6 

Installed:
  links.x86_64 1:2.6-6.fc20                                                                              

Dependency Installed:
  gpm-libs.x86_64 0:1.20.7-3.fc20   libX11.x86_64 0:1.6.1-1.fc20   libX11-common.noarch 0:1.6.1-1.fc20  
  libXau.x86_64 0:1.0.8-2.fc20      libxcb.x86_64 0:1.9.1-3.fc20  

Complete!


 

RDO Havana Neutron&GRE L3&L2 Layer Troubleshooting

This post follows up http://lxer.com/module/newswire/view/200975/index.html
For Basic Neutron Networking concepts,please, view developer.rackspace.com

Qrouters namespaces external and internal interfaces are mapped to `ovs-vsctl show` output. Qdhcps namespaces interfaces are also mapped to `ovs-vsctl show` output.  Traffic control is done via tcpdump on external and internal  network interfaces of Qrouter's namespace associated with Network attached via external gateway to router with interface for private sub net having Ubuntu Trusty Server instance running `apt-get install` commands.

[root@dfw02 ~(keystone_admin)]$ neutron router-list
+--------------------------------------+---------+-----------------------------------------------------------------------------+
| id                                   | name    | external_gateway_info                                                       |
+--------------------------------------+---------+-----------------------------------------------------------------------------+
| 86b3008c-297f-4301-9bdc-766b839785f1 | router2 | {"network_id": "780ce2f3-2e6e-4881-bbac-857813f9a8e0", "enable_snat": true} |
| bf360d81-79fb-4636-8241-0a843f228fc8 | router1 | {"network_id": "780ce2f3-2e6e-4881-bbac-857813f9a8e0", "enable_snat": true} |
+--------------------------------------+---------+-----------------------------------------------------------------------------+

[root@dfw02 ~(keystone_admin)]$ ip netns list | grep 86b3008c-297f-4301-9bdc-766b839785f1
qrouter-86b3008c-297f-4301-9bdc-766b839785f1

[root@dfw02 ~(keystone_admin)]$ ip netns exec qrouter-86b3008c-297f-4301-9bdc-766b839785f1 ip a

1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever

2: qr-e031db6b-d0: mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether fa:16:3e:83:90:90 brd ff:ff:ff:ff:ff:ff
    inet 40.0.0.1/24 brd 40.0.0.255 scope global qr-e031db6b-d0
       valid_lft forever preferred_lft forever
    inet6 fe80::f816:3eff:fe83:9090/64 scope link
       valid_lft forever preferred_lft forever

3: qg-9c090153-08: mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether fa:16:3e:b2:24:bc brd ff:ff:ff:ff:ff:ff
    inet 192.168.1.114/24 brd 192.168.1.255 scope global qg-9c090153-08
       valid_lft forever preferred_lft forever
    inet 192.168.1.105/32 brd 192.168.1.105 scope global qg-9c090153-08
       valid_lft forever preferred_lft forever
    inet 192.168.1.104/32 brd 192.168.1.104 scope global qg-9c090153-08
       valid_lft forever preferred_lft forever
    inet 192.168.1.102/32 brd 192.168.1.102 scope global qg-9c090153-08
       valid_lft forever preferred_lft forever
    inet 192.168.1.106/32 brd 192.168.1.106 scope global qg-9c090153-08
       valid_lft forever preferred_lft forever
    inet6 fe80::f816:3eff:feb2:24bc/64 scope link
       valid_lft forever preferred_lft forever
 
Getting list of all namespaces on Neutron Server

[root@dfw02 ~(keystone_admin)]$ ip netns list

qrouter-86b3008c-297f-4301-9bdc-766b839785f1
qrouter-bf360d81-79fb-4636-8241-0a843f228fc8
qdhcp-1eea88bb-4952-4aa4-9148-18b61c22d5b7
qdhcp-426bb226-0ab9-440d-ba14-05634a17fb2b

[root@dfw02 ~(keystone_admin)]$ ip netns exec qrouter-86b3008c-297f-4301-9bdc-766b839785f1 ip a

1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever

Router's internal interface

2: qr-e031db6b-d0: mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether fa:16:3e:83:90:90 brd ff:ff:ff:ff:ff:ff
    inet 40.0.0.1/24 brd 40.0.0.255 scope global qr-e031db6b-d0
       valid_lft forever preferred_lft forever
    inet6 fe80::f816:3eff:fe83:9090/64 scope link
       valid_lft forever preferred_lft forever

Router's  external  interface with the floating IPs

3: qg-9c090153-08: mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether fa:16:3e:b2:24:bc brd ff:ff:ff:ff:ff:ff
    inet 192.168.1.114/24 brd 192.168.1.255 scope global qg-9c090153-08
       valid_lft forever preferred_lft forever
    inet 192.168.1.104/32 brd 192.168.1.104 scope global qg-9c090153-08
       valid_lft forever preferred_lft forever
    inet 192.168.1.102/32 brd 192.168.1.102 scope global qg-9c090153-08
       valid_lft forever preferred_lft forever
    inet 192.168.1.105/32 brd 192.168.1.105 scope global qg-9c090153-08
       valid_lft forever preferred_lft forever
    inet6 fe80::f816:3eff:feb2:24bc/64 scope link
       valid_lft forever preferred_lft forever

[root@dfw02 ~(keystone_admin)]$ ovs-vsctl show| grep e031db6b-d0
        Port "tape031db6b-d0"
            Interface "tape031db6b-d0"

[root@dfw02 ~(keystone_admin)]$ ovs-vsctl show| grep 9c090153-08
        Port "tap9c090153-08"
            Interface "tap9c090153-08"

[root@dfw02 ~(keystone_admin)]$ ip netns exec qrouter-bf360d81-79fb-4636-8241-0a843f228fc8 ip a

1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
 
Router's internal interface

2: qr-f933e768-42: mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether fa:16:3e:6a:d3:f0 brd ff:ff:ff:ff:ff:ff
    inet 10.0.0.1/24 brd 10.0.0.255 scope global qr-f933e768-42
       valid_lft forever preferred_lft forever
    inet6 fe80::f816:3eff:fe6a:d3f0/64 scope link
       valid_lft forever preferred_lft forever
 
 Router's external   interface with the floating IPs

3: qg-54e34740-87: mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether fa:16:3e:00:9a:0d brd ff:ff:ff:ff:ff:ff
    inet 192.168.1.100/24 brd 192.168.1.255 scope global qg-54e34740-87
       valid_lft forever preferred_lft forever
    inet 192.168.1.103/32 brd 192.168.1.103 scope global qg-54e34740-87
       valid_lft forever preferred_lft forever
    inet6 fe80::f816:3eff:fe00:9a0d/64 scope link
       valid_lft forever preferred_lft forever

[root@dfw02 ~(keystone_admin)]$ ovs-vsctl show| grep f933e768-42
        Port "tapf933e768-42"
            Interface "tapf933e768-42"
 
[root@dfw02 ~(keystone_admin)]$ ovs-vsctl show| grep  54e34740-87
        Port "tap54e34740-87"
            Interface "tap54e34740-87"

[root@dfw02 ~(keystone_admin)]$ ip netns list
qrouter-86b3008c-297f-4301-9bdc-766b839785f1
qrouter-bf360d81-79fb-4636-8241-0a843f228fc8
qdhcp-1eea88bb-4952-4aa4-9148-18b61c22d5b7
qdhcp-426bb226-0ab9-440d-ba14-05634a17fb2b

[root@dfw02 ~(keystone_admin)]$ ip netns exec qdhcp-1eea88bb-4952-4aa4-9148-18b61c22d5b7 ip a

1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever

2: ns-40dd712c-e4: mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether fa:16:3e:93:44:f8 brd ff:ff:ff:ff:ff:ff
    inet 10.0.0.3/24 brd 10.0.0.255 scope global ns-40dd712c-e4
       valid_lft forever preferred_lft forever
    inet6 fe80::f816:3eff:fe93:44f8/64 scope link
       valid_lft forever preferred_lft forever

[root@dfw02 ~(keystone_admin)]$ ovs-vsctl show| grep  40dd712c-e4
        Port "tap40dd712c-e4"
            Interface "tap40dd712c-e4"
[root@dfw02 ~(keystone_admin)]$ ip netns exec qdhcp-426bb226-0ab9-440d-ba14-05634a17fb2b  ip a
1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever

2: ns-343b0090-24: mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether fa:16:3e:01:8b:55 brd ff:ff:ff:ff:ff:ff
    inet 40.0.0.3/24 brd 40.0.0.255 scope global ns-343b0090-24
       valid_lft forever povs-vsctl showreferred_lft forever
    inet6 fe80::f816:3eff:fe01:8b55/64 scope link
       valid_lft forever preferred_lft forever

[root@dfw02 ~(keystone_admin)]$ ovs-vsctl show| grep  343b0090-24
        Port "tap343b0090-24"
            Interface "tap343b0090-24"

***************************************
Now get `ovs-vsctl show` output
***************************************

[root@dfw02 ~(keystone_admin)]$ ovs-vsctl show
7d78d536-3612-416e-bce6-24605088212f
    Bridge br-ex
        Port "p37p1"
            Interface "p37p1"
        Port "tap54e34740-87"  <= qrouter-bf360d81-79fb-4636-8241-0a843f228fc8       <=Router's  external  interface attached here
            Interface "tap54e34740-87"
        Port br-ex
            Interface br-ex
                type: internal
        Port "tap9c090153-08" <= qrouter-86b3008c-297f-4301-9bdc-766b839785f1        <= Router's  external  interface   attached here                    
            Interface "tap9c090153-08"
    Bridge br-int
        Port br-int
            Interface br-int
                type: internal
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "tape031db6b-d0" <= qrouter-86b3008c-297f-4301-9bdc-766b839785f1                     <= Routers internal interface attached here
            tag: 1
            Interface "tape031db6b-d0"
        Port "tapf933e768-42"   <= qrouter-bf360d81-79fb-4636-8241-0a843f228fc8      <= Routers internal interface attached here
             tag: 2
            Interface "tapf933e768-42"
        Port "tap40dd712c-e4"   <= qdhcp-1eea88bb-4952-4aa4-9148-18b61c22d5b7
            tag: 2
            Interface "tap40dd712c-e4"
        Port "tap343b0090-24"   <= qdhcp-426bb226-0ab9-440d-ba14-05634a17fb2b
            tag: 1
            Interface "tap343b0090-24"
    Bridge br-tun
        Port "gre-2"
            Interface "gre-2"
                type: gre
                options: {in_key=flow, local_ip="192.168.1.127", out_key=flow, remote_ip="192.168.1.137"}
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
        Port br-tun
            Interface br-tun
                type: internal
    ovs_version: "2.0.1"

******************************************************************************
Testing qrouter-86b3008c-297f-4301-9bdc-766b839785f1 namespace
******************************************************************************
L3 routing test on running  instance of Ubuntu 14.04 (private IP 40.0.0.5 , floating IP 192.168.1.105) doing  ping 83.221.202.254

   

Capturing packages on interfaces

# tcpdump -i tape031db6b-d0 -vv | tee logdump.int
# tcpdump -i tap9c090153-08 -vv | tee logdump.ext

*********************
Content logdump.int
*********************

19:35:02.343287 IP (tos 0x0, ttl 64, id 11964, offset 0, flags [DF], proto ICMP (1), length 84)
    40.0.0.5 > ns2.donpac.ru: ICMP echo request, id 11817, seq 64, length 64
19:35:02.344662 IP (tos 0x0, ttl 59, id 43283, offset 0, flags [DF], proto ICMP (1), length 84)
    ns2.donpac.ru > 40.0.0.5: ICMP echo reply, id 11817, seq 64, length 64
19:35:03.345161 IP (tos 0x0, ttl 64, id 11965, offset 0, flags [DF], proto ICMP (1), length 84)
    40.0.0.5 > ns2.donpac.ru: ICMP echo request, id 11817, seq 65, length 64
19:35:03.346490 IP (tos 0x0, ttl 59, id 44767, offset 0, flags [DF], proto ICMP (1), length 84)
    ns2.donpac.ru > 40.0.0.5: ICMP echo reply, id 11817, seq 65, length 64
19:35:04.347239 IP (tos 0x0, ttl 64, id 11966, offset 0, flags [DF], proto ICMP (1), length 84)
    40.0.0.5 > ns2.donpac.ru: ICMP echo request, id 11817, seq 66, length 64
19:35:04.348662 IP (tos 0x0, ttl 59, id 46565, offset 0, flags [DF], proto ICMP (1), length 84)
    ns2.donpac.ru > 40.0.0.5: ICMP echo reply, id 11817, seq 66, length 64
19:35:05.353605 IP (tos 0x0, ttl 64, id 11967, offset 0, flags [DF], proto ICMP (1), length 84)

*********************
Content logdump.ext
*********************

19:35:51.434781 IP (tos 0x0, ttl 63, id 12013, offset 0, flags [DF], proto ICMP (1), length 84)
    192.168.1.105 > ns2.donpac.ru: ICMP echo request, id 11817, seq 113, length 64
19:35:51.435691 IP (tos 0x0, ttl 60, id 59160, offset 0, flags [DF], proto ICMP (1), length 84)
    ns2.donpac.ru > 192.168.1.105: ICMP echo reply, id 11817, seq 113, length 64
19:35:52.436598 IP (tos 0x0, ttl 63, id 12014, offset 0, flags [DF], proto ICMP (1), length 84)
    192.168.1.105 > ns2.donpac.ru: ICMP echo request, id 11817, seq 114, length 64
19:35:52.437575 IP (tos 0x0, ttl 60, id 60728, offset 0, flags [DF], proto ICMP (1), length 84)
    ns2.donpac.ru > 192.168.1.105: ICMP echo reply, id 11817, seq 114, length 64
19:35:53.438083 IP (tos 0x0, ttl 63, id 12015, offset 0, flags [DF], proto ICMP (1), length 84)
    192.168.1.105 > ns2.donpac.ru: ICMP echo request, id 11817, seq 115, length 64
19:35:53.439045 IP (tos 0x0, ttl 60, id 62490, offset 0, flags [DF], proto ICMP (1), length 84)
    ns2.donpac.ru > 192.168.1.105: ICMP echo reply, id 11817, seq 115, length 64
19:35:53.883506 IP (tos 0x10, ttl 63, id 31584, offset 0, flags [DF], proto TCP (6), length 152)
**********************************************************************************
L3 routing test on running  instance of Ubuntu 14.04 (private IP 40.0.0.4 , floating IP 192.168.1.106) doing `apt-get install xorg fluxbox gnome-terminal`


# tcpdump -i tape031db6b-d0 -vv | tee logdump01.int
**********************************************************************************
22:33:12.066339 IP (tos 0x0, ttl 64, id 12998, offset 0, flags [DF], proto TCP (6), length 52)
    40.0.0.4.40608 > ragana.canonical.com.http: Flags [.], cksum 0xe131 (correct), seq 16007, ack 20856182, win 6835, options [nop,nop,TS val 1137497 ecr 1286593365], length 0
22:33:12.066347 IP (tos 0x0, ttl 48, id 40522, offset 0, flags [DF], proto TCP (6), length 1454)
    ragana.canonical.com.http > 40.0.0.4.40608: Flags [.], cksum 0x81c5 (correct), seq 20868800:20870202, ack 16007, win 434, options [nop,nop,TS val 1286593365 ecr 1137460], length 1402
22:33:12.066465 IP (tos 0x0, ttl 48, id 40523, offset 0, flags [DF], proto TCP (6), length 1454)
    ragana.canonical.com.http > 40.0.0.4.40608: Flags [.], cksum 0x9e67 (correct), seq 20870202:20871604, ack 16007, win 434, options [nop,nop,TS val 1286593365 ecr 1137460], length 1402
22:33:12.066585 IP (tos 0x0, ttl 48, id 40524, offset 0, flags [DF], proto TCP (6), length 1454)
    ragana.canonical.com.http > 40.0.0.4.40608: Flags [.], cksum 0xada9 (correct), seq 20871604:20873006, ack 16007, win 434, options [nop,nop,TS val 1286593365 ecr 1137460], length 1402
22:33:12.066709 IP (tos 0x0, ttl 64, id 12999, offset 0, flags [DF], proto TCP (6), length 52)
    40.0.0.4.40608 > ragana.canonical.com.http: Flags [.], cksum 0xd63d (correct), seq 16007, ack 20858986, win 6835, options [nop,nop,TS val 1137497 ecr 1286593365], length 0
22:33:12.066716 IP (tos 0x0, ttl 48, id 40525, offset 0, flags [DF], proto TCP (6), length 1454)
    ragana.canonical.com.http > 40.0.0.4.40608: Flags [.], cksum 0x466b (correct), seq 20873006:20874408, ack 16007, win 434, options [nop,nop,TS val 1286593365 ecr 1137460], length 1402
22:33:12.066835 IP (tos 0x0, ttl 48, id 40526, offset 0, flags [DF], proto TCP (6), length 1454)
    ragana.canonical.com.http > 40.0.0.4.40608: Flags [.], cksum 0xbd64 (correct), seq 20874408:20875810, ack 16007, win 434, options [nop,nop,TS val 1286593365 ecr 1137460], length 1402
22:33:12.066956 IP (tos 0x0, ttl 64, id 13000, offset 0, flags [DF], proto TCP (6), length 52)
    40.0.0.4.40608 > ragana.canonical.com.http: Flags [.], cksum 0xcb49 (correct), seq 16007, ack 20861790, win 6835, options [nop,nop,TS val 1137497 ecr 1286593365], length 0
22:33:12.066963 IP (tos 0x0, ttl 48, id 40527, offset 0, flags [DF], proto TCP (6), length 1454)
    ragana.canonical.com.http > 40.0.0.4.40608: Flags [.], cksum 0x6470 (correct), seq 20875810:20877212, ack 16007, win 434, options [nop,nop,TS val 1286593365 ecr 1137460], length 1402
22:33:12.067087 IP (tos 0x0, ttl 64, id 13001, offset 0, flags [DF], proto TCP (6), length 52)
    40.0.0.4.40608 > ragana.canonical.com.http: Flags [.], cksum 0xc055 (correct), seq 16007, ack 20864594, win 6835, options [nop,nop,TS val 1137497 ecr 1286593365], length 0
22:33:12.067094 IP (tos 0x0, ttl 48, id 40528, offset 0, flags [DF], proto TCP (6), length 1454)
    ragana.canonical.com.http > 40.0.0.4.40608: Flags [.], cksum 0xa2d0 (correct), seq 20877212:20878614, ack 16007, win 434, options [nop,nop,TS val 1286593365 ecr 1137460], length 1402
22:33:12.067299 IP (tos 0x0, ttl 64, id 13002, offset 0, flags [DF], proto TCP (6), length 52)
    40.0.0.4.40608 > ragana.canonical.com.http: Flags [.], cksum 0xb561 (correct), seq 16007, ack 20867398, win 6835, options [nop,nop,TS val 1137497 ecr 1286593365], length 0
22:33:12.067548 IP (tos 0x0, ttl 64, id 13003, offset 0, flags [DF], proto TCP (6), length 52)
    40.0.0.4.40608 > ragana.canonical.com.http: Flags [.], cksum 0xaa6d (correct), seq 16007, ack 20870202, win 6835, options [nop,nop,TS val 1137497 ecr 1286593365], length 0
22:33:12.067799 IP (tos 0x0, ttl 64, id 13004, offset 0, flags [DF], proto TCP (6), length 52)


# tcpdump -i tap9c090153-08 -vv | tee logdump01.ext


22:33:06.010493 IP (tos 0x0, ttl 49, id 39783, offset 0, flags [DF], proto TCP (6), length 1454)
    ragana.canonical.com.http > 192.168.1.106.40608: Flags [.], cksum 0x5fc1 (correct), seq 19841501:19842903, ack 14014, win 392, options [nop,nop,TS val 1286591851 ecr 1135946], length 1402
22:33:06.010612 IP (tos 0x0, ttl 49, id 39784, offset 0, flags [DF], proto TCP (6), length 1454)
    ragana.canonical.com.http > 192.168.1.106.40608: Flags [.], cksum 0x3d2b (correct), seq 19842903:19844305, ack 14014, win 392, options [nop,nop,TS val 1286591851 ecr 1135946], length 1402
22:33:06.010734 IP (tos 0x0, ttl 49, id 39785, offset 0, flags [DF], proto TCP (6), length 1454)
    ragana.canonical.com.http > 192.168.1.106.40608: Flags [.], cksum 0x6be5 (correct), seq 19844305:19845707, ack 14014, win 392, options [nop,nop,TS val 1286591851 ecr 1135946], length 1402
22:33:06.010756 IP (tos 0x0, ttl 63, id 12619, offset 0, flags [DF], proto TCP (6), length 52)
    192.168.1.106.40608 > ragana.canonical.com.http: Flags [.], cksum 0xf743 (correct), seq 14014, ack 19833089, win 6835, options [nop,nop,TS val 1135984 ecr 1286591851], length 0
22:33:06.010862 IP (tos 0x0, ttl 49, id 39786, offset 0, flags [DF], proto TCP (6), length 1454)
    ragana.canonical.com.http > 192.168.1.106.40608: Flags [.], cksum 0xbe54 (correct), seq 19845707:19847109, ack 14014, win 392, options [nop,nop,TS val 1286591851 ecr 1135946], length 1402
22:33:06.010984 IP (tos 0x0, ttl 49, id 39787, offset 0, flags [DF], proto TCP (6), length 1454)
    ragana.canonical.com.http > 192.168.1.106.40608: Flags [.], cksum 0x1401 (correct), seq 19847109:19848511, ack 14014, win 392, options [nop,nop,TS val 1286591851 ecr 1135946], length 1402
22:33:06.011005 IP (tos 0x0, ttl 63, id 12620, offset 0, flags [DF], proto TCP (6), length 52)
    192.168.1.106.40608 > ragana.canonical.com.http: Flags [.], cksum 0xec4f (correct), seq 14014, ack 19835893, win 6835, options [nop,nop,TS val 1135984 ecr 1286591851], length 0
22:33:06.011111 IP (tos 0x0, ttl 49, id 39788, offset 0, flags [DF], proto TCP (6), length 1454)
    ragana.canonical.com.http > 192.168.1.106.40608: Flags [.], cksum 0x6967 (correct), seq 19848511:19849913, ack 14014, win 392, options [nop,nop,TS val 1286591851 ecr 1135946], length 1402
22:33:06.011234 IP (tos 0x0, ttl 49, id 39789, offset 0, flags [DF], proto TCP (6), length 1454)
    ragana.canonical.com.http > 192.168.1.106.40608: Flags [.], cksum 0xa08b (correct), seq 19849913:19851315, ack 14014, win 392, options [nop,nop,TS val 1286591851 ecr 1135946], length 1402
22:33:06.011256 IP (tos 0x0, ttl 63, id 12621, offset 0, flags [DF], proto TCP (6), length 52)
    192.168.1.106.40608 > ragana.canonical.com.http: Flags [.], cksum 0xe15b (correct), seq 14014, ack 19838697, win 6835, options [nop,nop,TS val 1135984 ecr 1286591851], length 0
22:33:06.011360 IP (tos 0x0, ttl 49, id 39790, offset 0, flags [DF], proto TCP (6), length 1454)
    ragana.canonical.com.http > 192.168.1.106.40608: Flags [.], cksum 0x651d (correct), seq 19851315:19852717, ack 14014, win 392, options [nop,nop,TS val 1286591851 ecr 1135946], length 1402
22:33:06.011483 IP (tos 0x0, ttl 49, id 39791, offset 0, flags [DF], proto TCP (6), length 1454)

What ovs-vswitchd does ? (bringing up tap interfaces & activating br-int, br-tun)

[root@dfw02 openvswitch]# ps -ef | grep ovs-vswitchd

root       943     1  0 07:00 ?        00:00:01 ovs-vswitchd: monitoring pid 944 (healthy)
root       944   943  0 07:00 ?        00:00:03 ovs-vswitchd unix:/var/run/openvswitch/db.sock -vconsole:emer -vsyslog:err -vfile:info --mlockall --no-chdir --log-file=/var/log/openvswitch/ovs-vswitchd.log --pidfile=/var/run/openvswitch/ovs-vswitchd.pid --detach --monitor
root     22018 20819  0 08:07 pts/1    00:00:00 grep --color=auto ovs-vswitchd

[root@dfw02 openvswitch]# tail -20 ovs-vswitchd.log

2014-04-24T03:00:40.588Z|00061|bridge|WARN|could not configure network device patch-tun (Invalid argument)
2014-04-24T03:00:40.758Z|00062|bridge|INFO|bridge br-int: added interface patch-tun on port 1
2014-04-24T03:00:40.999Z|00063|bridge|WARN|could not open network device patch-int (No such device)
2014-04-24T03:00:41.096Z|00064|netdev_vport|ERR|patch-int: patch type requires valid 'peer' argument
2014-04-24T03:00:41.096Z|00065|bridge|WARN|could not configure network device patch-int (Invalid argument)
2014-04-24T03:00:41.265Z|00066|bridge|INFO|bridge br-tun: added interface patch-int on port 1
2014-04-24T03:00:49.254Z|00067|ofproto|INFO|br-int: 2 flow_mods 10 s ago (1 adds, 1 deletes)
2014-04-24T03:00:52.083Z|00068|ofproto|INFO|br-tun: 10 flow_mods 10 s ago (9 adds, 1 deletes)
2014-04-24T03:01:33.213Z|00069|bridge|WARN|could not open network device gre-2 (No such device)
2014-04-24T03:01:33.271Z|00070|netdev_vport|ERR|gre-2: gre type requires valid 'remote_ip' argument
2014-04-24T03:01:33.271Z|00071|bridge|WARN|could not configure network device gre-2 (Invalid argument)
2014-04-24T03:01:33.329Z|00072|bridge|INFO|bridge br-tun: added interface gre-2 on port 2
2014-04-24T03:01:39.727Z|00073|bridge|INFO|bridge br-int: added interface tap343b0090-24 on port 2
2014-04-24T03:01:39.793Z|00074|bridge|INFO|bridge br-int: added interface tap40dd712c-e4 on port 3
2014-04-24T03:01:42.346Z|00075|bridge|INFO|bridge br-int: added interface tapf933e768-42 on port 4
2014-04-24T03:01:44.078Z|00076|bridge|INFO|bridge br-ex: added interface tap54e34740-87 on port 2
2014-04-24T03:01:46.789Z|00077|bridge|INFO|bridge br-int: added interface tape031db6b-d0 on port 5
2014-04-24T03:01:47.896Z|00078|bridge|INFO|bridge br-ex: added interface tap9c090153-08 on port 3
2014-04-24T03:01:52.132Z|00079|ofproto|INFO|br-int: 6 flow_mods in the 7 s starting 10 s ago (6 deletes)
2014-04-24T03:01:52.132Z|00080|ofproto|INFO|br-tun: 8 flow_mods in the 12 s starting 18 s ago (4 adds, 4 modifications)

********************************************************************************
Troubleshooting via qdhcp-xxxxxxxxxxx namespace (DHCP on private network)
********************************************************************************

[root@dfw02 ~(keystone_andrew)]$ neutron net-list
+--------------------------------------+------+--------------------------------------------------+
| id                                   | name | subnets                                          |
+--------------------------------------+------+--------------------------------------------------+
| 426bb226-0ab9-440d-ba14-05634a17fb2b | int1 | 9e0d457b-c4c4-45cf-84e2-4ac7550f3b06 40.0.0.0/24 |
| 780ce2f3-2e6e-4881-bbac-857813f9a8e0 | ext  | f30e5a16-a055-4388-a6ea-91ee142efc3d             |
| 971fd74b-66e4-4016-b997-433e124a22ad | int2 | caca1f54-3211-4395-ba3c-a0c26d9605c0 10.0.0.0/24 |
+--------------------------------------+------+--------------------------------------------------+

[root@dfw02 ~(keystone_andrew)]$ ip netns | grep 971fd74b-66e4-4016-b997-433e124a22ad
qdhcp-971fd74b-66e4-4016-b997-433e124a22ad

[root@dfw02 ~(keystone_andrew)]$ ip netns exec qdhcp-971fd74b-66e4-4016-b997-433e124a22ad ifconfig

lo: flags=73  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10
        loop  txqueuelen 0  (Local Loopback)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

ns-a2f724d1-74: flags=4163  mtu 1500
        inet 10.0.0.12  netmask 255.255.255.0  broadcast 10.0.0.255
        inet6 fe80::f816:3eff:fe75:956c  prefixlen 64  scopeid 0x20
        ether fa:16:3e:75:95:6c  txqueuelen 1000  (Ethernet)
        RX packets 27  bytes 3464 (3.3 KiB)
        RX errors 0  dropped 5  overruns 0  frame 0
        TX packets 18  bytes 2992 (2.9 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

[root@dfw02 ~(keystone_andrew)]$ ip netns exec qdhcp-971fd74b-66e4-4016-b997-433e124a22ad tcpdump -ln -i  ns-a2f724d1-74

tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on ns-a2f724d1-74, link-type EN10MB (Ethernet), capture size 65535 bytes
09:35:05.337881 IP 10.0.0.11.bootpc > 10.0.0.12.bootps: BOOTP/DHCP, Request from fa:16:3e:b2:50:44, length 300
09:35:05.338468 IP 10.0.0.12.bootps > 10.0.0.11.bootpc: BOOTP/DHCP, Reply, length 328
09:35:10.348322 ARP, Request who-has 10.0.0.12 tell 10.0.0.11, length 28
09:35:10.348342 ARP, Reply 10.0.0.12 is-at fa:16:3e:75:95:6c, length 28
09:35:44.517803 IP 10.0.0.11.bootpc > 10.0.0.12.bootps: BOOTP/DHCP, Request from fa:16:3e:b2:50:44, length 300
09:35:44.518373 IP 10.0.0.12.bootps > 10.0.0.11.bootpc: BOOTP/DHCP, Reply, length 328
09:35:49.522353 ARP, Request who-has 10.0.0.11 tell 10.0.0.12, length 28
09:35:49.523370 ARP, Reply 10.0.0.11 is-at fa:16:3e:b2:50:44, length 28
09:36:35.222754 IP 10.0.0.11.bootpc > 10.0.0.12.bootps: BOOTP/DHCP, Request from fa:16:3e:b2:50:44, length 300
09:36:35.223247 IP 10.0.0.12.bootps > 10.0.0.11.bootpc: BOOTP/DHCP, Reply, length 328
09:36:40.226298 ARP, Request who-has 10.0.0.11 tell 10.0.0.12, length 28
09:36:40.227281 ARP, Reply 10.0.0.11 is-at fa:16:3e:b2:50:44, length 28
09:37:22.469672 IP 10.0.0.11.bootpc > 10.0.0.12.bootps: BOOTP/DHCP, Request from fa:16:3e:b2:50:44, length 300
09:37:22.470165 IP 10.0.0.12.bootps > 10.0.0.11.bootpc: BOOTP/DHCP, Reply, length 328
09:37:27.474303 ARP, Request who-has 10.0.0.11 tell 10.0.0.12, length 28
09:37:27.475363 ARP, Reply 10.0.0.11 is-at fa:16:3e:b2:50:44, length 28

*****************************************
 Compute Node Troubleshooting
 ****************************************

For instance , qvo3451c4d3-94  and qvb3451c4d3-94  are two ends of veth pair. The qvo prefix stands for neutron veth Open vSwitch side and the qvb prefix stands for quantum veth linux bridge side.

# brctl show
bridge name                    bridge id        STP enabled    interfaces

qbr3451c4d3-94       8000.6ee716bf7cbe    no               qvb3451c4d3-94
qbr429fe7fa-21         8000.b607105e684b   no              qvb429fe7fa-21
qbr96790278-14       8000.b27a3f1d6cdb    no              qvb96790278-14
qbrd23c7356-bd       8000.5ab59cfa8036    no              qvbd23c7356-bda

#ovs-vsctl show
9a811052-e7a5-437c-b4ee-1eb9178cd144
    Bridge br-int
        Port "qvo429fe7fa-21"
            tag: 1
            Interface "qvo429fe7fa-21"
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "qvo3451c4d3-94"
            tag: 2
            Interface "qvo3451c4d3-94"
        Port "qvod23c7356-bd"
            tag: 2
            Interface "qvod23c7356-bd"
        Port br-int
            Interface br-int
                type: internal
        Port "qvo96790278-14"
            tag: 1
            Interface "qvo96790278-14"
    Bridge br-tun
        Port "gre-1"
            Interface "gre-1"
                type: gre
                options: {in_key=flow, local_ip="192.168.1.137", out_key=flow, remote_ip="192.168.1.127"}
        Port br-tun
            Interface br-tun
                type: internal
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
    ovs_version: "2.0.1"

***********************
XML definitions of VMs
***********************
<!--
WARNING: THIS IS AN AUTO-GENERATED FILE. CHANGES TO IT ARE LIKELY TO BE
OVERWRITTEN AND LOST. Changes to this xml configuration should be made using:
  virsh edit instance-00000104
or other application using the libvirt API.
-->

<domain type='qemu'>
  <name>instance-00000104</name>
  <uuid>bea1d407-2163-4e5a-99f1-d83aa96435d1</uuid>
  <memory unit='KiB'>2097152</memory>
  <currentMemory unit='KiB'>2097152</currentMemory>
  <vcpu placement='static'>1</vcpu>
  <sysinfo type='smbios'>
    <system>
      <entry name='manufacturer'>Fedora Project</entry>
      <entry name='product'>OpenStack Nova</entry>
      <entry name='version'>2013.2.3-1.fc20</entry>
      <entry name='serial'>6050001e-8c00-00ac-818a-90e6ba2d11eb</entry>
      <entry name='uuid'>bea1d407-2163-4e5a-99f1-d83aa96435d1</entry>
    </system>
  </sysinfo>
  <os>
    <type arch='x86_64' machine='pc-i440fx-1.6'>hvm</type>
    <boot dev='hd'/>
    <smbios mode='sysinfo'/>
  </os>
  <features>
    <acpi/>
    <apic/>
  </features>
  <cpu mode='host-model'>
    <model fallback='allow'/>
  </cpu>
  <clock offset='utc'/>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>destroy</on_crash>
  <devices>
    <emulator>/usr/bin/qemu-system-x86_64</emulator>
    <disk type='file' device='disk'>
      <driver name='qemu' type='raw' cache='none'/>
      <source file='/var/lib/nova/mnt/acdeb4a4d53d1bdae1a2eddc405ba43d/volume-5c7194e0-479e-4e6e-83a9-1d30568363ab'/>
      <target dev='vda' bus='virtio'/>
      <serial>5c7194e0-479e-4e6e-83a9-1d30568363ab</serial>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
    </disk>
    <disk type='file' device='disk'>
      <driver name='qemu' type='raw' cache='none'/>
      <source file='/var/lib/nova/mnt/acdeb4a4d53d1bdae1a2eddc405ba43d/volume-5ddc0cc1-3c72-46b7-81e2-b701363990cf'/>
      <target dev='vdb' bus='virtio'/>
      <serial>5ddc0cc1-3c72-46b7-81e2-b701363990cf</serial>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>
    </disk>
    <controller type='usb' index='0'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
    </controller>
    <controller type='pci' index='0' model='pci-root'/>
    <interface type='bridge'>
      <mac address='fa:16:3e:20:38:fc'/>
      <source bridge='qbr429fe7fa-21'/>
      <target dev='tap429fe7fa-21'/>
      <model type='virtio'/>
      <driver name='qemu'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
    </interface>
    <serial type='file'>
      <source path='/var/lib/nova/instances/bea1d407-2163-4e5a-99f1-d83aa96435d1/console.log'/>
      <target port='0'/>
    </serial>
    <serial type='pty'>
      <target port='1'/>
    </serial>
    <console type='file'>
      <source path='/var/lib/nova/instances/bea1d407-2163-4e5a-99f1-d83aa96435d1/console.log'/>
      <target type='serial' port='0'/>
    </console>
    <input type='tablet' bus='usb'/>
    <input type='mouse' bus='ps2'/>
    <graphics type='vnc' port='-1' autoport='yes' listen='127.0.0.1' keymap='en-us'>
      <listen type='address' address='127.0.0.1'/>
    </graphics>
    <video>
      <model type='cirrus' vram='9216' heads='1'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
    </video>
    <memballoon model='virtio'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
    </memballoon>
  </devices>
</domain>
<!--
WARNING: THIS IS AN AUTO-GENERATED FILE. CHANGES TO IT ARE LIKELY TO BE
OVERWRITTEN AND LOST. Changes to this xml configuration should be made using:
  virsh edit instance-0000010c
or other application using the libvirt API.
-->

<domain type='qemu'>
  <name>instance-0000010c</name>
  <uuid>19cb7aeb-e0b5-4e21-9793-f72e250d1524</uuid>
  <memory unit='KiB'>2097152</memory>
  <currentMemory unit='KiB'>2097152</currentMemory>
  <vcpu placement='static'>1</vcpu>
  <sysinfo type='smbios'>
    <system>
      <entry name='manufacturer'>Fedora Project</entry>
      <entry name='product'>OpenStack Nova</entry>
      <entry name='version'>2013.2.3-1.fc20</entry>
      <entry name='serial'>6050001e-8c00-00ac-818a-90e6ba2d11eb</entry>
      <entry name='uuid'>19cb7aeb-e0b5-4e21-9793-f72e250d1524</entry>
    </system>
  </sysinfo>
  <os>
    <type arch='x86_64' machine='pc-i440fx-1.6'>hvm</type>
    <boot dev='hd'/>
    <smbios mode='sysinfo'/>
  </os>
  <features>
    <acpi/>
    <apic/>
  </features>
  <cpu mode='host-model'>
    <model fallback='allow'/>
  </cpu>
  <clock offset='utc'/>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>destroy</on_crash>
  <devices>
    <emulator>/usr/bin/qemu-system-x86_64</emulator>
    <disk type='file' device='disk'>
      <driver name='qemu' type='raw' cache='none'/>
      <source file='/var/lib/nova/mnt/acdeb4a4d53d1bdae1a2eddc405ba43d/volume-416f77fe-b262-4a9d-b2f4-4bc1f217f574'/>
      <target dev='vda' bus='virtio'/>
      <serial>416f77fe-b262-4a9d-b2f4-4bc1f217f574</serial>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
    </disk>
    <controller type='usb' index='0'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
    </controller>
    <controller type='pci' index='0' model='pci-root'/>
    <interface type='bridge'>
      <mac address='fa:16:3e:74:06:8c'/>
      <source bridge='qbr3451c4d3-94'/>
      <target dev='tap3451c4d3-94'/>
      <model type='virtio'/>
      <driver name='qemu'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
    </interface>
    <serial type='file'>
      <source path='/var/lib/nova/instances/19cb7aeb-e0b5-4e21-9793-f72e250d1524/console.log'/>
      <target port='0'/>
    </serial>
    <serial type='pty'>
      <target port='1'/>
    </serial>
    <console type='file'>
      <source path='/var/lib/nova/instances/19cb7aeb-e0b5-4e21-9793-f72e250d1524/console.log'/>
      <target type='serial' port='0'/>
    </console>
    <input type='tablet' bus='usb'/>
    <input type='mouse' bus='ps2'/>
    <graphics type='vnc' port='-1' autoport='yes' listen='127.0.0.1' keymap='en-us'>
      <listen type='address' address='127.0.0.1'/>
    </graphics>
    <video>
      <model type='cirrus' vram='9216' heads='1'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
    </video>
    <memballoon model='virtio'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
    </memballoon>
  </devices>
</domain>
<!--
WARNING: THIS IS AN AUTO-GENERATED FILE. CHANGES TO IT ARE LIKELY TO BE
OVERWRITTEN AND LOST. Changes to this xml configuration should be made using:
  virsh edit instance-0000010d
or other application using the libvirt API.
-->

<domain type='qemu'>
  <name>instance-0000010d</name>
  <uuid>29546a14-a731-4eb3-acbc-d118e5b6dba6</uuid>
  <memory unit='KiB'>2097152</memory>
  <currentMemory unit='KiB'>2097152</currentMemory>
  <vcpu placement='static'>1</vcpu>
  <sysinfo type='smbios'>
    <system>
      <entry name='manufacturer'>Fedora Project</entry>
      <entry name='product'>OpenStack Nova</entry>
      <entry name='version'>2013.2.3-1.fc20</entry>
      <entry name='serial'>6050001e-8c00-00ac-818a-90e6ba2d11eb</entry>
      <entry name='uuid'>29546a14-a731-4eb3-acbc-d118e5b6dba6</entry>
    </system>
  </sysinfo>
  <os>
    <type arch='x86_64' machine='pc-i440fx-1.6'>hvm</type>
    <boot dev='hd'/>
    <smbios mode='sysinfo'/>
  </os>
  <features>
    <acpi/>
    <apic/>
  </features>
  <cpu mode='host-model'>
    <model fallback='allow'/>
  </cpu>
  <clock offset='utc'/>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>destroy</on_crash>
  <devices>
    <emulator>/usr/bin/qemu-system-x86_64</emulator>
    <disk type='file' device='disk'>
      <driver name='qemu' type='raw' cache='none'/>
      <source file='/var/lib/nova/mnt/acdeb4a4d53d1bdae1a2eddc405ba43d/volume-66c1fa6b-c012-4df4-94f0-626cb98a2dd0'/>
      <target dev='vda' bus='virtio'/>
      <serial>66c1fa6b-c012-4df4-94f0-626cb98a2dd0</serial>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
    </disk>
    <controller type='usb' index='0'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
    </controller>
    <controller type='pci' index='0' model='pci-root'/>
    <interface type='bridge'>
      <mac address='fa:16:3e:b2:50:44'/>
      <source bridge='qbrd23c7356-bd'/>
      <target dev='tapd23c7356-bd'/>
      <model type='virtio'/>
      <driver name='qemu'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
    </interface>
    <serial type='file'>
      <source path='/var/lib/nova/instances/29546a14-a731-4eb3-acbc-d118e5b6dba6/console.log'/>
      <target port='0'/>
    </serial>
    <serial type='pty'>
      <target port='1'/>
    </serial>
    <console type='file'>
      <source path='/var/lib/nova/instances/29546a14-a731-4eb3-acbc-d118e5b6dba6/console.log'/>
      <target type='serial' port='0'/>
    </console>
    <input type='tablet' bus='usb'/>
    <input type='mouse' bus='ps2'/>
    <graphics type='vnc' port='-1' autoport='yes' listen='127.0.0.1' keymap='en-us'>
      <listen type='address' address='127.0.0.1'/>
    </graphics>
    <video>
      <model type='cirrus' vram='9216' heads='1'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
    </video>
    <memballoon model='virtio'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
    </memballoon>
  </devices>
</domain>
<!--
WARNING: THIS IS AN AUTO-GENERATED FILE. CHANGES TO IT ARE LIKELY TO BE
OVERWRITTEN AND LOST. Changes to this xml configuration should be made using:
  virsh edit instance-0000010e
or other application using the libvirt API.
-->

<domain type='qemu'>
  <name>instance-0000010e</name>
  <uuid>e8b01199-a7ed-4ceb-99c8-ec8f9026f827</uuid>
  <memory unit='KiB'>2097152</memory>
  <currentMemory unit='KiB'>2097152</currentMemory>
  <vcpu placement='static'>1</vcpu>
  <sysinfo type='smbios'>
    <system>
      <entry name='manufacturer'>Fedora Project</entry>
      <entry name='product'>OpenStack Nova</entry>
      <entry name='version'>2013.2.3-1.fc20</entry>
      <entry name='serial'>6050001e-8c00-00ac-818a-90e6ba2d11eb</entry>
      <entry name='uuid'>e8b01199-a7ed-4ceb-99c8-ec8f9026f827</entry>
    </system>
  </sysinfo>
  <os>
    <type arch='x86_64' machine='pc-i440fx-1.6'>hvm</type>
    <boot dev='hd'/>
    <smbios mode='sysinfo'/>
  </os>
  <features>
    <acpi/>
    <apic/>
  </features>
  <cpu mode='host-model'>
    <model fallback='allow'/>
  </cpu>
  <clock offset='utc'/>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>destroy</on_crash>
  <devices>
    <emulator>/usr/bin/qemu-system-x86_64</emulator>
    <disk type='file' device='disk'>
      <driver name='qemu' type='raw' cache='none'/>
      <source file='/var/lib/nova/mnt/acdeb4a4d53d1bdae1a2eddc405ba43d/volume-06cc36eb-bbdb-4242-aa4e-1be01311744d'/>
      <target dev='vda' bus='virtio'/>
      <serial>06cc36eb-bbdb-4242-aa4e-1be01311744d</serial>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
    </disk>
    <controller type='usb' index='0'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
    </controller>
    <controller type='pci' index='0' model='pci-root'/>
    <interface type='bridge'>
      <mac address='fa:16:3e:b4:a1:6f'/>
      <source bridge='qbr96790278-14'/>
      <target dev='tap96790278-14'/>
      <model type='virtio'/>
      <driver name='qemu'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
    </interface>
    <serial type='file'>
      <source path='/var/lib/nova/instances/e8b01199-a7ed-4ceb-99c8-ec8f9026f827/console.log'/>
      <target port='0'/>
    </serial>
    <serial type='pty'>
      <target port='1'/>
    </serial>
    <console type='file'>
      <source path='/var/lib/nova/instances/e8b01199-a7ed-4ceb-99c8-ec8f9026f827/console.log'/>
      <target type='serial' port='0'/>
    </console>
    <input type='tablet' bus='usb'/>
    <input type='mouse' bus='ps2'/>
    <graphics type='vnc' port='-1' autoport='yes' listen='127.0.0.1' keymap='en-us'>
      <listen type='address' address='127.0.0.1'/>
    </graphics>
    <video>
      <model type='cirrus' vram='9216' heads='1'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
    </video>
    <memballoon model='virtio'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
    </memballoon>
  </devices>
</domain>

****************************************
GATEWAY TROUBLESHOOTING
****************************************

Every qrouter namespace has qr-* interface shown up by `ip netns exec qrouter-xxxxx ifconfig` command. It's IP matches IP corresponding qdhcp namespace shown up by `ip netns exec qdhcp-yyyyyy route -n` command.
This IP should be ping able inside both qrouter-xxxxx and corresponding  qdhcp-yyyyyy namespaces. Samples bellow are supposed to demonstrate statement above


[root@dfw02 ~(keystone_andrew)]$ neutron net-list
+--------------------------------------+------+--------------------------------------------------+
| id                                   | name | subnets                                          |
+--------------------------------------+------+--------------------------------------------------+
| 426bb226-0ab9-440d-ba14-05634a17fb2b | int1 | 9e0d457b-c4c4-45cf-84e2-4ac7550f3b06 40.0.0.0/24 |
| 780ce2f3-2e6e-4881-bbac-857813f9a8e0 | ext  | f30e5a16-a055-4388-a6ea-91ee142efc3d             |
| 971fd74b-66e4-4016-b997-433e124a22ad | int2 | caca1f54-3211-4395-ba3c-a0c26d9605c0 10.0.0.0/24 |
+--------------------------------------+------+--------------------------------------------------+

[root@dfw02 ~(keystone_andrew)]$ ip netns | grep 971fd74b-66e4-4016-b997-433e124a22ad
qdhcp-971fd74b-66e4-4016-b997-433e124a22ad


+--------------------------------------+---------+-----------------------------------------------------------------------------+
| id                                   | name    | external_gateway_info                                                       |
+--------------------------------------+---------+-----------------------------------------------------------------------------+
| 0686e526-0584-4004-a8b0-7d64b376f0ce | router5 | {"network_id": "780ce2f3-2e6e-4881-bbac-857813f9a8e0", "enable_snat": true} |
+--------------------------------------+---------+-----------------------------------------------------------------------------+

[root@dfw02 ~(keystone_andrew)]$ ip netns | grep 0686e526-0584-4004-a8b0-7d64b376f0ce
qrouter-0686e526-0584-4004-a8b0-7d64b376f0ce


[root@dfw02 ~(keystone_admin)]$ ip netns exec qrouter-0686e526-0584-4004-a8b0-7d64b376f0ce ifconfig
lo: flags=73  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10
        loop  txqueuelen 0  (Local Loopback)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

qg-edd58124-43: flags=4163  mtu 1500
        inet 192.168.1.107  netmask 255.255.255.0  broadcast 192.168.1.255
        inet6 fe80::f816:3eff:fec4:1408  prefixlen 64  scopeid 0x20
        ether fa:16:3e:c4:14:08  txqueuelen 1000  (Ethernet)
        RX packets 5701  bytes 2190585 (2.0 MiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 2650  bytes 265910 (259.6 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

qr-52247b0b-d9: flags=4163  mtu 1500
        inet 10.0.0.5  netmask 255.255.255.0  broadcast 10.0.0.255
        inet6 fe80::f816:3eff:fec8:4b56  prefixlen 64  scopeid 0x20
        ether fa:16:3e:c8:4b:56  txqueuelen 1000  (Ethernet)
        RX packets 2806  bytes 274752 (268.3 KiB)
        RX errors 0  dropped 5  overruns 0  frame 0
        TX packets 3095  bytes 1936194 (1.8 MiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0


[root@dfw02 ~(keystone_admin)]$ ip netns exec qdhcp-971fd74b-66e4-4016-b997-433e124a22ad route -n

Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.0.0.5        0.0.0.0         UG    0      0        0 ns-a2f724d1-74
10.0.0.0        0.0.0.0         255.255.255.0   U     0      0        0 ns-a2f724d1-74

[root@dfw02 ~(keystone_admin)]$ ip netns exec qdhcp-971fd74b-66e4-4016-b997-433e124a22ad ping -c 5 10.0.0.5

PING 10.0.0.5 (10.0.0.5) 56(84) bytes of data.
64 bytes from 10.0.0.5: icmp_seq=1 ttl=64 time=0.211 ms
64 bytes from 10.0.0.5: icmp_seq=2 ttl=64 time=0.052 ms
64 bytes from 10.0.0.5: icmp_seq=3 ttl=64 time=0.052 ms
64 bytes from 10.0.0.5: icmp_seq=4 ttl=64 time=0.075 ms
64 bytes from 10.0.0.5: icmp_seq=5 ttl=64 time=0.056 ms

--- 10.0.0.5 ping statistics ---
5 packets transmitted, 5 received, 0% packet loss, time 4000ms
rtt min/avg/max/mdev = 0.052/0.089/0.211/0.061 ms

[root@dfw02 ~(keystone_andrew)]$ ip netns exec qrouter-0686e526-0584-4004-a8b0-7d64b376f0ce  ping -c 5   10.0.0.5
PING 10.0.0.5 (10.0.0.5) 56(84) bytes of data.
64 bytes from 10.0.0.5: icmp_seq=1 ttl=64 time=0.046 ms
64 bytes from 10.0.0.5: icmp_seq=2 ttl=64 time=0.055 ms
64 bytes from 10.0.0.5: icmp_seq=3 ttl=64 time=0.044 ms
64 bytes from 10.0.0.5: icmp_seq=4 ttl=64 time=0.051 ms
64 bytes from 10.0.0.5: icmp_seq=5 ttl=64 time=0.036 ms

--- 10.0.0.5 ping statistics ---
5 packets transmitted, 5 received, 0% packet loss, time 3999ms
rtt min/avg/max/mdev = 0.036/0.046/0.055/0.008 ms


[root@dfw02 ~(keystone_boris)]$ neutron net-list
+--------------------------------------+------+--------------------------------------------------+
| id                                   | name | subnets                                          |
+--------------------------------------+------+--------------------------------------------------+
| 426bb226-0ab9-440d-ba14-05634a17fb2b | int1 | 9e0d457b-c4c4-45cf-84e2-4ac7550f3b06 40.0.0.0/24 |
| 780ce2f3-2e6e-4881-bbac-857813f9a8e0 | ext  | f30e5a16-a055-4388-a6ea-91ee142efc3d             |
+--------------------------------------+------+--------------------------------------------------+


[root@dfw02 ~(keystone_boris)]$ ip netns exec qdhcp-426bb226-0ab9-440d-ba14-05634a17fb2b route -n

Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         40.0.0.1        0.0.0.0         UG    0      0        0 ns-343b0090-24
40.0.0.0        0.0.0.0         255.255.255.0   U     0      0        0 ns-343b0090-24


[root@dfw02 ~(keystone_boris)]$ neutron router-list
+--------------------------------------+---------+-----------------------------------------------------------------------------+
| id                                   | name    | external_gateway_info                                                       |
+--------------------------------------+---------+-----------------------------------------------------------------------------+
| 86b3008c-297f-4301-9bdc-766b839785f1 | router2 | {"network_id": "780ce2f3-2e6e-4881-bbac-857813f9a8e0", "enable_snat": true} |
+--------------------------------------+---------+-----------------------------------------------------------------------------+

[root@dfw02 ~(keystone_boris)]$ ip netns exec qrouter-86b3008c-297f-4301-9bdc-766b839785f1 ifconfig

lo: flags=73  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10
        loop  txqueuelen 0  (Local Loopback)
        RX packets 42  bytes 3528 (3.4 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 42  bytes 3528 (3.4 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

qg-9c090153-08: flags=4163  mtu 1500
        inet 192.168.1.114  netmask 255.255.255.0  broadcast 192.168.1.255
        inet6 fe80::f816:3eff:feb2:24bc  prefixlen 64  scopeid 0x20
        ether fa:16:3e:b2:24:bc  txqueuelen 1000  (Ethernet)
        RX packets 89454  bytes 120036239 (114.4 MiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 46881  bytes 3729107 (3.5 MiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

qr-e031db6b-d0: flags=4163  mtu 1500
        inet 40.0.0.1  netmask 255.255.255.0  broadcast 40.0.0.255
        inet6 fe80::f816:3eff:fe83:9090  prefixlen 64  scopeid 0x20
        ether fa:16:3e:83:90:90  txqueuelen 1000  (Ethernet)
        RX packets 47244  bytes 3764649 (3.5 MiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 87137  bytes 119819932 (114.2 MiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0