Friday, June 13, 2014

RDO Setup Two Real Node (Controller+Compute) IceHouse Neutron ML2&OVS&GRE Cluster on Fedora 20

Finally I've designed answer-file creating  ml2_conf.ini -> /etc/neutron/plugins/ml2/ml2_conf.ini, but plugin.ini -> /etc/neutron/plugins/openvswitch/ovs_neutron_plugin.ini has been created manually exactly the same as ml2_conf.ini following http://kashyapc.fedorapeople.org/virt/openstack/rdo/IceHouse-Nova-Neutron-ML2-GRE-OVS.txt
Similar file has been created on Compute Node.
Metadata_agent.ini are the same on Controller and on Compute Nodes

Two boxes  have been setup , each one having 2  NICs (p37p1,p4p1) for
Controller && Compute Nodes setup. Before running
`packstack --answer-file= TwoNodeML2&OVS&GRE.txt` SELINUX set to permissive on both nodes.Both p4p1's assigned IPs and set to support GRE tunnel  (192.168.0.127, 192.168.0.137 ) between Controller and Compute Nodes. Services firewalld and NetworkManager disabled (after packstack completion), IPv4 firewall with iptables and service network are enabled and running. Packstack is bind to public IP of interface p37p1 192.169.1.127, Compute Node is 192.169.1.137 ( view answer-file ).

Setup configuration

- Controller node: Nova, Keystone, Cinder, Glance, Neutron (using Open vSwitch plugin && GRE )
- Compute node: Nova (nova-compute), Neutron (openvswitch-agent)


icehouse1.localdomain   -  Controller (192.168.1.127)
icehouse2.localdomain   -  Compute   (192.168.1.137)


********************************
Metadata access verification
********************************

[root@icehouse1 ~(keystone_admin)]# iptables-save | grep 8775
-A INPUT -p tcp -m multiport --dports 8773,8774,8775 -m comment --comment "001 novaapi incoming" -j ACCEPT
-A nova-api-INPUT -d 192.168.1.127/32 -p tcp -m tcp --dport 8775 -j ACCEPT

[root@icehouse1 ~(keystone_admin)]# netstat -antp | grep 8775
tcp        0      0 0.0.0.0:8775            0.0.0.0:*               LISTEN      1181/python        

[root@icehouse1 ~(keystone_admin)]# ps -ef| grep 1181
nova      1181     1  0 06:30 ?        00:00:25 /usr/bin/python /usr/bin/nova-api
nova      3478  1181  0 06:31 ?        00:00:00 /usr/bin/python /usr/bin/nova-api
nova      3479  1181  0 06:31 ?        00:00:00 /usr/bin/python /usr/bin/nova-api
nova      3524  1181  0 06:31 ?        00:00:04 /usr/bin/python /usr/bin/nova-api
nova      3525  1181  0 06:31 ?        00:00:04 /usr/bin/python /usr/bin/nova-api
nova      3549  1181  0 06:31 ?        00:00:00 /usr/bin/python /usr/bin/nova-api
nova      3555  1181  0 06:31 ?        00:00:00 /usr/bin/python /usr/bin/nova-api
root     11803  4686  0 07:48 pts/0    00:00:00 grep --color=auto 1181

[root@icehouse1 ~(keystone_admin)]# ip netns
qdhcp-8b22b262-c9c1-4138-8092-0581195f0889
qrouter-ecf9ee4e-b92c-4a5b-a884-d753a184764b

[root@icehouse1 ~(keystone_admin)]# ip netns exec qrouter-ecf9ee4e-b92c-4a5b-a884-d753a184764b iptables -S -t nat | grep 169.254

-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 9697

[root@icehouse1 ~(keystone_admin)]# ip netns exec qrouter-ecf9ee4e-b92c-4a5b-a884-d753a184764b netstat -antp

Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name   
tcp        0      0 0.0.0.0:9697            0.0.0.0:*               LISTEN      3821/python        

[root@icehouse1 ~(keystone_admin)]# ps -ef| grep 3821
root      3821     1  0 06:31 ?        00:00:00 /usr/bin/python /bin/neutron-ns-metadata-proxy --pid_file=/var/lib/neutron/external/pids/ecf9ee4e-b92c-4a5b-a884-d753a184764b.pid --metadata_proxy_socket=/var/lib/neutron/metadata_proxy --router_id=ecf9ee4e-b92c-4a5b-a884-d753a184764b --state_path=/var/lib/neutron --metadata_port=9697 --verbose --log-file=neutron-ns-metadata-proxy-ecf9ee4e-b92c-4a5b-a884-d753a184764b.log --log-dir=/var/log/neutron
root     11908  4686  0 07:50 pts/0    00:00:00 grep --color=auto 3821



***********************************************
Status nova && neutron services after install
***********************************************

[root@icehouse1 ~(keystone_admin)]# neutron agent-list
+--------------------------------------+--------------------+-----------------------+-------+----------------+
| id                                   | agent_type         | host                  | alive | admin_state_up |
+--------------------------------------+--------------------+-----------------------+-------+----------------+
| 43fa28fb-46fa-4030-9f25-5da92847754f | Open vSwitch agent | icehouse2.localdomain | :-)   | True           |
| 471ab637-49eb-424b-b63e-3d03539150ac | Open vSwitch agent | icehouse1.localdomain | :-)   | True           |
| 495056c8-bb69-4bb4-b954-2398f49dd57a | Metadata agent     | icehouse1.localdomain | :-)   | True           |
| 76eb528d-2673-4ac2-936f-70157d46c566 | L3 agent           | icehouse1.localdomain | :-)   | True           |
| 8f1b4d6b-81df-4903-8a35-df9250143a8b | DHCP agent         | icehouse1.localdomain | :-)   | True           |
+--------------------------------------+--------------------+-----------------------+-------+----------------+

[root@icehouse1 ~(keystone_admin)]# nova-manage service list
Binary           Host                                 Zone             Status     State Updated_At
nova-consoleauth icehouse1.localdomain                internal         enabled    :-)   2014-06-14 17:44:56
nova-scheduler   icehouse1.localdomain                internal         enabled    :-)   2014-06-14 17:44:56
nova-conductor   icehouse1.localdomain                internal         enabled    :-)   2014-06-14 17:44:47
nova-cert        icehouse1.localdomain                internal         enabled    :-)   2014-06-14 17:44:46
nova-compute     icehouse2.localdomain                nova             enabled    :-)   2014-06-14 17:44:47

******************************************************
Routines tables on Controller && Compute Nodes
******************************************************

[root@icehouse1 ~(keystone_admin)]# route -n


Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.1.1     0.0.0.0         UG    0      0        0 br-ex
169.254.0.0     0.0.0.0         255.255.0.0     U     1003   0        0 p37p1
169.254.0.0     0.0.0.0         255.255.0.0     U     1004   0        0 p4p1
169.254.0.0     0.0.0.0         255.255.0.0     U     1018   0        0 br-ex
192.168.0.0     0.0.0.0         255.255.255.0   U     0      0        0 p4p1
192.168.1.0     0.0.0.0         255.255.255.0   U     0      0        0 br-ex

[root@icehouse1 ~(keystone_admin)]# ssh 192.168.1.137
Last login: Thu Oct  2 16:10:58 2014
[root@icehouse2 ~]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.1.1     0.0.0.0         UG    0      0        0 p37p1
169.254.0.0     0.0.0.0         255.255.0.0     U     1002   0        0 p37p1
169.254.0.0     0.0.0.0         255.255.0.0     U     1003   0        0 p4p1
192.168.0.0     0.0.0.0         255.255.255.0   U     0      0        0 p4p1
192.168.1.0     0.0.0.0         255.255.255.0   U     0      0        0 p37p1
192.168.122.0   0.0.0.0       255.255.255.0   U     0      0        0 virbr0


****************************************
Neutron database status after install
****************************************


[root@icehouse1 ~(keystone_admin)]# mysql -u root -p
Enter password:
Welcome to the MariaDB monitor.  Commands end with ; or \g.
Your MariaDB connection id is 1588
Server version: 5.5.36-MariaDB-wsrep MariaDB Server, wsrep_25.9.r3961

Copyright (c) 2000, 2014, Oracle, Monty Program Ab and others.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

MariaDB [(none)]> show databases ;
+--------------------+
| Database           |
+--------------------+
| information_schema |
| cinder             |
| glance             |
| keystone           |
| mysql              |
| neutron            |
| nova               |
| performance_schema |
| test               |
+--------------------+
9 rows in set (0.00 sec)

MariaDB [(none)]> use neutron ;
Reading table information for completion of table and column names
You can turn off this feature to get a quicker startup with -A

Database changed
MariaDB [neutron]> show tables ;
+------------------------------+
| Tables_in_neutron            |
+------------------------------+
| agents                       |
| alembic_version              |
| allowedaddresspairs          |
| arista_provisioned_nets      |
| arista_provisioned_tenants   |
| arista_provisioned_vms       |
| cisco_ml2_credentials        |
| cisco_ml2_nexusport_bindings |
| consistencyhashes            |
| dnsnameservers               |
| externalnetworks             |
| extradhcpopts                |
| floatingips                  |
| ipallocationpools            |
| ipallocations                |
| ipavailabilityranges         |
| ml2_brocadenetworks          |
| ml2_brocadeports             |
| ml2_flat_allocations         |
| ml2_gre_allocations          |
| ml2_gre_endpoints            |
| ml2_network_segments         |
| ml2_port_bindings            |
| ml2_vlan_allocations         |
| ml2_vxlan_allocations        |
| ml2_vxlan_endpoints          |
| networkdhcpagentbindings     |
| networks                     |
| ports                        |
| quotas                       |
| routerl3agentbindings        |
| routerroutes                 |
| routers                      |
| securitygroupportbindings    |
| securitygrouprules           |
| securitygroups               |
| servicedefinitions           |
| servicetypes                 |
| subnetroutes                 |
| subnets                      |
+------------------------------+
40 rows in set (0.00 sec)


*******************************************************************************
System completely functional, however packstack picked up several undesired gre_endpoints showing up in `ovs-vsctl show` reports
*******************************************************************************
Removing not needed gre_endpoints via databases deleting 1 record from
ml2_gre_endpoints
*******************************************************************************


MariaDB [neutron]> select * from ml2_gre_endpoints ;
+---------------+
| ip_address    |
+---------------+
| 192.168.1.137 |
| 192.168.0.127 |
| 192.168.0.137 |
+---------------+
3 rows in set (0.00 sec)

MariaDB [neutron]> delete from ml2_gre_endpoints where ip_address='192.168.1.137' ;
Query OK, 1 row affected (0.01 sec)

MariaDB [neutron]> select * from ml2_gre_endpoints ;
+---------------+
| ip_address    |
+---------------+
| 192.168.0.127 |
| 192.168.0.137 |
+---------------+
2 rows in set (0.00 sec)

MariaDB [neutron]> quit
 

Restart neutron-openvswitch-agent service on both nodes


 [root@icehouse1 neutron(keystone_admin)]# ls -l
total 72
-rw-r--r--. 1 root root      193 Sep 30 17:08 api-paste.ini
-rw-r-----. 1 root neutron  3901 Sep 30 19:19 dhcp_agent.ini
-rw-r--r--. 1 root root       86 Sep 30 19:20 dnsmasq.conf
-rw-r-----. 1 root neutron   208 Sep 30 17:08 fwaas_driver.ini
-rw-r-----. 1 root neutron  3431 Sep 30 17:08 l3_agent.ini
-rw-r-----. 1 root neutron  1400 Aug  8 02:56 lbaas_agent.ini
-rw-r-----. 1 root neutron  1863 Sep 30 17:08 metadata_agent.ini
lrwxrwxrwx. 1 root root       37 Sep 30 18:41 ml2_conf.ini -> /etc/neutron/plugins/ml2/ml2_conf.ini
-rw-r-----. 1 root neutron 19187 Sep 30 17:08 neutron.conf
lrwxrwxrwx. 1 root root       55 Sep 30 18:40 plugin.ini -> /etc/neutron/plugins/openvswitch/ovs_neutron_plugin.ini
-rw-r--r--. 1 root root      211 Sep 30 17:30 plugin.out
drwxr-xr-x. 4 root root     4096 Sep 30 17:08 plugins
-rw-r-----. 1 root neutron  6148 Aug  8 02:56 policy.json
-rw-r--r--. 1 root root       79 Aug 11 15:27 release
-rw-r--r--. 1 root root     1216 Aug  8 02:56 rootwrap.conf

[root@icehouse1 neutron(keystone_admin)]# cat ml2_conf.ini
[ml2]
type_drivers = gre
tenant_network_types = gre
mechanism_drivers = openvswitch
[ml2_type_flat]
[ml2_type_vlan]
[ml2_type_gre]
tunnel_id_ranges = 1:1000
[ml2_type_vxlan]
[securitygroup]
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
enable_security_group = True
[ovs]
local_ip = 192.168.0.127
[agent]
tunnel_types = gre
root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf

[root@icehouse1 neutron(keystone_admin)]# cat plugin.ini
[ml2]
type_drivers = gre
tenant_network_types = gre
mechanism_drivers = openvswitch
[ml2_type_flat]
[ml2_type_vlan]
[ml2_type_gre]
tunnel_id_ranges = 1:1000
[ml2_type_vxlan]
[securitygroup]
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
enable_security_group = True
[ovs]
local_ip = 192.168.0.127
[agent]
tunnel_types = gre
root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf




On Controller:-

 [root@icehouse1 ~(keystone_admin)]# ovs-vsctl show
50a2dcb7-9502-4c08-b175-563eec368db9
    Bridge br-int
        Port "qr-19f312c1-cb"
            tag: 1
            Interface "qr-19f312c1-cb"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port "tap707ec6ff-71"
            tag: 1
            Interface "tap707ec6ff-71"
                type: internal
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
    Bridge br-tun
        Port "gre-c0a80189"
            Interface "gre-c0a80189"
                type: gre
                options: {in_key=flow, local_ip="192.168.0.127", out_key=flow, remote_ip="192.168.0.137"}
        Port br-tun
            Interface br-tun
                type: internal
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
        Port "qg-908c1363-66"
            Interface "qg-908c1363-66"
                type: internal
        Port "p37p1"
            Interface "p37p1"
    ovs_version: "2.1.2"

On Compute:-

[root@icehouse1 ~(keystone_admin)]# ssh 192.168.1.137
Last login: Sat Jun 14 12:47:57 2014
[root@icehouse2 ~]# ovs-vsctl show
bd17e782-fc1b-4c75-8a9a-0bd11ca90dbc
    Bridge br-int
        Port "qvo1e52ffe0-c9"
            tag: 1
            Interface "qvo1e52ffe0-c9"
        Port "qvo897b91ae-71"
            tag: 1
            Interface "qvo897b91ae-71"
        Port "qvo67962cf3-c8"
            tag: 1
            Interface "qvo67962cf3-c8"
        Port br-int
            Interface br-int
                type: internal
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "qvo7e0bdbb7-4e"
            tag: 1
            Interface "qvo7e0bdbb7-4e"
    Bridge br-tun
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
        Port br-tun
            Interface br-tun
                type: internal
        Port "gre-c0a8017f"
            Interface "gre-c0a8017f"
                type: gre
                options: {in_key=flow, local_ip="192.168.0.137", out_key=flow, remote_ip="192.168.0.127"}
    ovs_version: "2.1.2"


 [root@icehouse1 ~(keystone_admin)]# ovs-ofctl show br-tun && ovs-ofctl dump-flows br-tun

OFPT_FEATURES_REPLY (xid=0x2): dpid:00001ecc77fbb64c
n_tables:254, n_buffers:256
capabilities: FLOW_STATS TABLE_STATS PORT_STATS QUEUE_STATS ARP_MATCH_IP
actions: OUTPUT SET_VLAN_VID SET_VLAN_PCP STRIP_VLAN SET_DL_SRC SET_DL_DST SET_NW_SRC SET_NW_DST SET_NW_TOS SET_TP_SRC SET_TP_DST ENQUEUE
 1(patch-int): addr:0a:f9:4e:af:fe:c6
     config:     0
     state:      0
     speed: 0 Mbps now, 0 Mbps max
 2(gre-c0a80089): addr:32:c5:59:d7:4c:8b
     config:     0
     state:      0
     speed: 0 Mbps now, 0 Mbps max
 LOCAL(br-tun): addr:1e:cc:77:fb:b6:4c
     config:     PORT_DOWN
     state:      LINK_DOWN
     speed: 0 Mbps now, 0 Mbps max
OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0
NXST_FLOW reply (xid=0x4):
 cookie=0x0, duration=15350.220s, table=0, n_packets=0, n_bytes=0, idle_age=15350, priority=0 actions=drop
 cookie=0x0, duration=15350.290s, table=0, n_packets=712066, n_bytes=983698886, idle_age=62, priority=1,in_port=1 actions=resubmit(,1)
 cookie=0x0, duration=13862.653s, table=0, n_packets=428887, n_bytes=34296128, idle_age=63, priority=1,in_port=2 actions=resubmit(,2)
 cookie=0x0, duration=15350.131s, table=1, n_packets=712019, n_bytes=983695552, idle_age=62, priority=1,dl_dst=00:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,20)
 cookie=0x0, duration=15350.025s, table=1, n_packets=47, n_bytes=3334, idle_age=9071, priority=1,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,21)
 cookie=0x0, duration=15349.909s, table=2, n_packets=0, n_bytes=0, idle_age=15349, priority=0 actions=drop
 cookie=0x0, duration=13583.119s, table=2, n_packets=360519, n_bytes=28901782, idle_age=9071, priority=1,tun_id=0x4 actions=mod_vlan_vid:3,resubmit(,10)
 cookie=0x0, duration=15346.715s, table=2, n_packets=68542, n_bytes=5413601, idle_age=63, priority=1,tun_id=0x3 actions=mod_vlan_vid:1,resubmit(,10)
 cookie=0x0, duration=15345.408s, table=2, n_packets=0, n_bytes=0, idle_age=15345, priority=1,tun_id=0x2 actions=mod_vlan_vid:2,resubmit(,10)
 cookie=0x0, duration=15349.797s, table=3, n_packets=0, n_bytes=0, idle_age=15349, priority=0 actions=drop
 cookie=0x0, duration=15349.663s, table=10, n_packets=429061, n_bytes=34315383, idle_age=63, priority=1

actions=learn(table=20,hard_timeout=300,priority=1,NXM_OF_VLAN_TCI[0..11],
NXM_OF_ETH_DST[]=NXM_OF_ETH_SRC[],load:0->NXM_OF_VLAN_TCI[],
load:NXM_NX_TUN_ID[]->NXM_NX_TUN_ID[],output:NXM_OF_IN_PORT[]),output:1

 cookie=0x0, duration=15349.575s, table=20, n_packets=2, n_bytes=204, idle_age=752, priority=0 actions=resubmit(,21)

 cookie=0x0, duration=752.794s, table=20, n_packets=25787, n_bytes=34340181, hard_timeout=300, idle_age=62, hard_age=62, priority=1,vlan_tci=0x0001/0x0fff,dl_dst=fa:16:3e:00:5e:64 actions=load:0->NXM_OF_VLAN_TCI[],load:0x3->NXM_NX_TUN_ID[],output:2

 cookie=0x0, duration=15349.503s, table=21, n_packets=28, n_bytes=2084, idle_age=13454, priority=0 actions=drop
 cookie=0x0, duration=13583.174s, table=21, n_packets=10, n_bytes=656, idle_age=9071, dl_vlan=3 actions=strip_vlan,set_tunnel:0x4,output:2
 cookie=0x0, duration=15345.489s, table=21, n_packets=3, n_bytes=210, idle_age=15337, hard_age=13862, dl_vlan=2 actions=strip_vlan,set_tunnel:0x2,output:2
 cookie=0x0, duration=15346.806s, table=21, n_packets=7, n_bytes=498, idle_age=752, hard_age=13862, dl_vlan=1 actions=strip_vlan,set_tunnel:0x3,output:2


Samples here

[root@icehouse1 neutron(keystone_admin)]# cat neutron.conf
[DEFAULT]
verbose = True
debug = False
use_syslog = False
log_dir =/var/log/neutron
bind_host = 0.0.0.0
bind_port = 9696
core_plugin = ml2
service_plugins = router
auth_strategy = keystone
base_mac = fa:16:3e:00:00:00
mac_generation_retries = 16
dhcp_lease_duration = 86400
allow_bulk = True
allow_pagination = False
allow_sorting = False
allow_overlapping_ips = True
rpc_backend = neutron.openstack.common.rpc.impl_kombu
control_exchange = neutron
rabbit_host = 192.168.0.127
rabbit_password = guest
rabbit_port = 5672
rabbit_hosts = 192.168.1.127:5672
rabbit_userid = guest
rabbit_virtual_host = /
rabbit_ha_queues = False
agent_down_time = 75
router_scheduler_driver = neutron.scheduler.l3_agent_scheduler.ChanceScheduler
dhcp_agents_per_network = 1
api_workers = 0
notify_nova_on_port_status_changes = True
notify_nova_on_port_data_changes = True
nova_url = http://192.168.1.127:8774/v2
nova_region_name =RegionOne
nova_admin_username =nova
nova_admin_tenant_id =f4e7985ae16d4fac9166b41c394614af
nova_admin_password =aaf8cf4c60224150
nova_admin_auth_url =http://192.168.1.127:35357/v2.0
send_events_interval = 2
[quotas]
[agent]
root_helper = sudo neutron-rootwrap /etc/neutron/rootwrap.conf
report_interval = 30
[keystone_authtoken]
auth_host = 192.168.1.127
auth_port = 35357
auth_protocol = http
admin_tenant_name = services
admin_user = neutron
admin_password = 5f11f559abc94440
auth_uri=http://192.168.1.127:5000/
[database]
connection = mysql://neutron:0302dcfeb69e439f@192.168.1.127/neutron
max_retries = 10
retry_interval = 10
idle_timeout = 3600
[service_providers]
service_provider=VPN:openswan:neutron.services.vpn.service_drivers.ipsec.IPsecVPNDriver:default

[root@icehouse1 neutron(keystone_admin)]# cat plugin.ini
[ml2]
type_drivers = gre
tenant_network_types = gre
mechanism_drivers = openvswitch
[ml2_type_flat]
[ml2_type_vlan]
[ml2_type_gre]
tunnel_id_ranges = 1:1000
[ml2_type_vxlan]
[securitygroup]
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
enable_security_group = True
[ovs]
local_ip = 192.168.0.127
[agent]
tunnel_types = gre
root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf