Sunday, December 13, 2015

AIO RDO Liberty && several external networks VLAN provider setup

UPDATE 05/04/2016
 I got back to this writing due to
 https://ask.openstack.org/en/question/91611/how-to-configure-multiple-external-networks-in-rdo-libertymitaka/
 in answer field  contains several misleading steps  in configuration  vlan enabled
 OVS bridges.
END UPDATE

Post bellow is addressing the question when AIO RDO Liberty Node has to have external networks of VLAN type with predefined vlan tags. Straight forward packstack --allinone install doesn't  allow to achieve desired network configuration. External network provider of vlan type appears to be required. In particular case, office networks 10.10.10.0/24 vlan tagged (157) ,10.10.57.0/24 vlan tagged (172), 10.10.32.0/24 vlan tagged (200) already exists when RDO install is running. If demo_provision was "y" , then delete router1 and created external network of VXLAN type

First
***********************************************************
Update /etc/neutron/plugins/ml2/ml2_conf.ini
***********************************************************
[root@ip-192-169-142-52 ml2(keystone_demo)]# cat ml2_conf.ini
[ml2]
type_drivers = flat,vlan,vxlan
tenant_network_types = vlan,vxlan
mechanism_drivers =openvswitch
path_mtu = 0
[ml2_type_flat]
[ml2_type_vlan]
network_vlan_ranges = vlan157:157:157,vlan172:172:172,vlan200:200:200
[ml2_type_gre]
[ml2_type_vxlan]
vni_ranges =10:100
vxlan_group =224.0.0.1
[ml2_type_geneve]
[securitygroup]
enable_security_group = True

**************
Then
**************

# openstack-service restart neutron


***************************************************
Invoke external network provider
***************************************************

[root@ip-192-169-142-52 ~(keystone_admin]#neutron net-create vlan157 --shared --provider:network_type vlan --provider:segmentation_id 157 --provider:physical_network vlan157 --router:external

[root@ip-192-169-142-52 ~(keystone_admin]# neutron subnet-create --name sub-vlan157 --gateway 10.10.10.1  --allocation-pool start=10.10.10.100,end=10.10.10.200 vlan157 10.10.10.0/24


***********************************************
Create second external network
***********************************************

[root@ip-192-169-142-52 ~(keystone_admin]# neutron net-create vlan172 --shared --provider:network_type vlan --provider:segmentation_id 172 --provider:physical_network vlan172  --router:external

[root@ip-192-169-142-52 ~(keystone_admin]# neutron subnet-create --name sub-vlan172 --gateway 10.10.57.1 --allocation-pool start=10.10.57.100,end=10.10.57.200 vlan172 10.10.57.0/24


***********************************************
Create third external network
***********************************************

[root@ip-192-169-142-52 ~(keystone_admin]# neutron net-create vlan200 --shared --provider:network_type vlan --provider:segmentation_id 200 --provider:physical_network vlan200  --router:external
[root@ip-192-169-142-52 ~(keystone_admin]# neutron subnet-create --name sub-vlan200 --gateway 10.10.32.1 --allocation-pool start=10.10.32.100,end=10.10.57.200 vlan172 10.10.32.0/24

***********************************************************************
No need to update sub-net (vs [ 1 ]). No switch to "enable_isolataed_metadata=True"
Neutron L3 agent configuration results attaching qg-<port-id> interfaces to br-int
***********************************************************************


[root@ip-192-169-142-52 ~(keystone_admin)]# neutron net-show vlan157
+---------------------------+--------------------------------------+
| Field                     | Value                                |
+---------------------------+--------------------------------------+
| admin_state_up            | True                                 |
| id                        | b41e4d36-9a63-4631-abb0-6436f2f50e2e |
| mtu                       | 0                                    |
| name                      | vlan157                              |
| provider:network_type     | vlan                                 |
| provider:physical_network | vlan157                              |
| provider:segmentation_id  | 157                                  |
| router:external           | True                                 |
| shared                    | True                                 |
| status                    | ACTIVE                               |
| subnets                   | bb753fc3-f257-4ce5-aa7c-56648648056b |
| tenant_id                 | b18d25d66bbc48b1ad4b855a9c14da70     |
+---------------------------+--------------------------------------+

[root@ip-192-169-142-52 ~(keystone_admin)]# neutron subnet-show sub-vlan157
+-------------------+------------------------------------------------------------------+
| Field             | Value                                                            |
+-------------------+------------------------------------------------------------------+
| allocation_pools  | {"start": "10.10.10.100", "end": "10.10.10.200"}                 |
| cidr              | 10.10.10.0/24                                                    |
| dns_nameservers   |                                                                  |
| enable_dhcp       | True                                                             |
| gateway_ip        | 10.10.10.1                                                       |
| host_routes       | {"destination": "169.254.169.254/32", "nexthop": "10.10.10.151"} |
| id                | bb753fc3-f257-4ce5-aa7c-56648648056b                             |
| ip_version        | 4                                                                |
| ipv6_address_mode |                                                                  |
| ipv6_ra_mode      |                                                                  |
| name              | sub-vlan157                                                      |
| network_id        | b41e4d36-9a63-4631-abb0-6436f2f50e2e                             |
| subnetpool_id     |                                                                  |
| tenant_id         | b18d25d66bbc48b1ad4b855a9c14da70                                 |
+-------------------+------------------------------------------------------------------+

[root@ip-192-169-142-52 ~(keystone_admin)]# neutron net-show vlan172
+---------------------------+--------------------------------------+
| Field                     | Value                                |
+---------------------------+--------------------------------------+
| admin_state_up            | True                                 |
| id                        | 3714adc9-ab17-4f96-9df2-48a6c0b64513 |
| mtu                       | 0                                    |
| name                      | vlan172                              |
| provider:network_type     | vlan                                 |
| provider:physical_network | vlan172                              |
| provider:segmentation_id  | 172                                  |
| router:external           | True                                 |
| shared                    | True                                 |
| status                    | ACTIVE                               |
| subnets                   | 21419f2f-212b-409a-8021-2b4a2ba6532f |
| tenant_id                 | b18d25d66bbc48b1ad4b855a9c14da70     |
+---------------------------+--------------------------------------+

[root@ip-192-169-142-52 ~(keystone_admin)]# neutron subnet-show sub-vlan172
+-------------------+------------------------------------------------------------------+
| Field             | Value                                                            |
+-------------------+------------------------------------------------------------------+
| allocation_pools  | {"start": "10.10.57.100", "end": "10.10.57.200"}                 |
| cidr              | 10.10.57.0/24                                                    |
| dns_nameservers   |                                                                  |
| enable_dhcp       | True                                                             |
| gateway_ip        | 10.10.57.1                                                       |
| host_routes       | {"destination": "169.254.169.254/32", "nexthop": "10.10.57.151"} |
| id                | 21419f2f-212b-409a-8021-2b4a2ba6532f                             |
| ip_version        | 4                                                                |
| ipv6_address_mode |                                                                  |
| ipv6_ra_mode      |                                                                  |
| name              | sub-vlan172                                                      |
| network_id        | 3714adc9-ab17-4f96-9df2-48a6c0b64513                             |
| subnetpool_id     |                                                                  |
| tenant_id         | b18d25d66bbc48b1ad4b855a9c14da70                                 |
+-------------------+------------------------------------------------------------------+

[root@ip-192-169-142-52 ~(keystone_admin)]# neutron net-show vlan200
+---------------------------+--------------------------------------+
| Field                     | Value                                |
+---------------------------+--------------------------------------+
| admin_state_up            | True                                 |
| id                        | 3dc90ff7-b1df-4079-aca1-cceedb23f440 |
| mtu                       | 0                                    |
| name                      | vlan200                              |
| provider:network_type     | vlan                                 |
| provider:physical_network | vlan200                              |
| provider:segmentation_id  | 200                                  |
| router:external           | True                                 |
| shared                    | True                                 |
| status                    | ACTIVE                               |
| subnets                   | 60181211-ea36-4e4e-8781-f13f743baa19 |
| tenant_id                 | b18d25d66bbc48b1ad4b855a9c14da70     |
+---------------------------+--------------------------------------+

[root@ip-192-169-142-52 ~(keystone_admin)]# neutron subnet-show sub-vlan200
+-------------------+--------------------------------------------------+
| Field             | Value                                            |
+-------------------+--------------------------------------------------+
| allocation_pools  | {"start": "10.10.32.100", "end": "10.10.32.200"} |
| cidr              | 10.10.32.0/24                                    |
| dns_nameservers   |                                                  |
| enable_dhcp       | True                                             |
| gateway_ip        | 10.10.32.1                                       |
| host_routes       |                                                  |
| id                | 60181211-ea36-4e4e-8781-f13f743baa19             |
| ip_version        | 4                                                |
| ipv6_address_mode |                                                  |
| ipv6_ra_mode      |                                                  |
| name              | sub-vlan200                                      |
| network_id        | 3dc90ff7-b1df-4079-aca1-cceedb23f440             |
| subnetpool_id     |                                                  |
| tenant_id         | b18d25d66bbc48b1ad4b855a9c14da70                 |
+-------------------+--------------------------------------------------+


**************
Next Step
**************

# modprobe 8021q
# ovs-vsctl add-br br-vlan
# ovs-vsctl add-port br-vlan eth1
# vconfig add br-vlan 157

# ovs-vsctl add-br br-vlan2
# ovs-vsctl add-port br-vlan2 eth2
# vconfig add br-vlan2 172

# ovs-vsctl add-br br-vlan3
# ovs-vsctl add-port br-vlan3 eth3
# vconfig add br-vlan3  200



******************************
Update l3_agent.ini file
******************************
external_network_bridge =
gateway_external_network_id =


**********************************************
/etc/neutron/plugins/ml2/openvswitch_agent.ini
**********************************************
bridge_mappings = vlan157:br-vlan,vlan172:br-vlan2,vlan200:br-vlan3

*************************************
Update Neutron Configuration
*************************************

# openstack-service restart neutron


*******************************************
Set up config persistent between reboots
*******************************************
/etc/sysconfig/network-scripts/ifcfg-eth1

DEVICE="eth1"
ONBOOT=yes
OVS_BRIDGE=br-vlan
TYPE=OVSPort
DEVICETYPE="ovs"

/etc/sysconfig/network-scripts/ifcfg-br-vlan

DEVICE=br-vlan
BOOTPROTO=none
ONBOOT=yes
TYPE=OVSBridge
DEVICETYPE="ovs"

/etc/sysconfig/network-scripts/ifcfg-br-vlan.157

BOOTPROTO="none"
DEVICE="br-vlan.157"
ONBOOT="yes"
IPADDR="10.10.10.150"
PREFIX="24"
GATEWAY="10.10.10.1"
DNS1="83.221.202.254"
VLAN=yes
NOZEROCONF=yes
USERCTL=no


/etc/sysconfig/network-scripts/ifcfg-eth2

DEVICE="eth2"
ONBOOT=yes
OVS_BRIDGE=br-vlan2
TYPE=OVSPort
DEVICETYPE="ovs"

/etc/sysconfig/network-scripts/ifcfg-br-vlan2

DEVICE=br-vlan2
BOOTPROTO=none
ONBOOT=yes
TYPE=OVSBridge
DEVICETYPE="ovs"

/etc/sysconfig/network-scripts/ifcfg-br-vlan2.172

BOOTPROTO="none"
DEVICE="br-vlan2.172"
ONBOOT="yes"
IPADDR="10.10.57.150"
PREFIX="24"
GATEWAY="10.10.57.1"
DNS1="83.221.202.254"
VLAN=yes
NOZEROCONF=yes


/etc/sysconfig/network-scripts/ifcfg-br-vlan3
DEVICE=br-vlan3
BOOTPROTO=none
ONBOOT=yes
TYPE=OVSBridge
DEVICETYPE="ovs"

/etc/sysconfig/network-scripts/ifcfg-br-vlan3.200

BOOTPROTO="none"
DEVICE="br-vlan3.200"
ONBOOT="yes"
IPADDR="10.10.32.150"
PREFIX="24"
GATEWAY="10.10.32.1"
DNS1="83.221.202.254"
VLAN=yes
NOZEROCONF=yes
USERCTL=no

/etc/sysconfig/network-scripts/ifcfg-eth3

DEVICE="eth3"
ONBOOT=yes
OVS_BRIDGE=br-vlan3
TYPE=OVSPort
DEVICETYPE="ovs"


********************************************
Routing table on AIO RDO Liberty Node
********************************************
[root@ip-192-169-142-52 ~(keystone_admin)]# ip route
default via 10.10.10.1 dev br-vlan.157
10.10.10.0/24 dev br-vlan.157  proto kernel  scope link  src 10.10.10.150
10.10.32.0/24 dev br-vlan3.200  proto kernel  scope link  src 10.10.32.150
10.10.57.0/24 dev br-vlan2.172  proto kernel  scope link  src 10.10.57.150
169.254.0.0/16 dev eth0  scope link  metric 1002
169.254.0.0/16 dev eth1  scope link  metric 1003
169.254.0.0/16 dev eth2  scope link  metric 1004
169.254.0.0/16 dev eth3  scope link  metric 1005
169.254.0.0/16 dev br-vlan3  scope link  metric 1008
169.254.0.0/16 dev br-vlan2  scope link  metric 1009
169.254.0.0/16 dev br-vlan  scope link  metric 1011
192.169.142.0/24 dev eth0  proto kernel  scope link  src 192.169.142.52

****************************************************************************
Notice that both qrouter-namespaces are attached to br-int.
No switch to "enable_isolated_metadata=True" vs  [ 1 ]

*****************************************************************************
[root@ip-192-169-142-52 ~(keystone_admin)]# neutron net-list | grep vlan
| 3dc90ff7-b1df-4079-aca1-cceedb23f440 | vlan200   | 60181211-ea36-4e4e-8781-f13f743baa19 10.10.32.0/24 |
| 235c8173-d3f8-407e-ad6a-c1d3d423c763 | vlan172   | c7588239-4941-419b-8d27-ccd970acc4ce 10.10.57.0/24 |
| b41e4d36-9a63-4631-abb0-6436f2f50e2e | vlan157   | bb753fc3-f257-4ce5-aa7c-56648648056b 10.10.10.0/24 |

[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-vsctl show
40286423-e174-4714-9c82-32d026ef47ca
    Bridge br-vlan
        Port "eth1"
            Interface "eth1"
        Port br-vlan
            Interface br-vlan
                type: internal
        Port phy-br-vlan
            Interface phy-br-vlan
                type: patch
                options: {peer=int-br-vlan}
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
    Bridge br-tun
        fail_mode: secure
        Port br-tun
            Interface br-tun
                type: internal
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
    Bridge "br-vlan2"
        Port "phy-br-vlan2"
            Interface "phy-br-vlan2"
                type: patch
                options: {peer="int-br-vlan2"}
        Port "eth2"
            Interface "eth2"
        Port "br-vlan2"
            Interface "br-vlan2"
                type: internal
    Bridge "br-vlan3"
        Port "br-vlan3"
            Interface "br-vlan3"
                type: internal
        Port "phy-br-vlan3"
            Interface "phy-br-vlan3"
                type: patch
                options: {peer="int-br-vlan3"}
        Port "eth3"
            Interface "eth3"
    Bridge br-int
        fail_mode: secure
        Port "qr-4e77c7a3-b5"
            tag: 3
            Interface "qr-4e77c7a3-b5"
                type: internal
        Port "int-br-vlan3"
            Interface "int-br-vlan3"
                type: patch
                options: {peer="phy-br-vlan3"}
        Port "tap8e684c78-a3"
            tag: 2
            Interface "tap8e684c78-a3"
                type: internal
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "qvoe2761636-b5"
            tag: 4
            Interface "qvoe2761636-b5"
        Port "tap6cd6fadf-31"
            tag: 1
            Interface "tap6cd6fadf-31"
                type: internal
        Port "qg-02f7ff0d-6d"
            tag: 2
            Interface "qg-02f7ff0d-6d"
                type: internal
        Port "qg-943f7831-46"
            tag: 1
            Interface "qg-943f7831-46"
                type: internal
        Port "tap4ef27b41-be"
            tag: 5
            Interface "tap4ef27b41-be"
                type: internal
        Port "qr-f0fd3793-4e"
            tag: 8
            Interface "qr-f0fd3793-4e"
                type: internal
        Port "tapb1435e62-8b"
            tag: 7
            Interface "tapb1435e62-8b"
                type: internal
        Port "qvo1bb76476-05"
            tag: 3
            Interface "qvo1bb76476-05"
        Port "qvocf68fcd8-68"
            tag: 8
            Interface "qvocf68fcd8-68"
        Port "qvo8605f075-25"
            tag: 4
            Interface "qvo8605f075-25"
        Port "qg-08ccc224-1e"
            tag: 7
            Interface "qg-08ccc224-1e"
                type: internal
        Port "tapbb485628-0b"
            tag: 4
            Interface "tapbb485628-0b"
                type: internal
        Port "int-br-vlan2"
            Interface "int-br-vlan2"
                type: patch
                options: {peer="phy-br-vlan2"}
        Port "tapee030534-da"
            tag: 8
            Interface "tapee030534-da"
                type: internal
        Port "qr-4d679697-39"
            tag: 4
            Interface "qr-4d679697-39"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port "tap9b38c69e-46"
            tag: 6
            Interface "tap9b38c69e-46"
                type: internal
        Port "tapc166022a-54"
            tag: 3
            Interface "tapc166022a-54"
                type: internal
        Port "qvo66d8f235-d4"
            tag: 8
            Interface "qvo66d8f235-d4"
        Port int-br-vlan
            Interface int-br-vlan
                type: patch
                options: {peer=phy-br-vlan}
    ovs_version: "2.4.0"

[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns
qdhcp-e826aa22-dee0-478d-8bd7-721336e3824a
qrouter-a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b
qdhcp-3dc90ff7-b1df-4079-aca1-cceedb23f440
qdhcp-4481aee1-ef86-4997-bf52-e435aafb9c20
qdhcp-eda69965-c6ee-42be-944f-2d61498e4bea
qdhcp-6768214b-b71c-4178-a0fc-774b2a5d59ef
qdhcp-b41e4d36-9a63-4631-abb0-6436f2f50e2e
qdhcp-03812cc9-69c5-492a-9995-985bf6e1ff13
qdhcp-235c8173-d3f8-407e-ad6a-c1d3d423c763
qdhcp-d958a059-f7bd-4f9f-93a3-3499d20a1fe2
qrouter-c1900dab-447f-4f87-80e7-b4c8ca087d28
qrouter-71237c84-59ca-45dc-a6ec-23eb94c4249d

********************************************************************************
Access to Nova Metadata Server provided via neutron-ns-metadata-proxy
running in corresponding qrouter namespaces  (Neutron L3 Configuration)
********************************************************************************

[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns exec qrouter-a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b netstat -antp
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name

tcp        0      0 0.0.0.0:9697            0.0.0.0:*               LISTEN      12548/python2    

[root@ip-192-169-142-52 ~(keystone_admin)]# ps aux | grep 12548
neutron  12548  0.0  0.4 281028 35992 ?        S    18:34   0:00 /usr/bin/python2 /bin/neutron-ns-metadata-proxy --pid_file=/var/lib/neutron/external/pids/a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b.pid --metadata_proxy_socket=/var/lib/neutron/metadata_proxy --router_id=a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b --state_path=/var/lib/neutron --metadata_port=9697 --metadata_proxy_user=990 --metadata_proxy_group=988 --verbose --log-file=neutron-ns-metadata-proxy-a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b.log --log-dir=/var/log/neutron
root     32665  0.0  0.0 112644   960 pts/8    S+   19:29   0:00 grep --color=auto 12548

******************************************************************************
OVS flow verification on br-vlan3,br-vlan2. On each external network  vlan172,
vlan200 two VMs (on each one of vlan networks) are pinging each other
****************************************************************************** 

[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-flows br-vlan3 | grep NORMAL
 cookie=0x0, duration=3554.739s, table=0, n_packets=33, n_bytes=2074, idle_age=2137, priority=4,in_port=2,dl_vlan=7 actions=mod_vlan_vid:200,NORMAL
 cookie=0x0, duration=4204.459s, table=0, n_packets=2102, n_bytes=109304, idle_age=1, priority=0 actions=NORMAL
[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-flows br-vlan3 | grep NORMAL
 cookie=0x0, duration=3557.643s, table=0, n_packets=33, n_bytes=2074, idle_age=2140, priority=4,in_port=2,dl_vlan=7 actions=mod_vlan_vid:200,NORMAL
 cookie=0x0, duration=4207.363s, table=0, n_packets=2103, n_bytes=109356, idle_age=2, priority=0 actions=NORMAL
[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-flows br-vlan3 | grep NORMAL
 cookie=0x0, duration=3568.225s, table=0, n_packets=33, n_bytes=2074, idle_age=2151, priority=4,in_port=2,dl_vlan=7 actions=mod_vlan_vid:200,NORMAL
 cookie=0x0, duration=4217.945s, table=0, n_packets=2109, n_bytes=109668, idle_age=0, priority=0 actions=NORMAL


[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-flows br-vlan2 | grep NORMAL
 cookie=0x0, duration=4140.528s, table=0, n_packets=11, n_bytes=642, idle_age=695, priority=4,in_port=2,dl_vlan=1 actions=mod_vlan_vid:172,NORMAL
 cookie=0x0, duration=4225.918s, table=0, n_packets=2113, n_bytes=109876, idle_age=1, priority=0 actions=NORMAL
[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-flows br-vlan2 | grep NORMAL
 cookie=0x0, duration=4143.600s, table=0, n_packets=11, n_bytes=642, idle_age=698, priority=4,in_port=2,dl_vlan=1 actions=mod_vlan_vid:172,NORMAL
 cookie=0x0, duration=4228.990s, table=0, n_packets=2115, n_bytes=109980, idle_age=0, priority=0 actions=NORMAL
[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-flows br-vlan2 | grep NORMAL
 cookie=0x0, duration=4145.912s, table=0, n_packets=11, n_bytes=642, idle_age=700, priority=4,in_port=2,dl_vlan=1 actions=mod_vlan_vid:172,NORMAL
 cookie=0x0, duration=4231.302s, table=0, n_packets=2116, n_bytes=110032, idle_age=0, priority=0 actions=NORMAL


********************************************************************************
Next question how local vlan tag 7 gets generated
Run following commands :-
********************************************************************************

 [root@ip-192-169-142-52 ~(keystone_admin)]# neutron net-show vlan200
+---------------------------+--------------------------------------+
| Field                     | Value                                |
+---------------------------+--------------------------------------+
| admin_state_up            | True                                 |
| id                        | 3dc90ff7-b1df-4079-aca1-cceedb23f440 |
| mtu                       | 0                                    |
| name                      | vlan200                              |
| provider:network_type     | vlan                                 |
| provider:physical_network | vlan200                              |
| provider:segmentation_id  | 200                                  |
| router:external           | True                                 |
| shared                    | True                                 |
| status                    | ACTIVE                               |
| subnets                   | 60181211-ea36-4e4e-8781-f13f743baa19 |
| tenant_id                 | b18d25d66bbc48b1ad4b855a9c14da70     |
+---------------------------+--------------------------------------+

[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns | grep 3dc90ff7-b1df-4079-aca1-cceedb23f440
qdhcp-3dc90ff7-b1df-4079-aca1-cceedb23f440

[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns exec qdhcp-3dc90ff7-b1df-4079-aca1-cceedb23f440 ifconfig
lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10<host>
        loop  txqueuelen 0  (Local Loopback)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

tapb1435e62-8b: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 10.10.32.100  netmask 255.255.255.0  broadcast 10.10.32.255
        inet6 fe80::f816:3eff:fee3:19f2  prefixlen 64  scopeid 0x20<link>
        ether fa:16:3e:e3:19:f2  txqueuelen 0  (Ethernet)
        RX packets 27  bytes 1526 (1.4 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 8  bytes 648 (648.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns exec qdhcp-3dc90ff7-b1df-4079-aca1-cceedb23f440 route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.10.32.1      0.0.0.0         UG    0      0        0 tapb1435e62-8b
10.10.32.0      0.0.0.0         255.255.255.0   U     0      0        0 tapb1435e62-8b

[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-vsctl show | grep b1435e62-8b
        Port "tapb1435e62-8b"
            Interface "tapb1435e62-8b"
**************************************************************************
Actually, directives mentioned in  [ 1 ]
**************************************************************************
# neutron subnet-create --name vlan100 --gateway 192.168.0.1 --allocation-pool \
start=192.168.0.150,end=192.168.0.200 --enable-dhcp \
--dns-nameserver 192.168.0.1 vlan100 192.168.0.0/24
# neutron subnet-update --host-route destination=169.254.169.254/32,nexthop=192.168.0.151 vlan100

along with switch to "enable_isolated_metadata=True" are targeting launching VMs to external_fixed_ip pool in qdhcp-3dc90ff7-b1df-4079-aca1-cceedb23f440 without creating Neutron router, spiting tenants with vlan tag IDs. I might be missing somesing , but 1 ] configures system where each vlan(XXX) external network would belong the only one tenant supposed identified by tag (XXX).
Unless RBAC policies will be created to control who has access to the provider network.

That is not what I intend to do. Neutron work flow on br-int won't touch mentioned qdhcp-namespace at all. Any  external vlan(XXX) network might be used by several tenants each one having it ownVXLAN subnet (identified in system by VXLAN ID)  and it's own neutron router(XXX) to external network vlan(XXX). AIO RDO set up is just a sample, I am talking about Network Node in multi node RDO Liberty depoyment.
*********************************************
Fragment from `ovs-vsct show `
*********************************************
Port "tapb1435e62-8b"
            tag: 7
            Interface "tapb1435e62-8b"


*************************************************************************
Next appearance of vlan tag 7, as expected is qg-08ccc224-1e.
Outgoing interface of  qrouter-a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b
namespace.
*************************************************************************
[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns exec qrouter-a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b ifconfig
lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10<host>
        loop  txqueuelen 0  (Local Loopback)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

qg-08ccc224-1e: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 10.10.32.101  netmask 255.255.255.0  broadcast 10.10.32.255
        inet6 fe80::f816:3eff:fed4:e7d  prefixlen 64  scopeid 0x20<link>
        ether fa:16:3e:d4:0e:7d  txqueuelen 0  (Ethernet)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 28  bytes 1704 (1.6 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

qr-f0fd3793-4e: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 30.0.0.1  netmask 255.255.255.0  broadcast 30.0.0.255
        inet6 fe80::f816:3eff:fea9:5422  prefixlen 64  scopeid 0x20<link>
        ether fa:16:3e:a9:54:22  txqueuelen 0  (Ethernet)
        RX packets 68948  bytes 7192868 (6.8 MiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 68859  bytes 7185051 (6.8 MiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns exec qrouter-a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.10.32.1      0.0.0.0         UG    0      0        0 qg-08ccc224-1e
10.10.32.0      0.0.0.0         255.255.255.0   U     0      0        0 qg-08ccc224-1e
30.0.0.0        0.0.0.0         255.255.255.0   U     0      0        0 qr-f0fd3793-4e

*******************************************************************************************************
Now verify Neutron router connecting qrouter-namespace, having interface with tag 7 and qdhcp namespace, been create to launch the instances.
RoutesDSA has been created with external gateway to vlan200 and internal interface to subnet private07 (30.0.0.0/24) having dhcp enabled and DNS server defined.
vlan157,vlan172 are configured as external networks for theirs coresponding routers as well.
*******************************************************************************************************

[root@ip-192-169-142-52 ~(keystone_admin)]# neutron router-list | grep RoutesDSA
| a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b | RoutesDSA  | {"network_id": "3dc90ff7-b1df-4079-aca1-cceedb23f440", "enable_snat": true, "external_fixed_ips": [{"subnet_id": "60181211-ea36-4e4e-8781-f13f743baa19", "ip_address": "10.10.32.101"}]} | False       | False |

[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns | grep a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b
qrouter-a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b

[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns | grep 3dc90ff7-b1df-4079-aca1-cceedb23f440
qdhcp-3dc90ff7-b1df-4079-aca1-cceedb23f440

[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns exec qdhcp-3dc90ff7-b1df-4079-aca1-cceedb23f440 ifconfig
lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10<host>
        loop  txqueuelen 0  (Local Loopback)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

tapb1435e62-8b: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 10.10.32.100  netmask 255.255.255.0  broadcast 10.10.32.255
        inet6 fe80::f816:3eff:fee3:19f2  prefixlen 64  scopeid 0x20<link>
        ether fa:16:3e:e3:19:f2  txqueuelen 0  (Ethernet)
        RX packets 27  bytes 1526 (1.4 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 8  bytes 648 (648.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

**************************
Finally run:-
**************************
[root@ip-192-169-142-52 ~(keystone_admin)]# neutron router-port-list RoutesDSA
+--------------------------------------+------+-------------------+-------------------------------------------------------------------------------------+
| id                                   | name | mac_address       | fixed_ips                                                                           |
+--------------------------------------+------+-------------------+-------------------------------------------------------------------------------------+
| 08ccc224-1e23-491a-8eec-c4db0ec00f02 |      | fa:16:3e:d4:0e:7d | {"subnet_id": "60181211-ea36-4e4e-8781-f13f743baa19", "ip_address": "10.10.32.101"} |
| f0fd3793-4e5a-467a-bd3c-e87bc9063d26 |      | fa:16:3e:a9:54:22 | {"subnet_id": "0c962484-3e48-4d86-a17f-16b0b1e5fc4d", "ip_address": "30.0.0.1"}     |
+--------------------------------------+------+-------------------+-------------------------------------------------------------------------------------+
[root@ip-192-169-142-52 ~(keystone_admin)]# neutron subnet-list | grep 0c962484-3e48-4d86-a17f-16b0b1e5fc4d
| 0c962484-3e48-4d86-a17f-16b0b1e5fc4d |               | 30.0.0.0/24   | {"start": "30.0.0.2", "end": "30.0.0.254"}       |

[root@ip-192-169-142-52 ~(keystone_admin)]# neutron subnet-list | grep 60181211-ea36-4e4e-8781-f13f743baa19
| 60181211-ea36-4e4e-8781-f13f743baa19 | sub-vlan200   | 10.10.32.0/24 | {"start": "10.10.32.100", "end": "10.10.32.200"} |



************************************
OVS Flows at br-vlan3
************************************


[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-flows br-vlan3 | grep NORMAL

cookie=0x0, duration=15793.182s, table=0, n_packets=33, n_bytes=2074, idle_age=14376, priority=4,in_port=2,dl_vlan=7 actions=mod_vlan_vid:200,NORMAL
 cookie=0x0, duration=16442.902s, table=0, n_packets=8221, n_bytes=427492, idle_age=1, priority=0 actions=NORMAL

[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-flows br-vlan3 | grep NORMAL
cookie=0x0, duration=15796.300s, table=0, n_packets=33, n_bytes=2074, idle_age=14379, priority=4,in_port=2,dl_vlan=7 actions=mod_vlan_vid:200,NORMAL
 cookie=0x0, duration=16446.020s, table=0, n_packets=8223, n_bytes=427596, idle_age=0, priority=0 actions=NORMAL

************************************************************
OVS Flow for {phy-br-vlan3,in-br-vlan3} veth pair
************************************************************

[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl show br-vlan3 | grep phy-br-vlan3
 2(phy-br-vlan3): addr:da:e4:fb:ba:8b:1a


[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl show br-int | grep int-br-vlan3
 19(int-br-vlan3): addr:b2:a9:9e:89:07:1b


[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-ports br-vlan3 2
OFPST_PORT reply (xid=0x2): 1 ports
  port  2: rx pkts=6977, bytes=304270, drop=0, errs=0, frame=0, over=0, crc=0
           tx pkts=55, bytes=7037, drop=0, errs=0, coll=0


[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-ports br-vlan3 2
OFPST_PORT reply (xid=0x2): 1 ports
  port  2: rx pkts=6979, bytes=304354, drop=0, errs=0, frame=0, over=0, crc=0
           tx pkts=55, bytes=7037, drop=0, errs=0, coll=0

[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-ports br-vlan3 2
OFPST_PORT reply (xid=0x2): 1 ports
  port  2: rx pkts=6981, bytes=304438, drop=0, errs=0, frame=0, over=0, crc=0
           tx pkts=55, bytes=7037, drop=0, errs=0, coll=0

[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-ports br-int 19
OFPST_PORT reply (xid=0x2): 1 ports
  port 19: rx pkts=55, bytes=7037, drop=0, errs=0, frame=0, over=0, crc=0
           tx pkts=6991, bytes=304858, drop=0, errs=0, coll=0


[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-ports br-int 19
OFPST_PORT reply (xid=0x2): 1 ports
  port 19: rx pkts=55, bytes=7037, drop=0, errs=0, frame=0, over=0, crc=0
           tx pkts=6994, bytes=304984, drop=0, errs=0, coll=0

[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-ports br-int 19
OFPST_PORT reply (xid=0x2): 1 ports
  port 19: rx pkts=55, bytes=7037, drop=0, errs=0, frame=0, over=0, crc=0
           tx pkts=7450, bytes=324136, drop=0, errs=0, coll=0



****************************************************************
Another OVS flow on test br-int for vlan157
****************************************************************

[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns exec qdhcp-4481aee1-ef86-4997-bf52-e435aafb9c20 ssh -i oskeyvls.pem cirros@10.10.10.101
$ ping -c 5 10.10.10.108
PING 10.10.10.108 (10.10.10.108): 56 data bytes
64 bytes from 10.10.10.108: seq=0 ttl=63 time=0.706 ms
64 bytes from 10.10.10.108: seq=1 ttl=63 time=0.772 ms
64 bytes from 10.10.10.108: seq=2 ttl=63 time=0.734 ms
64 bytes from 10.10.10.108: seq=3 ttl=63 time=0.740 ms
64 bytes from 10.10.10.108: seq=4 ttl=63 time=0.785 ms

--- 10.10.10.108 ping statistics ---
5 packets transmitted, 5 packets received, 0% packet loss
round-trip min/avg/max = 0.706/0.747/0.785 ms



  
  

******************************************************************************
   Testing VM1<=>VM2 via floating IPs on external vlan net 10.10.10.0/24
*******************************************************************************

[root@ip-192-169-142-52 ~(keystone_admin)]# nova list --all
+--------------------------------------+--------------+----------------------------------+--------+------------+-------------+---------------------------------+
| ID                                   | Name         | Tenant ID                        | Status | Task State | Power State | Networks                        |
+--------------------------------------+--------------+----------------------------------+--------+------------+-------------+---------------------------------+
| a3d5ecf6-0fdb-4aa3-815f-171871eccb77 | CirrOSDevs01 | f16de8f8497d4f92961018ed836dee19 | ACTIVE | -          | Running     | private=40.0.0.17, 10.10.10.101 |
| 1b65f5db-d7d5-4e92-9a7c-60e7866ff8e5 | CirrOSDevs02 | f16de8f8497d4f92961018ed836dee19 | ACTIVE | -          | Running     | private=40.0.0.18, 10.10.10.110 |
| 46b7dad1-3a7d-4d94-8407-a654cca42750 | VF23Devs01   | f16de8f8497d4f92961018ed836dee19 | ACTIVE | -          | Running     | private=40.0.0.19, 10.10.10.111 |
+--------------------------------------+--------------+----------------------------------+--------+------------+-------------+---------------------------------+

[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns
qdhcp-4481aee1-ef86-4997-bf52-e435aafb9c20
qdhcp-b41e4d36-9a63-4631-abb0-6436f2f50e2e
qrouter-c1900dab-447f-4f87-80e7-b4c8ca087d28

[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns exec qdhcp-4481aee1-ef86-4997-bf52-e435aafb9c20 ssh cirros@10.10.10.110
The authenticity of host '10.10.10.110 (10.10.10.110)' can't be established.
RSA key fingerprint is b8:d3:ec:10:70:a7:da:d4:50:13:a8:2d:01:ba:e4:83.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '10.10.10.110' (RSA) to the list of known hosts.
cirros@10.10.10.110's password:
$ ifconfig
eth0      Link encap:Ethernet  HWaddr FA:16:3E:F1:6E:E5 
          inet addr:40.0.0.18  Bcast:40.0.0.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fef1:6ee5/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1400  Metric:1
          RX packets:367 errors:0 dropped:0 overruns:0 frame:0
          TX packets:291 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:36442 (35.5 KiB)  TX bytes:32019 (31.2 KiB)

lo        Link encap:Local Loopback 
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:16436  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

$ curl http://169.254.169.254/latest/meta-data/public-ipv4
10.10.10.110$

$ ssh fedora@10.10.10.111
Host '10.10.10.111' is not in the trusted hosts file.
(fingerprint md5 23:c0:fb:fd:74:80:2f:12:d3:09:2f:9e:dd:19:f1:74)
Do you want to continue connecting? (y/n) y
fedora@10.10.10.111's password:
Last login: Sun Dec 13 15:52:43 2015 from 10.10.10.101

[fedora@vf23devs01 ~]$ ifconfig
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1400
        inet 40.0.0.19  netmask 255.255.255.0  broadcast 40.0.0.255
        inet6 fe80::f816:3eff:fea4:1a52  prefixlen 64  scopeid 0x20<link>
        ether fa:16:3e:a4:1a:52  txqueuelen 1000  (Ethernet)
        RX packets 283  bytes 30213 (29.5 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 303  bytes 35022 (34.2 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10<host>
        loop  txqueuelen 0  (Local Loopback)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

[fedora@vf23devs01 ~]$ curl http://169.254.169.254/latest/meta-data/public-ipv4
10.10.10.111[fedora@vf23devs01 ~]$
[fedora@vf23devs01 ~]$ curl http://169.254.169.254/latest/meta-data/instance-id
i-00000009[fedora@vf23devs01 ~]$
[fedora@vf23devs01 ~]$

No comments:

Post a Comment