cloudstack-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From martin kolly <martin.ko...@senselan.ch>
Subject Issue: CLOUDSTACK-9255 Unable to start VM DomainRouter due to error in finalizeStart
Date Fri, 18 Mar 2016 10:58:38 GMT
Hi All

We are facing the same issue as reported by Milamber (Ticket 9255) 
https://issues.apache.org/jira/browse/CLOUDSTACK-9255. When deploying a 
couple of VMs or Port Forwarding's the re-deployment of the router with 
cleanup fails.

We found that iptables configuration takes a lot of time, this 
eventually leads to a timeout on the management server "Unable to start 
VM DomainRouter due to error in finalizeStart, not retrying"

Environment:
- Cloudstack 4.8
- KVM (local storage)
- hosts/mgr on Ubuntu 14.04

We tested with a simple set of four forwarding rules, here the setup:

root@r-96-VM:~# cat /etc/cloudstack/forwardingrules.json
{
     "185.20.146.56": [
         {
             "internal_ip": "10.100.1.95",
             "internal_ports": "22:22",
             "protocol": "tcp",
             "public_ip": "185.20.146.56",
             "public_ports": "22:22",
             "type": "forward"
         }
     ],
     "185.20.146.79": [
         {
             "internal_ip": "10.100.1.42",
             "internal_ports": "22:22",
             "protocol": "tcp",
             "public_ip": "185.20.146.79",
             "public_ports": "22:22",
             "type": "forward"
         },
         {
             "internal_ip": "10.100.1.42",
             "internal_ports": "8443:8443",
             "protocol": "tcp",
             "public_ip": "185.20.146.79",
             "public_ports": "8443:8443",
             "type": "forward"
         },
         {
             "internal_ip": "10.100.1.42",
             "internal_ports": "53:53",
             "protocol": "udp",
             "public_ip": "185.20.146.79",
             "public_ports": "53:53",
             "type": "forward"
         }
     ],
     "id": "forwardingrules"

The definition for every port forwarding seems to take at ~1.5 seconds.

python /opt/cloud/bin/configure.py.timed 
/etc/cloudstack/forwardingrules.json

-A PREROUTING -d 185.20.146.79/32 -i eth2 -p tcp -m tcp --dport 22 -j 
DNAT --to-destination 10.100.1.42:22
time : 0.000965118408203
-A PREROUTING -d 185.20.146.79/32 -i eth0 -p tcp -m tcp --dport 22 -j 
DNAT --to-destination 10.100.1.42:22
time : 0.395485162735
-A OUTPUT -d 185.20.146.79/32 -p tcp -m tcp --dport 22 -j DNAT 
--to-destination 10.100.1.42:22
time : 0.395533084869
-j SNAT --to-source 10.100.1.1 -A POSTROUTING -s 10.100.1.0/24 -d 
10.100.1.42/32 -o eth0 -p tcp -m tcp --dport 22
time : 1.16180706024
-A PREROUTING -d 185.20.146.79/32 -i eth2 -p tcp -m tcp --dport 22 -j 
MARK --set-xmark 0x2/0xffffffff
time : 1.16329216957
-A PREROUTING -d 185.20.146.79/32 -i eth2 -p tcp -m tcp --dport 22 -m 
state --state NEW -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask 
0xffffffff
time : 1.16407108307
-A FORWARD -i eth2 -o eth0 -p tcp -m tcp --dport 22 -m state --state 
NEW,ESTABLISHED -j ACCEPT
Total time for creating Policy : 1.53959512711
----------------------------------------------
-A PREROUTING -d 185.20.146.79/32 -i eth2 -p tcp -m tcp --dport 8443 -j 
DNAT --to-destination 10.100.1.42:8443
time : 0.000781059265137
-A PREROUTING -d 185.20.146.79/32 -i eth0 -p tcp -m tcp --dport 8443 -j 
DNAT --to-destination 10.100.1.42:8443
time : 0.378201007843
-A OUTPUT -d 185.20.146.79/32 -p tcp -m tcp --dport 8443 -j DNAT 
--to-destination 10.100.1.42:8443
time : 0.37822508812
-j SNAT --to-source 10.100.1.1 -A POSTROUTING -s 10.100.1.0/24 -d 
10.100.1.42/32 -o eth0 -p tcp -m tcp --dport 8443
time : 1.14627504349
-A PREROUTING -d 185.20.146.79/32 -i eth2 -p tcp -m tcp --dport 8443 -j 
MARK --set-xmark 0x2/0xffffffff
time : 1.1477329731
-A PREROUTING -d 185.20.146.79/32 -i eth2 -p tcp -m tcp --dport 8443 -m 
state --state NEW -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask 
0xffffffff
time : 1.14850592613
-A FORWARD -i eth2 -o eth0 -p tcp -m tcp --dport 8443 -m state --state 
NEW,ESTABLISHED -j ACCEPT
Total time for creating Policy : 1.52321791649
----------------------------------------------
-A PREROUTING -d 185.20.146.79/32 -i eth2 -p udp -m udp --dport 53 -j 
DNAT --to-destination 10.100.1.42:53
time : 0.000754117965698
-A PREROUTING -d 185.20.146.79/32 -i eth0 -p udp -m udp --dport 53 -j 
DNAT --to-destination 10.100.1.42:53
time : 0.383729934692
-A OUTPUT -d 185.20.146.79/32 -p udp -m udp --dport 53 -j DNAT 
--to-destination 10.100.1.42:53
time : 0.383754968643
-j SNAT --to-source 10.100.1.1 -A POSTROUTING -s 10.100.1.0/24 -d 
10.100.1.42/32 -o eth0 -p udp -m udp --dport 53
time : 1.14376091957
-A PREROUTING -d 185.20.146.79/32 -i eth2 -p udp -m udp --dport 53 -j 
MARK --set-xmark 0x2/0xffffffff
time : 1.14526605606
-A PREROUTING -d 185.20.146.79/32 -i eth2 -p udp -m udp --dport 53 -m 
state --state NEW -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask 
0xffffffff
time : 1.14599299431
-A FORWARD -i eth2 -o eth0 -p udp -m udp --dport 53 -m state --state 
NEW,ESTABLISHED -j ACCEPT
Total time for creating Policy : 1.52742600441
----------------------------------------------
-A PREROUTING -d 185.20.146.56/32 -i eth2 -p tcp -m tcp --dport 22 -j 
DNAT --to-destination 10.100.1.95:22
time : 0.000700950622559
-A PREROUTING -d 185.20.146.56/32 -i eth0 -p tcp -m tcp --dport 22 -j 
DNAT --to-destination 10.100.1.95:22
time : 0.382349014282
-A OUTPUT -d 185.20.146.56/32 -p tcp -m tcp --dport 22 -j DNAT 
--to-destination 10.100.1.95:22
time : 0.382384061813
-j SNAT --to-source 10.100.1.1 -A POSTROUTING -s 10.100.1.0/24 -d 
10.100.1.95/32 -o eth0 -p tcp -m tcp --dport 22
time : 1.1425909996
-A PREROUTING -d 185.20.146.56/32 -i eth2 -p tcp -m tcp --dport 22 -j 
MARK --set-xmark 0x2/0xffffffff
time : 1.14400196075
-A PREROUTING -d 185.20.146.56/32 -i eth2 -p tcp -m tcp --dport 22 -m 
state --state NEW -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask 
0xffffffff
time : 1.14468812943
-A FORWARD -i eth2 -o eth0 -p tcp -m tcp --dport 22 -m state --state 
NEW,ESTABLISHED -j ACCEPT
Total time for creating Policy : 1.52619600296
----------------------------------------------

Having a closer look at configure.py how the iptables rules are defined. 
We think that it is not efficient to lookup these values for every policy:

def forward_vr(self, rule):

fw1 = "-A PREROUTING -d %s/32 -i %s -p %s -m %s --dport %s -j DNAT 
--to-destination %s:%s" % \
               (
                 rule['public_ip'],
*self.getDeviceByIp(rule['public_ip']),*
                 rule['protocol'],
                 rule['protocol'],
*self.portsToString(rule['public_ports'], ':'),*
                 rule['internal_ip'],
*self.portsToString(rule['internal_ports'], '-')*
               )
fw2 = "-A PREROUTING -d %s/32 -i %s -p %s -m %s --dport %s -j DNAT 
--to-destination %s:%s" % \
               (
                 rule['public_ip'],
*             self.getDeviceByIp(rule['internal_ip']),*
                 rule['protocol'],
                 rule['protocol'],
*                self.portsToString(rule['public_ports'], ':'),*
                 rule['internal_ip'],
*             self.portsToString(rule['internal_ports'], '-')
.....
*

Defining these values once at the beginning would be much more 
efficient, no ?

def forward_vr(self, rule):

*       pub_interface = self.getDeviceByIp(rule['public_ip'])**
**       int_interface = self.getDeviceByIp(rule['internal_ip'])**
**       pub_ports = self.portsToString(rule['public_ports'], ':')**
**       int_ports = self.portsToString(rule['internal_ports'], '-')**
**       int_network = self.getNetworkByIp(rule['internal_ip'])

* fw1 = "-A PREROUTING -d %s/32 -i %s -p %s -m %s --dport %s -j DNAT 
--to-destination %s:%s" % \
               (
                 rule['public_ip'],
                 pub_interface,
                 rule['protocol'],
                 rule['protocol'],
                 pub_ports,
                 rule['internal_ip'],
                 int_ports
               )

  fw2 = "-A PREROUTING -d %s/32 -i %s -p %s -m %s --dport %s -j DNAT 
--to-destination %s:%s" % \
               (
                 rule['public_ip'],
                 int_interface,
                 rule['protocol'],
                 rule['protocol'],
                 pub_ports,
                 rule['internal_ip'],
                 int_ports
               )
.....

If we run the configure.py with these modifications we have the following:

root@r-96-VM:~#  python /opt/cloud/bin/configure_modified.py 
/etc/cloudstack/forwardingrules.json
-A PREROUTING -d 185.20.146.79/32 -i eth2 -p tcp -m tcp --dport 22 -j 
DNAT --to-destination 10.100.1.42:22
time : 0.000349044799805
-A PREROUTING -d 185.20.146.79/32 -i eth0 -p tcp -m tcp --dport 22 -j 
DNAT --to-destination 10.100.1.42:22
time : 0.000686883926392
-A OUTPUT -d 185.20.146.79/32 -p tcp -m tcp --dport 22 -j DNAT 
--to-destination 10.100.1.42:22
time : 0.000943899154663
-j SNAT --to-source 10.100.1.1 -A POSTROUTING -s 10.100.1.0/24 -d 
10.100.1.42/32 -o eth0 -p tcp -m tcp --dport 22
time : 0.00131487846375
-A PREROUTING -d 185.20.146.79/32 -i eth2 -p tcp -m tcp --dport 22 -j 
MARK --set-xmark 0x2/0xffffffff
time : 0.00161194801331
-A PREROUTING -d 185.20.146.79/32 -i eth2 -p tcp -m tcp --dport 22 -m 
state --state NEW -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask 
0xffffffff
time : 0.00186896324158
-A FORWARD -i eth2 -o eth0 -p tcp -m tcp --dport 22 -m state --state 
NEW,ESTABLISHED -j ACCEPT
Total time for creating Policy : 0.00216102600098
----------------------------------------------
-A PREROUTING -d 185.20.146.79/32 -i eth2 -p tcp -m tcp --dport 8443 -j 
DNAT --to-destination 10.100.1.42:8443
time : 0.000232934951782
-A PREROUTING -d 185.20.146.79/32 -i eth0 -p tcp -m tcp --dport 8443 -j 
DNAT --to-destination 10.100.1.42:8443
time : 0.000478029251099
-A OUTPUT -d 185.20.146.79/32 -p tcp -m tcp --dport 8443 -j DNAT 
--to-destination 10.100.1.42:8443
time : 0.00071907043457
-j SNAT --to-source 10.100.1.1 -A POSTROUTING -s 10.100.1.0/24 -d 
10.100.1.42/32 -o eth0 -p tcp -m tcp --dport 8443
time : 0.000991106033325
-A PREROUTING -d 185.20.146.79/32 -i eth2 -p tcp -m tcp --dport 8443 -j 
MARK --set-xmark 0x2/0xffffffff
time : 0.00136613845825
-A PREROUTING -d 185.20.146.79/32 -i eth2 -p tcp -m tcp --dport 8443 -m 
state --state NEW -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask 
0xffffffff
time : 0.00174498558044
-A FORWARD -i eth2 -o eth0 -p tcp -m tcp --dport 8443 -m state --state 
NEW,ESTABLISHED -j ACCEPT
Total time for creating Policy : 0.00219202041626
----------------------------------------------
-A PREROUTING -d 185.20.146.79/32 -i eth2 -p udp -m udp --dport 53 -j 
DNAT --to-destination 10.100.1.42:53
time : 0.000226974487305
-A PREROUTING -d 185.20.146.79/32 -i eth0 -p udp -m udp --dport 53 -j 
DNAT --to-destination 10.100.1.42:53
time : 0.000502824783325
-A OUTPUT -d 185.20.146.79/32 -p udp -m udp --dport 53 -j DNAT 
--to-destination 10.100.1.42:53
time : 0.000762939453125
-j SNAT --to-source 10.100.1.1 -A POSTROUTING -s 10.100.1.0/24 -d 
10.100.1.42/32 -o eth0 -p udp -m udp --dport 53
time : 0.00103092193604
-A PREROUTING -d 185.20.146.79/32 -i eth2 -p udp -m udp --dport 53 -j 
MARK --set-xmark 0x2/0xffffffff
time : 0.00134587287903
-A PREROUTING -d 185.20.146.79/32 -i eth2 -p udp -m udp --dport 53 -m 
state --state NEW -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask 
0xffffffff
time : 0.00158596038818
-A FORWARD -i eth2 -o eth0 -p udp -m udp --dport 53 -m state --state 
NEW,ESTABLISHED -j ACCEPT
Total time for creating Policy : 0.00182485580444
----------------------------------------------
-A PREROUTING -d 185.20.146.56/32 -i eth2 -p tcp -m tcp --dport 22 -j 
DNAT --to-destination 10.100.1.95:22
time : 0.000264167785645
-A PREROUTING -d 185.20.146.56/32 -i eth0 -p tcp -m tcp --dport 22 -j 
DNAT --to-destination 10.100.1.95:22
time : 0.000508069992065
-A OUTPUT -d 185.20.146.56/32 -p tcp -m tcp --dport 22 -j DNAT 
--to-destination 10.100.1.95:22
time : 0.000750064849854
-j SNAT --to-source 10.100.1.1 -A POSTROUTING -s 10.100.1.0/24 -d 
10.100.1.95/32 -o eth0 -p tcp -m tcp --dport 22
time : 0.00102114677429
-A PREROUTING -d 185.20.146.56/32 -i eth2 -p tcp -m tcp --dport 22 -j 
MARK --set-xmark 0x2/0xffffffff
time : 0.00138115882874
-A PREROUTING -d 185.20.146.56/32 -i eth2 -p tcp -m tcp --dport 22 -m 
state --state NEW -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask 
0xffffffff
time : 0.00165915489197
-A FORWARD -i eth2 -o eth0 -p tcp -m tcp --dport 22 -m state --state 
NEW,ESTABLISHED -j ACCEPT
Total time for creating Policy : 0.00196814537048
----------------------------------------------

Location of configure.py:
https://github.com/apache/cloudstack/blob/master/systemvm/patches/debian/config/opt/cloud/bin/configure.py

The modified scripts are attached. Thanks for your feedback.

regards
Martin


Mime
View raw message