cloudstack-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bhais...@apache.org
Subject [cloudstack] 01/01: CLOUDSTACK-10013: Make the generated VR/json files unique (ports #1470)
Date Sat, 02 Dec 2017 19:26:49 GMT
This is an automated email from the ASF dual-hosted git repository.

bhaisaab pushed a commit to branch debian9-systemvmtemplate
in repository https://gitbox.apache.org/repos/asf/cloudstack.git

commit f590e653c24ce54307f2f7c15ee63877f7b79879
Author: Remi Bergsma <github@remi.nl>
AuthorDate: Sat Dec 2 23:19:14 2017 +0530

    CLOUDSTACK-10013: Make the generated VR/json files unique (ports #1470)
    
    This ports PR #1470 by @remibergsma.
    
    Make the generated json files unique to prevent concurrency issues:
    The json files now have UUIDs to prevent them from getting overwritten
    before they've been executed. Prevents config to be pushed to the wrong
    router.
    
    2016-02-25 18:32:23,797 DEBUG [c.c.a.t.Request] (AgentManager-Handler-1:null) (logid:)
Seq 2-4684025087442026584: Processing:  { Ans: , MgmtId: 90520732674657, via: 2, Ver: v1,
Flags: 10, [{"com.cloud.agent.api.routing.GroupA
    nswer":{"results":["null - success: null","null - success: [INFO] update_config.py ::
Processing incoming file => vm_dhcp_entry.json.4ea45061-2efb-4467-8eaa-db3d77fb0a7b\n[INFO]
Processing JSON file vm_dhcp_entry.json.4ea4506
    1-2efb-4467-8eaa-db3d77fb0a7b\n"],"result":true,"wait":0}}] }
    
    On the router:
    2016-02-25 18:32:23,416  merge.py __moveFile:298 Processed file written to /var/cache/cloud/processed/vm_dhcp_entry.json.4ea45061-2efb-4467-8eaa-db3d77fb0a7b.gz
    
    Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>
---
 .../facade/AbstractConfigItemFacade.java           |  21 ++-
 .../kvm/resource/LibvirtComputingResource.java     |   5 +
 systemvm/patches/debian/opt/cloud/bin/configure.py | 210 ++++++++++-----------
 systemvm/patches/debian/opt/cloud/bin/merge.py     |  37 ++--
 .../patches/debian/opt/cloud/bin/update_config.py  |  18 +-
 5 files changed, 153 insertions(+), 138 deletions(-)

diff --git a/core/src/com/cloud/agent/resource/virtualnetwork/facade/AbstractConfigItemFacade.java
b/core/src/com/cloud/agent/resource/virtualnetwork/facade/AbstractConfigItemFacade.java
index f017384..a083012 100644
--- a/core/src/com/cloud/agent/resource/virtualnetwork/facade/AbstractConfigItemFacade.java
+++ b/core/src/com/cloud/agent/resource/virtualnetwork/facade/AbstractConfigItemFacade.java
@@ -22,6 +22,8 @@ package com.cloud.agent.resource.virtualnetwork.facade;
 import java.util.Hashtable;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.UUID;
+import org.apache.log4j.Logger;
 
 import com.cloud.agent.api.BumpUpPriorityCommand;
 import com.cloud.agent.api.SetupGuestNetworkCommand;
@@ -58,6 +60,8 @@ import com.google.gson.GsonBuilder;
 
 public abstract class AbstractConfigItemFacade {
 
+    private static final Logger s_logger = Logger.getLogger(AbstractConfigItemFacade.class);
+
     private final static Gson gson;
 
     private static Hashtable<Class<? extends NetworkElementCommand>, AbstractConfigItemFacade>
flyweight = new Hashtable<Class<? extends NetworkElementCommand>, AbstractConfigItemFacade>();
@@ -104,13 +108,26 @@ public abstract class AbstractConfigItemFacade {
         return instance;
     }
 
+    private static String appendUuidToJsonFiles(final String filename) {
+        String remoteFileName = new String(filename);
+        if (remoteFileName.endsWith("json")) {
+            remoteFileName += "." + UUID.randomUUID().toString();
+        }
+        return remoteFileName;
+    }
+
     protected List<ConfigItem> generateConfigItems(final ConfigBase configuration)
{
         final List<ConfigItem> cfg = new LinkedList<>();
 
-        final ConfigItem configFile = new FileConfigItem(VRScripts.CONFIG_PERSIST_LOCATION,
destinationFile, gson.toJson(configuration));
+        final String remoteFilename = appendUuidToJsonFiles(destinationFile);
+        if (s_logger.isDebugEnabled()) {
+            s_logger.debug("Transformed filename: " + destinationFile + " to: " + remoteFilename);
+        }
+
+        final ConfigItem configFile = new FileConfigItem(VRScripts.CONFIG_PERSIST_LOCATION,
remoteFilename, gson.toJson(configuration));
         cfg.add(configFile);
 
-        final ConfigItem updateCommand = new ScriptConfigItem(VRScripts.UPDATE_CONFIG, destinationFile);
+        final ConfigItem updateCommand = new ScriptConfigItem(VRScripts.UPDATE_CONFIG, remoteFilename);
         cfg.add(updateCommand);
 
         return cfg;
diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
index 7794249..e139ecb 100644
--- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
+++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
@@ -332,6 +332,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements
Serv
         if (details == null) {
             details = parser.getLines();
         }
+
+        s_logger.debug("Executing script in VR: " + script);
+
         return new ExecutionResult(command.getExitValue() == 0, details);
     }
 
@@ -340,6 +343,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements
Serv
         final File permKey = new File("/root/.ssh/id_rsa.cloud");
         String error = null;
 
+        s_logger.debug("Creating file in VR, with ip: " + routerIp + ", file: " + filename);
+
         try {
             SshHelper.scpTo(routerIp, 3922, "root", permKey, null, path, content.getBytes(),
filename, null);
         } catch (final Exception e) {
diff --git a/systemvm/patches/debian/opt/cloud/bin/configure.py b/systemvm/patches/debian/opt/cloud/bin/configure.py
index b8a3e02..1301df0 100755
--- a/systemvm/patches/debian/opt/cloud/bin/configure.py
+++ b/systemvm/patches/debian/opt/cloud/bin/configure.py
@@ -17,33 +17,29 @@
 # specific language governing permissions and limitations
 # under the License.
 import sys
-import os
 import base64
 
-from merge import DataBag
-from pprint import pprint
-import subprocess
+from collections import OrderedDict
+
 import logging
 import re
-import time
-import shutil
+
 import os.path
 import os
 from fcntl import flock, LOCK_EX, LOCK_UN
 
-from cs.CsDatabag import CsDataBag, CsCmdLine
-import cs.CsHelper
+from cs.CsDatabag import CsDataBag
 from cs.CsNetfilter import CsNetfilters
 from cs.CsDhcp import CsDhcp
 from cs.CsRedundant import *
 from cs.CsFile import CsFile
-from cs.CsApp import CsApache, CsDnsmasq
 from cs.CsMonitor import CsMonitor
 from cs.CsLoadBalancer import CsLoadBalancer
 from cs.CsConfig import CsConfig
 from cs.CsProcess import CsProcess
 from cs.CsStaticRoutes import CsStaticRoutes
 
+OCCURRENCES = 1
 
 class CsPassword(CsDataBag):
 
@@ -960,13 +956,49 @@ class CsForwardingRules(CsDataBag):
         self.fw.append(["nat", "front", "-A POSTROUTING -s %s -d %s -j SNAT -o eth0 --to-source
%s" % (self.getNetworkByIp(rule['internal_ip']),rule["internal_ip"], self.getGuestIp())])
 
 
+class IpTablesExecutor:
+
+    config = None
+
+    def __init__(self, config):
+        self.config = config
+
+    def process(self):
+        acls = CsAcl('networkacl', self.config)
+        acls.process()
+
+        acls = CsAcl('firewallrules', self.config)
+        acls.process()
+
+        fwd = CsForwardingRules("forwardingrules", self.config)
+        fwd.process()
+
+        vpns = CsSite2SiteVpn("site2sitevpn", self.config)
+        vpns.process()
+
+        rvpn = CsRemoteAccessVpn("remoteaccessvpn", self.config)
+        rvpn.process()
+
+        lb = CsLoadBalancer("loadbalancer", self.config)
+        lb.process()
+
+        logging.debug("Configuring iptables rules")
+        nf = CsNetfilters(True)
+        nf.compare(self.config.get_fw())
+
+        logging.debug("Configuring iptables rules done ...saving rules")
+
+        # Save iptables configuration - will be loaded on reboot by the iptables-restore
that is configured on /etc/rc.local
+        CsHelper.save_iptables("iptables-save", "/etc/iptables/router_rules.v4")
+        CsHelper.save_iptables("ip6tables-save", "/etc/iptables/router_rules.v6")
+
 def main(argv):
     # The file we are currently processing, if it is "cmd_line.json" everything will be processed.
     process_file = argv[1]
 
-    # process_file can be None, if so assume cmd_line.json
     if process_file is None:
-        process_file = "cmd_line.json"
+        logging.debug("No file was received, do not go on processing the other actions. Just
leave for now.")
+        return
 
     # Track if changes need to be committed to NetFilter
     iptables_change = False
@@ -977,108 +1009,60 @@ def main(argv):
     logging.basicConfig(filename=config.get_logger(),
                         level=config.get_level(),
                         format=config.get_format())
-    try:
-        # Load stored ip adresses from disk to CsConfig()
-        config.set_address()
-
-        logging.debug("Configuring ip addresses")
-        config.address().compare()
-        config.address().process()
-
-        if process_file in ["cmd_line.json", "guest_network.json"]:
-            logging.debug("Configuring Guest Network")
-            iptables_change = True
-
-        if process_file in ["cmd_line.json", "vm_password.json"]:
-            logging.debug("Configuring vmpassword")
-            password = CsPassword("vmpassword", config)
-            password.process()
-
-        if process_file in ["cmd_line.json", "vm_metadata.json"]:
-            logging.debug("Configuring vmdata")
-            metadata = CsVmMetadata('vmdata', config)
-            metadata.process()
-
-        if process_file in ["cmd_line.json", "network_acl.json"]:
-            logging.debug("Configuring networkacl")
-            iptables_change = True
-
-        if process_file in ["cmd_line.json", "firewall_rules.json"]:
-            logging.debug("Configuring firewall rules")
-            iptables_change = True
-
-        if process_file in ["cmd_line.json", "forwarding_rules.json", "staticnat_rules.json"]:
-            logging.debug("Configuring PF rules")
-            iptables_change = True
-
-        if process_file in ["cmd_line.json", "site_2_site_vpn.json"]:
-            logging.debug("Configuring s2s vpn")
-            iptables_change = True
-
-        if process_file in ["cmd_line.json", "remote_access_vpn.json"]:
-            logging.debug("Configuring remote access vpn")
-            iptables_change = True
-
-        if process_file in ["cmd_line.json", "vpn_user_list.json"]:
-            logging.debug("Configuring vpn users list")
-            vpnuser = CsVpnUser("vpnuserlist", config)
-            vpnuser.process()
-
-        if process_file in ["cmd_line.json", "vm_dhcp_entry.json", "dhcp.json"]:
-            logging.debug("Configuring dhcp entry")
-            dhcp = CsDhcp("dhcpentry", config)
-            dhcp.process()
-
-        if process_file in ["cmd_line.json", "load_balancer.json"]:
-            logging.debug("Configuring load balancer")
-            iptables_change = True
-
-        if process_file in ["cmd_line.json", "monitor_service.json"]:
-            logging.debug("Configuring monitor service")
-            mon = CsMonitor("monitorservice", config)
-            mon.process()
-
-        # If iptable rules have changed, apply them.
-        if iptables_change:
-            acls = CsAcl('networkacl', config)
-            acls.process()
-
-            acls = CsAcl('firewallrules', config)
-            acls.flushAllowAllEgressRules()
-            acls.process()
-
-            fwd = CsForwardingRules("forwardingrules", config)
-            fwd.process()
-
-            vpns = CsSite2SiteVpn("site2sitevpn", config)
-            vpns.process()
-
-            rvpn = CsRemoteAccessVpn("remoteaccessvpn", config)
-            rvpn.process()
-
-            lb = CsLoadBalancer("loadbalancer", config)
-            lb.process()
-
-            logging.debug("Configuring iptables rules")
-            nf = CsNetfilters()
-            nf.compare(config.get_fw())
-
-            logging.debug("Configuring iptables rules done ...saving rules")
-
-            # Save iptables configuration - will be loaded on reboot by the iptables-restore
that is configured on /etc/rc.local
-            CsHelper.save_iptables("iptables-save", "/etc/iptables/router_rules.v4")
-            CsHelper.save_iptables("ip6tables-save", "/etc/iptables/router_rules.v6")
-
-        red = CsRedundant(config)
-        red.set()
-
-        if process_file in ["cmd_line.json", "static_routes.json"]:
-            logging.debug("Configuring static routes")
-            static_routes = CsStaticRoutes("staticroutes", config)
-            static_routes.process()
-    except Exception:
-        logging.exception("Exception while configuring router")
-        return 1
+    # Load stored ip adresses from disk to CsConfig()
+    config.set_address()
+
+    logging.debug("Configuring ip addresses")
+    config.address().compare()
+    config.address().process()
+
+    databag_map = OrderedDict([("guest_network.json", {"process_iptables" : True, "executor"
: IpTablesExecutor(config)}),
+                               ("vm_password.json", {"process_iptables" : False, "executor"
: CsPassword("vmpassword", config)}),
+                               ("vm_metadata.json", {"process_iptables" : False, "executor"
: CsVmMetadata('vmdata', config)}),
+                               ("network_acl.json", {"process_iptables" : True, "executor"
: IpTablesExecutor(config)}),
+                               ("firewall_rules.json", {"process_iptables" : True, "executor"
: IpTablesExecutor(config)}),
+                               ("forwarding_rules.json", {"process_iptables" : True, "executor"
: IpTablesExecutor(config)}),
+                               ("staticnat_rules.json", {"process_iptables" : True, "executor"
: IpTablesExecutor(config)}),
+                               ("site_2_site_vpn.json", {"process_iptables" : True, "executor"
: IpTablesExecutor(config)}),
+                               ("remote_access_vpn.json", {"process_iptables" : True, "executor"
: IpTablesExecutor(config)}),
+                               ("vpn_user_list.json", {"process_iptables" : False, "executor"
: CsVpnUser("vpnuserlist", config)}),
+                               ("vm_dhcp_entry.json", {"process_iptables" : False, "executor"
: CsDhcp("dhcpentry", config)}),
+                               ("dhcp.json", {"process_iptables" : False, "executor" : CsDhcp("dhcpentry",
config)}),
+                               ("load_balancer.json", {"process_iptables" : True, "executor"
: IpTablesExecutor(config)}),
+                               ("monitor_service.json", {"process_iptables" : False, "executor"
: CsMonitor("monitorservice", config)}),
+                               ("static_routes.json", {"process_iptables" : False, "executor"
: CsStaticRoutes("staticroutes", config)})
+                               ])
+
+    if process_file.count("cmd_line.json") == OCCURRENCES:
+        logging.debug("cmd_line.json changed. All other files will be processed as well.")
+
+        while databag_map:
+            item = databag_map.popitem(last = False)
+            item_name = item[0]
+            item_dict = item[1]
+            if not item_dict["process_iptables"]:
+                executor = item_dict["executor"]
+                executor.process()
+
+        iptables_executor = IpTablesExecutor(config)
+        iptables_executor.process()
+    else:
+        while databag_map:
+            item = databag_map.popitem(last = False)
+            item_name = item[0]
+            item_dict = item[1]
+            if process_file.count(item_name) == OCCURRENCES:
+                executor = item_dict["executor"]
+                executor.process()
+
+                if item_dict["process_iptables"]:
+                    iptables_executor = IpTablesExecutor(config)
+                    iptables_executor.process()
+
+                break
+
+    red = CsRedundant(config)
+    red.set()
 
 if __name__ == "__main__":
     main(sys.argv)
diff --git a/systemvm/patches/debian/opt/cloud/bin/merge.py b/systemvm/patches/debian/opt/cloud/bin/merge.py
index 1c9adf2..ef20d0b 100755
--- a/systemvm/patches/debian/opt/cloud/bin/merge.py
+++ b/systemvm/patches/debian/opt/cloud/bin/merge.py
@@ -18,8 +18,10 @@
 
 import json
 import os
-import time
+import uuid
 import logging
+import gzip
+import shutil
 import cs_ip
 import cs_guestnetwork
 import cs_cmdline
@@ -36,8 +38,6 @@ import cs_remoteaccessvpn
 import cs_vpnusers
 import cs_staticroutes
 
-from pprint import pprint
-
 
 class DataBag:
 
@@ -282,22 +282,26 @@ class QueueFile:
         if data is not None:
             self.data = data
             self.type = self.data["type"]
-            proc = updateDataBag(self)
+            updateDataBag(self)
             return
-        fn = self.configCache + '/' + self.fileName
+        filename = '{cache_location}/{json_file}'.format(cache_location = self.configCache,
json_file = self.fileName)
         try:
-            handle = open(fn)
-        except IOError:
-            logging.error("Could not open %s", fn)
+            handle = open(filename)
+        except IOError as exception:
+            error_message = ("Exception occurred with the following exception error '{error}'.
Could not open '{file}'. "
+                             "It seems that the file has already been moved.".format(error
= exception, file = filename))
+            logging.error(error_message)
         else:
+            logging.info("Continuing with the processing of file '{file}'".format(file =
filename))
+
             self.data = json.load(handle)
             self.type = self.data["type"]
             handle.close()
             if self.keep:
-                self.__moveFile(fn, self.configCache + "/processed")
+                self.__moveFile(filename, self.configCache + "/processed")
             else:
-                os.remove(fn)
-            proc = updateDataBag(self)
+                os.remove(filename)
+            updateDataBag(self)
 
     def setFile(self, name):
         self.fileName = name
@@ -314,8 +318,15 @@ class QueueFile:
     def __moveFile(self, origPath, path):
         if not os.path.exists(path):
             os.makedirs(path)
-        timestamp = str(int(round(time.time())))
-        os.rename(origPath, path + "/" + self.fileName + "." + timestamp)
+        originalName = os.path.basename(origPath)
+        if originalName.count(".") == 1:
+            originalName += "." + str(uuid.uuid4())
+        zipped_file_name = path + "/" + originalName + ".gz"
+        with open(origPath, 'rb') as f_in, gzip.open(zipped_file_name, 'wb') as f_out:
+            shutil.copyfileobj(f_in, f_out)
+        os.remove(origPath)
+
+        logging.debug("Processed file written to %s", zipped_file_name)
 
 
 class PrivateGatewayHack:
diff --git a/systemvm/patches/debian/opt/cloud/bin/update_config.py b/systemvm/patches/debian/opt/cloud/bin/update_config.py
index ab08e03..82e4320 100755
--- a/systemvm/patches/debian/opt/cloud/bin/update_config.py
+++ b/systemvm/patches/debian/opt/cloud/bin/update_config.py
@@ -24,8 +24,11 @@ from subprocess import PIPE, STDOUT
 import os
 import os.path
 import configure
+import glob
 import json
 
+OCCURRENCES = 1
+
 logging.basicConfig(filename='/var/log/cloud.log', level=logging.INFO, format='%(asctime)s
 %(filename)s %(funcName)s:%(lineno)d %(message)s')
 
 # first commandline argument should be the file to process
@@ -72,7 +75,8 @@ def is_guestnet_configured(guestnet_dict, keys):
         print "[WARN] update_config.py :: Reconfiguring guest network..."
         return False
 
-    file = open(jsonCmdConfigPath)
+    fn = min(glob.iglob(jsonCmdConfigPath + '*'), key=os.path.getctime)
+    file = open(fn)
     new_guestnet_dict = json.load(file)
 
     if not new_guestnet_dict['add']:
@@ -111,19 +115,13 @@ def is_guestnet_configured(guestnet_dict, keys):
 
     return exists
 
-if not (os.path.isfile(jsonCmdConfigPath) and os.access(jsonCmdConfigPath, os.R_OK)):
+filename = min(glob.iglob(jsonCmdConfigPath + '*'), key=os.path.getctime)
+if not (os.path.isfile(filename) and os.access(filename, os.R_OK)):
     print "[ERROR] update_config.py :: You are telling me to process %s, but i can't access
it" % jsonCmdConfigPath
     sys.exit(1)
 
-# If the command line json file is unprocessed process it
-# This is important or, the control interfaces will get deleted!
-if os.path.isfile(jsonPath % "cmd_line.json"):
-    qf = QueueFile()
-    qf.setFile("cmd_line.json")
-    qf.load(None)
-
 # If the guest network is already configured and have the same IP, do not try to configure
it again otherwise it will break
-if sys.argv[1] == "guest_network.json":
+if sys.argv[1] and sys.argv[1].count("guest_network.json") == OCCURRENCES:
     if os.path.isfile(currentGuestNetConfig):
         file = open(currentGuestNetConfig)
         guestnet_dict = json.load(file)

-- 
To stop receiving notification emails like this one, please contact
"commits@cloudstack.apache.org" <commits@cloudstack.apache.org>.

Mime
View raw message