cloudstack-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mlsoren...@apache.org
Subject [14/50] [abbrv] git commit: updated refs/heads/resize-root to 0eb9967
Date Sat, 08 Mar 2014 00:22:47 GMT
CLOUDSTACK-6203: KVM Migration fixes. Moved migration to a thread
so we can monitor it and potentially take action to make migration
complete if admin has defined such.


Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/e5449e29
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/e5449e29
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/e5449e29

Branch: refs/heads/resize-root
Commit: e5449e29c9bdc03c93fe8eb361f32878c9a1b980
Parents: 63e3eea
Author: Marcus Sorensen <marcus@betterservers.com>
Authored: Wed Mar 5 12:24:04 2014 -0700
Committer: Marcus Sorensen <marcus@betterservers.com>
Committed: Wed Mar 5 12:24:04 2014 -0700

----------------------------------------------------------------------
 agent/conf/agent.properties                     | 11 +++
 .../kvm/resource/LibvirtComputingResource.java  | 97 ++++++++++++++++++--
 2 files changed, 101 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e5449e29/agent/conf/agent.properties
----------------------------------------------------------------------
diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties
index a7376b6..e7e3b9c 100644
--- a/agent/conf/agent.properties
+++ b/agent/conf/agent.properties
@@ -70,6 +70,17 @@ domr.scripts.dir=scripts/network/domr/kvm
 # In MegaBytes per second
 #vm.migrate.speed=0
 
+# set target downtime at end of livemigration, the 'hiccup' for final copy. Higher numbers
+# make livemigration easier, lower numbers may cause migration to never complete. Less than
1
+# means hypervisor default (20ms).
+#vm.migrate.downtime=0
+
+# Busy VMs may never finish migrating, depending on environment. When its available, we will
+# want to add support for autoconvergence migration flag which should fix this. Set an upper
+# limit in seconds for how long live migration should wait, at which point VM is paused and
+# migration will finish quickly.  Less than 1 means disabled.
+#vm.migrate.pauseafter=0
+
 # set the type of bridge used on the hypervisor, this defines what commands the resource

 # will use to setup networking. Currently supported NATIVE, OPENVSWITCH
 #network.bridge.type=native

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e5449e29/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
----------------------------------------------------------------------
diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
index 1056bcf..3f6abfc 100755
--- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
+++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
@@ -254,11 +254,14 @@ import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
 import java.util.UUID;
+import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.TimeUnit;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -306,6 +309,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements
Serv
     private String _pod;
     private String _clusterId;
     private int _migrateSpeed;
+    private int _migrateDowntime;
+    private int _migratePauseAfter;
 
     private long _hvVersion;
     private long _kernelVersion;
@@ -889,6 +894,12 @@ public class LibvirtComputingResource extends ServerResourceBase implements
Serv
             _mountPoint = "/mnt";
         }
 
+        value = (String) params.get("vm.migrate.downtime");
+        _migrateDowntime = NumbersUtil.parseInt(value, -1);
+
+        value = (String) params.get("vm.migrate.pauseafter");
+        _migratePauseAfter = NumbersUtil.parseInt(value, -1);
+
         value = (String)params.get("vm.migrate.speed");
         _migrateSpeed = NumbersUtil.parseInt(value, -1);
         if (_migrateSpeed == -1) {
@@ -2986,7 +2997,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements
Serv
         Connect conn = null;
         String xmlDesc = null;
         try {
-            conn = LibvirtConnection.getConnectionByVmName(cmd.getVmName());
+            conn = LibvirtConnection.getConnectionByVmName(vmName);
             ifaces = getInterfaces(conn, vmName);
             disks = getDisks(conn, vmName);
             dm = conn.domainLookupByName(vmName);
@@ -3006,17 +3017,65 @@ public class LibvirtComputingResource extends ServerResourceBase implements
Serv
             xmlDesc = dm.getXMLDesc(0).replace(_privateIp, cmd.getDestinationIp());
 
             dconn = new Connect("qemu+tcp://" + cmd.getDestinationIp() + "/system");
-            /*
-             * Hard code lm flag: VIR_MIGRATE_LIVE(1<<0)
-             */
-            destDomain = dm.migrate(dconn, (1 << 0), xmlDesc, vmName, "tcp:" + cmd.getDestinationIp(),
_migrateSpeed);
 
-            for (DiskDef disk : disks) {
-                cleanupDisk(disk);
+            //run migration in thread so we can monitor it
+            s_logger.info("Live migration of instance " + vmName + " initiated");
+            ExecutorService executor = Executors.newFixedThreadPool(1);
+            Callable<Domain> worker = new MigrateKVMAsync(dm, dconn, vmName, cmd.getDestinationIp());
+            Future<Domain> migrateThread = executor.submit(worker);
+            executor.shutdown();
+            long sleeptime = 0;
+            while (!executor.isTerminated()) {
+                Thread.sleep(100);
+                sleeptime += 100;
+                if (sleeptime == 1000) { // wait 1s before attempting to set downtime on
migration, since I don't know of a VIR_DOMAIN_MIGRATING state
+                    if (_migrateDowntime > 0 ) {
+                        try {
+                            int setDowntime = dm.migrateSetMaxDowntime(_migrateDowntime);
+                            if (setDowntime == 0 ) {
+                                s_logger.debug("Set max downtime for migration of " + vmName
+ " to " + String.valueOf(_migrateDowntime) + "ms");
+                            }
+                        } catch (LibvirtException e) {
+                            s_logger.debug("Failed to set max downtime for migration, perhaps
migration completed? Error: " + e.getMessage());
+                        }
+                    }
+                }
+                if ((sleeptime % 1000) == 0) {
+                    s_logger.info("Waiting for migration of " + vmName + " to complete, waited
" + sleeptime + "ms");
+                }
+
+                // pause vm if we meet the vm.migrate.pauseafter threshold and not already
paused
+                if (_migratePauseAfter > 0 && sleeptime > _migratePauseAfter
&& dm.getInfo().state == DomainInfo.DomainState.VIR_DOMAIN_RUNNING ) {
+                    s_logger.info("Pausing VM " + vmName + " due to property vm.migrate.pauseafter
setting to " + _migratePauseAfter+ "ms to complete migration");
+                    try {
+                        dm.suspend();
+                    } catch (LibvirtException e) {
+                        // pause could be racy if it attempts to pause right when vm is finished,
simply warn
+                        s_logger.info("Failed to pause vm " + vmName + " : " + e.getMessage());
+                    }
+                }
+            }
+            s_logger.info("Migration thread for " + vmName + " is done");
+
+            destDomain = migrateThread.get(10, TimeUnit.SECONDS);
+
+            if (destDomain != null) {
+                for (DiskDef disk : disks) {
+                    cleanupDisk(disk);
+                }
             }
         } catch (LibvirtException e) {
             s_logger.debug("Can't migrate domain: " + e.getMessage());
             result = e.getMessage();
+        } catch (InterruptedException e) {
+            s_logger.debug("Interrupted while migrating domain: " + e.getMessage());
+            result = e.getMessage();
+        } catch (ExecutionException e) {
+            s_logger.debug("Failed to execute while migrating domain: " + e.getMessage());
+            result = e.getMessage();
+        } catch (TimeoutException e) {
+            s_logger.debug("Timed out while migrating domain: " + e.getMessage());
+            result = e.getMessage();
         } finally {
             try {
                 if (dm != null) {
@@ -3054,6 +3113,30 @@ public class LibvirtComputingResource extends ServerResourceBase implements
Serv
         return new MigrateAnswer(cmd, result == null, result, null);
     }
 
+    private class MigrateKVMAsync implements Callable<Domain> {
+        Domain dm = null;
+        Connect dconn = null;
+        String vmName = "";
+        String destIp = "";
+
+        MigrateKVMAsync(Domain dm, Connect dconn, String vmName, String destIp) {
+            this.dm = dm;
+            this.dconn = dconn;
+            this.vmName = vmName;
+            this.destIp = destIp;
+        }
+
+        @Override
+        public Domain call() throws LibvirtException {
+            // set compression flag for migration if libvirt version supports it
+            if (dconn.getLibVirVersion() < 1003000) {
+                return dm.migrate(dconn, (1 << 0), vmName, "tcp:" + destIp, _migrateSpeed);
+            } else {
+                return dm.migrate(dconn, (1 << 0)|(1 << 11), vmName, "tcp:" +
destIp, _migrateSpeed);
+            }
+        }
+    }
+
     private synchronized Answer execute(PrepareForMigrationCommand cmd) {
 
         VirtualMachineTO vm = cmd.getVirtualMachine();


Mime
View raw message