cloudstack-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ahu...@apache.org
Subject [36/50] [abbrv] Merged
Date Fri, 10 May 2013 23:33:19 GMT
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/30479293/services/secondary-storage/src/org/apache/cloudstack/storage/resource/SecondaryStorageDiscoverer.java
----------------------------------------------------------------------
diff --cc services/secondary-storage/src/org/apache/cloudstack/storage/resource/SecondaryStorageDiscoverer.java
index 0000000,d3af792..62b9813
mode 000000,100755..100755
--- a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/SecondaryStorageDiscoverer.java
+++ b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/SecondaryStorageDiscoverer.java
@@@ -1,0 -1,320 +1,317 @@@
+ // Licensed to the Apache Software Foundation (ASF) under one
+ // or more contributor license agreements.  See the NOTICE file
+ // distributed with this work for additional information
+ // regarding copyright ownership.  The ASF licenses this file
+ // to you under the Apache License, Version 2.0 (the
+ // "License"); you may not use this file except in compliance
+ // with the License.  You may obtain a copy of the License at
+ //
+ //   http://www.apache.org/licenses/LICENSE-2.0
+ //
+ // Unless required by applicable law or agreed to in writing,
+ // software distributed under the License is distributed on an
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ // KIND, either express or implied.  See the License for the
+ // specific language governing permissions and limitations
+ // under the License.
+ package org.apache.cloudstack.storage.resource;
+ 
+ import java.io.File;
+ import java.lang.reflect.Constructor;
+ import java.lang.reflect.InvocationTargetException;
+ import java.net.URI;
+ import java.util.Date;
+ import java.util.HashMap;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Random;
+ 
+ import javax.ejb.Local;
+ import javax.inject.Inject;
+ import javax.naming.ConfigurationException;
+ 
+ import org.apache.log4j.Logger;
+ 
+ import com.cloud.agent.AgentManager;
+ import com.cloud.configuration.dao.ConfigurationDao;
+ import com.cloud.host.HostVO;
+ import com.cloud.host.Status.Event;
+ import com.cloud.host.dao.HostDao;
+ import com.cloud.hypervisor.Hypervisor;
+ import com.cloud.resource.Discoverer;
+ import com.cloud.resource.DiscovererBase;
+ import com.cloud.resource.ServerResource;
+ import com.cloud.storage.VMTemplateVO;
+ import com.cloud.storage.VMTemplateZoneVO;
+ import com.cloud.storage.dao.VMTemplateDao;
 -import com.cloud.storage.dao.VMTemplateHostDao;
+ import com.cloud.storage.dao.VMTemplateZoneDao;
+ import com.cloud.storage.resource.DummySecondaryStorageResource;
+ import com.cloud.utils.component.ComponentContext;
+ import com.cloud.utils.net.NfsUtils;
+ import com.cloud.utils.script.Script;
+ 
+ /**
+  * SecondaryStorageDiscoverer is used to discover secondary
+  * storage servers and make sure everything it can do is
+  * correct.
+  */
+ @Local(value=Discoverer.class)
+ public class SecondaryStorageDiscoverer extends DiscovererBase implements Discoverer {
+     private static final Logger s_logger = Logger.getLogger(SecondaryStorageDiscoverer.class);
 -    
++
+     long _timeout = 2 * 60 * 1000; // 2 minutes
+     String _mountParent;
+     boolean _useServiceVM = false;
 -    
++
+     Random _random = new Random(System.currentTimeMillis());
+     @Inject
+     protected HostDao _hostDao = null;
+     @Inject
+     protected VMTemplateDao _tmpltDao = null;
+     @Inject
 -    protected VMTemplateHostDao _vmTemplateHostDao = null;
 -    @Inject
+     protected VMTemplateZoneDao _vmTemplateZoneDao = null;
+     @Inject
+     protected VMTemplateDao _vmTemplateDao = null;
+     @Inject
+     protected ConfigurationDao _configDao = null;
+     @Inject
+     protected AgentManager _agentMgr = null;
 -    
++
+     protected SecondaryStorageDiscoverer() {
+     }
 -    
++
+     @Override
+     public Map<? extends ServerResource, Map<String, String>> find(long dcId, Long podId, Long clusterId, URI uri, String username, String password, List<String> hostTags) {
+         if (!uri.getScheme().equalsIgnoreCase("nfs") && !uri.getScheme().equalsIgnoreCase("file")
+                 && !uri.getScheme().equalsIgnoreCase("iso") && !uri.getScheme().equalsIgnoreCase("dummy")) {
+             s_logger.debug("It's not NFS or file or ISO, so not a secondary storage server: " + uri.toString());
+             return null;
+         }
+ 
+         if (uri.getScheme().equalsIgnoreCase("nfs") || uri.getScheme().equalsIgnoreCase("iso")) {
+             return createNfsSecondaryStorageResource(dcId, podId, uri);
+         } else if (uri.getScheme().equalsIgnoreCase("file")) {
+             return createLocalSecondaryStorageResource(dcId, podId, uri);
+         } else if (uri.getScheme().equalsIgnoreCase("dummy")) {
+             return createDummySecondaryStorageResource(dcId, podId, uri);
+         } else {
+             return null;
+         }
+     }
 -    
++
+     protected Map<? extends ServerResource, Map<String, String>> createNfsSecondaryStorageResource(long dcId, Long podId, URI uri) {
 -        
++
+     	if (_useServiceVM) {
+     	    return createDummySecondaryStorageResource(dcId, podId, uri);
+     	}
+         String mountStr = NfsUtils.uri2Mount(uri);
 -        
++
+         Script script = new Script(true, "mount", _timeout, s_logger);
+         String mntPoint = null;
+         File file = null;
+         do {
+             mntPoint = _mountParent + File.separator + Integer.toHexString(_random.nextInt(Integer.MAX_VALUE));
+             file = new File(mntPoint);
+         } while (file.exists());
 -                
++
+         if (!file.mkdirs()) {
+             s_logger.warn("Unable to make directory: " + mntPoint);
+             return null;
+         }
 -        
++
+         script.add(mountStr, mntPoint);
+         String result = script.execute();
+         if (result != null && !result.contains("already mounted")) {
+             s_logger.warn("Unable to mount " + uri.toString() + " due to " + result);
+             file.delete();
+             return null;
+         }
 -        
++
+         script = new Script(true, "umount", 0, s_logger);
+         script.add(mntPoint);
+         script.execute();
 -        
++
+         file.delete();
 -        
++
+         Map<NfsSecondaryStorageResource, Map<String, String>> srs = new HashMap<NfsSecondaryStorageResource, Map<String, String>>();
 -        
++
+         NfsSecondaryStorageResource storage;
+         if(_configDao.isPremium()) {
+             Class<?> impl;
+             String name = "com.cloud.storage.resource.PremiumSecondaryStorageResource";
+             try {
+                 impl = Class.forName(name);
+                 final Constructor<?> constructor = impl.getDeclaredConstructor();
+                 constructor.setAccessible(true);
+                 storage = (NfsSecondaryStorageResource)constructor.newInstance();
+             } catch (final ClassNotFoundException e) {
+             	s_logger.error("Unable to load com.cloud.storage.resource.PremiumSecondaryStorageResource due to ClassNotFoundException");
+             	return null;
+             } catch (final SecurityException e) {
+             	s_logger.error("Unable to load com.cloud.storage.resource.PremiumSecondaryStorageResource due to SecurityException");
+             	return null;
+             } catch (final NoSuchMethodException e) {
+             	s_logger.error("Unable to load com.cloud.storage.resource.PremiumSecondaryStorageResource due to NoSuchMethodException");
+             	return null;
+             } catch (final IllegalArgumentException e) {
+             	s_logger.error("Unable to load com.cloud.storage.resource.PremiumSecondaryStorageResource due to IllegalArgumentException");
+             	return null;
+             } catch (final InstantiationException e) {
+             	s_logger.error("Unable to load com.cloud.storage.resource.PremiumSecondaryStorageResource due to InstantiationException");
+             	return null;
+             } catch (final IllegalAccessException e) {
+             	s_logger.error("Unable to load com.cloud.storage.resource.PremiumSecondaryStorageResource due to IllegalAccessException");
+             	return null;
+             } catch (final InvocationTargetException e) {
+             	s_logger.error("Unable to load com.cloud.storage.resource.PremiumSecondaryStorageResource due to InvocationTargetException");
+             	return null;
+             }
+         } else {
+         	storage = new NfsSecondaryStorageResource();
+         }
 -        
++
+         Map<String, String> details = new HashMap<String, String>();
+         details.put("mount.path", mountStr);
+         details.put("orig.url", uri.toString());
+         details.put("mount.parent", _mountParent);
 -        
++
+         Map<String, Object> params = new HashMap<String, Object>();
+         params.putAll(details);
+         params.put("zone", Long.toString(dcId));
+         if (podId != null) {
+             params.put("pod", podId.toString());
+         }
+         params.put("guid", uri.toString());
+         params.put("secondary.storage.vm", "false");
+         params.put("max.template.iso.size", _configDao.getValue("max.template.iso.size"));
 -        
++
+         try {
+             storage.configure("Storage", params);
+         } catch (ConfigurationException e) {
+             s_logger.warn("Unable to configure the storage ", e);
+             return null;
+         }
+         srs.put(storage, details);
 -        
++
+         return srs;
+     }
 -    
++
+     protected Map<? extends ServerResource, Map<String, String>> createLocalSecondaryStorageResource(long dcId, Long podId, URI uri) {
+         Map<LocalSecondaryStorageResource, Map<String, String>> srs = new HashMap<LocalSecondaryStorageResource, Map<String, String>>();
 -        
++
+         LocalSecondaryStorageResource storage = new LocalSecondaryStorageResource();
+         storage = ComponentContext.inject(storage);
 -        
++
+         Map<String, String> details = new HashMap<String, String>();
 -        
++
+         File file = new File(uri);
+         details.put("mount.path", file.getAbsolutePath());
+         details.put("orig.url", uri.toString());
 -        
++
+         Map<String, Object> params = new HashMap<String, Object>();
+         params.putAll(details);
+         params.put("zone", Long.toString(dcId));
+         if (podId != null) {
+             params.put("pod", podId.toString());
+         }
+         params.put("guid", uri.toString());
 -        
++
+         try {
+             storage.configure("Storage", params);
+         } catch (ConfigurationException e) {
+             s_logger.warn("Unable to configure the storage ", e);
+             return null;
+         }
+         srs.put(storage, details);
 -        
++
+         return srs;
+     }
 -    
++
+     protected Map<ServerResource, Map<String, String>> createDummySecondaryStorageResource(long dcId, Long podId, URI uri) {
+         Map<ServerResource, Map<String, String>> srs = new HashMap<ServerResource, Map<String, String>>();
 -        
++
+         DummySecondaryStorageResource storage = new DummySecondaryStorageResource(_useServiceVM);
+         storage = ComponentContext.inject(storage);
 -        
++
+         Map<String, String> details = new HashMap<String, String>();
 -        
++
+         details.put("mount.path", uri.toString());
+         details.put("orig.url", uri.toString());
 -        
++
+         Map<String, Object> params = new HashMap<String, Object>();
+         params.putAll(details);
+         params.put("zone", Long.toString(dcId));
+         if (podId != null) {
+             params.put("pod", podId.toString());
+         }
+         params.put("guid", uri.toString());
 -        
++
+         try {
+             storage.configure("Storage", params);
+         } catch (ConfigurationException e) {
+             s_logger.warn("Unable to configure the storage ", e);
+             return null;
+         }
+         srs.put(storage, details);
 -        
++
+         return srs;
+     }
 -    
++
+     @Override
+     public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
+         super.configure(name, params);
 -        
++
+         _mountParent = _params.get("mount.parent");
+         if (_mountParent == null) {
+             _mountParent = "/mnt";
+         }
 -        
++
+         String useServiceVM = _params.get("secondary.storage.vm");
+         if ("true".equalsIgnoreCase(useServiceVM)){
+         	_useServiceVM = true;
+         }
+         return true;
+     }
 -    
++
+     @Override
+ 	public boolean matchHypervisor(String hypervisor) {
+         if( hypervisor.equals("SecondaryStorage")) {
+             return true;
+         } else {
+             return false;
+         }
+     }
+ 
+     @Override
+ 	public Hypervisor.HypervisorType getHypervisorType() {
+     	return Hypervisor.HypervisorType.None;
+     }
 -    
++
+ 	@Override
+ 	public void postDiscovery(List<HostVO> hosts, long msId) {
+ 		if (_useServiceVM) {
+ 			for (HostVO h: hosts) {
+ 				_agentMgr.agentStatusTransitTo(h, Event.AgentDisconnected, msId);
+ 			}
+ 		}
+ 		for (HostVO h: hosts) {
+ 			associateTemplatesToZone(h.getId(), h.getDataCenterId());
+ 		}
 -		
++
+ 	}
 -	
++
+     private void associateTemplatesToZone(long hostId, long dcId){
+     	VMTemplateZoneVO tmpltZone;
+ 
+     	List<VMTemplateVO> allTemplates = _vmTemplateDao.listAll();
+     	for (VMTemplateVO vt: allTemplates){
+     		if (vt.isCrossZones()) {
+     			tmpltZone = _vmTemplateZoneDao.findByZoneTemplate(dcId, vt.getId());
+     			if (tmpltZone == null) {
+     				VMTemplateZoneVO vmTemplateZone = new VMTemplateZoneVO(dcId, vt.getId(), new Date());
+     				_vmTemplateZoneDao.persist(vmTemplateZone);
+     			}
+     		}
+     	}
+     }
+ }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/30479293/services/secondary-storage/src/org/apache/cloudstack/storage/template/DownloadManager.java
----------------------------------------------------------------------
diff --cc services/secondary-storage/src/org/apache/cloudstack/storage/template/DownloadManager.java
index 0000000,3e5072a..9f14d90
mode 000000,100644..100644
--- a/services/secondary-storage/src/org/apache/cloudstack/storage/template/DownloadManager.java
+++ b/services/secondary-storage/src/org/apache/cloudstack/storage/template/DownloadManager.java
@@@ -1,0 -1,105 +1,112 @@@
+ // Licensed to the Apache Software Foundation (ASF) under one
+ // or more contributor license agreements.  See the NOTICE file
+ // distributed with this work for additional information
+ // regarding copyright ownership.  The ASF licenses this file
+ // to you under the Apache License, Version 2.0 (the
+ // "License"); you may not use this file except in compliance
+ // with the License.  You may obtain a copy of the License at
+ //
+ //   http://www.apache.org/licenses/LICENSE-2.0
+ //
+ // Unless required by applicable law or agreed to in writing,
+ // software distributed under the License is distributed on an
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ // KIND, either express or implied.  See the License for the
+ // specific language governing permissions and limitations
+ // under the License.
+ package org.apache.cloudstack.storage.template;
+ 
+ import java.util.Map;
+ 
 -import org.apache.cloudstack.storage.resource.SecondaryStorageResource;
++import org.apache.cloudstack.storage.command.DownloadCommand;
++import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
+ 
+ import com.cloud.agent.api.storage.DownloadAnswer;
 -import com.cloud.agent.api.storage.DownloadCommand;
 -import com.cloud.agent.api.storage.DownloadCommand.Proxy;
 -import com.cloud.agent.api.storage.DownloadCommand.ResourceType;
++import com.cloud.agent.api.storage.Proxy;
++import com.cloud.agent.api.to.S3TO;
++import com.cloud.storage.VMTemplateHostVO;
++
++import org.apache.cloudstack.storage.resource.SecondaryStorageResource;
++
+ import com.cloud.storage.Storage.ImageFormat;
+ import com.cloud.storage.VMTemplateHostVO;
+ import com.cloud.storage.template.TemplateDownloader;
 -import com.cloud.storage.template.TemplateInfo;
++import com.cloud.storage.template.TemplateProp;
++
+ import com.cloud.utils.component.Manager;
+ 
+ public interface DownloadManager extends Manager {
+ 
+ 	/**
+ 	 * Initiate download of a public template
+ 	 * @param id unique id.
+ 	 * @param url  the url from where to download from
+      * @param hvm  whether the template is a hardware virtual machine
+      * @param accountId the accountId of the iso owner (null if public iso)
+ 	 * @param descr	description of the template
+ 	 * @param user username used for authentication to the server
+ 	 * @param password password used for authentication to the server
+ 	 * @param maxDownloadSizeInBytes (optional) max download size for the template, in bytes.
 -	 * @param resourceType signifying the type of resource like template, volume etc. 
++	 * @param resourceType signifying the type of resource like template, volume etc.
+ 	 * @return job-id that can be used to interrogate the status of the download.
+ 	 */
 -	public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, String userName, String passwd, long maxDownloadSizeInBytes, Proxy proxy, ResourceType resourceType);
 -	
 -	
++	public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, String templatePath, String userName, String passwd, long maxDownloadSizeInBytes, Proxy proxy, ResourceType resourceType);
++
++    public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType);
++
++
+ 	/**
+ 	 * Get the status of a download job
+ 	 * @param jobId job Id
+ 	 * @return status of the download job
+ 	 */
+ 	public TemplateDownloader.Status getDownloadStatus(String jobId);
 -	
++
+ 	/**
+ 	 * Get the status of a download job
+ 	 * @param jobId job Id
+ 	 * @return status of the download job
+ 	 */
+ 	public VMTemplateHostVO.Status getDownloadStatus2(String jobId);
+ 
+ 	/**
+ 	 * Get the download percent of a download job
+ 	 * @param jobId job Id
+ 	 * @return
+ 	 */
+ 	public int getDownloadPct(String jobId);
+ 
+ 	/**
+ 	 * Get the download error if any
+ 	 * @param jobId job Id
+ 	 * @return
+ 	 */
+ 	public String getDownloadError(String jobId);
+ 
+ 	/**
+ 	 * Get the local path for the download
+ 	 * @param jobId job Id
+ 	 * @return
+ 	public String getDownloadLocalPath(String jobId);
+      */
 -	
++
+ 	/** Handle download commands from the management server
+ 	 * @param cmd cmd from server
+ 	 * @return answer representing status of download.
+ 	 */
+ 	public DownloadAnswer handleDownloadCommand(SecondaryStorageResource resource, DownloadCommand cmd);
 -		
++
+ 	/**
+ 	/**
+ 	 * @return list of template info for installed templates
+ 	 */
 -	public Map<String, TemplateInfo> gatherTemplateInfo(String templateDir);
 -	
++	public Map<String, TemplateProp> gatherTemplateInfo(String templateDir);
++
+ 	/**
+ 	/**
+ 	 * @return list of volume info for installed volumes
+ 	 */
 -	public Map<Long, TemplateInfo> gatherVolumeInfo(String volumeDir);
++	public Map<Long, TemplateProp> gatherVolumeInfo(String volumeDir);
+ 
+ 
+ }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/30479293/services/secondary-storage/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
----------------------------------------------------------------------
diff --cc services/secondary-storage/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
index 0000000,a9d23cb..b1cebf4
mode 000000,100755..100755
--- a/services/secondary-storage/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
+++ b/services/secondary-storage/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
@@@ -1,0 -1,1074 +1,1147 @@@
+ // Licensed to the Apache Software Foundation (ASF) under one
+ // or more contributor license agreements.  See the NOTICE file
+ // distributed with this work for additional information
+ // regarding copyright ownership.  The ASF licenses this file
+ // to you under the Apache License, Version 2.0 (the
+ // "License"); you may not use this file except in compliance
+ // with the License.  You may obtain a copy of the License at
+ //
+ //   http://www.apache.org/licenses/LICENSE-2.0
+ //
+ // Unless required by applicable law or agreed to in writing,
+ // software distributed under the License is distributed on an
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ // KIND, either express or implied.  See the License for the
+ // specific language governing permissions and limitations
+ // under the License.
+ package org.apache.cloudstack.storage.template;
+ 
+ import java.io.BufferedReader;
+ import java.io.File;
+ import java.io.FileInputStream;
+ import java.io.IOException;
+ import java.io.InputStream;
+ import java.math.BigInteger;
+ import java.net.URI;
+ import java.net.URISyntaxException;
+ import java.security.MessageDigest;
+ import java.security.NoSuchAlgorithmException;
+ import java.text.SimpleDateFormat;
+ import java.util.ArrayList;
+ import java.util.Date;
+ import java.util.HashMap;
+ import java.util.Iterator;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.UUID;
+ import java.util.concurrent.ConcurrentHashMap;
+ import java.util.concurrent.ExecutorService;
+ import java.util.concurrent.Executors;
+ 
+ import javax.ejb.Local;
+ import javax.naming.ConfigurationException;
+ 
++import org.apache.cloudstack.storage.command.DownloadCommand;
++import org.apache.cloudstack.storage.command.DownloadProgressCommand;
++import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
++import org.apache.cloudstack.storage.command.DownloadProgressCommand.RequestType;
+ import org.apache.cloudstack.storage.resource.SecondaryStorageResource;
++
+ import org.apache.log4j.Logger;
+ 
+ import com.cloud.agent.api.storage.DownloadAnswer;
 -import com.cloud.agent.api.storage.DownloadCommand;
 -import com.cloud.agent.api.storage.DownloadCommand.Proxy;
 -import com.cloud.agent.api.storage.DownloadCommand.ResourceType;
 -import com.cloud.agent.api.storage.DownloadProgressCommand;
 -import com.cloud.agent.api.storage.DownloadProgressCommand.RequestType;
++import com.cloud.agent.api.storage.Proxy;
++import com.cloud.agent.api.to.DataStoreTO;
++import com.cloud.agent.api.to.NfsTO;
++import com.cloud.agent.api.to.S3TO;
+ import com.cloud.exception.InternalErrorException;
+ import com.cloud.storage.Storage.ImageFormat;
+ import com.cloud.storage.StorageLayer;
+ import com.cloud.storage.VMTemplateHostVO;
+ import com.cloud.storage.VMTemplateStorageResourceAssoc;
+ import com.cloud.storage.template.HttpTemplateDownloader;
+ import com.cloud.storage.template.IsoProcessor;
+ import com.cloud.storage.template.LocalTemplateDownloader;
+ import com.cloud.storage.template.Processor;
+ import com.cloud.storage.template.Processor.FormatInfo;
+ import com.cloud.storage.template.QCOW2Processor;
+ import com.cloud.storage.template.RawImageProcessor;
++import com.cloud.storage.template.S3TemplateDownloader;
+ import com.cloud.storage.template.ScpTemplateDownloader;
+ import com.cloud.storage.template.TemplateConstants;
+ import com.cloud.storage.template.TemplateDownloader;
+ import com.cloud.storage.template.TemplateDownloader.DownloadCompleteCallback;
+ import com.cloud.storage.template.TemplateDownloader.Status;
 -import com.cloud.storage.template.TemplateInfo;
+ import com.cloud.storage.template.TemplateLocation;
++import com.cloud.storage.template.TemplateProp;
+ import com.cloud.storage.template.VhdProcessor;
+ import com.cloud.storage.template.VmdkProcessor;
+ import com.cloud.utils.NumbersUtil;
+ import com.cloud.utils.component.ManagerBase;
+ import com.cloud.utils.exception.CloudRuntimeException;
+ import com.cloud.utils.script.OutputInterpreter;
+ import com.cloud.utils.script.Script;
+ 
+ @Local(value = DownloadManager.class)
+ public class DownloadManagerImpl extends ManagerBase implements DownloadManager {
+     private String _name;
+     StorageLayer _storage;
+     Map<String, Processor> _processors;
+ 
+     public class Completion implements DownloadCompleteCallback {
+         private final String jobId;
+ 
+         public Completion(String jobId) {
+             this.jobId = jobId;
+         }
+ 
+         @Override
+         public void downloadComplete(Status status) {
+             setDownloadStatus(jobId, status);
+         }
+     }
+ 
+     private static class DownloadJob {
+         private final TemplateDownloader td;
+         private final String jobId;
+         private final String tmpltName;
+         private final boolean hvm;
+         private final ImageFormat format;
+         private String tmpltPath;
+         private final String description;
+         private String checksum;
+         private final Long accountId;
+         private final String installPathPrefix;
+         private long templatesize;
+         private long templatePhysicalSize;
+         private final long id;
+         private final ResourceType resourceType;
+ 
 -        public DownloadJob(TemplateDownloader td, String jobId, long id, String tmpltName, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, ResourceType resourceType) {
++        public DownloadJob(TemplateDownloader td, String jobId, long id, String tmpltName, ImageFormat format, boolean hvm, Long accountId,
++                String descr, String cksum, String installPathPrefix, ResourceType resourceType) {
+             super();
+             this.td = td;
+             this.jobId = jobId;
+             this.tmpltName = tmpltName;
+             this.format = format;
+             this.hvm = hvm;
+             this.accountId = accountId;
+             this.description = descr;
+             this.checksum = cksum;
+             this.installPathPrefix = installPathPrefix;
+             this.templatesize = 0;
+             this.id = id;
+             this.resourceType = resourceType;
+         }
+ 
+         public TemplateDownloader getTd() {
+             return td;
+         }
+ 
+         public String getDescription() {
+             return description;
+         }
+ 
+         public String getChecksum() {
+             return checksum;
+         }
+ 
+         public TemplateDownloader getTemplateDownloader() {
+             return td;
+         }
+ 
+         public String getJobId() {
+             return jobId;
+         }
+ 
+         public String getTmpltName() {
+             return tmpltName;
+         }
+ 
+         public ImageFormat getFormat() {
+             return format;
+         }
+ 
+         public boolean isHvm() {
+             return hvm;
+         }
+ 
+         public Long getAccountId() {
+             return accountId;
+         }
+ 
+         public long getId() {
+             return id;
+         }
+ 
+         public ResourceType getResourceType() {
+             return resourceType;
+         }
+ 
+         public void setTmpltPath(String tmpltPath) {
+             this.tmpltPath = tmpltPath;
+         }
+ 
+         public String getTmpltPath() {
+             return tmpltPath;
+         }
+ 
+         public String getInstallPathPrefix() {
+             return installPathPrefix;
+         }
+ 
+         public void cleanup() {
+             if (td != null) {
+                 String dnldPath = td.getDownloadLocalPath();
+                 if (dnldPath != null) {
+                     File f = new File(dnldPath);
+                     File dir = f.getParentFile();
+                     f.delete();
+                     if (dir != null) {
+                         dir.delete();
+                     }
+                 }
+             }
+ 
+         }
+ 
+         public void setTemplatesize(long templatesize) {
+             this.templatesize = templatesize;
+         }
+ 
+         public long getTemplatesize() {
+             return templatesize;
+         }
+ 
+         public void setTemplatePhysicalSize(long templatePhysicalSize) {
+             this.templatePhysicalSize = templatePhysicalSize;
+         }
+ 
+         public long getTemplatePhysicalSize() {
+             return templatePhysicalSize;
+         }
+ 
+         public void setCheckSum(String checksum) {
+             this.checksum = checksum;
+         }
+     }
+ 
+     public static final Logger s_logger = Logger.getLogger(DownloadManagerImpl.class);
+     private String _templateDir;
+     private String _volumeDir;
+     private String createTmpltScr;
+     private String createVolScr;
+     private List<Processor> processors;
+ 
+     private ExecutorService threadPool;
+ 
+     private final Map<String, DownloadJob> jobs = new ConcurrentHashMap<String, DownloadJob>();
+     private String listTmpltScr;
+     private String listVolScr;
+     private int installTimeoutPerGig = 180 * 60 * 1000;
+     private boolean _sslCopy;
+ 
++
++    public void setThreadPool(ExecutorService threadPool) {
++        this.threadPool = threadPool;
++    }
++
++    public void setStorageLayer(StorageLayer storage){
++        this._storage = storage;
++    }
++
+     /**
 -     * Get notified of change of job status. Executed in context of downloader thread
 -     * 
++     * Get notified of change of job status. Executed in context of downloader
++     * thread
++     *
+      * @param jobId
+      *            the id of the job
+      * @param status
+      *            the status of the job
+      */
+     public void setDownloadStatus(String jobId, Status status) {
+         DownloadJob dj = jobs.get(jobId);
+         if (dj == null) {
+             s_logger.warn("setDownloadStatus for jobId: " + jobId + ", status=" + status + " no job found");
+             return;
+         }
+         TemplateDownloader td = dj.getTemplateDownloader();
+         s_logger.info("Download Completion for jobId: " + jobId + ", status=" + status);
 -        s_logger.info("local: " + td.getDownloadLocalPath() + ", bytes=" + td.getDownloadedBytes() + ", error=" + td.getDownloadError() + ", pct=" + td.getDownloadPercent());
++        s_logger.info("local: " + td.getDownloadLocalPath() + ", bytes=" + td.getDownloadedBytes() + ", error=" + td.getDownloadError() + ", pct="
++                + td.getDownloadPercent());
+ 
+         switch (status) {
+         case ABORTED:
+         case NOT_STARTED:
+         case UNRECOVERABLE_ERROR:
+             // TODO
+             dj.cleanup();
+             break;
+         case UNKNOWN:
+             return;
+         case IN_PROGRESS:
+             s_logger.info("Resuming jobId: " + jobId + ", status=" + status);
+             td.setResume(true);
+             threadPool.execute(td);
+             break;
+         case RECOVERABLE_ERROR:
+             threadPool.execute(td);
+             break;
+         case DOWNLOAD_FINISHED:
 -            td.setDownloadError("Download success, starting install ");
 -            String result = postDownload(jobId);
 -            if (result != null) {
 -                s_logger.error("Failed post download script: " + result);
 -                td.setStatus(Status.UNRECOVERABLE_ERROR);
 -                td.setDownloadError("Failed post download script: " + result);
 -            } else {
++            if (!(td instanceof S3TemplateDownloader)) {
++                // we currently only create template.properties for NFS by
++                // running some post download script
++                td.setDownloadError("Download success, starting install ");
++                String result = postDownload(jobId);
++                if (result != null) {
++                    s_logger.error("Failed post download script: " + result);
++                    td.setStatus(Status.UNRECOVERABLE_ERROR);
++                    td.setDownloadError("Failed post download script: " + result);
++                } else {
++                    td.setStatus(Status.POST_DOWNLOAD_FINISHED);
++                    td.setDownloadError("Install completed successfully at " + new SimpleDateFormat().format(new Date()));
++                }
++            } else{
++                // for s3 and swift, we skip post download step and just set status to trigger callback.
+                 td.setStatus(Status.POST_DOWNLOAD_FINISHED);
 -                td.setDownloadError("Install completed successfully at " + new SimpleDateFormat().format(new Date()));
+             }
+             dj.cleanup();
+             break;
+         default:
+             break;
+         }
+     }
+ 
+     private String computeCheckSum(File f) {
+         byte[] buffer = new byte[8192];
+         int read = 0;
+         MessageDigest digest;
 -        String checksum = null;        
++        String checksum = null;
+         InputStream is = null;
+         try {
 -            digest = MessageDigest.getInstance("MD5");           
 -            is = new FileInputStream(f);     
 -            while( (read = is.read(buffer)) > 0) {
++            digest = MessageDigest.getInstance("MD5");
++            is = new FileInputStream(f);
++            while ((read = is.read(buffer)) > 0) {
+                 digest.update(buffer, 0, read);
 -            }       
++            }
+             byte[] md5sum = digest.digest();
+             BigInteger bigInt = new BigInteger(1, md5sum);
 -            checksum = String.format("%032x",bigInt);
++            checksum = String.format("%032x", bigInt);
+             return checksum;
 -        }catch(IOException e) {
++        } catch (IOException e) {
+             return null;
 -        }catch (NoSuchAlgorithmException e) {         
++        } catch (NoSuchAlgorithmException e) {
+             return null;
 -        }
 -        finally {
++        } finally {
+             try {
 -                if(is != null)
++                if (is != null)
+                     is.close();
+             } catch (IOException e) {
+                 return null;
 -            }                        
++            }
+         }
+     }
+ 
+     /**
 -     * Post download activity (install and cleanup). Executed in context of downloader thread
 -     * 
++     * Post download activity (install and cleanup). Executed in context of
++     * downloader thread
++     *
+      * @throws IOException
+      */
+     private String postDownload(String jobId) {
+         DownloadJob dnld = jobs.get(jobId);
+         TemplateDownloader td = dnld.getTemplateDownloader();
 -        String resourcePath = null;               
++        String resourcePath = dnld.getInstallPathPrefix(); // path with mount directory
++        String finalResourcePath = dnld.getTmpltPath(); // template download path on secondary storage
+         ResourceType resourceType = dnld.getResourceType();
+ 
 -        // once template path is set, remove the parent dir so that the template is installed with a relative path
++        /*
++        // once template path is set, remove the parent dir so that the template
++        // is installed with a relative path
+         String finalResourcePath = "";
 -        if (resourceType == ResourceType.TEMPLATE){
++        if (resourceType == ResourceType.TEMPLATE) {
+             finalResourcePath += _templateDir + File.separator + dnld.getAccountId() + File.separator + dnld.getId() + File.separator;
+             resourcePath = dnld.getInstallPathPrefix() + dnld.getAccountId() + File.separator + dnld.getId() + File.separator;// dnld.getTmpltName();
 -        }else {
++        } else {
+             finalResourcePath += _volumeDir + File.separator + dnld.getId() + File.separator;
+             resourcePath = dnld.getInstallPathPrefix() + dnld.getId() + File.separator;// dnld.getTmpltName();
+         }
+ 
+         _storage.mkdirs(resourcePath);
+         dnld.setTmpltPath(finalResourcePath);
++        */
+ 
+         File originalTemplate = new File(td.getDownloadLocalPath());
+         String checkSum = computeCheckSum(originalTemplate);
+         if (checkSum == null) {
+             s_logger.warn("Something wrong happened when try to calculate the checksum of downloaded template!");
+         }
+         dnld.setCheckSum(checkSum);
+ 
+         int imgSizeGigs = (int) Math.ceil(_storage.getSize(td.getDownloadLocalPath()) * 1.0d / (1024 * 1024 * 1024));
+         imgSizeGigs++; // add one just in case
+         long timeout = imgSizeGigs * installTimeoutPerGig;
+         Script scr = null;
+         String script = resourceType == ResourceType.TEMPLATE ? createTmpltScr : createVolScr;
+         scr = new Script(script, timeout, s_logger);
+         scr.add("-s", Integer.toString(imgSizeGigs));
+         scr.add("-S", Long.toString(td.getMaxTemplateSizeInBytes()));
+         if (dnld.getDescription() != null && dnld.getDescription().length() > 1) {
+             scr.add("-d", dnld.getDescription());
+         }
+         if (dnld.isHvm()) {
+             scr.add("-h");
+         }
+ 
+         // add options common to ISO and template
+         String extension = dnld.getFormat().getFileExtension();
+         String templateName = "";
 -        if( extension.equals("iso")) {
++        if (extension.equals("iso")) {
+             templateName = jobs.get(jobId).getTmpltName().trim().replace(" ", "_");
+         } else {
+             templateName = java.util.UUID.nameUUIDFromBytes((jobs.get(jobId).getTmpltName() + System.currentTimeMillis()).getBytes()).toString();
+         }
+ 
++        // run script to mv the temporary template file to the final template file
+         String templateFilename = templateName + "." + extension;
+         dnld.setTmpltPath(finalResourcePath + "/" + templateFilename);
+         scr.add("-n", templateFilename);
+ 
+         scr.add("-t", resourcePath);
 -        scr.add("-f", td.getDownloadLocalPath());
++        scr.add("-f", td.getDownloadLocalPath()); // this is the temporary template file downloaded
+         if (dnld.getChecksum() != null && dnld.getChecksum().length() > 1) {
+             scr.add("-c", dnld.getChecksum());
+         }
+         scr.add("-u"); // cleanup
+         String result;
+         result = scr.execute();
+ 
+         if (result != null) {
+             return result;
+         }
+ 
+         // Set permissions for the downloaded template
+         File downloadedTemplate = new File(resourcePath + "/" + templateFilename);
+         _storage.setWorldReadableAndWriteable(downloadedTemplate);
+ 
+         // Set permissions for template/volume.properties
+         String propertiesFile = resourcePath;
 -        if (resourceType == ResourceType.TEMPLATE){
++        if (resourceType == ResourceType.TEMPLATE) {
+             propertiesFile += "/template.properties";
 -        }else{
++        } else {
+             propertiesFile += "/volume.properties";
+         }
+         File templateProperties = new File(propertiesFile);
+         _storage.setWorldReadableAndWriteable(templateProperties);
+ 
+         TemplateLocation loc = new TemplateLocation(_storage, resourcePath);
+         try {
+             loc.create(dnld.getId(), true, dnld.getTmpltName());
+         } catch (IOException e) {
+             s_logger.warn("Something is wrong with template location " + resourcePath, e);
+             loc.purge();
+             return "Unable to download due to " + e.getMessage();
+         }
+ 
+         Iterator<Processor> en = _processors.values().iterator();
+         while (en.hasNext()) {
+             Processor processor = en.next();
+ 
+             FormatInfo info = null;
+             try {
+                 info = processor.process(resourcePath, null, templateName);
+             } catch (InternalErrorException e) {
+                 s_logger.error("Template process exception ", e);
+                 return e.toString();
+             }
+             if (info != null) {
+                 loc.addFormat(info);
+                 dnld.setTemplatesize(info.virtualSize);
+                 dnld.setTemplatePhysicalSize(info.size);
+                 break;
+             }
+         }
+ 
+         if (!loc.save()) {
+             s_logger.warn("Cleaning up because we're unable to save the formats");
+             loc.purge();
+         }
+ 
+         return null;
+     }
+ 
+     @Override
+     public Status getDownloadStatus(String jobId) {
+         DownloadJob job = jobs.get(jobId);
+         if (job != null) {
+             TemplateDownloader td = job.getTemplateDownloader();
+             if (td != null) {
+                 return td.getStatus();
+             }
+         }
+         return Status.UNKNOWN;
+     }
+ 
+     @Override
 -    public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) {
++    public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr,
++            String cksum, String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) {
+         UUID uuid = UUID.randomUUID();
+         String jobId = uuid.toString();
 -        String tmpDir = "";
 -        if(resourceType == ResourceType.TEMPLATE){
 -            tmpDir = installPathPrefix + File.separator + accountId + File.separator + id;
 -        }else {
 -            tmpDir = installPathPrefix + File.separator + id;
++
++        URI uri;
++        try {
++            uri = new URI(url);
++        } catch (URISyntaxException e) {
++            throw new CloudRuntimeException("URI is incorrect: " + url);
++        }
++        TemplateDownloader td;
++        if ((uri != null) && (uri.getScheme() != null)) {
++            if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) {
++                td = new S3TemplateDownloader(s3, url, installPathPrefix, new Completion(jobId), maxTemplateSizeInBytes, user, password, proxy,
++                        resourceType);
++            } else {
++                throw new CloudRuntimeException("Scheme is not supported " + url);
++            }
++        } else {
++            throw new CloudRuntimeException("Unable to download from URL: " + url);
+         }
++        DownloadJob dj = new DownloadJob(td, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix, resourceType);
++        dj.setTmpltPath(installPathPrefix);
++        jobs.put(jobId, dj);
++        threadPool.execute(td);
++
++        return jobId;
++    }
++
++    @Override
++    public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr,
++            String cksum, String installPathPrefix, String templatePath,  String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) {
++        UUID uuid = UUID.randomUUID();
++        String jobId = uuid.toString();
++        String tmpDir = installPathPrefix;
+ 
+         try {
+ 
+             if (!_storage.mkdirs(tmpDir)) {
+                 s_logger.warn("Unable to create " + tmpDir);
+                 return "Unable to create " + tmpDir;
+             }
 -            //	TO DO - define constant for volume properties.
 -            File file = ResourceType.TEMPLATE == resourceType ? _storage.getFile(tmpDir + File.separator + TemplateLocation.Filename) : 
 -                _storage.getFile(tmpDir + File.separator + "volume.properties");
 -            if ( file.exists() ) {
++            // TO DO - define constant for volume properties.
++            File file = ResourceType.TEMPLATE == resourceType ? _storage.getFile(tmpDir + File.separator + TemplateLocation.Filename) : _storage
++                    .getFile(tmpDir + File.separator + "volume.properties");
++            if (file.exists()) {
+                 file.delete();
+             }
+ 
+             if (!file.createNewFile()) {
+                 s_logger.warn("Unable to create new file: " + file.getAbsolutePath());
+                 return "Unable to create new file: " + file.getAbsolutePath();
+             }
+ 
+             URI uri;
+             try {
+                 uri = new URI(url);
+             } catch (URISyntaxException e) {
+                 throw new CloudRuntimeException("URI is incorrect: " + url);
+             }
+             TemplateDownloader td;
+             if ((uri != null) && (uri.getScheme() != null)) {
+                 if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) {
 -                    td = new HttpTemplateDownloader(_storage, url, tmpDir, new Completion(jobId), maxTemplateSizeInBytes, user, password, proxy, resourceType);
++                    td = new HttpTemplateDownloader(_storage, url, tmpDir, new Completion(jobId), maxTemplateSizeInBytes, user, password, proxy,
++                            resourceType);
+                 } else if (uri.getScheme().equalsIgnoreCase("file")) {
+                     td = new LocalTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes, new Completion(jobId));
+                 } else if (uri.getScheme().equalsIgnoreCase("scp")) {
+                     td = new ScpTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes, new Completion(jobId));
+                 } else if (uri.getScheme().equalsIgnoreCase("nfs")) {
+                     td = null;
+                     // TODO: implement this.
+                     throw new CloudRuntimeException("Scheme is not supported " + url);
+                 } else {
+                     throw new CloudRuntimeException("Scheme is not supported " + url);
+                 }
+             } else {
+                 throw new CloudRuntimeException("Unable to download from URL: " + url);
+             }
++            // NOTE the difference between installPathPrefix and templatePath here. instalPathPrefix is the absolute path for template including mount directory
++            // on ssvm, while templatePath is the final relative path on secondary storage.
+             DownloadJob dj = new DownloadJob(td, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix, resourceType);
++            dj.setTmpltPath(templatePath);
+             jobs.put(jobId, dj);
+             threadPool.execute(td);
+ 
+             return jobId;
+         } catch (IOException e) {
+             s_logger.warn("Unable to download to " + tmpDir, e);
+             return null;
+         }
+     }
+ 
+     @Override
+     public String getDownloadError(String jobId) {
+         DownloadJob dj = jobs.get(jobId);
+         if (dj != null) {
+             return dj.getTemplateDownloader().getDownloadError();
+         }
+         return null;
+     }
+ 
+     public long getDownloadTemplateSize(String jobId) {
+         DownloadJob dj = jobs.get(jobId);
+         if (dj != null) {
+             return dj.getTemplatesize();
+         }
+         return 0;
+     }
+ 
+     public String getDownloadCheckSum(String jobId) {
+         DownloadJob dj = jobs.get(jobId);
+         if (dj != null) {
+             return dj.getChecksum();
+         }
+         return null;
+     }
+ 
+     public long getDownloadTemplatePhysicalSize(String jobId) {
+         DownloadJob dj = jobs.get(jobId);
+         if (dj != null) {
+             return dj.getTemplatePhysicalSize();
+         }
+         return 0;
+     }
+ 
+     // @Override
+     public String getDownloadLocalPath(String jobId) {
+         DownloadJob dj = jobs.get(jobId);
+         if (dj != null) {
+             return dj.getTemplateDownloader().getDownloadLocalPath();
+         }
+         return null;
+     }
+ 
+     @Override
+     public int getDownloadPct(String jobId) {
+         DownloadJob dj = jobs.get(jobId);
+         if (dj != null) {
+             return dj.getTemplateDownloader().getDownloadPercent();
+         }
+         return 0;
+     }
+ 
+     public static VMTemplateHostVO.Status convertStatus(Status tds) {
+         switch (tds) {
+         case ABORTED:
+             return VMTemplateHostVO.Status.NOT_DOWNLOADED;
+         case DOWNLOAD_FINISHED:
+             return VMTemplateHostVO.Status.DOWNLOAD_IN_PROGRESS;
+         case IN_PROGRESS:
+             return VMTemplateHostVO.Status.DOWNLOAD_IN_PROGRESS;
+         case NOT_STARTED:
+             return VMTemplateHostVO.Status.NOT_DOWNLOADED;
+         case RECOVERABLE_ERROR:
+             return VMTemplateHostVO.Status.NOT_DOWNLOADED;
+         case UNKNOWN:
+             return VMTemplateHostVO.Status.UNKNOWN;
+         case UNRECOVERABLE_ERROR:
+             return VMTemplateHostVO.Status.DOWNLOAD_ERROR;
+         case POST_DOWNLOAD_FINISHED:
+             return VMTemplateHostVO.Status.DOWNLOADED;
+         default:
+             return VMTemplateHostVO.Status.UNKNOWN;
+         }
+     }
+ 
+     @Override
+     public com.cloud.storage.VMTemplateHostVO.Status getDownloadStatus2(String jobId) {
+         return convertStatus(getDownloadStatus(jobId));
+     }
+ 
+     @Override
+     public DownloadAnswer handleDownloadCommand(SecondaryStorageResource resource, DownloadCommand cmd) {
+         ResourceType resourceType = cmd.getResourceType();
+         if (cmd instanceof DownloadProgressCommand) {
 -            return handleDownloadProgressCmd( resource, (DownloadProgressCommand) cmd);
++            return handleDownloadProgressCmd(resource, (DownloadProgressCommand) cmd);
+         }
+ 
+         if (cmd.getUrl() == null) {
 -            return new DownloadAnswer(resourceType.toString() + " is corrupted on storage due to an invalid url , cannot download", VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR);
++            return new DownloadAnswer(resourceType.toString() + " is corrupted on storage due to an invalid url , cannot download",
++                    VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR);
+         }
+ 
+         if (cmd.getName() == null) {
+             return new DownloadAnswer("Invalid Name", VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR);
+         }
+ 
 -        String installPathPrefix = null;
 -        if (ResourceType.TEMPLATE == resourceType){
 -            installPathPrefix = resource.getRootDir(cmd) + File.separator + _templateDir;
 -        }else {
 -            installPathPrefix = resource.getRootDir(cmd) + File.separator + _volumeDir;
++        DataStoreTO dstore = cmd.getDataStore();
++        String installPathPrefix = cmd.getInstallPath();
++        // for NFS, we need to get mounted path
++        if (dstore instanceof NfsTO) {
++            if (ResourceType.TEMPLATE == resourceType) {
++                installPathPrefix = resource.getRootDir(cmd) + File.separator + installPathPrefix;
++            } else {
++                installPathPrefix = resource.getRootDir(cmd) + File.separator + installPathPrefix;
++            }
++        } else if (dstore instanceof S3TO ){
++            // S3 key has template name inside to help template sync
++            installPathPrefix = installPathPrefix + File.separator + cmd.getName();
+         }
 -
+         String user = null;
+         String password = null;
+         if (cmd.getAuth() != null) {
+             user = cmd.getAuth().getUserName();
+             password = new String(cmd.getAuth().getPassword());
+         }
 -        //TO DO - Define Volume max size as well
 -        long maxDownloadSizeInBytes = (cmd.getMaxDownloadSizeInBytes() == null) ? TemplateDownloader.DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES : (cmd.getMaxDownloadSizeInBytes());
 -        String jobId = downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(), installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType);
++        // TO DO - Define Volume max size as well
++        long maxDownloadSizeInBytes = (cmd.getMaxDownloadSizeInBytes() == null) ? TemplateDownloader.DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES : (cmd
++                .getMaxDownloadSizeInBytes());
++        String jobId = null;
++        if (dstore instanceof S3TO) {
++            jobId = downloadS3Template((S3TO) dstore, cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(),
++                    cmd.getDescription(), cmd.getChecksum(), installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType);
++        } else {
++            jobId = downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(),
++                    cmd.getDescription(), cmd.getChecksum(), installPathPrefix, cmd.getInstallPath(), user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType);
++        }
+         sleep();
+         if (jobId == null) {
+             return new DownloadAnswer("Internal Error", VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR);
+         }
 -        return new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId), getDownloadStatus2(jobId), getDownloadLocalPath(jobId), getInstallPath(jobId),
 -                getDownloadTemplateSize(jobId), getDownloadTemplateSize(jobId), getDownloadCheckSum(jobId));
++        return new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId), getDownloadStatus2(jobId), getDownloadLocalPath(jobId),
++                getInstallPath(jobId), getDownloadTemplateSize(jobId), getDownloadTemplateSize(jobId), getDownloadCheckSum(jobId));
+     }
+ 
+     private void sleep() {
+         try {
+             Thread.sleep(3000);
+         } catch (InterruptedException e) {
+             // ignore
+         }
+     }
+ 
+     private DownloadAnswer handleDownloadProgressCmd(SecondaryStorageResource resource, DownloadProgressCommand cmd) {
+         String jobId = cmd.getJobId();
+         DownloadAnswer answer;
+         DownloadJob dj = null;
+         if (jobId != null) {
+             dj = jobs.get(jobId);
+         }
+         if (dj == null) {
+             if (cmd.getRequest() == RequestType.GET_OR_RESTART) {
+                 DownloadCommand dcmd = new DownloadCommand(cmd);
+                 return handleDownloadCommand(resource, dcmd);
+             } else {
+                 return new DownloadAnswer("Cannot find job", VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR.UNKNOWN);
+             }
+         }
+         TemplateDownloader td = dj.getTemplateDownloader();
+         switch (cmd.getRequest()) {
+         case GET_STATUS:
+             break;
+         case ABORT:
+             td.stopDownload();
+             sleep();
+             break;
+         case RESTART:
+             td.stopDownload();
+             sleep();
+             threadPool.execute(td);
+             break;
+         case PURGE:
+             td.stopDownload();
 -            answer = new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId), getDownloadStatus2(jobId), getDownloadLocalPath(jobId),
 -                    getInstallPath(jobId), getDownloadTemplateSize(jobId), getDownloadTemplatePhysicalSize(jobId), getDownloadCheckSum(jobId));
++            answer = new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId), getDownloadStatus2(jobId),
++                    getDownloadLocalPath(jobId), getInstallPath(jobId), getDownloadTemplateSize(jobId), getDownloadTemplatePhysicalSize(jobId),
++                    getDownloadCheckSum(jobId));
+             jobs.remove(jobId);
+             return answer;
+         default:
+             break; // TODO
+         }
+         return new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId), getDownloadStatus2(jobId), getDownloadLocalPath(jobId),
+                 getInstallPath(jobId), getDownloadTemplateSize(jobId), getDownloadTemplatePhysicalSize(jobId), getDownloadCheckSum(jobId));
+     }
+ 
+     private String getInstallPath(String jobId) {
+         DownloadJob dj = jobs.get(jobId);
+         if (dj != null) {
+             return dj.getTmpltPath();
+         }
+         return null;
+     }
+ 
+     private String createTempDir(File rootDir, String name) throws IOException {
+ 
+         File f = File.createTempFile(name, "", rootDir);
+         f.delete();
+         f.mkdir();
+         _storage.setWorldReadableAndWriteable(f);
+         return f.getAbsolutePath();
+ 
+     }
+ 
 -
+     private List<String> listVolumes(String rootdir) {
+         List<String> result = new ArrayList<String>();
+ 
+         Script script = new Script(listVolScr, s_logger);
+         script.add("-r", rootdir);
+         ZfsPathParser zpp = new ZfsPathParser(rootdir);
+         script.execute(zpp);
+         result.addAll(zpp.getPaths());
+         s_logger.info("found " + zpp.getPaths().size() + " volumes" + zpp.getPaths());
+         return result;
+     }
+ 
 -
 -
+     private List<String> listTemplates(String rootdir) {
+         List<String> result = new ArrayList<String>();
+ 
+         Script script = new Script(listTmpltScr, s_logger);
+         script.add("-r", rootdir);
+         ZfsPathParser zpp = new ZfsPathParser(rootdir);
+         script.execute(zpp);
+         result.addAll(zpp.getPaths());
+         s_logger.info("found " + zpp.getPaths().size() + " templates" + zpp.getPaths());
+         return result;
+     }
+ 
+     @Override
 -    public Map<String, TemplateInfo> gatherTemplateInfo(String rootDir) {
 -        Map<String, TemplateInfo> result = new HashMap<String, TemplateInfo>();
++    public Map<String, TemplateProp> gatherTemplateInfo(String rootDir) {
++        Map<String, TemplateProp> result = new HashMap<String, TemplateProp>();
+         String templateDir = rootDir + File.separator + _templateDir;
+ 
 -        if (! _storage.exists(templateDir)) {
++        if (!_storage.exists(templateDir)) {
+             _storage.mkdirs(templateDir);
+         }
+ 
+         List<String> publicTmplts = listTemplates(templateDir);
+         for (String tmplt : publicTmplts) {
+             String path = tmplt.substring(0, tmplt.lastIndexOf(File.separator));
+             TemplateLocation loc = new TemplateLocation(_storage, path);
+             try {
+                 if (!loc.load()) {
+                     s_logger.warn("Post download installation was not completed for " + path);
 -                    //loc.purge();
++                    // loc.purge();
+                     _storage.cleanup(path, templateDir);
+                     continue;
+                 }
+             } catch (IOException e) {
+                 s_logger.warn("Unable to load template location " + path, e);
+                 continue;
+             }
+ 
 -            TemplateInfo tInfo = loc.getTemplateInfo();
++            TemplateProp tInfo = loc.getTemplateInfo();
+ 
+ 			if ((tInfo.getSize() == tInfo.getPhysicalSize())
+ 					&& (tInfo.getInstallPath().endsWith(ImageFormat.OVA.getFileExtension()))) {
+                 try {
+                     Processor processor = _processors.get("VMDK Processor");
 -                    VmdkProcessor vmdkProcessor = (VmdkProcessor)processor;
 -					long vSize =
 -							vmdkProcessor.getTemplateVirtualSize(
 -									path,
 -									tInfo.getInstallPath().substring(
 -											tInfo.getInstallPath().lastIndexOf(File.separator) + 1));
 -					tInfo.setSize(vSize);
++                    VmdkProcessor vmdkProcessor = (VmdkProcessor) processor;
++                    long vSize =
++                            vmdkProcessor.getTemplateVirtualSize(
++                                    path,
++                                    tInfo.getInstallPath().substring(
++                                            tInfo.getInstallPath().lastIndexOf(File.separator) + 1));
++                    tInfo.setSize(vSize);
+                     loc.updateVirtualSize(vSize);
+                     loc.save();
+                 } catch (Exception e) {
+ 					s_logger.error("Unable to get the virtual size of the template: " + tInfo.getInstallPath()
+ 							+ " due to " + e.getMessage());
+                 }
+             }
+ 
+ 			result.put(tInfo.getTemplateName(), tInfo);
+ 			s_logger.debug("Added template name: " + tInfo.getTemplateName() + ", path: " + tmplt);
+         }
+         /*
+         for (String tmplt : isoTmplts) {
+             String tmp[];
+             tmp = tmplt.split("/");
+             String tmpltName = tmp[tmp.length - 2];
+             tmplt = tmplt.substring(tmplt.lastIndexOf("iso/"));
+             TemplateInfo tInfo = new TemplateInfo(tmpltName, tmplt, false);
+             s_logger.debug("Added iso template name: " + tmpltName + ", path: " + tmplt);
+             result.put(tmpltName, tInfo);
+         }
+          */
+         return result;
+     }
+ 
+     @Override
 -    public Map<Long, TemplateInfo> gatherVolumeInfo(String rootDir) {	
 -        Map<Long, TemplateInfo> result = new HashMap<Long, TemplateInfo>();
++    public Map<Long, TemplateProp> gatherVolumeInfo(String rootDir) {
++        Map<Long, TemplateProp> result = new HashMap<Long, TemplateProp>();
+         String volumeDir = rootDir + File.separator + _volumeDir;
+ 
 -        if (! _storage.exists(volumeDir)) {
++        if (!_storage.exists(volumeDir)) {
+             _storage.mkdirs(volumeDir);
+         }
+ 
+         List<String> vols = listVolumes(volumeDir);
+         for (String vol : vols) {
+             String path = vol.substring(0, vol.lastIndexOf(File.separator));
+             TemplateLocation loc = new TemplateLocation(_storage, path);
+             try {
+                 if (!loc.load()) {
+                     s_logger.warn("Post download installation was not completed for " + path);
 -                    //loc.purge();
++                    // loc.purge();
+                     _storage.cleanup(path, volumeDir);
+                     continue;
+                 }
+             } catch (IOException e) {
+                 s_logger.warn("Unable to load volume location " + path, e);
+                 continue;
+             }
+ 
 -            TemplateInfo vInfo = loc.getTemplateInfo();
++            TemplateProp vInfo = loc.getTemplateInfo();
+ 
+ 			if ((vInfo.getSize() == vInfo.getPhysicalSize())
+ 					&& (vInfo.getInstallPath().endsWith(ImageFormat.OVA.getFileExtension()))) {
+                 try {
+                     Processor processor = _processors.get("VMDK Processor");
+                     VmdkProcessor vmdkProcessor = (VmdkProcessor)processor;
+ 					long vSize =
+ 							vmdkProcessor.getTemplateVirtualSize(
+ 									path,
+ 									vInfo.getInstallPath().substring(
+ 											vInfo.getInstallPath().lastIndexOf(File.separator) + 1));
+ 					vInfo.setSize(vSize);
+                     loc.updateVirtualSize(vSize);
+                     loc.save();
+                 } catch (Exception e) {
+ 					s_logger.error("Unable to get the virtual size of the volume: " + vInfo.getInstallPath()
+ 							+ " due to " + e.getMessage());
+                 }
+             }
+ 
+             result.put(vInfo.getId(), vInfo);
+ 			s_logger.debug("Added volume name: " + vInfo.getTemplateName() + ", path: " + vol);
+         }
+         return result;
+     }
+ 
+     private int deleteDownloadDirectories(File downloadPath, int deleted) {
+         try {
+             if (downloadPath.exists()) {
+                 File[] files = downloadPath.listFiles();
+                 for (int i = 0; i < files.length; i++) {
+                     if (files[i].isDirectory()) {
+                         deleteDownloadDirectories(files[i], deleted);
+                         files[i].delete();
+                         deleted++;
+                     } else {
+                         files[i].delete();
+                         deleted++;
+                     }
+                 }
+             }
+         } catch (Exception ex) {
+             s_logger.info("Failed to clean up template downloads directory " + ex.toString());
+         }
+         return deleted;
+     }
+ 
+     public static class ZfsPathParser extends OutputInterpreter {
+         String _parent;
+         List<String> paths = new ArrayList<String>();
+ 
+         public ZfsPathParser(String parent) {
+             _parent = parent;
+         }
+ 
+         @Override
+         public String interpret(BufferedReader reader) throws IOException {
+             String line = null;
+             while ((line = reader.readLine()) != null) {
+                 paths.add(line);
+             }
+             return null;
+         }
+ 
+         public List<String> getPaths() {
+             return paths;
+         }
+ 
+         @Override
+         public boolean drain() {
+             return true;
+         }
+     }
+ 
+     public DownloadManagerImpl() {
+     }
+ 
+     @Override
+     @SuppressWarnings("unchecked")
+     public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
+         _name = name;
+ 
+         String value = null;
+ 
 -        _storage = (StorageLayer)params.get(StorageLayer.InstanceConfigKey);
++        _storage = (StorageLayer) params.get(StorageLayer.InstanceConfigKey);
+         if (_storage == null) {
+             value = (String) params.get(StorageLayer.ClassConfigKey);
+             if (value == null) {
+                 throw new ConfigurationException("Unable to find the storage layer");
+             }
+ 
+             Class<StorageLayer> clazz;
+             try {
+                 clazz = (Class<StorageLayer>) Class.forName(value);
+                 _storage = clazz.newInstance();
+             } catch (ClassNotFoundException e) {
+                 throw new ConfigurationException("Unable to instantiate " + value);
+             } catch (InstantiationException e) {
+                 throw new ConfigurationException("Unable to instantiate " + value);
+             } catch (IllegalAccessException e) {
+                 throw new ConfigurationException("Unable to instantiate " + value);
+             }
+         }
 -        String useSsl = (String)params.get("sslcopy");
++        String useSsl = (String) params.get("sslcopy");
+         if (useSsl != null) {
+             _sslCopy = Boolean.parseBoolean(useSsl);
+ 
+         }
 -        String inSystemVM = (String)params.get("secondary.storage.vm");
++        String inSystemVM = (String) params.get("secondary.storage.vm");
+         if (inSystemVM != null && "true".equalsIgnoreCase(inSystemVM)) {
+             s_logger.info("DownloadManager: starting additional services since we are inside system vm");
+             startAdditionalServices();
+             blockOutgoingOnPrivate();
+         }
+ 
+         value = (String) params.get("install.timeout.pergig");
+         this.installTimeoutPerGig = NumbersUtil.parseInt(value, 15 * 60) * 1000;
+ 
+         value = (String) params.get("install.numthreads");
+         final int numInstallThreads = NumbersUtil.parseInt(value, 10);
+ 
+         String scriptsDir = (String) params.get("template.scripts.dir");
+         if (scriptsDir == null) {
+             scriptsDir = "scripts/storage/secondary";
+         }
+ 
+         listTmpltScr = Script.findScript(scriptsDir, "listvmtmplt.sh");
+         if (listTmpltScr == null) {
+             throw new ConfigurationException("Unable to find the listvmtmplt.sh");
+         }
+         s_logger.info("listvmtmplt.sh found in " + listTmpltScr);
+ 
+         createTmpltScr = Script.findScript(scriptsDir, "createtmplt.sh");
+         if (createTmpltScr == null) {
+             throw new ConfigurationException("Unable to find createtmplt.sh");
+         }
+         s_logger.info("createtmplt.sh found in " + createTmpltScr);
+ 
+         listVolScr = Script.findScript(scriptsDir, "listvolume.sh");
+         if (listVolScr == null) {
+             throw new ConfigurationException("Unable to find the listvolume.sh");
+         }
+         s_logger.info("listvolume.sh found in " + listVolScr);
+ 
+         createVolScr = Script.findScript(scriptsDir, "createvolume.sh");
+         if (createVolScr == null) {
+             throw new ConfigurationException("Unable to find createvolume.sh");
+         }
+         s_logger.info("createvolume.sh found in " + createVolScr);
+ 
+         _processors = new HashMap<String, Processor>();
+ 
+         Processor processor = new VhdProcessor();
+         processor.configure("VHD Processor", params);
+         _processors.put("VHD Processor", processor);
+ 
+         processor = new IsoProcessor();
+         processor.configure("ISO Processor", params);
+         _processors.put("ISO Processor", processor);
+ 
+         processor = new QCOW2Processor();
+         processor.configure("QCOW2 Processor", params);
+         _processors.put("QCOW2 Processor", processor);
+ 
+         processor = new VmdkProcessor();
+         processor.configure("VMDK Processor", params);
+         _processors.put("VMDK Processor", processor);
+ 
+         processor = new RawImageProcessor();
+         processor.configure("Raw Image Processor", params);
+         _processors.put("Raw Image Processor", processor);
+ 
+         _templateDir = (String) params.get("public.templates.root.dir");
+         if (_templateDir == null) {
+             _templateDir = TemplateConstants.DEFAULT_TMPLT_ROOT_DIR;
+         }
+         _templateDir += File.separator + TemplateConstants.DEFAULT_TMPLT_FIRST_LEVEL_DIR;
+         _volumeDir = TemplateConstants.DEFAULT_VOLUME_ROOT_DIR + File.separator;
+         // Add more processors here.
+         threadPool = Executors.newFixedThreadPool(numInstallThreads);
+         return true;
+     }
+ 
+     private void blockOutgoingOnPrivate() {
+         Script command = new Script("/bin/bash", s_logger);
+         String intf = "eth1";
+         command.add("-c");
 -        command.add("iptables -A OUTPUT -o " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "80" + " -j REJECT;" +
 -                "iptables -A OUTPUT -o " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j REJECT;");
++        command.add("iptables -A OUTPUT -o " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "80" + " -j REJECT;"
++                + "iptables -A OUTPUT -o " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j REJECT;");
+ 
+         String result = command.execute();
+         if (result != null) {
 -            s_logger.warn("Error in blocking outgoing to port 80/443 err=" + result );
++            s_logger.warn("Error in blocking outgoing to port 80/443 err=" + result);
+             return;
+         }
+     }
+ 
+     @Override
+     public String getName() {
+         return _name;
+     }
+ 
+     @Override
+     public boolean start() {
+         return true;
+     }
+ 
+     @Override
+     public boolean stop() {
+         return true;
+     }
+ 
+     private void startAdditionalServices() {
+ 
+         Script command = new Script("/bin/bash", s_logger);
+         command.add("-c");
+         command.add("if [ -d /etc/apache2 ] ; then service apache2 stop; else service httpd stop; fi ");
+         String result = command.execute();
+         if (result != null) {
 -            s_logger.warn("Error in stopping httpd service err=" + result );
++            s_logger.warn("Error in stopping httpd service err=" + result);
+         }
+         String port = Integer.toString(TemplateConstants.DEFAULT_TMPLT_COPY_PORT);
+         String intf = TemplateConstants.DEFAULT_TMPLT_COPY_INTF;
+ 
+         command = new Script("/bin/bash", s_logger);
+         command.add("-c");
 -        command.add("iptables -I INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + port + " -j ACCEPT;" +
 -                "iptables -I INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j ACCEPT;");
++        command.add("iptables -I INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + port + " -j ACCEPT;" + "iptables -I INPUT -i "
++                + intf + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j ACCEPT;");
+ 
+         result = command.execute();
+         if (result != null) {
 -            s_logger.warn("Error in opening up httpd port err=" + result );
++            s_logger.warn("Error in opening up httpd port err=" + result);
+             return;
+         }
+ 
+         command = new Script("/bin/bash", s_logger);
+         command.add("-c");
+         command.add("if [ -d /etc/apache2 ] ; then service apache2 start; else service httpd start; fi ");
+         result = command.execute();
+         if (result != null) {
 -            s_logger.warn("Error in starting httpd service err=" + result );
++            s_logger.warn("Error in starting httpd service err=" + result);
+             return;
+         }
+         command = new Script("mkdir", s_logger);
+         command.add("-p");
+         command.add("/var/www/html/copy/template");
+         result = command.execute();
+         if (result != null) {
 -            s_logger.warn("Error in creating directory =" + result );
++            s_logger.warn("Error in creating directory =" + result);
+             return;
+         }
+     }
+ 
+ }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/30479293/setup/db/db/schema-410to420.sql
----------------------------------------------------------------------
diff --cc setup/db/db/schema-410to420.sql
index 75d29f6,c8ac1ec..85b5c83
--- a/setup/db/db/schema-410to420.sql
+++ b/setup/db/db/schema-410to420.sql
@@@ -522,213 -1115,31 +1246,228 @@@ CREATE VIEW `cloud`.`account_view` A
              and async_job.instance_type = 'Account'
              and async_job.job_status = 0;
  
- ALTER TABLE `cloud`.`remote_access_vpn` ADD COLUMN `id` bigint unsigned NOT NULL UNIQUE AUTO_INCREMENT COMMENT 'id';
- ALTER TABLE `cloud`.`remote_access_vpn` ADD COLUMN `uuid` varchar(40) UNIQUE;
  
- -- START: support for LXC
-  
- INSERT IGNORE INTO `cloud`.`hypervisor_capabilities`(hypervisor_type, hypervisor_version, max_guests_limit, security_group_enabled) VALUES ('LXC', 'default', 50, 1);
- ALTER TABLE `cloud`.`physical_network_traffic_types` ADD COLUMN `lxc_network_label` varchar(255) DEFAULT 'cloudbr0' COMMENT 'The network name label of the physical device dedicated to this traffic on a LXC host';
-  
- UPDATE configuration SET value='KVM,XenServer,VMware,BareMetal,Ovm,LXC' WHERE name='hypervisor.list';
-  
- INSERT INTO `cloud`.`vm_template` (id, unique_name, name, public, created, type, hvm, bits, account_id, url, checksum, enable_password, display_text, format, guest_os_id, featured, cross_zones, hypervisor_type)
-      VALUES (10, 'routing-10', 'SystemVM Template (LXC)', 0, now(), 'SYSTEM', 0, 64, 1, 'http://download.cloud.com/templates/acton/acton-systemvm-02062012.qcow2.bz2', '2755de1f9ef2ce4d6f2bee2efbb4da92', 0, 'SystemVM Template (LXC)', 'QCOW2', 15, 0, 1, 'LXC');
+ alter table `cloud_usage`.`usage_network_offering` add column nic_id bigint(20) unsigned NOT NULL;
+ ALTER TABLE `cloud`.`data_center_details` MODIFY value varchar(1024);
+ ALTER TABLE `cloud`.`cluster_details` MODIFY value varchar(255);
+ ALTER TABLE `cloud`.`storage_pool_details` MODIFY value varchar(255);
+ ALTER TABLE `cloud`.`account_details` MODIFY value varchar(255);
  
 +-- END: support for LXC
 +
 +DROP VIEW IF EXISTS `cloud`.`template_view`;
 +CREATE VIEW `cloud`.`template_view` AS
 +    select 
 +        vm_template.id,
 +        vm_template.uuid,
 +        vm_template.unique_name,
 +        vm_template.name,
 +        vm_template.public,
 +        vm_template.featured,
 +        vm_template.type,
 +        vm_template.hvm,
 +        vm_template.bits,
 +        vm_template.url,
 +        vm_template.format,
 +        vm_template.created,
 +        vm_template.checksum,
 +        vm_template.display_text,
 +        vm_template.enable_password,
 +        vm_template.guest_os_id,
 +        guest_os.uuid guest_os_uuid,
 +        guest_os.display_name guest_os_name,
 +        vm_template.bootable,
 +        vm_template.prepopulate,
 +        vm_template.cross_zones,
 +        vm_template.hypervisor_type,
 +        vm_template.extractable,
 +        vm_template.template_tag,
 +        vm_template.sort_key,
 +        vm_template.removed,
 +        vm_template.enable_sshkey,
 +        source_template.id source_template_id,
 +        source_template.uuid source_template_uuid,
 +        account.id account_id,
 +        account.uuid account_uuid,
 +        account.account_name account_name,
 +        account.type account_type,
 +        domain.id domain_id,
 +        domain.uuid domain_uuid,
 +        domain.name domain_name,
 +        domain.path domain_path,
 +        projects.id project_id,
 +        projects.uuid project_uuid,
 +        projects.name project_name,        
 +        data_center.id data_center_id,
 +        data_center.uuid data_center_uuid,
 +        data_center.name data_center_name,
 +        launch_permission.account_id lp_account_id,
 +        template_store_ref.download_state,
 +        template_store_ref.download_pct,
 +        template_store_ref.error_str,
 +        template_store_ref.size,
 +        template_store_ref.destroyed,
 +        template_store_ref.created created_on_store,
 +        vm_template_details.name detail_name,
 +        vm_template_details.value detail_value,
 +        resource_tags.id tag_id,
 +        resource_tags.uuid tag_uuid,
 +        resource_tags.key tag_key,
 +        resource_tags.value tag_value,
 +        resource_tags.domain_id tag_domain_id,
 +        resource_tags.account_id tag_account_id,
 +        resource_tags.resource_id tag_resource_id,
 +        resource_tags.resource_uuid tag_resource_uuid,
 +        resource_tags.resource_type tag_resource_type,
 +        resource_tags.customer tag_customer
 +    from
 +        `cloud`.`vm_template`
 +            inner join
 +        `cloud`.`guest_os` ON guest_os.id = vm_template.guest_os_id        
 +            inner join
 +        `cloud`.`account` ON account.id = vm_template.account_id
 +            inner join
 +        `cloud`.`domain` ON domain.id = account.domain_id
 +            left join
 +        `cloud`.`projects` ON projects.project_account_id = account.id    
 +            left join
 +        `cloud`.`vm_template_details` ON vm_template_details.template_id = vm_template.id         
 +            left join
 +        `cloud`.`vm_template` source_template ON source_template.id = vm_template.source_template_id            
 +            left join
 +        `cloud`.`template_zone_ref` ON template_zone_ref.template_id = vm_template.id
 +            AND template_zone_ref.removed is null
 +            left join
 +        `cloud`.`data_center` ON template_zone_ref.zone_id = data_center.id
 +            left join
 +        `cloud`.`image_store` ON image_store.data_center_id = data_center.id OR image_store.scope = 'REGION'
 +            left join
 +        `cloud`.`template_store_ref` ON template_store_ref.template_id = vm_template.id AND template_store_ref.store_id = image_store.id
 +            left join
 +        `cloud`.`launch_permission` ON launch_permission.template_id = vm_template.id
 +            left join
 +        `cloud`.`resource_tags` ON resource_tags.resource_id = vm_template.id
 +            and (resource_tags.resource_type = 'Template' or resource_tags.resource_type='ISO');
 +
 +
 +DROP VIEW IF EXISTS `cloud`.`volume_view`;
 +CREATE VIEW `cloud`.`volume_view` AS
 +    select 
 +        volumes.id,
 +        volumes.uuid,
 +        volumes.name,
 +        volumes.device_id,
 +        volumes.volume_type,
 +        volumes.size,
 +        volumes.created,
 +        volumes.state,
 +        volumes.attached,
 +        volumes.removed,
 +        volumes.pod_id,
 +        account.id account_id,
 +        account.uuid account_uuid,
 +        account.account_name account_name,
 +        account.type account_type,
 +        domain.id domain_id,
 +        domain.uuid domain_uuid,
 +        domain.name domain_name,
 +        domain.path domain_path,
 +        projects.id project_id,
 +        projects.uuid project_uuid,
 +        projects.name project_name,
 +        data_center.id data_center_id,
 +        data_center.uuid data_center_uuid,
 +        data_center.name data_center_name,
 +        vm_instance.id vm_id,
 +        vm_instance.uuid vm_uuid,
 +        vm_instance.name vm_name,
 +        vm_instance.state vm_state,
 +        vm_instance.vm_type,
 +        user_vm.display_name vm_display_name,
 +        volume_store_ref.size volume_host_size,
 +        volume_store_ref.created volume_host_created,
 +        volume_store_ref.format,
 +        volume_store_ref.download_pct,
 +        volume_store_ref.download_state,
 +        volume_store_ref.error_str,
 +        disk_offering.id disk_offering_id,
 +        disk_offering.uuid disk_offering_uuid,
 +        disk_offering.name disk_offering_name,
 +        disk_offering.display_text disk_offering_display_text,
 +        disk_offering.use_local_storage,
 +        disk_offering.system_use,
 +        storage_pool.id pool_id,
 +        storage_pool.uuid pool_uuid,
 +        storage_pool.name pool_name,
 +        cluster.hypervisor_type,
 +        vm_template.id template_id,
 +        vm_template.uuid template_uuid,
 +        vm_template.extractable,
 +        vm_template.type template_type,
 +        resource_tags.id tag_id,
 +        resource_tags.uuid tag_uuid,
 +        resource_tags.key tag_key,
 +        resource_tags.value tag_value,
 +        resource_tags.domain_id tag_domain_id,
 +        resource_tags.account_id tag_account_id,
 +        resource_tags.resource_id tag_resource_id,
 +        resource_tags.resource_uuid tag_resource_uuid,
 +        resource_tags.resource_type tag_resource_type,
 +        resource_tags.customer tag_customer,
 +        async_job.id job_id,
 +        async_job.uuid job_uuid,
 +        async_job.job_status job_status,
 +        async_job.account_id job_account_id
 +    from
 +        `cloud`.`volumes`
 +            inner join
 +        `cloud`.`account` ON volumes.account_id = account.id
 +            inner join
 +        `cloud`.`domain` ON volumes.domain_id = domain.id
 +            left join
 +        `cloud`.`projects` ON projects.project_account_id = account.id
 +            left join
 +        `cloud`.`data_center` ON volumes.data_center_id = data_center.id
 +            left join
 +        `cloud`.`vm_instance` ON volumes.instance_id = vm_instance.id
 +            left join
 +        `cloud`.`user_vm` ON user_vm.id = vm_instance.id
 +            left join
 +        `cloud`.`volume_store_ref` ON volumes.id = volume_store_ref.volume_id
 +            and volumes.data_center_id = volume_store_ref.zone_id
 +            left join
 +        `cloud`.`disk_offering` ON volumes.disk_offering_id = disk_offering.id
 +            left join
 +        `cloud`.`storage_pool` ON volumes.pool_id = storage_pool.id
 +            left join
 +        `cloud`.`cluster` ON storage_pool.cluster_id = cluster.id
 +            left join
 +        `cloud`.`vm_template` ON volumes.template_id = vm_template.id
 +            left join
 +        `cloud`.`resource_tags` ON resource_tags.resource_id = volumes.id
 +            and resource_tags.resource_type = 'Volume'
 +            left join
 +        `cloud`.`async_job` ON async_job.instance_id = volumes.id
 +            and async_job.instance_type = 'Volume'
 +            and async_job.job_status = 0;     
+ INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Network', 'DEFAULT', 'management-server', 'midonet.apiserver.address', 'http://localhost:8081', 'Specify the address at which the Midonet API server can be contacted (if using Midonet)');
+ INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Network', 'DEFAULT', 'management-server', 'midonet.providerrouter.id', 'd7c5e6a3-e2f4-426b-b728-b7ce6a0448e5', 'Specifies the UUID of the Midonet provider router (if using Midonet)');
+ 
+ alter table cloud.vpc_gateways add column source_nat boolean default false;
+ alter table cloud.private_ip_address add column source_nat boolean default false;
+ 
+ CREATE TABLE `cloud`.`account_vnet_map` (
+   `id` bigint unsigned NOT NULL UNIQUE AUTO_INCREMENT,
+   `uuid` varchar(255) UNIQUE,
+   `vnet_range` varchar(255) NOT NULL COMMENT 'dedicated guest vlan range',
+   `account_id` bigint unsigned NOT NULL COMMENT 'account id. foreign key to account table',
+   `physical_network_id` bigint unsigned NOT NULL COMMENT 'physical network id. foreign key to the the physical network table',
+   PRIMARY KEY (`id`),
+   CONSTRAINT `fk_account_vnet_map__physical_network_id` FOREIGN KEY (`physical_network_id`) REFERENCES `physical_network` (`id`) ON DELETE CASCADE,
+   INDEX `i_account_vnet_map__physical_network_id`(`physical_network_id`),
+   CONSTRAINT `fk_account_vnet_map__account_id` FOREIGN KEY (`account_id`) REFERENCES `account` (`id`) ON DELETE CASCADE,
+   INDEX `i_account_vnet_map__account_id`(`account_id`)
+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+ 
+ ALTER TABLE `cloud`.`op_dc_vnet_alloc` ADD COLUMN account_vnet_map_id bigint unsigned;
+ ALTER TABLE `cloud`.`op_dc_vnet_alloc` ADD CONSTRAINT `fk_op_dc_vnet_alloc__account_vnet_map_id` FOREIGN KEY `fk_op_dc_vnet_alloc__account_vnet_map_id` (`account_vnet_map_id`) REFERENCES `account_vnet_map` (`id`);

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/30479293/tools/devcloud/devcloud.cfg
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/30479293/tools/marvin/marvin/configGenerator.py
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/30479293/ui/scripts/cloudStack.js
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/30479293/ui/scripts/zoneWizard.js
----------------------------------------------------------------------


Mime
View raw message