cloudstack-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mc...@apache.org
Subject git commit: updated refs/heads/object_store to 23db72d
Date Fri, 03 May 2013 21:59:40 GMT
Updated Branches:
  refs/heads/object_store df7a56d63 -> 23db72d8b


S3 template download test is working except state transition part.

Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/23db72d8
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/23db72d8
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/23db72d8

Branch: refs/heads/object_store
Commit: 23db72d8b40903e306a3e5953b7e39155829d81d
Parents: df7a56d
Author: Min Chen <min.chen@citrix.com>
Authored: Fri May 3 14:59:13 2013 -0700
Committer: Min Chen <min.chen@citrix.com>
Committed: Fri May 3 14:59:13 2013 -0700

----------------------------------------------------------------------
 .../resource/LocalNfsSecondaryStorageResource.java |    4 -
 .../resource/NfsSecondaryStorageResource.java      |    4 +-
 .../storage/template/DownloadManagerImpl.java      |  145 ++++++++-------
 .../storage/template/S3TemplateDownloader.java     |   37 +++-
 .../MockLocalNfsSecondaryStorageResource.java      |  103 ++++++++++
 .../cloudstack/storage/test/S3TemplateTest.java    |    8 +-
 .../integration-test/test/resource/testng.xml      |   12 +-
 .../cloudstack/storage/LocalHostEndpoint.java      |    8 +
 8 files changed, 237 insertions(+), 84 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cloudstack/blob/23db72d8/core/src/com/cloud/storage/resource/LocalNfsSecondaryStorageResource.java
----------------------------------------------------------------------
diff --git a/core/src/com/cloud/storage/resource/LocalNfsSecondaryStorageResource.java b/core/src/com/cloud/storage/resource/LocalNfsSecondaryStorageResource.java
index 06933cc..a2cf757 100644
--- a/core/src/com/cloud/storage/resource/LocalNfsSecondaryStorageResource.java
+++ b/core/src/com/cloud/storage/resource/LocalNfsSecondaryStorageResource.java
@@ -32,10 +32,6 @@ import com.cloud.utils.exception.CloudRuntimeException;
 public class LocalNfsSecondaryStorageResource extends
 		NfsSecondaryStorageResource {
 
-    public LocalNfsSecondaryStorageResource(){
-        _dlMgr = new DownloadManagerImpl();
-    }
-
     @Override
     public Answer executeRequest(Command cmd) {
         if (cmd instanceof DownloadSystemTemplateCommand){

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/23db72d8/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java
----------------------------------------------------------------------
diff --git a/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java b/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java
index 47d5ce7..0345f6b 100755
--- a/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java
+++ b/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java
@@ -144,8 +144,8 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements
S
     boolean _inSystemVM = false;
     boolean _sslCopy = false;
 
-    DownloadManager _dlMgr;
-    UploadManager _upldMgr;
+    protected DownloadManager _dlMgr;
+    protected UploadManager _upldMgr;
     private String _configSslScr;
     private String _configAuthScr;
     private String _configIpFirewallScr;

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/23db72d8/core/src/com/cloud/storage/template/DownloadManagerImpl.java
----------------------------------------------------------------------
diff --git a/core/src/com/cloud/storage/template/DownloadManagerImpl.java b/core/src/com/cloud/storage/template/DownloadManagerImpl.java
index 02a6256..cd6165d 100755
--- a/core/src/com/cloud/storage/template/DownloadManagerImpl.java
+++ b/core/src/com/cloud/storage/template/DownloadManagerImpl.java
@@ -116,7 +116,8 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
         private final long id;
         private final ResourceType resourceType;
 
-        public DownloadJob(TemplateDownloader td, String jobId, long id, String tmpltName,
ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix,
ResourceType resourceType) {
+        public DownloadJob(TemplateDownloader td, String jobId, long id, String tmpltName,
ImageFormat format, boolean hvm, Long accountId,
+                String descr, String cksum, String installPathPrefix, ResourceType resourceType)
{
             super();
             this.td = td;
             this.jobId = jobId;
@@ -239,8 +240,14 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
     private int installTimeoutPerGig = 180 * 60 * 1000;
     private boolean _sslCopy;
 
+
+    public void setThreadPool(ExecutorService threadPool) {
+        this.threadPool = threadPool;
+    }
+
     /**
-     * Get notified of change of job status. Executed in context of downloader thread
+     * Get notified of change of job status. Executed in context of downloader
+     * thread
      *
      * @param jobId
      *            the id of the job
@@ -255,7 +262,8 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
         }
         TemplateDownloader td = dj.getTemplateDownloader();
         s_logger.info("Download Completion for jobId: " + jobId + ", status=" + status);
-        s_logger.info("local: " + td.getDownloadLocalPath() + ", bytes=" + td.getDownloadedBytes()
+ ", error=" + td.getDownloadError() + ", pct=" + td.getDownloadPercent());
+        s_logger.info("local: " + td.getDownloadLocalPath() + ", bytes=" + td.getDownloadedBytes()
+ ", error=" + td.getDownloadError() + ", pct="
+                + td.getDownloadPercent());
 
         switch (status) {
         case ABORTED:
@@ -276,7 +284,8 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
             break;
         case DOWNLOAD_FINISHED:
             if (!(td instanceof S3TemplateDownloader)) {
-                // we currently only create template.properties for NFS by running some post
download script
+                // we currently only create template.properties for NFS by
+                // running some post download script
                 td.setDownloadError("Download success, starting install ");
                 String result = postDownload(jobId);
                 if (result != null) {
@@ -304,21 +313,20 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
         try {
             digest = MessageDigest.getInstance("MD5");
             is = new FileInputStream(f);
-            while( (read = is.read(buffer)) > 0) {
+            while ((read = is.read(buffer)) > 0) {
                 digest.update(buffer, 0, read);
             }
             byte[] md5sum = digest.digest();
             BigInteger bigInt = new BigInteger(1, md5sum);
-            checksum = String.format("%032x",bigInt);
+            checksum = String.format("%032x", bigInt);
             return checksum;
-        }catch(IOException e) {
+        } catch (IOException e) {
             return null;
-        }catch (NoSuchAlgorithmException e) {
+        } catch (NoSuchAlgorithmException e) {
             return null;
-        }
-        finally {
+        } finally {
             try {
-                if(is != null)
+                if (is != null)
                     is.close();
             } catch (IOException e) {
                 return null;
@@ -327,7 +335,8 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
     }
 
     /**
-     * Post download activity (install and cleanup). Executed in context of downloader thread
+     * Post download activity (install and cleanup). Executed in context of
+     * downloader thread
      *
      * @throws IOException
      */
@@ -337,12 +346,13 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
         String resourcePath = null;
         ResourceType resourceType = dnld.getResourceType();
 
-        // once template path is set, remove the parent dir so that the template is installed
with a relative path
+        // once template path is set, remove the parent dir so that the template
+        // is installed with a relative path
         String finalResourcePath = "";
-        if (resourceType == ResourceType.TEMPLATE){
+        if (resourceType == ResourceType.TEMPLATE) {
             finalResourcePath += _templateDir + File.separator + dnld.getAccountId() + File.separator
+ dnld.getId() + File.separator;
             resourcePath = dnld.getInstallPathPrefix() + dnld.getAccountId() + File.separator
+ dnld.getId() + File.separator;// dnld.getTmpltName();
-        }else {
+        } else {
             finalResourcePath += _volumeDir + File.separator + dnld.getId() + File.separator;
             resourcePath = dnld.getInstallPathPrefix() + dnld.getId() + File.separator;//
dnld.getTmpltName();
         }
@@ -375,7 +385,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
         // add options common to ISO and template
         String extension = dnld.getFormat().getFileExtension();
         String templateName = "";
-        if( extension.equals("iso")) {
+        if (extension.equals("iso")) {
             templateName = jobs.get(jobId).getTmpltName().trim().replace(" ", "_");
         } else {
             templateName = java.util.UUID.nameUUIDFromBytes((jobs.get(jobId).getTmpltName()
+ System.currentTimeMillis()).getBytes()).toString();
@@ -404,9 +414,9 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
 
         // Set permissions for template/volume.properties
         String propertiesFile = resourcePath;
-        if (resourceType == ResourceType.TEMPLATE){
+        if (resourceType == ResourceType.TEMPLATE) {
             propertiesFile += "/template.properties";
-        }else{
+        } else {
             propertiesFile += "/volume.properties";
         }
         File templateProperties = new File(propertiesFile);
@@ -461,7 +471,8 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
     }
 
     @Override
-    public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat
format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix,
String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType)
{
+    public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat
format, boolean hvm, Long accountId, String descr,
+            String cksum, String installPathPrefix, String user, String password, long maxTemplateSizeInBytes,
Proxy proxy, ResourceType resourceType) {
         UUID uuid = UUID.randomUUID();
         String jobId = uuid.toString();
 
@@ -489,15 +500,15 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
         return jobId;
     }
 
-
     @Override
-    public String downloadPublicTemplate(long id, String url, String name, ImageFormat format,
boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, String
user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType)
{
+    public String downloadPublicTemplate(long id, String url, String name, ImageFormat format,
boolean hvm, Long accountId, String descr,
+            String cksum, String installPathPrefix, String user, String password, long maxTemplateSizeInBytes,
Proxy proxy, ResourceType resourceType) {
         UUID uuid = UUID.randomUUID();
         String jobId = uuid.toString();
         String tmpDir = "";
-        if(resourceType == ResourceType.TEMPLATE){
+        if (resourceType == ResourceType.TEMPLATE) {
             tmpDir = installPathPrefix + File.separator + accountId + File.separator + id;
-        }else {
+        } else {
             tmpDir = installPathPrefix + File.separator + id;
         }
 
@@ -507,10 +518,10 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
                 s_logger.warn("Unable to create " + tmpDir);
                 return "Unable to create " + tmpDir;
             }
-            //	TO DO - define constant for volume properties.
-            File file = ResourceType.TEMPLATE == resourceType ? _storage.getFile(tmpDir +
File.separator + TemplateLocation.Filename) :
-                _storage.getFile(tmpDir + File.separator + "volume.properties");
-            if ( file.exists() ) {
+            // TO DO - define constant for volume properties.
+            File file = ResourceType.TEMPLATE == resourceType ? _storage.getFile(tmpDir +
File.separator + TemplateLocation.Filename) : _storage
+                    .getFile(tmpDir + File.separator + "volume.properties");
+            if (file.exists()) {
                 file.delete();
             }
 
@@ -528,7 +539,8 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
             TemplateDownloader td;
             if ((uri != null) && (uri.getScheme() != null)) {
                 if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https"))
{
-                    td = new HttpTemplateDownloader(_storage, url, tmpDir, new Completion(jobId),
maxTemplateSizeInBytes, user, password, proxy, resourceType);
+                    td = new HttpTemplateDownloader(_storage, url, tmpDir, new Completion(jobId),
maxTemplateSizeInBytes, user, password, proxy,
+                            resourceType);
                 } else if (uri.getScheme().equalsIgnoreCase("file")) {
                     td = new LocalTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes,
new Completion(jobId));
                 } else if (uri.getScheme().equalsIgnoreCase("scp")) {
@@ -633,16 +645,16 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
         return convertStatus(getDownloadStatus(jobId));
     }
 
-
     @Override
     public DownloadAnswer handleDownloadCommand(SecondaryStorageResource resource, DownloadCommand
cmd) {
         ResourceType resourceType = cmd.getResourceType();
         if (cmd instanceof DownloadProgressCommand) {
-            return handleDownloadProgressCmd( resource, (DownloadProgressCommand) cmd);
+            return handleDownloadProgressCmd(resource, (DownloadProgressCommand) cmd);
         }
 
         if (cmd.getUrl() == null) {
-            return new DownloadAnswer(resourceType.toString() + " is corrupted on storage
due to an invalid url , cannot download", VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR);
+            return new DownloadAnswer(resourceType.toString() + " is corrupted on storage
due to an invalid url , cannot download",
+                    VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR);
         }
 
         if (cmd.getName() == null) {
@@ -657,21 +669,23 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
             user = cmd.getAuth().getUserName();
             password = new String(cmd.getAuth().getPassword());
         }
-        //TO DO - Define Volume max size as well
-        long maxDownloadSizeInBytes = (cmd.getMaxDownloadSizeInBytes() == null) ? TemplateDownloader.DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES
: (cmd.getMaxDownloadSizeInBytes());
+        // TO DO - Define Volume max size as well
+        long maxDownloadSizeInBytes = (cmd.getMaxDownloadSizeInBytes() == null) ? TemplateDownloader.DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES
: (cmd
+                .getMaxDownloadSizeInBytes());
         String jobId = null;
-        if (dstore instanceof S3TO){
-            jobId = downloadS3Template((S3TO)dstore, cmd.getId(), cmd.getUrl(), cmd.getName(),
cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(),
installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType);
-        }
-        else{
-            jobId = downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(),
cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(), installPathPrefix,
user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType);
+        if (dstore instanceof S3TO) {
+            jobId = downloadS3Template((S3TO) dstore, cmd.getId(), cmd.getUrl(), cmd.getName(),
cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(),
+                    cmd.getDescription(), cmd.getChecksum(), installPathPrefix, user, password,
maxDownloadSizeInBytes, cmd.getProxy(), resourceType);
+        } else {
+            jobId = downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(),
cmd.isHvm(), cmd.getAccountId(),
+                    cmd.getDescription(), cmd.getChecksum(), installPathPrefix, user, password,
maxDownloadSizeInBytes, cmd.getProxy(), resourceType);
         }
         sleep();
         if (jobId == null) {
             return new DownloadAnswer("Internal Error", VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR);
         }
-        return new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId),
getDownloadStatus2(jobId), getDownloadLocalPath(jobId), getInstallPath(jobId),
-                getDownloadTemplateSize(jobId), getDownloadTemplateSize(jobId), getDownloadCheckSum(jobId));
+        return new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId),
getDownloadStatus2(jobId), getDownloadLocalPath(jobId),
+                getInstallPath(jobId), getDownloadTemplateSize(jobId), getDownloadTemplateSize(jobId),
getDownloadCheckSum(jobId));
     }
 
     private void sleep() {
@@ -712,8 +726,9 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
             break;
         case PURGE:
             td.stopDownload();
-            answer = new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId),
getDownloadStatus2(jobId), getDownloadLocalPath(jobId),
-                    getInstallPath(jobId), getDownloadTemplateSize(jobId), getDownloadTemplatePhysicalSize(jobId),
getDownloadCheckSum(jobId));
+            answer = new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId),
getDownloadStatus2(jobId),
+                    getDownloadLocalPath(jobId), getInstallPath(jobId), getDownloadTemplateSize(jobId),
getDownloadTemplatePhysicalSize(jobId),
+                    getDownloadCheckSum(jobId));
             jobs.remove(jobId);
             return answer;
         default:
@@ -741,7 +756,6 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
 
     }
 
-
     private List<String> listVolumes(String rootdir) {
         List<String> result = new ArrayList<String>();
 
@@ -754,8 +768,6 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
         return result;
     }
 
-
-
     private List<String> listTemplates(String rootdir) {
         List<String> result = new ArrayList<String>();
 
@@ -773,7 +785,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
         Map<String, TemplateProp> result = new HashMap<String, TemplateProp>();
         String templateDir = rootDir + File.separator + _templateDir;
 
-        if (! _storage.exists(templateDir)) {
+        if (!_storage.exists(templateDir)) {
             _storage.mkdirs(templateDir);
         }
 
@@ -784,7 +796,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
             try {
                 if (!loc.load()) {
                     s_logger.warn("Post download installation was not completed for " + path);
-                    //loc.purge();
+                    // loc.purge();
                     _storage.cleanup(path, templateDir);
                     continue;
                 }
@@ -798,8 +810,9 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
             if ((tInfo.size == tInfo.physicalSize) && (tInfo.installPath.endsWith(ImageFormat.OVA.getFileExtension())))
{
                 try {
                     Processor processor = _processors.get("VMDK Processor");
-                    VmdkProcessor vmdkProcessor = (VmdkProcessor)processor;
-                    long vSize = vmdkProcessor.getTemplateVirtualSize(path, tInfo.installPath.substring(tInfo.installPath.lastIndexOf(File.separator)
+ 1));
+                    VmdkProcessor vmdkProcessor = (VmdkProcessor) processor;
+                    long vSize = vmdkProcessor.getTemplateVirtualSize(path,
+                            tInfo.installPath.substring(tInfo.installPath.lastIndexOf(File.separator)
+ 1));
                     tInfo.size = vSize;
                     loc.updateVirtualSize(vSize);
                     loc.save();
@@ -830,7 +843,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
         Map<Long, TemplateProp> result = new HashMap<Long, TemplateProp>();
         String volumeDir = rootDir + File.separator + _volumeDir;
 
-        if (! _storage.exists(volumeDir)) {
+        if (!_storage.exists(volumeDir)) {
             _storage.mkdirs(volumeDir);
         }
 
@@ -841,7 +854,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
             try {
                 if (!loc.load()) {
                     s_logger.warn("Post download installation was not completed for " + path);
-                    //loc.purge();
+                    // loc.purge();
                     _storage.cleanup(path, volumeDir);
                     continue;
                 }
@@ -855,8 +868,9 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
             if ((vInfo.size == vInfo.physicalSize) && (vInfo.installPath.endsWith(ImageFormat.OVA.getFileExtension())))
{
                 try {
                     Processor processor = _processors.get("VMDK Processor");
-                    VmdkProcessor vmdkProcessor = (VmdkProcessor)processor;
-                    long vSize = vmdkProcessor.getTemplateVirtualSize(path, vInfo.installPath.substring(vInfo.installPath.lastIndexOf(File.separator)
+ 1));
+                    VmdkProcessor vmdkProcessor = (VmdkProcessor) processor;
+                    long vSize = vmdkProcessor.getTemplateVirtualSize(path,
+                            vInfo.installPath.substring(vInfo.installPath.lastIndexOf(File.separator)
+ 1));
                     vInfo.size = vSize;
                     loc.updateVirtualSize(vSize);
                     loc.save();
@@ -929,7 +943,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
 
         String value = null;
 
-        _storage = (StorageLayer)params.get(StorageLayer.InstanceConfigKey);
+        _storage = (StorageLayer) params.get(StorageLayer.InstanceConfigKey);
         if (_storage == null) {
             value = (String) params.get(StorageLayer.ClassConfigKey);
             if (value == null) {
@@ -948,12 +962,12 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
                 throw new ConfigurationException("Unable to instantiate " + value);
             }
         }
-        String useSsl = (String)params.get("sslcopy");
+        String useSsl = (String) params.get("sslcopy");
         if (useSsl != null) {
             _sslCopy = Boolean.parseBoolean(useSsl);
 
         }
-        String inSystemVM = (String)params.get("secondary.storage.vm");
+        String inSystemVM = (String) params.get("secondary.storage.vm");
         if (inSystemVM != null && "true".equalsIgnoreCase(inSystemVM)) {
             s_logger.info("DownloadManager: starting additional services since we are inside
system vm");
             startAdditionalServices();
@@ -1032,12 +1046,12 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
         Script command = new Script("/bin/bash", s_logger);
         String intf = "eth1";
         command.add("-c");
-        command.add("iptables -A OUTPUT -o " + intf + " -p tcp -m state --state NEW -m tcp
--dport " + "80" + " -j REJECT;" +
-                "iptables -A OUTPUT -o " + intf + " -p tcp -m state --state NEW -m tcp --dport
" + "443" + " -j REJECT;");
+        command.add("iptables -A OUTPUT -o " + intf + " -p tcp -m state --state NEW -m tcp
--dport " + "80" + " -j REJECT;"
+                + "iptables -A OUTPUT -o " + intf + " -p tcp -m state --state NEW -m tcp
--dport " + "443" + " -j REJECT;");
 
         String result = command.execute();
         if (result != null) {
-            s_logger.warn("Error in blocking outgoing to port 80/443 err=" + result );
+            s_logger.warn("Error in blocking outgoing to port 80/443 err=" + result);
             return;
         }
     }
@@ -1064,19 +1078,19 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
         command.add("if [ -d /etc/apache2 ] ; then service apache2 stop; else service httpd
stop; fi ");
         String result = command.execute();
         if (result != null) {
-            s_logger.warn("Error in stopping httpd service err=" + result );
+            s_logger.warn("Error in stopping httpd service err=" + result);
         }
         String port = Integer.toString(TemplateConstants.DEFAULT_TMPLT_COPY_PORT);
         String intf = TemplateConstants.DEFAULT_TMPLT_COPY_INTF;
 
         command = new Script("/bin/bash", s_logger);
         command.add("-c");
-        command.add("iptables -I INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp
--dport " + port + " -j ACCEPT;" +
-                "iptables -I INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport
" + "443" + " -j ACCEPT;");
+        command.add("iptables -I INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp
--dport " + port + " -j ACCEPT;" + "iptables -I INPUT -i "
+                + intf + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j ACCEPT;");
 
         result = command.execute();
         if (result != null) {
-            s_logger.warn("Error in opening up httpd port err=" + result );
+            s_logger.warn("Error in opening up httpd port err=" + result);
             return;
         }
 
@@ -1085,7 +1099,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
         command.add("if [ -d /etc/apache2 ] ; then service apache2 start; else service httpd
start; fi ");
         result = command.execute();
         if (result != null) {
-            s_logger.warn("Error in starting httpd service err=" + result );
+            s_logger.warn("Error in starting httpd service err=" + result);
             return;
         }
         command = new Script("mkdir", s_logger);
@@ -1093,10 +1107,9 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
         command.add("/var/www/html/copy/template");
         result = command.execute();
         if (result != null) {
-            s_logger.warn("Error in creating directory =" + result );
+            s_logger.warn("Error in creating directory =" + result);
             return;
         }
     }
 
-
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/23db72d8/core/src/com/cloud/storage/template/S3TemplateDownloader.java
----------------------------------------------------------------------
diff --git a/core/src/com/cloud/storage/template/S3TemplateDownloader.java b/core/src/com/cloud/storage/template/S3TemplateDownloader.java
index 84b2b10..eac9bbf 100644
--- a/core/src/com/cloud/storage/template/S3TemplateDownloader.java
+++ b/core/src/com/cloud/storage/template/S3TemplateDownloader.java
@@ -40,14 +40,18 @@ import org.apache.commons.httpclient.HttpClient;
 import org.apache.commons.httpclient.HttpException;
 import org.apache.commons.httpclient.HttpMethod;
 import org.apache.commons.httpclient.HttpMethodRetryHandler;
+import org.apache.commons.httpclient.HttpStatus;
 import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
 import org.apache.commons.httpclient.NoHttpResponseException;
 import org.apache.commons.httpclient.UsernamePasswordCredentials;
 import org.apache.commons.httpclient.auth.AuthScope;
 import org.apache.commons.httpclient.methods.GetMethod;
 import org.apache.commons.httpclient.params.HttpMethodParams;
+import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
 
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.AmazonServiceException;
 import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.amazonaws.services.s3.model.ProgressEvent;
 import com.amazonaws.services.s3.model.ProgressListener;
@@ -176,8 +180,7 @@ public class S3TemplateDownloader implements TemplateDownloader {
 				port = 80;
 			}
 
-            URL urlObj = new URL(url);
-            this.fileName = urlObj.getFile();
+            this.fileName = StringUtils.substringAfterLast(url, "/");
 
 			String host = uri.getHost();
 			try {
@@ -199,10 +202,7 @@ public class S3TemplateDownloader implements TemplateDownloader {
 		} catch (URISyntaxException use) {
 			s_logger.warn("Failed uri syntax check: " + use.getMessage());
 			throw new IllegalArgumentException(use.getMessage());
-		} catch (MalformedURLException e) {
-            s_logger.warn("Failed url syntax check: " + e.getMessage());
-            throw new IllegalArgumentException(e.getMessage());
-        }
+		}
 	}
 
 	@Override
@@ -215,8 +215,17 @@ public class S3TemplateDownloader implements TemplateDownloader {
 		default:
 
 		}
+
+
         int bytes=0;
 		try {
+	        // execute get method
+	        int responseCode = HttpStatus.SC_OK;
+	        if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) {
+	            status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
+	            errorString = " HTTP Server returned " + responseCode + " (expected 200 OK)
";
+	            return 0; //FIXME: retry?
+	        }
 		    // get the total size of file
             Header contentLengthHeader = request.getResponseHeader("Content-Length");
             boolean chunked = false;
@@ -277,14 +286,23 @@ public class S3TemplateDownloader implements TemplateDownloader {
                             if (progressEvent.getEventCode() == ProgressEvent.COMPLETED_EVENT_CODE)
{
                                 s_logger.info("download completed");
                                 status = TemplateDownloader.Status.DOWNLOAD_FINISHED;
-                            } else {
+                            } else if (progressEvent.getEventCode() == ProgressEvent.FAILED_EVENT_CODE){
+                                status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
+                            } else if (progressEvent.getEventCode() == ProgressEvent.CANCELED_EVENT_CODE){
+                                status = TemplateDownloader.Status.ABORTED;
+                            } else{
                                 status = TemplateDownloader.Status.IN_PROGRESS;
                             }
-
                         }
 
                     });
             S3Utils.putObject(s3, putObjectRequest);
+            while (status != TemplateDownloader.Status.DOWNLOAD_FINISHED &&
+                    status != TemplateDownloader.Status.UNRECOVERABLE_ERROR &&
+                    status != TemplateDownloader.Status.ABORTED ){
+                // wait for completion
+            }
+            // finished or aborted
             Date finish = new Date();
             String downloaded = "(incomplete download)";
             if (totalBytes >= remoteSize) {
@@ -300,6 +318,9 @@ public class S3TemplateDownloader implements TemplateDownloader {
 		} catch (IOException ioe) {
 			status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; //probably a file write error?
 			errorString = ioe.getMessage();
+		} catch (AmazonClientException ex) {
+            status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; // S3 api exception
+            errorString = ex.getMessage();
 		} finally {
 		    // close input stream
 			request.releaseConnection();

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/23db72d8/engine/storage/integration-test/test/org/apache/cloudstack/storage/MockLocalNfsSecondaryStorageResource.java
----------------------------------------------------------------------
diff --git a/engine/storage/integration-test/test/org/apache/cloudstack/storage/MockLocalNfsSecondaryStorageResource.java
b/engine/storage/integration-test/test/org/apache/cloudstack/storage/MockLocalNfsSecondaryStorageResource.java
new file mode 100644
index 0000000..a5c0af1
--- /dev/null
+++ b/engine/storage/integration-test/test/org/apache/cloudstack/storage/MockLocalNfsSecondaryStorageResource.java
@@ -0,0 +1,103 @@
+package org.apache.cloudstack.storage;
+
+import static com.cloud.utils.StringUtils.join;
+import static java.util.Arrays.asList;
+
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.List;
+import java.util.concurrent.Executors;
+
+import org.apache.cloudstack.storage.command.DownloadSystemTemplateCommand;
+import org.springframework.stereotype.Component;
+
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.Command;
+import com.cloud.agent.api.storage.DownloadAnswer;
+import com.cloud.agent.api.to.DataStoreTO;
+import com.cloud.agent.api.to.NfsTO;
+import com.cloud.agent.api.to.S3TO;
+import com.cloud.agent.api.to.SwiftTO;
+import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
+import com.cloud.storage.resource.NfsSecondaryStorageResource;
+import com.cloud.storage.template.DownloadManagerImpl;
+import com.cloud.utils.S3Utils;
+import com.cloud.utils.UriUtils;
+import com.cloud.utils.exception.CloudRuntimeException;
+
+@Component
+public class MockLocalNfsSecondaryStorageResource extends
+		NfsSecondaryStorageResource {
+
+    public MockLocalNfsSecondaryStorageResource(){
+        _dlMgr = new DownloadManagerImpl();
+        ((DownloadManagerImpl)_dlMgr).setThreadPool(Executors.newFixedThreadPool(10));
+    }
+
+    @Override
+    public Answer executeRequest(Command cmd) {
+        if (cmd instanceof DownloadSystemTemplateCommand){
+            return execute((DownloadSystemTemplateCommand)cmd);
+        } else {
+            //return Answer.createUnsupportedCommandAnswer(cmd);
+            return super.executeRequest(cmd);
+        }
+    }
+
+    private Answer execute(DownloadSystemTemplateCommand cmd){
+        DataStoreTO dstore = cmd.getDataStore();
+        if ( dstore instanceof S3TO ){
+            //TODO: how to handle download progress for S3
+            S3TO s3 = (S3TO)cmd.getDataStore();
+            String url = cmd.getUrl();
+            String user = null;
+            String password = null;
+            if (cmd.getAuth() != null) {
+                user = cmd.getAuth().getUserName();
+                password = new String(cmd.getAuth().getPassword());
+            }
+            // get input stream from the given url
+            InputStream in = UriUtils.getInputStreamFromUrl(url, user, password);
+            URI uri;
+            URL urlObj;
+            try {
+                uri = new URI(url);
+                urlObj = new URL(url);
+            } catch (URISyntaxException e) {
+                throw new CloudRuntimeException("URI is incorrect: " + url);
+            } catch (MalformedURLException e) {
+                throw new CloudRuntimeException("URL is incorrect: " + url);
+            }
+
+            final String bucket = s3.getBucketName();
+            // convention is no / in the end for install path based on S3Utils implementation.
+            String path = determineS3TemplateDirectory(cmd.getAccountId(), cmd.getResourceId(),
cmd.getName());
+            // template key is
+            // TEMPLATE_ROOT_DIR/account_id/template_id/template_name
+            String key = join(asList(path, urlObj.getFile()), S3Utils.SEPARATOR);
+            S3Utils.putObject(s3, in, bucket, key);
+            List<S3ObjectSummary> s3Obj = S3Utils.getDirectory(s3, bucket, path);
+            if (s3Obj == null || s3Obj.size() == 0) {
+                return new Answer(cmd, false, "Failed to download to S3 bucket: " + bucket
+ " with key: " + key);
+            } else {
+                return new DownloadAnswer(null, 100, null, Status.DOWNLOADED, path, path,
s3Obj.get(0).getSize(), s3Obj.get(0).getSize(), s3Obj.get(0)
+                        .getETag());
+            }
+        }
+        else if ( dstore instanceof NfsTO ){
+            return new Answer(cmd, false, "Nfs needs to be pre-installed with system vm templates");
+        }
+        else if ( dstore instanceof SwiftTO ){
+            //TODO: need to move code from execute(uploadTemplateToSwiftFromSecondaryStorageCommand)
here, but we need to handle
+            // source is url, most likely we need to modify our existing swiftUpload python
script.
+            return new Answer(cmd, false, "Swift is not currently support DownloadCommand");
+        }
+        else{
+            return new Answer(cmd, false, "Unsupported image data store: " + dstore);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/23db72d8/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/S3TemplateTest.java
----------------------------------------------------------------------
diff --git a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/S3TemplateTest.java
b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/S3TemplateTest.java
index 4d4d037..9b9f03d 100644
--- a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/S3TemplateTest.java
+++ b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/S3TemplateTest.java
@@ -18,6 +18,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService.TemplateApiResult;
 import org.apache.cloudstack.framework.async.AsyncCallFuture;
 import org.apache.cloudstack.storage.LocalHostEndpoint;
+import org.apache.cloudstack.storage.MockLocalNfsSecondaryStorageResource;
 import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
 import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailVO;
 import org.apache.cloudstack.storage.datastore.db.ImageStoreVO;
@@ -108,8 +109,11 @@ public class S3TemplateTest extends CloudStackTestNGBase {
 		image = templateDao.persist(image);
 		templateId = image.getId();
 
-		Mockito.when(epSelector.select(Mockito.any(DataObject.class))).thenReturn(new LocalHostEndpoint());
-		Mockito.when(epSelector.select(Mockito.any(DataStore.class))).thenReturn(new LocalHostEndpoint());
+		// inject mockito
+		LocalHostEndpoint ep = new LocalHostEndpoint();
+		ep.setResource(new MockLocalNfsSecondaryStorageResource());
+		Mockito.when(epSelector.select(Mockito.any(DataObject.class))).thenReturn(ep);
+		Mockito.when(epSelector.select(Mockito.any(DataStore.class))).thenReturn(ep);
 	}
 
 	@Test

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/23db72d8/engine/storage/integration-test/test/resource/testng.xml
----------------------------------------------------------------------
diff --git a/engine/storage/integration-test/test/resource/testng.xml b/engine/storage/integration-test/test/resource/testng.xml
index 55f7edf..ad4829b 100644
--- a/engine/storage/integration-test/test/resource/testng.xml
+++ b/engine/storage/integration-test/test/resource/testng.xml
@@ -29,8 +29,16 @@
   <parameter name="devcloud-host-uuid" value="759ee4c9-a15a-297b-67c6-ac267d8aa429"/>
   <parameter name="devcloud-secondary-storage" value="nfs://192.168.56.10/opt/storage/secondary"/>
 
- 
-   <listeners>
+  <parameter name="s3-accesskey" value="OYAZXCAFUC1DAFOXNJWI"/>
+  <parameter name="s3-secretkey" value="YHpRtzNDwl12DtrQmwRvdpnf2xK2AeVM30rXxQ=="/>
+  <parameter name="s3-endpoint"  value="10.223.89.7:8080" />
+  <parameter name="s3-template-bucket" value="cstemplates"/>
+  <parameter name="s3-usehttps"  value="false" />
+
+  <parameter name="image-install-path" value="template/tmpl/1/5/"/>
+
+   
+  <listeners>
     <listener class-name="org.apache.cloudstack.storage.test.TestNGAop" />
   </listeners>
  

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/23db72d8/engine/storage/src/org/apache/cloudstack/storage/LocalHostEndpoint.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/LocalHostEndpoint.java b/engine/storage/src/org/apache/cloudstack/storage/LocalHostEndpoint.java
index ea93560..1707725 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/LocalHostEndpoint.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/LocalHostEndpoint.java
@@ -92,5 +92,13 @@ public class LocalHostEndpoint implements EndPoint {
 			executor.schedule(new CmdRunner2(cmd, listener.getCallback()), 10, TimeUnit.SECONDS);
 		}
 	}
+    public ServerResource getResource() {
+        return resource;
+    }
+    public void setResource(ServerResource resource) {
+        this.resource = resource;
+    }
+
+
 
 }


Mime
View raw message