cloudstack-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mc...@apache.org
Subject git commit: updated refs/heads/object_store to 5933375
Date Mon, 15 Apr 2013 21:45:02 GMT
Updated Branches:
  refs/heads/object_store 2dd8e2cb9 -> 593337565


Add S3 code to handle DownloadCommand for registerTemplate.

Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/59333756
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/59333756
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/59333756

Branch: refs/heads/object_store
Commit: 593337565e75f114075053dc06f51165bb862508
Parents: 2dd8e2c
Author: Min Chen <min.chen@citrix.com>
Authored: Mon Apr 15 14:44:39 2013 -0700
Committer: Min Chen <min.chen@citrix.com>
Committed: Mon Apr 15 14:44:39 2013 -0700

----------------------------------------------------------------------
 .../cloud/agent/api/storage/DownloadCommand.java   |   29 ++++-
 api/src/com/cloud/agent/api/to/NfsTO.java          |   60 +++++++++
 .../resource/NfsSecondaryStorageResource.java      |   97 +++++++++++----
 .../storage/template/DownloadManagerImpl.java      |   40 +++++--
 .../com/cloud/agent/transport/RequestTest.java     |   18 ++--
 .../storage/motion/AncientDataMotionStrategy.java  |   41 +++---
 .../storage/image/store/ImageStoreImpl.java        |    2 +-
 .../storage/datastore/PrimaryDataStoreImpl.java    |   14 +-
 .../driver/CloudStackImageStoreDriverImpl.java     |    6 +-
 .../storage/download/DownloadMonitorImpl.java      |    2 +-
 utils/src/com/cloud/utils/S3Utils.java             |   21 +++
 utils/src/com/cloud/utils/UriUtils.java            |   81 ++++++++++++
 12 files changed, 336 insertions(+), 75 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cloudstack/blob/59333756/api/src/com/cloud/agent/api/storage/DownloadCommand.java
----------------------------------------------------------------------
diff --git a/api/src/com/cloud/agent/api/storage/DownloadCommand.java b/api/src/com/cloud/agent/api/storage/DownloadCommand.java
index c6ffe45..77e7ae3 100644
--- a/api/src/com/cloud/agent/api/storage/DownloadCommand.java
+++ b/api/src/com/cloud/agent/api/storage/DownloadCommand.java
@@ -20,6 +20,7 @@ import java.net.URI;
 
 import org.apache.cloudstack.api.InternalIdentity;
 
+import com.cloud.agent.api.to.DataStoreTO;
 import com.cloud.storage.Storage.ImageFormat;
 import com.cloud.storage.Volume;
 import com.cloud.template.VirtualMachineTemplate;
@@ -105,6 +106,8 @@ public class DownloadCommand extends AbstractDownloadCommand implements
Internal
 	private Long maxDownloadSizeInBytes = null;
 	private long id;
 	private ResourceType resourceType = ResourceType.TEMPLATE;
+	private DataStoreTO _store;
+    private Long resourceId;
 
 	protected DownloadCommand() {
 	}
@@ -122,14 +125,16 @@ public class DownloadCommand extends AbstractDownloadCommand implements
Internal
 	    this.resourceType = that.resourceType;
 	}
 
-	public DownloadCommand(String secUrl, VirtualMachineTemplate template, Long maxDownloadSizeInBytes)
{
+	public DownloadCommand(DataStoreTO store, String secUrl, VirtualMachineTemplate template,
Long maxDownloadSizeInBytes) {
 	    super(template.getUniqueName(), template.getUrl(), template.getFormat(), template.getAccountId());
+	    this._store = store;
 	    this.hvm = template.isRequiresHvm();
 	    this.checksum = template.getChecksum();
 	    this.id = template.getId();
 	    this.description = template.getDisplayText();
 	    this.setSecUrl(secUrl);
 	    this.maxDownloadSizeInBytes = maxDownloadSizeInBytes;
+	    this.resourceId = template.getId();
 	}
 
 	public DownloadCommand(String secUrl, Volume volume, Long maxDownloadSizeInBytes, String
checkSum, String url, ImageFormat format) {
@@ -216,4 +221,26 @@ public class DownloadCommand extends AbstractDownloadCommand implements
Internal
 	public void setResourceType(ResourceType resourceType) {
 		this.resourceType = resourceType;
 	}
+
+
+    public DataStoreTO getDataStore() {
+        return _store;
+    }
+
+
+    public void setDataStore(DataStoreTO _store) {
+        this._store = _store;
+    }
+
+
+    public Long getResourceId() {
+        return resourceId;
+    }
+
+
+    public void setResourceId(Long resourceId) {
+        this.resourceId = resourceId;
+    }
+
+
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/59333756/api/src/com/cloud/agent/api/to/NfsTO.java
----------------------------------------------------------------------
diff --git a/api/src/com/cloud/agent/api/to/NfsTO.java b/api/src/com/cloud/agent/api/to/NfsTO.java
new file mode 100644
index 0000000..5490fd1
--- /dev/null
+++ b/api/src/com/cloud/agent/api/to/NfsTO.java
@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.agent.api.to;
+
+import com.cloud.storage.DataStoreRole;
+
+public final class NfsTO implements DataStoreTO {
+
+    private String _url;
+    private DataStoreRole _role;
+
+    public NfsTO() {
+
+        super();
+
+    }
+
+    public NfsTO(String url, DataStoreRole role) {
+
+        super();
+
+        this._url = url;
+        this._role = role;
+
+    }
+
+    public String getUrl() {
+        return _url;
+    }
+
+    public void setUrl(String _url) {
+        this._url = _url;
+    }
+
+    @Override
+    public DataStoreRole getRole() {
+        return _role;
+    }
+
+    public void setRole(DataStoreRole _role) {
+        this._role = _role;
+    }
+
+
+
+}

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/59333756/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java
----------------------------------------------------------------------
diff --git a/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java b/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java
index a8788c6..776ce29 100755
--- a/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java
+++ b/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java
@@ -34,7 +34,10 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.math.BigInteger;
 import java.net.InetAddress;
+import java.net.MalformedURLException;
 import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
@@ -90,6 +93,8 @@ import com.cloud.agent.api.storage.ListVolumeAnswer;
 import com.cloud.agent.api.storage.ListVolumeCommand;
 import com.cloud.agent.api.storage.UploadCommand;
 import com.cloud.agent.api.storage.ssCommand;
+import com.cloud.agent.api.to.DataStoreTO;
+import com.cloud.agent.api.to.NfsTO;
 import com.cloud.agent.api.to.S3TO;
 import com.cloud.agent.api.to.SwiftTO;
 import com.cloud.exception.InternalErrorException;
@@ -106,6 +111,7 @@ import com.cloud.storage.template.UploadManager;
 import com.cloud.storage.template.UploadManagerImpl;
 import com.cloud.utils.NumbersUtil;
 import com.cloud.utils.S3Utils;
+import com.cloud.utils.UriUtils;
 import com.cloud.utils.S3Utils.FileNamingStrategy;
 import com.cloud.utils.S3Utils.ObjectNamingStrategy;
 import com.cloud.utils.component.ComponentContext;
@@ -126,7 +132,7 @@ SecondaryStorageResource {
 
     int _timeout;
 
-    String _instance;  
+    String _instance;
     String _dc;
     String _pod;
     String _guid;
@@ -162,8 +168,8 @@ SecondaryStorageResource {
         if (cmd instanceof DownloadProgressCommand) {
             return _dlMgr.handleDownloadCommand(this, (DownloadProgressCommand)cmd);
         } else if (cmd instanceof DownloadCommand) {
-            return _dlMgr.handleDownloadCommand(this, (DownloadCommand)cmd);
-        } else if (cmd instanceof UploadCommand) {        	
+            return execute((DownloadCommand)cmd);
+        } else if (cmd instanceof UploadCommand) {
             return _upldMgr.handleUploadCommand(this, (UploadCommand)cmd);
         } else if (cmd instanceof CreateEntityDownloadURLCommand){
             return _upldMgr.handleCreateEntityURLCommand((CreateEntityDownloadURLCommand)cmd);
@@ -325,6 +331,51 @@ SecondaryStorageResource {
         }
     }
 
+    private Answer execute(DownloadCommand cmd){
+        DataStoreTO dstore = cmd.getDataStore();
+        if ( dstore instanceof NfsTO ){
+            return _dlMgr.handleDownloadCommand(this, cmd);
+        }
+        else if ( dstore instanceof S3TO ){
+            //TODO: how to handle download progress for S3
+            S3TO s3 = (S3TO)cmd.getDataStore();
+            String url = cmd.getUrl();
+            String user = null;
+            String password = null;
+            if (cmd.getAuth() != null) {
+                user = cmd.getAuth().getUserName();
+                password = new String(cmd.getAuth().getPassword());
+            }
+            // get input stream from the given url
+            InputStream in = UriUtils.getInputStreamFromUrl(url, user, password);
+            URI uri;
+            URL urlObj;
+            try {
+                uri = new URI(url);
+                urlObj = new URL(url);
+            } catch (URISyntaxException e) {
+                throw new CloudRuntimeException("URI is incorrect: " + url);
+            } catch (MalformedURLException e) {
+                throw new CloudRuntimeException("URL is incorrect: " + url);
+            }
+
+            final String bucket = s3.getBucketName();
+            String key = join(asList(determineS3TemplateDirectory(cmd.getAccountId(), cmd.getResourceId()),
urlObj.getFile()), S3Utils.SEPARATOR);
+            S3Utils.putObject(s3, in, bucket, key);
+            return new Answer(cmd, true, format("Uploaded the contents of input stream from
%1$s for template id %2$s to S3 bucket %3$s", url,
+                    cmd.getResourceId(), bucket));
+        }
+        else if ( dstore instanceof SwiftTO ){
+            //TODO: need to move code from execute(uploadTemplateToSwiftFromSecondaryStorageCommand)
here, but we need to handle
+            // source is url, most likely we need to modify our existing swiftUpload python
script.
+            return new Answer(cmd, false, "Swift is not currently support DownloadCommand");
+        }
+        else{
+            return new Answer(cmd, false, "Unsupport image data store: " + dstore);
+        }
+
+    }
+
     private Answer execute(uploadTemplateToSwiftFromSecondaryStorageCommand cmd) {
         SwiftTO swift = cmd.getSwift();
         String secondaryStorageUrl = cmd.getSecondaryStorageUrl();
@@ -834,33 +885,33 @@ SecondaryStorageResource {
         String absoluteTemplatePath = parent + relativeTemplatePath;
         MessageDigest digest;
         String checksum = null;
-        File f = new File(absoluteTemplatePath);   
+        File f = new File(absoluteTemplatePath);
         InputStream is = null;
         byte[] buffer = new byte[8192];
         int read = 0;
         if(s_logger.isDebugEnabled()){
-            s_logger.debug("parent path " +parent+ " relative template path " +relativeTemplatePath
);   
+            s_logger.debug("parent path " +parent+ " relative template path " +relativeTemplatePath
);
         }
 
 
         try {
-            digest = MessageDigest.getInstance("MD5");           
-            is = new FileInputStream(f);     
+            digest = MessageDigest.getInstance("MD5");
+            is = new FileInputStream(f);
             while( (read = is.read(buffer)) > 0) {
                 digest.update(buffer, 0, read);
-            }       
+            }
             byte[] md5sum = digest.digest();
             BigInteger bigInt = new BigInteger(1, md5sum);
             checksum = bigInt.toString(16);
             if(s_logger.isDebugEnabled()){
-                s_logger.debug("Successfully calculated checksum for file " +absoluteTemplatePath+
" - " +checksum );   
+                s_logger.debug("Successfully calculated checksum for file " +absoluteTemplatePath+
" - " +checksum );
             }
 
         }catch(IOException e) {
             String logMsg = "Unable to process file for MD5 - " + absoluteTemplatePath;
             s_logger.error(logMsg);
-            return new Answer(cmd, false, checksum); 
-        }catch (NoSuchAlgorithmException e) {         
+            return new Answer(cmd, false, checksum);
+        }catch (NoSuchAlgorithmException e) {
             return new Answer(cmd, false, checksum);
         }
         finally {
@@ -869,10 +920,10 @@ SecondaryStorageResource {
                     is.close();
             } catch (IOException e) {
                 if(s_logger.isDebugEnabled()){
-                    s_logger.debug("Could not close the file " +absoluteTemplatePath);  

+                    s_logger.debug("Could not close the file " +absoluteTemplatePath);
                 }
-                return new Answer(cmd, false, checksum);   
-            }                        
+                return new Answer(cmd, false, checksum);
+            }
         }
 
         return new Answer(cmd, true, checksum);
@@ -1141,7 +1192,7 @@ SecondaryStorageResource {
             if (nfsIps.contains(cidr)) {
                 /*
                  * if the internal download ip is the same with secondary storage ip, adding
internal sites will flush
-                 * ip route to nfs through storage ip. 
+                 * ip route to nfs through storage ip.
                  */
                 continue;
             }
@@ -1243,7 +1294,7 @@ SecondaryStorageResource {
 
         for (PortConfig pCfg:cmd.getPortConfigs()){
             if (pCfg.isAdd()) {
-                ipList.add(pCfg.getSourceIp());		
+                ipList.add(pCfg.getSourceIp());
             }
         }
         boolean success = true;
@@ -1401,7 +1452,7 @@ SecondaryStorageResource {
             String nfsPath = nfsHostIp + ":" + uri.getPath();
             String dir = UUID.nameUUIDFromBytes(nfsPath.getBytes()).toString();
             String root = _parent + "/" + dir;
-            mount(root, nfsPath);    
+            mount(root, nfsPath);
             return root;
         } catch (Exception e) {
             String msg = "GetRootDir for " + secUrl + " failed due to " + e.toString();
@@ -1470,7 +1521,7 @@ SecondaryStorageResource {
         String eth2ip = (String) params.get("eth2ip");
         if (eth2ip != null) {
             params.put("public.network.device", "eth2");
-        }         
+        }
         _publicIp = (String) params.get("eth2ip");
         _hostname = (String) params.get("name");
 
@@ -1677,7 +1728,7 @@ SecondaryStorageResource {
         command.add(String.valueOf(isAppend));
         for (String ip : ipList){
             command.add(ip);
-        }		
+        }
 
         String result = command.execute();
         if (result != null) {
@@ -1693,7 +1744,7 @@ SecondaryStorageResource {
                 s_logger.debug("create mount point: " + root);
             } else {
                 s_logger.debug("Unable to create mount point: " + root);
-                return null;       
+                return null;
             }
         }
 
@@ -1813,13 +1864,13 @@ SecondaryStorageResource {
 	@Override
 	public void setName(String name) {
 		// TODO Auto-generated method stub
-		
+
 	}
 
 	@Override
 	public void setConfigParams(Map<String, Object> params) {
 		// TODO Auto-generated method stub
-		
+
 	}
 
 	@Override
@@ -1837,6 +1888,6 @@ SecondaryStorageResource {
 	@Override
 	public void setRunLevel(int level) {
 		// TODO Auto-generated method stub
-		
+
 	}
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/59333756/core/src/com/cloud/storage/template/DownloadManagerImpl.java
----------------------------------------------------------------------
diff --git a/core/src/com/cloud/storage/template/DownloadManagerImpl.java b/core/src/com/cloud/storage/template/DownloadManagerImpl.java
index db264c1..7eb6841 100755
--- a/core/src/com/cloud/storage/template/DownloadManagerImpl.java
+++ b/core/src/com/cloud/storage/template/DownloadManagerImpl.java
@@ -16,9 +16,15 @@
 // under the License.
 package com.cloud.storage.template;
 
+import static com.cloud.utils.S3Utils.putDirectory;
+import static com.cloud.utils.StringUtils.join;
+import static java.lang.String.format;
+import static java.util.Arrays.asList;
+
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FilenameFilter;
 import java.io.IOException;
 import java.io.InputStream;
 import java.math.BigInteger;
@@ -41,14 +47,21 @@ import java.util.concurrent.Executors;
 import javax.ejb.Local;
 import javax.naming.ConfigurationException;
 
+import org.apache.commons.httpclient.Credentials;
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
+import org.apache.commons.httpclient.UsernamePasswordCredentials;
+import org.apache.commons.httpclient.auth.AuthScope;
 import org.apache.log4j.Logger;
 
+import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.storage.DownloadAnswer;
 import com.cloud.agent.api.storage.DownloadCommand;
 import com.cloud.agent.api.storage.DownloadCommand.Proxy;
 import com.cloud.agent.api.storage.DownloadCommand.ResourceType;
 import com.cloud.agent.api.storage.DownloadProgressCommand;
 import com.cloud.agent.api.storage.DownloadProgressCommand.RequestType;
+import com.cloud.agent.api.to.S3TO;
 import com.cloud.exception.InternalErrorException;
 import com.cloud.storage.Storage.ImageFormat;
 import com.cloud.storage.StorageLayer;
@@ -59,6 +72,9 @@ import com.cloud.storage.template.Processor.FormatInfo;
 import com.cloud.storage.template.TemplateDownloader.DownloadCompleteCallback;
 import com.cloud.storage.template.TemplateDownloader.Status;
 import com.cloud.utils.NumbersUtil;
+import com.cloud.utils.S3Utils;
+import com.cloud.utils.UriUtils;
+import com.cloud.utils.S3Utils.ObjectNamingStrategy;
 import com.cloud.utils.component.ManagerBase;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.script.OutputInterpreter;
@@ -224,7 +240,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
 
     /**
      * Get notified of change of job status. Executed in context of downloader thread
-     * 
+     *
      * @param jobId
      *            the id of the job
      * @param status
@@ -279,21 +295,21 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
         byte[] buffer = new byte[8192];
         int read = 0;
         MessageDigest digest;
-        String checksum = null;        
+        String checksum = null;
         InputStream is = null;
         try {
-            digest = MessageDigest.getInstance("MD5");           
-            is = new FileInputStream(f);     
+            digest = MessageDigest.getInstance("MD5");
+            is = new FileInputStream(f);
             while( (read = is.read(buffer)) > 0) {
                 digest.update(buffer, 0, read);
-            }       
+            }
             byte[] md5sum = digest.digest();
             BigInteger bigInt = new BigInteger(1, md5sum);
             checksum = String.format("%032x",bigInt);
             return checksum;
         }catch(IOException e) {
             return null;
-        }catch (NoSuchAlgorithmException e) {         
+        }catch (NoSuchAlgorithmException e) {
             return null;
         }
         finally {
@@ -302,19 +318,19 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
                     is.close();
             } catch (IOException e) {
                 return null;
-            }                        
+            }
         }
     }
 
     /**
      * Post download activity (install and cleanup). Executed in context of downloader thread
-     * 
+     *
      * @throws IOException
      */
     private String postDownload(String jobId) {
         DownloadJob dnld = jobs.get(jobId);
         TemplateDownloader td = dnld.getTemplateDownloader();
-        String resourcePath = null;               
+        String resourcePath = null;
         ResourceType resourceType = dnld.getResourceType();
 
         // once template path is set, remove the parent dir so that the template is installed
with a relative path
@@ -458,7 +474,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
                 return "Unable to create " + tmpDir;
             }
             //	TO DO - define constant for volume properties.
-            File file = ResourceType.TEMPLATE == resourceType ? _storage.getFile(tmpDir +
File.separator + TemplateLocation.Filename) : 
+            File file = ResourceType.TEMPLATE == resourceType ? _storage.getFile(tmpDir +
File.separator + TemplateLocation.Filename) :
                 _storage.getFile(tmpDir + File.separator + "volume.properties");
             if ( file.exists() ) {
                 file.delete();
@@ -583,6 +599,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
         return convertStatus(getDownloadStatus(jobId));
     }
 
+
     @Override
     public DownloadAnswer handleDownloadCommand(SecondaryStorageResource resource, DownloadCommand
cmd) {
         ResourceType resourceType = cmd.getResourceType();
@@ -774,7 +791,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
     }
 
     @Override
-    public Map<Long, TemplateProp> gatherVolumeInfo(String rootDir) {	
+    public Map<Long, TemplateProp> gatherVolumeInfo(String rootDir) {
         Map<Long, TemplateProp> result = new HashMap<Long, TemplateProp>();
         String volumeDir = rootDir + File.separator + _volumeDir;
 
@@ -1046,4 +1063,5 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
         }
     }
 
+
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/59333756/core/test/com/cloud/agent/transport/RequestTest.java
----------------------------------------------------------------------
diff --git a/core/test/com/cloud/agent/transport/RequestTest.java b/core/test/com/cloud/agent/transport/RequestTest.java
index 64c1e0b..048bd21 100644
--- a/core/test/com/cloud/agent/transport/RequestTest.java
+++ b/core/test/com/cloud/agent/transport/RequestTest.java
@@ -31,19 +31,21 @@ import com.cloud.agent.api.SecStorageFirewallCfgCommand;
 import com.cloud.agent.api.UpdateHostPasswordCommand;
 import com.cloud.agent.api.storage.DownloadAnswer;
 import com.cloud.agent.api.storage.DownloadCommand;
+import com.cloud.agent.api.to.NfsTO;
 import com.cloud.exception.UnsupportedVersionException;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.serializer.GsonHelper;
 import com.cloud.storage.Storage.ImageFormat;
 import com.cloud.storage.Storage.TemplateType;
 import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
+import com.cloud.storage.DataStoreRole;
 import com.cloud.storage.VMTemplateVO;
 
 /**
- * 
- * 
- * 
- * 
+ *
+ *
+ *
+ *
  */
 
 public class RequestTest extends TestCase {
@@ -51,7 +53,7 @@ public class RequestTest extends TestCase {
 
     public void testSerDeser() {
         s_logger.info("Testing serializing and deserializing works as expected");
-        
+
         s_logger.info("UpdateHostPasswordCommand should have two parameters that doesn't
show in logging");
         UpdateHostPasswordCommand cmd1 = new UpdateHostPasswordCommand("abc", "def");
         s_logger.info("SecStorageFirewallCfgCommand has a context map that shouldn't show
up in debug level");
@@ -89,7 +91,7 @@ public class RequestTest extends TestCase {
         logger.setLevel(level);
 
         byte[] bytes = sreq.getBytes();
-        
+
         assert Request.getSequence(bytes) == 892403717;
         assert Request.getManagementServerId(bytes) == 3;
         assert Request.getAgentId(bytes) == 2;
@@ -130,7 +132,7 @@ public class RequestTest extends TestCase {
         s_logger.info("Testing Download answer");
         VMTemplateVO template = new VMTemplateVO(1, "templatename", ImageFormat.QCOW2, true,
true, true, TemplateType.USER, "url", true, 32, 1, "chksum", "displayText", true, 30, true,
                 HypervisorType.KVM, null);
-        DownloadCommand cmd = new DownloadCommand("secUrl", template, 30000000l);
+        DownloadCommand cmd = new DownloadCommand(new NfsTO("secUrl", DataStoreRole.Image),
"secUrl", template, 30000000l);
         Request req = new Request(1, 1, cmd, true);
 
         req.logD("Debug for Download");
@@ -161,7 +163,7 @@ public class RequestTest extends TestCase {
             }
         }
     }
-    
+
     public void testLogging() {
         s_logger.info("Testing Logging");
         GetHostStatsCommand cmd3 = new GetHostStatsCommand("hostguid", "hostname", 101);

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/59333756/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java
----------------------------------------------------------------------
diff --git a/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java
b/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java
index efc2f99..e72bb54 100644
--- a/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java
+++ b/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java
@@ -28,7 +28,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionStrategy;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectType;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreRole;
+
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
@@ -65,6 +65,7 @@ import com.cloud.configuration.dao.ConfigurationDao;
 import com.cloud.exception.StorageUnavailableException;
 import com.cloud.host.HostVO;
 import com.cloud.host.dao.HostDao;
+import com.cloud.storage.DataStoreRole;
 import com.cloud.storage.DiskOfferingVO;
 import com.cloud.storage.SnapshotVO;
 import com.cloud.storage.Storage.ImageFormat;
@@ -130,7 +131,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
     VolumeManager volumeMgr;
     @Inject
     private SwiftManager _swiftMgr;
-    @Inject 
+    @Inject
     private S3Manager _s3Mgr;
     @Inject
     StorageCacheManager cacheMgr;
@@ -314,12 +315,12 @@ public class AncientDataMotionStrategy implements DataMotionStrategy
{
             }
         }
     }
-    
+
     protected Answer cloneVolume(DataObject template, DataObject volume) {
         VolumeInfo volInfo = (VolumeInfo)volume;
         DiskOfferingVO offering = diskOfferingDao.findById(volInfo.getDiskOfferingId());
         VMTemplateStoragePoolVO  tmpltStoredOn =  templatePoolDao.findByPoolTemplate(template.getDataStore().getId(),
template.getId());
-        
+
         DiskProfile diskProfile = new DiskProfile(volInfo, offering,
                 null);
         CreateCommand cmd = new CreateCommand(diskProfile,
@@ -334,7 +335,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
             s_logger.debug("Failed to send to storage pool", e);
             throw new CloudRuntimeException("Failed to send to storage pool", e);
         }
-        
+
         if (answer.getResult()) {
             VolumeVO vol = this.volDao.findById(volume.getId());
             CreateAnswer createAnswer = (CreateAnswer) answer;
@@ -345,7 +346,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
             vol.setPoolId(pool.getId());
             vol.setPodId(pool.getPodId());
             this.volDao.update(vol.getId(), vol);
-           
+
         } else {
             if (tmpltStoredOn != null
                     && (answer instanceof CreateAnswer)
@@ -354,15 +355,15 @@ public class AncientDataMotionStrategy implements DataMotionStrategy
{
                 if (!templateMgr
                         .resetTemplateDownloadStateOnPool(tmpltStoredOn
                                 .getId())) {
-                   
+
                 }
             }
             errMsg = answer.getDetails();
         }
-        
+
         return answer;
     }
-    
+
     protected Answer copyVolumeBetweenPools(DataObject srcData, DataObject destData) {
         VolumeInfo volume = (VolumeInfo)srcData;
         VolumeInfo destVolume = (VolumeInfo)destData;
@@ -370,9 +371,9 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
                 .getDataCenterId());
         StoragePool srcPool = (StoragePool)this.dataStoreMgr.getDataStore(volume
                 .getPoolId(), DataStoreRole.Primary);
-        
+
         StoragePool destPool = (StoragePool)this.dataStoreMgr.getDataStore(destVolume.getPoolId(),
DataStoreRole.Primary);
-        
+
         String value = this.configDao.getValue(Config.CopyVolumeWait.toString());
         int _copyvolumewait = NumbersUtil.parseInt(value,
                 Integer.parseInt(Config.CopyVolumeWait.getDefaultValue()));
@@ -394,7 +395,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
         }
 
         String secondaryStorageVolumePath = cvAnswer.getVolumePath();
-        
+
         cvCmd = new CopyVolumeCommand(volume.getId(),
                 secondaryStorageVolumePath, destPool,
                 secondaryStorageURL, false, _copyvolumewait);
@@ -409,7 +410,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
             throw new CloudRuntimeException(
                     "Failed to copy the volume from secondary storage to the destination
primary storage pool.");
         }
-        
+
         VolumeVO destVol = this.volDao.findById(destVolume.getId());
         destVol.setPath(cvAnswer.getVolumePath());
         this.volDao.update(destVol.getId(), destVol);
@@ -437,7 +438,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
             } else if (srcData.getType() == DataObjectType.VOLUME
                     && destData.getType() == DataObjectType.TEMPLATE) {
             	answer = createTemplateFromVolume(srcData, destData);
-            } else if (srcData.getType() == DataObjectType.TEMPLATE 
+            } else if (srcData.getType() == DataObjectType.TEMPLATE
                     && destData.getType() == DataObjectType.VOLUME) {
             	answer = cloneVolume(srcData, destData);
             } else if (destData.getType() == DataObjectType.VOLUME
@@ -667,21 +668,21 @@ public class AncientDataMotionStrategy implements DataMotionStrategy
{
         return sendCommand(cmd, pool, template.getId(), zoneId,
                 secondaryStorageHost.getId());
     }
-    
+
     private HostVO getSecHost(long volumeId, long dcId) {
         Long id = snapshotDao.getSecHostId(volumeId);
-        if ( id != null) { 
+        if ( id != null) {
             return hostDao.findById(id);
         }
         return this.templateMgr.getSecondaryStorageHost(dcId);
     }
-    
+
     protected Answer copySnapshot(DataObject srcObject, DataObject destObject) {
     	SnapshotInfo srcSnapshot = (SnapshotInfo)srcObject;
     	VolumeInfo baseVolume = srcSnapshot.getBaseVolume();
     	 Long dcId = baseVolume.getDataCenterId();
          Long accountId = baseVolume.getAccountId();
-         
+
          HostVO secHost = getSecHost(baseVolume.getId(), baseVolume.getDataCenterId());
          Long secHostId = secHost.getId();
          String secondaryStoragePoolUrl = secHost.getStorageUrl();
@@ -696,12 +697,12 @@ public class AncientDataMotionStrategy implements DataMotionStrategy
{
 
          SwiftTO swift = _swiftMgr.getSwiftTO();
          S3TO s3 = _s3Mgr.getS3TO();
-         
+
          long prevSnapshotId = srcSnapshot.getPrevSnapshotId();
          if (prevSnapshotId > 0) {
              prevSnapshot = snapshotDao.findByIdIncludingRemoved(prevSnapshotId);
              if ( prevSnapshot.getBackupSnapshotId() != null && swift == null) {
-                 if (prevSnapshot.getVersion() != null && prevSnapshot.getVersion().equals("2.2"))
{                   
+                 if (prevSnapshot.getVersion() != null && prevSnapshot.getVersion().equals("2.2"))
{
                      prevBackupUuid = prevSnapshot.getBackupSnapshotId();
                      prevSnapshotUuid = prevSnapshot.getPath();
                  }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/59333756/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java
b/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java
index 4989b26..bc8d2e2 100644
--- a/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java
+++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java
@@ -175,7 +175,7 @@ public class ImageStoreImpl implements ImageStoreEntity {
 
     @Override
     public DataStoreTO getTO() {
-        return null;
+        return getDriver().getStoreTO(this);
     }
 
 

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/59333756/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java
b/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java
index 0feefd0..8be1c8a 100644
--- a/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java
+++ b/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java
@@ -83,10 +83,10 @@ public class PrimaryDataStoreImpl implements PrimaryDataStore {
 
     private VolumeDao volumeDao;
 
-    public	 PrimaryDataStoreImpl() {				
-       
+    public	 PrimaryDataStoreImpl() {
+
     }
-    
+
     public void configure(StoragePoolVO pdsv,
             PrimaryDataStoreDriver driver, DataStoreProvider provider) {
         this.pdsv = pdsv;
@@ -245,9 +245,9 @@ public class PrimaryDataStoreImpl implements PrimaryDataStore {
                     }
                 }
             }
-            
+
         }
-        
+
         return objectInStoreMgr.get(obj, this);
     }
 
@@ -341,9 +341,9 @@ public class PrimaryDataStoreImpl implements PrimaryDataStore {
     public String getStorageProviderName() {
         return this.pdsv.getStorageProviderName();
     }
-    
+
     @Override
     public DataStoreTO getTO() {
-        return null;
+        return getDriver().getStoreTO(this);
     }
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/59333756/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
----------------------------------------------------------------------
diff --git a/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
b/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
index af6a3b6..f902385 100644
--- a/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
+++ b/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
@@ -46,6 +46,7 @@ import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.DeleteSnapshotBackupCommand;
 import com.cloud.agent.api.storage.DeleteVolumeCommand;
 import com.cloud.agent.api.to.DataStoreTO;
+import com.cloud.agent.api.to.NfsTO;
 import com.cloud.agent.api.to.S3TO;
 import com.cloud.agent.api.to.SwiftTO;
 import com.cloud.host.HostVO;
@@ -106,10 +107,9 @@ public class CloudStackImageStoreDriverImpl implements ImageStoreDriver
{
     @Override
     public DataStoreTO getStoreTO(DataStore store) {
         ImageStoreImpl nfsStore = (ImageStoreImpl)store;
-        ImageStoreTO nfsTO = new ImageStoreTO();
-        nfsTO.setProviderName("CloudStack");
+        NfsTO nfsTO = new NfsTO();
         nfsTO.setRole(DataStoreRole.Image);
-        nfsTO.setUri(nfsStore.getUri());
+        nfsTO.setUrl(nfsStore.getUri());
         return nfsTO;
     }
 

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/59333756/server/src/com/cloud/storage/download/DownloadMonitorImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/storage/download/DownloadMonitorImpl.java b/server/src/com/cloud/storage/download/DownloadMonitorImpl.java
index 507d575..58477b4 100755
--- a/server/src/com/cloud/storage/download/DownloadMonitorImpl.java
+++ b/server/src/com/cloud/storage/download/DownloadMonitorImpl.java
@@ -366,7 +366,7 @@ public class DownloadMonitorImpl extends ManagerBase implements  DownloadMonitor
 		if(vmTemplateStore != null) {
 		    start();
 			DownloadCommand dcmd =
-             new DownloadCommand(secUrl, template, maxTemplateSizeInBytes);
+             new DownloadCommand(store.getTO(), secUrl, template, maxTemplateSizeInBytes);
 			dcmd.setProxy(getHttpProxy());
 	        if (downloadJobExists) {
 	            dcmd = new DownloadProgressCommand(dcmd, vmTemplateStore.getJobId(), RequestType.GET_OR_RESTART);

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/59333756/utils/src/com/cloud/utils/S3Utils.java
----------------------------------------------------------------------
diff --git a/utils/src/com/cloud/utils/S3Utils.java b/utils/src/com/cloud/utils/S3Utils.java
index b7273a1..423ea71 100644
--- a/utils/src/com/cloud/utils/S3Utils.java
+++ b/utils/src/com/cloud/utils/S3Utils.java
@@ -138,6 +138,24 @@ public final class S3Utils {
 
     }
 
+    public static void putObject(final ClientOptions clientOptions,
+            final InputStream sourceStream, final String bucketName, final String key) {
+
+        assert clientOptions != null;
+        assert sourceStream != null;
+        assert !isBlank(bucketName);
+        assert !isBlank(key);
+
+        if (LOGGER.isDebugEnabled()) {
+            LOGGER.debug(format("Sending stream as S3 object %1$s in "
+                    + "bucket %2$s", key, bucketName));
+        }
+
+        acquireClient(clientOptions).putObject(bucketName, key, sourceStream, null);
+
+    }
+
+
     @SuppressWarnings("unchecked")
     public static File getFile(final ClientOptions clientOptions,
             final String bucketName, final String key,
@@ -239,6 +257,7 @@ public final class S3Utils {
 
     }
 
+
     public static void putDirectory(final ClientOptions clientOptions,
             final String bucketName, final File directory,
             final FilenameFilter fileNameFilter,
@@ -284,6 +303,8 @@ public final class S3Utils {
 
     }
 
+
+
     public static void deleteObject(final ClientOptions clientOptions,
             final String bucketName, final String key) {
 

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/59333756/utils/src/com/cloud/utils/UriUtils.java
----------------------------------------------------------------------
diff --git a/utils/src/com/cloud/utils/UriUtils.java b/utils/src/com/cloud/utils/UriUtils.java
index 3bcee7a..ef1aba2 100644
--- a/utils/src/com/cloud/utils/UriUtils.java
+++ b/utils/src/com/cloud/utils/UriUtils.java
@@ -18,16 +18,34 @@ package com.cloud.utils;
 
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.HttpURLConnection;
+import java.net.Inet6Address;
+import java.net.InetAddress;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URLEncoder;
+import java.net.UnknownHostException;
 
 import javax.net.ssl.HttpsURLConnection;
 
+import org.apache.commons.httpclient.Credentials;
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.HttpStatus;
+import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
+import org.apache.commons.httpclient.UsernamePasswordCredentials;
+import org.apache.commons.httpclient.auth.AuthScope;
+import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.log4j.Logger;
+
 import com.cloud.utils.exception.CloudRuntimeException;
 
 public class UriUtils {
+
+    public static final Logger s_logger = Logger.getLogger(UriUtils.class.getName());
+
     public static String formNfsUri(String host, String path) {
         try {
             URI uri = new URI("nfs", host, path, null);
@@ -111,4 +129,67 @@ public class UriUtils {
         }
         return remoteSize;
     }
+
+    public static  Pair<String, Integer> validateUrl(String url) throws IllegalArgumentException
{
+        try {
+            URI uri = new URI(url);
+            if (!uri.getScheme().equalsIgnoreCase("http") && !uri.getScheme().equalsIgnoreCase("https")
) {
+                throw new IllegalArgumentException("Unsupported scheme for url");
+            }
+            int port = uri.getPort();
+            if (!(port == 80 || port == 443 || port == -1)) {
+                throw new IllegalArgumentException("Only ports 80 and 443 are allowed");
+            }
+
+            if (port == -1 && uri.getScheme().equalsIgnoreCase("https")) {
+                port = 443;
+            } else if (port == -1 && uri.getScheme().equalsIgnoreCase("http")) {
+                port = 80;
+            }
+
+            String host = uri.getHost();
+            try {
+                InetAddress hostAddr = InetAddress.getByName(host);
+                if (hostAddr.isAnyLocalAddress() || hostAddr.isLinkLocalAddress() || hostAddr.isLoopbackAddress()
|| hostAddr.isMulticastAddress()) {
+                    throw new IllegalArgumentException("Illegal host specified in url");
+                }
+                if (hostAddr instanceof Inet6Address) {
+                    throw new IllegalArgumentException("IPV6 addresses not supported (" +
hostAddr.getHostAddress() + ")");
+                }
+                return new Pair<String, Integer>(host, port);
+            } catch (UnknownHostException uhe) {
+                throw new IllegalArgumentException("Unable to resolve " + host);
+            }
+        } catch (URISyntaxException use) {
+            throw new IllegalArgumentException("Invalid URL: " + url);
+        }
+    }
+
+    public static InputStream getInputStreamFromUrl(String url, String user, String password)
{
+
+        try{
+          Pair<String, Integer> hostAndPort = validateUrl(url);
+          HttpClient httpclient = new HttpClient(new MultiThreadedHttpConnectionManager());
+          if ((user != null) && (password != null)) {
+              httpclient.getParams().setAuthenticationPreemptive(true);
+              Credentials defaultcreds = new UsernamePasswordCredentials(user, password);
+              httpclient.getState().setCredentials(new AuthScope(hostAndPort.first(), hostAndPort.second(),
AuthScope.ANY_REALM), defaultcreds);
+              s_logger.info("Added username=" + user + ", password=" + password + "for host
" + hostAndPort.first() + ":" + hostAndPort.second());
+          }
+          // Execute the method.
+          GetMethod method = new GetMethod(url);
+          int statusCode = httpclient.executeMethod(method);
+
+          if (statusCode != HttpStatus.SC_OK) {
+            s_logger.error("Failed to read from URL: " + url);
+            return null;
+          }
+
+          return method.getResponseBodyAsStream();
+        }
+        catch (Exception ex){
+            s_logger.error("Failed to read from URL: " + url);
+            return null;
+        }
+    }
 }


Mime
View raw message