Return-Path: X-Original-To: apmail-cloudstack-commits-archive@www.apache.org Delivered-To: apmail-cloudstack-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id D68D810F76 for ; Wed, 25 Sep 2013 10:43:38 +0000 (UTC) Received: (qmail 90250 invoked by uid 500); 25 Sep 2013 10:30:23 -0000 Delivered-To: apmail-cloudstack-commits-archive@cloudstack.apache.org Received: (qmail 89843 invoked by uid 500); 25 Sep 2013 10:29:33 -0000 Mailing-List: contact commits-help@cloudstack.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@cloudstack.apache.org Delivered-To: mailing list commits@cloudstack.apache.org Received: (qmail 89380 invoked by uid 99); 25 Sep 2013 10:28:58 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 25 Sep 2013 10:28:58 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id B496F90988E; Wed, 25 Sep 2013 10:28:57 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: widodh@apache.org To: commits@cloudstack.apache.org Date: Wed, 25 Sep 2013 10:29:33 -0000 Message-Id: <1ea3fd60eea9478b8efa57e099e02b9b@git.apache.org> In-Reply-To: <74771eb5b1e24edb9ecf13cb5701714f@git.apache.org> References: <74771eb5b1e24edb9ecf13cb5701714f@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [38/59] [abbrv] git commit: updated refs/heads/disk-cache to bbae6f8 Check all warnings in core and fix where applicable, also apply autoformatting on save. Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/4945d465 Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/4945d465 Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/4945d465 Branch: refs/heads/disk-cache Commit: 4945d465bff3d4941821124578fc9460932c744a Parents: bb8b734 Author: Hugo Trippaers Authored: Sat Sep 21 16:15:36 2013 +0800 Committer: Hugo Trippaers Committed: Sat Sep 21 16:15:36 2013 +0800 ---------------------------------------------------------------------- .../agent/api/MigrateWithStorageAnswer.java | 3 - .../api/MigrateWithStorageCompleteAnswer.java | 2 - core/src/com/cloud/agent/api/RebootCommand.java | 1 - .../api/routing/HealthCheckLBConfigAnswer.java | 1 - .../api/routing/HealthCheckLBConfigCommand.java | 1 - .../agent/api/routing/SetNetworkACLCommand.java | 2 - .../cloud/agent/api/storage/UploadCommand.java | 23 +- .../virtualnetwork/VirtualRoutingResource.java | 1 - core/src/com/cloud/agent/transport/Request.java | 47 +- .../cloud/exception/UsageServerException.java | 19 +- .../src/com/cloud/storage/JavaStorageLayer.java | 6 +- .../storage/resource/StoragePoolResource.java | 2 - .../storage/template/FtpTemplateUploader.java | 381 ++++++----- .../template/HttpTemplateDownloader.java | 643 ++++++++++--------- .../template/LocalTemplateDownloader.java | 57 +- .../storage/template/RawImageProcessor.java | 3 +- .../storage/template/S3TemplateDownloader.java | 28 +- .../storage/template/ScpTemplateDownloader.java | 4 +- .../storage/template/TemplateUploader.java | 3 - .../storage/command/DownloadCommand.java | 47 +- .../agent/test/BackupSnapshotCommandTest.java | 1 - 21 files changed, 640 insertions(+), 635 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4945d465/core/src/com/cloud/agent/api/MigrateWithStorageAnswer.java ---------------------------------------------------------------------- diff --git a/core/src/com/cloud/agent/api/MigrateWithStorageAnswer.java b/core/src/com/cloud/agent/api/MigrateWithStorageAnswer.java index d87a5f1..6468884 100644 --- a/core/src/com/cloud/agent/api/MigrateWithStorageAnswer.java +++ b/core/src/com/cloud/agent/api/MigrateWithStorageAnswer.java @@ -20,9 +20,6 @@ import java.util.List; import org.apache.cloudstack.storage.to.VolumeObjectTO; -import com.cloud.agent.api.to.DiskTO; -import com.cloud.agent.api.to.VolumeTO; - public class MigrateWithStorageAnswer extends Answer { List volumeTos; http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4945d465/core/src/com/cloud/agent/api/MigrateWithStorageCompleteAnswer.java ---------------------------------------------------------------------- diff --git a/core/src/com/cloud/agent/api/MigrateWithStorageCompleteAnswer.java b/core/src/com/cloud/agent/api/MigrateWithStorageCompleteAnswer.java index fd8f22f..ec8bd0f 100644 --- a/core/src/com/cloud/agent/api/MigrateWithStorageCompleteAnswer.java +++ b/core/src/com/cloud/agent/api/MigrateWithStorageCompleteAnswer.java @@ -20,8 +20,6 @@ import java.util.List; import org.apache.cloudstack.storage.to.VolumeObjectTO; -import com.cloud.agent.api.to.VolumeTO; - public class MigrateWithStorageCompleteAnswer extends Answer { List volumeTos; http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4945d465/core/src/com/cloud/agent/api/RebootCommand.java ---------------------------------------------------------------------- diff --git a/core/src/com/cloud/agent/api/RebootCommand.java b/core/src/com/cloud/agent/api/RebootCommand.java index 49712b6..299e61b 100755 --- a/core/src/com/cloud/agent/api/RebootCommand.java +++ b/core/src/com/cloud/agent/api/RebootCommand.java @@ -16,7 +16,6 @@ // under the License. package com.cloud.agent.api; -import com.cloud.hypervisor.Hypervisor; import com.cloud.vm.VirtualMachine; public class RebootCommand extends Command { http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4945d465/core/src/com/cloud/agent/api/routing/HealthCheckLBConfigAnswer.java ---------------------------------------------------------------------- diff --git a/core/src/com/cloud/agent/api/routing/HealthCheckLBConfigAnswer.java b/core/src/com/cloud/agent/api/routing/HealthCheckLBConfigAnswer.java index dfca4ab..ee8033a 100644 --- a/core/src/com/cloud/agent/api/routing/HealthCheckLBConfigAnswer.java +++ b/core/src/com/cloud/agent/api/routing/HealthCheckLBConfigAnswer.java @@ -20,7 +20,6 @@ import java.util.List; import com.cloud.agent.api.Answer; import com.cloud.agent.api.to.LoadBalancerTO; -import com.cloud.agent.api.to.NicTO; /** * LoadBalancerConfigCommand sends the load balancer configuration http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4945d465/core/src/com/cloud/agent/api/routing/HealthCheckLBConfigCommand.java ---------------------------------------------------------------------- diff --git a/core/src/com/cloud/agent/api/routing/HealthCheckLBConfigCommand.java b/core/src/com/cloud/agent/api/routing/HealthCheckLBConfigCommand.java index f705f6c..7206d2f 100644 --- a/core/src/com/cloud/agent/api/routing/HealthCheckLBConfigCommand.java +++ b/core/src/com/cloud/agent/api/routing/HealthCheckLBConfigCommand.java @@ -17,7 +17,6 @@ package com.cloud.agent.api.routing; import com.cloud.agent.api.to.LoadBalancerTO; -import com.cloud.agent.api.to.NicTO; /** * LoadBalancerConfigCommand sends the load balancer configuration http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4945d465/core/src/com/cloud/agent/api/routing/SetNetworkACLCommand.java ---------------------------------------------------------------------- diff --git a/core/src/com/cloud/agent/api/routing/SetNetworkACLCommand.java b/core/src/com/cloud/agent/api/routing/SetNetworkACLCommand.java index 236e8ea..ba4b4b4 100644 --- a/core/src/com/cloud/agent/api/routing/SetNetworkACLCommand.java +++ b/core/src/com/cloud/agent/api/routing/SetNetworkACLCommand.java @@ -20,9 +20,7 @@ package com.cloud.agent.api.routing; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; -import java.util.HashSet; import java.util.List; -import java.util.Set; import com.cloud.agent.api.to.NetworkACLTO; import com.cloud.agent.api.to.NicTO; http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4945d465/core/src/com/cloud/agent/api/storage/UploadCommand.java ---------------------------------------------------------------------- diff --git a/core/src/com/cloud/agent/api/storage/UploadCommand.java b/core/src/com/cloud/agent/api/storage/UploadCommand.java index 9b893e2..98eebe4 100644 --- a/core/src/com/cloud/agent/api/storage/UploadCommand.java +++ b/core/src/com/cloud/agent/api/storage/UploadCommand.java @@ -41,30 +41,30 @@ public class UploadCommand extends AbstractUploadCommand implements InternalIden this.template = new TemplateTO(template); this.url = url; this.installPath = installPath; - this.checksum = template.getChecksum(); - this.id = template.getId(); - this.templateSizeInBytes = sizeInBytes; + checksum = template.getChecksum(); + id = template.getId(); + templateSizeInBytes = sizeInBytes; } public UploadCommand(String url, long id, long sizeInBytes, String installPath, Type type){ - this.template = null; + template = null; this.url = url; this.installPath = installPath; this.id = id; this.type = type; - this.templateSizeInBytes = sizeInBytes; + templateSizeInBytes = sizeInBytes; } protected UploadCommand() { } public UploadCommand(UploadCommand that) { - this.template = that.template; - this.url = that.url; - this.installPath = that.installPath; - this.checksum = that.getChecksum(); - this.id = that.id; + template = that.template; + url = that.url; + installPath = that.installPath; + checksum = that.getChecksum(); + id = that.id; } public String getDescription() { @@ -114,7 +114,8 @@ public class UploadCommand extends AbstractUploadCommand implements InternalIden this.templateSizeInBytes = templateSizeInBytes; } - public long getId() { + @Override + public long getId() { return id; } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4945d465/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java ---------------------------------------------------------------------- diff --git a/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java b/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java index 9e6216f..0592a9b 100755 --- a/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java +++ b/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java @@ -108,7 +108,6 @@ public class VirtualRoutingResource implements Manager { private String _firewallPath; private String _loadbPath; private String _dhcpEntryPath; - private String _vmDataPath; private String _publicEthIf; private String _privateEthIf; private String _bumpUpPriorityPath; http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4945d465/core/src/com/cloud/agent/transport/Request.java ---------------------------------------------------------------------- diff --git a/core/src/com/cloud/agent/transport/Request.java b/core/src/com/cloud/agent/transport/Request.java index b0fa4cc..31076c0 100755 --- a/core/src/com/cloud/agent/transport/Request.java +++ b/core/src/com/cloud/agent/transport/Request.java @@ -31,14 +31,6 @@ import java.util.zip.GZIPOutputStream; import org.apache.log4j.Level; import org.apache.log4j.Logger; -import com.cloud.agent.api.Answer; -import com.cloud.agent.api.Command; -import com.cloud.agent.api.SecStorageFirewallCfgCommand.PortConfig; -import com.cloud.exception.UnsupportedVersionException; -import com.cloud.serializer.GsonHelper; -import com.cloud.utils.NumbersUtil; -import com.cloud.utils.Pair; -import com.cloud.utils.exception.CloudRuntimeException; import com.google.gson.Gson; import com.google.gson.JsonArray; import com.google.gson.JsonDeserializationContext; @@ -50,6 +42,15 @@ import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; import com.google.gson.stream.JsonReader; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.Command; +import com.cloud.agent.api.SecStorageFirewallCfgCommand.PortConfig; +import com.cloud.exception.UnsupportedVersionException; +import com.cloud.serializer.GsonHelper; +import com.cloud.utils.NumbersUtil; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; + /** * Request is a simple wrapper around command and answer to add sequencing, * versioning, and flags. Note that the version here represents the changes @@ -107,7 +108,7 @@ public class Request { protected long _agentId; protected Command[] _cmds; protected String _content; - + protected Request() { } @@ -158,14 +159,14 @@ public class Request { } protected Request(final Request that, final Command[] cmds) { - this._ver = that._ver; - this._seq = that._seq; + _ver = that._ver; + _seq = that._seq; setInSequence(that.executeInSequence()); setStopOnError(that.stopOnError()); - this._cmds = cmds; - this._mgmtId = that._mgmtId; - this._via = that._via; - this._agentId = that._agentId; + _cmds = cmds; + _mgmtId = that._mgmtId; + _via = that._via; + _agentId = that._agentId; setFromServer(!that.isFromServer()); } @@ -287,7 +288,7 @@ public class Request { retBuff.flip(); return retBuff; } - + public static ByteBuffer doCompress(ByteBuffer buffer, int length) { ByteArrayOutputStream byteOut = new ByteArrayOutputStream(length); byte[] array; @@ -307,11 +308,11 @@ public class Request { } return ByteBuffer.wrap(byteOut.toByteArray()); } - + public ByteBuffer[] toBytes() { final ByteBuffer[] buffers = new ByteBuffer[2]; ByteBuffer tmp; - + if (_content == null) { _content = s_gson.toJson(_cmds, _cmds.getClass()); } @@ -372,7 +373,7 @@ public class Request { } } } - + @Override public String toString() { return log("", true, Level.DEBUG); @@ -447,7 +448,7 @@ public class Request { if (version.ordinal() != Version.v1.ordinal() && version.ordinal() != Version.v3.ordinal()) { throw new UnsupportedVersionException("This version is no longer supported: " + version.toString(), UnsupportedVersionException.IncompatibleVersion); } - final byte reserved = buff.get(); // tossed away for now. + buff.get(); final short flags = buff.getShort(); final boolean isRequest = (flags & FLAG_REQUEST) > 0; @@ -456,7 +457,7 @@ public class Request { final int size = buff.getInt(); final long mgmtId = buff.getLong(); final long agentId = buff.getLong(); - + long via; if (version.ordinal() == Version.v1.ordinal()) { via = buff.getLong(); @@ -467,7 +468,7 @@ public class Request { if ((flags & FLAG_COMPRESSED) != 0) { buff = doDecompress(buff, size); } - + byte[] command = null; int offset = 0; if (buff.hasArray()) { @@ -519,7 +520,7 @@ public class Request { public static long getViaAgentId(final byte[] bytes) { return NumbersUtil.bytesToLong(bytes, 32); } - + public static boolean fromServer(final byte[] bytes) { return (bytes[3] & FLAG_FROM_SERVER) > 0; } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4945d465/core/src/com/cloud/exception/UsageServerException.java ---------------------------------------------------------------------- diff --git a/core/src/com/cloud/exception/UsageServerException.java b/core/src/com/cloud/exception/UsageServerException.java index 68f8377..924934f 100644 --- a/core/src/com/cloud/exception/UsageServerException.java +++ b/core/src/com/cloud/exception/UsageServerException.java @@ -18,15 +18,20 @@ package com.cloud.exception; public class UsageServerException extends CloudException { - public UsageServerException() { - - } + /** + * + */ + private static final long serialVersionUID = -8398313106067116466L; + + public UsageServerException() { + + } + + public UsageServerException(String message) { + super(message); + } - public UsageServerException(String message) { - super(message); - } - } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4945d465/core/src/com/cloud/storage/JavaStorageLayer.java ---------------------------------------------------------------------- diff --git a/core/src/com/cloud/storage/JavaStorageLayer.java b/core/src/com/cloud/storage/JavaStorageLayer.java index bfaa767..e2e28ee 100644 --- a/core/src/com/cloud/storage/JavaStorageLayer.java +++ b/core/src/com/cloud/storage/JavaStorageLayer.java @@ -17,7 +17,6 @@ package com.cloud.storage; import java.io.File; -import java.io.FileFilter; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -41,7 +40,7 @@ public class JavaStorageLayer implements StorageLayer { public JavaStorageLayer(boolean makeWorldWriteable) { this(); - this._makeWorldWriteable = makeWorldWriteable; + _makeWorldWriteable = makeWorldWriteable; } @Override @@ -171,7 +170,7 @@ public class JavaStorageLayer implements StorageLayer { File dir = new File(dirName); if (dir.exists()) { String uniqDirName = dir.getAbsolutePath() + File.separator + UUID.randomUUID().toString(); - if (this.mkdir(uniqDirName)) { + if (mkdir(uniqDirName)) { return new File(uniqDirName); } } @@ -219,6 +218,7 @@ public class JavaStorageLayer implements StorageLayer { return dirPaths; } + @Override public boolean setWorldReadableAndWriteable(File file) { return (file.setReadable(true, false) && file.setWritable(true, false)); } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4945d465/core/src/com/cloud/storage/resource/StoragePoolResource.java ---------------------------------------------------------------------- diff --git a/core/src/com/cloud/storage/resource/StoragePoolResource.java b/core/src/com/cloud/storage/resource/StoragePoolResource.java index 8dff97d..f6d7896 100644 --- a/core/src/com/cloud/storage/resource/StoragePoolResource.java +++ b/core/src/com/cloud/storage/resource/StoragePoolResource.java @@ -21,8 +21,6 @@ import com.cloud.agent.api.storage.CopyVolumeAnswer; import com.cloud.agent.api.storage.CopyVolumeCommand; import com.cloud.agent.api.storage.CreateAnswer; import com.cloud.agent.api.storage.CreateCommand; -import com.cloud.agent.api.storage.CreateVolumeOVAAnswer; -import com.cloud.agent.api.storage.CreateVolumeOVACommand; import com.cloud.agent.api.storage.DestroyCommand; import com.cloud.agent.api.storage.PrimaryStorageDownloadAnswer; import com.cloud.agent.api.storage.PrimaryStorageDownloadCommand; http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4945d465/core/src/com/cloud/storage/template/FtpTemplateUploader.java ---------------------------------------------------------------------- diff --git a/core/src/com/cloud/storage/template/FtpTemplateUploader.java b/core/src/com/cloud/storage/template/FtpTemplateUploader.java index 61b1984..c3c9f1e 100755 --- a/core/src/com/cloud/storage/template/FtpTemplateUploader.java +++ b/core/src/com/cloud/storage/template/FtpTemplateUploader.java @@ -30,203 +30,202 @@ import org.apache.log4j.Logger; public class FtpTemplateUploader implements TemplateUploader { - - public static final Logger s_logger = Logger.getLogger(FtpTemplateUploader.class.getName()); - public TemplateUploader.Status status = TemplateUploader.Status.NOT_STARTED; - public String errorString = ""; - public long totalBytes = 0; - public long entitySizeinBytes; - private String sourcePath; - private String ftpUrl; - private UploadCompleteCallback completionCallback; - private boolean resume; + + public static final Logger s_logger = Logger.getLogger(FtpTemplateUploader.class.getName()); + public TemplateUploader.Status status = TemplateUploader.Status.NOT_STARTED; + public String errorString = ""; + public long totalBytes = 0; + public long entitySizeinBytes; + private String sourcePath; + private String ftpUrl; + private UploadCompleteCallback completionCallback; private BufferedInputStream inputStream = null; private BufferedOutputStream outputStream = null; - private static final int CHUNK_SIZE = 1024*1024; //1M - - public FtpTemplateUploader(String sourcePath, String url, UploadCompleteCallback callback, long entitySizeinBytes){ - - this.sourcePath = sourcePath; - this.ftpUrl = url; - this.completionCallback = callback; - this.entitySizeinBytes = entitySizeinBytes; - - } - - public long upload(UploadCompleteCallback callback ) - { - - switch (status) { - case ABORTED: - case UNRECOVERABLE_ERROR: - case UPLOAD_FINISHED: - return 0; - default: - - } - - Date start = new Date(); - - StringBuffer sb = new StringBuffer(ftpUrl); - // check for authentication else assume its anonymous access. - /* if (user != null && password != null) + private static final int CHUNK_SIZE = 1024*1024; //1M + + public FtpTemplateUploader(String sourcePath, String url, UploadCompleteCallback callback, long entitySizeinBytes){ + + this.sourcePath = sourcePath; + ftpUrl = url; + completionCallback = callback; + this.entitySizeinBytes = entitySizeinBytes; + + } + + @Override + public long upload(UploadCompleteCallback callback ) + { + + switch (status) { + case ABORTED: + case UNRECOVERABLE_ERROR: + case UPLOAD_FINISHED: + return 0; + default: + + } + + new Date(); + + StringBuffer sb = new StringBuffer(ftpUrl); + // check for authentication else assume its anonymous access. + /* if (user != null && password != null) { sb.append( user ); sb.append( ':' ); sb.append( password ); sb.append( '@' ); - }*/ - /* - * type ==> a=ASCII mode, i=image (binary) mode, d= file directory - * listing - */ - sb.append( ";type=i" ); - - try - { - URL url = new URL( sb.toString() ); - URLConnection urlc = url.openConnection(); - File sourceFile = new File(sourcePath); - entitySizeinBytes = sourceFile.length(); - - outputStream = new BufferedOutputStream( urlc.getOutputStream() ); - inputStream = new BufferedInputStream( new FileInputStream(sourceFile) ); - - status = TemplateUploader.Status.IN_PROGRESS; - - int bytes = 0; - byte[] block = new byte[CHUNK_SIZE]; - boolean done=false; - while (!done && status != Status.ABORTED ) { - if ( (bytes = inputStream.read(block, 0, CHUNK_SIZE)) > -1) { - outputStream.write(block,0, bytes); - totalBytes += bytes; - } else { - done = true; - } - } - status = TemplateUploader.Status.UPLOAD_FINISHED; - return totalBytes; - } catch (MalformedURLException e) { - status = TemplateUploader.Status.UNRECOVERABLE_ERROR; - errorString = e.getMessage(); - s_logger.error(errorString); - } catch (IOException e) { - status = TemplateUploader.Status.UNRECOVERABLE_ERROR; - errorString = e.getMessage(); - s_logger.error(errorString); - } - finally - { - try - { - if (inputStream != null){ - inputStream.close(); - } - if (outputStream != null){ - outputStream.close(); - } - }catch (IOException ioe){ - s_logger.error(" Caught exception while closing the resources" ); - } - if (callback != null) { - callback.uploadComplete(status); - } - } - - return 0; - } - - @Override - public void run() { - try { - upload(completionCallback); - } catch (Throwable t) { - s_logger.warn("Caught exception during upload "+ t.getMessage(), t); - errorString = "Failed to install: " + t.getMessage(); - status = TemplateUploader.Status.UNRECOVERABLE_ERROR; - } - - } - - @Override - public Status getStatus() { - return status; - } - - @Override - public String getUploadError() { - return errorString; - } - - @Override - public String getUploadLocalPath() { - return sourcePath; - } - - @Override - public int getUploadPercent() { - if (entitySizeinBytes == 0) { - return 0; - } - return (int)(100.0*totalBytes/entitySizeinBytes); - } - - @Override - public long getUploadTime() { - // TODO - return 0; - } - - @Override - public long getUploadedBytes() { - return totalBytes; - } - - @Override - public void setResume(boolean resume) { - this.resume = resume; - - } - - @Override - public void setStatus(Status status) { - this.status = status; - } - - @Override - public void setUploadError(String string) { - errorString = string; - } - - @Override - public boolean stopUpload() { - switch (getStatus()) { - case IN_PROGRESS: - try { - if(outputStream != null) { - outputStream.close(); - } - if (inputStream != null){ - inputStream.close(); - } - } catch (IOException e) { - s_logger.error(" Caught exception while closing the resources" ); - } - status = TemplateUploader.Status.ABORTED; - return true; - case UNKNOWN: - case NOT_STARTED: - case RECOVERABLE_ERROR: - case UNRECOVERABLE_ERROR: - case ABORTED: - status = TemplateUploader.Status.ABORTED; - case UPLOAD_FINISHED: - return true; - - default: - return true; - } - } + }*/ + /* + * type ==> a=ASCII mode, i=image (binary) mode, d= file directory + * listing + */ + sb.append( ";type=i" ); + + try + { + URL url = new URL( sb.toString() ); + URLConnection urlc = url.openConnection(); + File sourceFile = new File(sourcePath); + entitySizeinBytes = sourceFile.length(); + + outputStream = new BufferedOutputStream( urlc.getOutputStream() ); + inputStream = new BufferedInputStream( new FileInputStream(sourceFile) ); + + status = TemplateUploader.Status.IN_PROGRESS; + + int bytes = 0; + byte[] block = new byte[CHUNK_SIZE]; + boolean done=false; + while (!done && status != Status.ABORTED ) { + if ( (bytes = inputStream.read(block, 0, CHUNK_SIZE)) > -1) { + outputStream.write(block,0, bytes); + totalBytes += bytes; + } else { + done = true; + } + } + status = TemplateUploader.Status.UPLOAD_FINISHED; + return totalBytes; + } catch (MalformedURLException e) { + status = TemplateUploader.Status.UNRECOVERABLE_ERROR; + errorString = e.getMessage(); + s_logger.error(errorString); + } catch (IOException e) { + status = TemplateUploader.Status.UNRECOVERABLE_ERROR; + errorString = e.getMessage(); + s_logger.error(errorString); + } + finally + { + try + { + if (inputStream != null){ + inputStream.close(); + } + if (outputStream != null){ + outputStream.close(); + } + }catch (IOException ioe){ + s_logger.error(" Caught exception while closing the resources" ); + } + if (callback != null) { + callback.uploadComplete(status); + } + } + + return 0; + } + + @Override + public void run() { + try { + upload(completionCallback); + } catch (Throwable t) { + s_logger.warn("Caught exception during upload "+ t.getMessage(), t); + errorString = "Failed to install: " + t.getMessage(); + status = TemplateUploader.Status.UNRECOVERABLE_ERROR; + } + + } + + @Override + public Status getStatus() { + return status; + } + + @Override + public String getUploadError() { + return errorString; + } + + @Override + public String getUploadLocalPath() { + return sourcePath; + } + + @Override + public int getUploadPercent() { + if (entitySizeinBytes == 0) { + return 0; + } + return (int)(100.0*totalBytes/entitySizeinBytes); + } + + @Override + public long getUploadTime() { + // TODO + return 0; + } + + @Override + public long getUploadedBytes() { + return totalBytes; + } + + @Override + public void setResume(boolean resume) { + + } + + @Override + public void setStatus(Status status) { + this.status = status; + } + + @Override + public void setUploadError(String string) { + errorString = string; + } + + @Override + public boolean stopUpload() { + switch (getStatus()) { + case IN_PROGRESS: + try { + if(outputStream != null) { + outputStream.close(); + } + if (inputStream != null){ + inputStream.close(); + } + } catch (IOException e) { + s_logger.error(" Caught exception while closing the resources" ); + } + status = TemplateUploader.Status.ABORTED; + return true; + case UNKNOWN: + case NOT_STARTED: + case RECOVERABLE_ERROR: + case UNRECOVERABLE_ERROR: + case ABORTED: + status = TemplateUploader.Status.ABORTED; + case UPLOAD_FINISHED: + return true; + + default: + return true; + } + } } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4945d465/core/src/com/cloud/storage/template/HttpTemplateDownloader.java ---------------------------------------------------------------------- diff --git a/core/src/com/cloud/storage/template/HttpTemplateDownloader.java b/core/src/com/cloud/storage/template/HttpTemplateDownloader.java index d87dd68..8789615 100644 --- a/core/src/com/cloud/storage/template/HttpTemplateDownloader.java +++ b/core/src/com/cloud/storage/template/HttpTemplateDownloader.java @@ -22,14 +22,9 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.RandomAccessFile; -import java.net.Inet6Address; -import java.net.InetAddress; -import java.net.URI; import java.net.URISyntaxException; -import java.net.UnknownHostException; import java.util.Date; -import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; import org.apache.commons.httpclient.ChunkedInputStream; import org.apache.commons.httpclient.Credentials; import org.apache.commons.httpclient.Header; @@ -46,9 +41,10 @@ import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.params.HttpMethodParams; import org.apache.log4j.Logger; +import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; + import com.cloud.agent.api.storage.Proxy; import com.cloud.storage.StorageLayer; -import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; @@ -57,144 +53,145 @@ import com.cloud.utils.UriUtils; * */ public class HttpTemplateDownloader implements TemplateDownloader { - public static final Logger s_logger = Logger.getLogger(HttpTemplateDownloader.class.getName()); + public static final Logger s_logger = Logger.getLogger(HttpTemplateDownloader.class.getName()); private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager(); - private static final int CHUNK_SIZE = 1024*1024; //1M - private String downloadUrl; - private String toFile; - public TemplateDownloader.Status status= TemplateDownloader.Status.NOT_STARTED; - public String errorString = " "; - private long remoteSize = 0; - public long downloadTime = 0; - public long totalBytes; - private final HttpClient client; - private GetMethod request; - private boolean resume = false; - private DownloadCompleteCallback completionCallback; - StorageLayer _storage; - boolean inited = true; - - private String toDir; - private long MAX_TEMPLATE_SIZE_IN_BYTES; - private ResourceType resourceType = ResourceType.TEMPLATE; - private final HttpMethodRetryHandler myretryhandler; - - - - public HttpTemplateDownloader (StorageLayer storageLayer, String downloadUrl, String toDir, DownloadCompleteCallback callback, long maxTemplateSizeInBytes, String user, String password, Proxy proxy, ResourceType resourceType) { - this._storage = storageLayer; - this.downloadUrl = downloadUrl; - this.setToDir(toDir); - this.status = TemplateDownloader.Status.NOT_STARTED; - this.resourceType = resourceType; - this.MAX_TEMPLATE_SIZE_IN_BYTES = maxTemplateSizeInBytes; - - this.totalBytes = 0; - this.client = new HttpClient(s_httpClientManager); - - myretryhandler = new HttpMethodRetryHandler() { - public boolean retryMethod( - final HttpMethod method, - final IOException exception, - int executionCount) { - if (executionCount >= 2) { - // Do not retry if over max retry count - return false; - } - if (exception instanceof NoHttpResponseException) { - // Retry if the server dropped connection on us - return true; - } - if (!method.isRequestSent()) { - // Retry if the request has not been sent fully or - // if it's OK to retry methods that have been sent - return true; - } - // otherwise do not retry - return false; - } - }; - - try { - this.request = new GetMethod(downloadUrl); - this.request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler); - this.completionCallback = callback; - //this.request.setFollowRedirects(false); - - File f = File.createTempFile("dnld", "tmp_", new File(toDir)); - - if (_storage != null) { - _storage.setWorldReadableAndWriteable(f); - } - - toFile = f.getAbsolutePath(); - Pair hostAndPort = UriUtils.validateUrl(downloadUrl); - - if (proxy != null) { - client.getHostConfiguration().setProxy(proxy.getHost(), proxy.getPort()); - if (proxy.getUserName() != null) { - Credentials proxyCreds = new UsernamePasswordCredentials(proxy.getUserName(), proxy.getPassword()); - client.getState().setProxyCredentials(AuthScope.ANY, proxyCreds); - } - } - if ((user != null) && (password != null)) { - client.getParams().setAuthenticationPreemptive(true); - Credentials defaultcreds = new UsernamePasswordCredentials(user, password); - client.getState().setCredentials(new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds); - s_logger.info("Added username=" + user + ", password=" + password + "for host " + hostAndPort.first() + ":" + hostAndPort.second()); - } else { - s_logger.info("No credentials configured for host=" + hostAndPort.first() + ":" + hostAndPort.second()); - } - } catch (IllegalArgumentException iae) { - errorString = iae.getMessage(); - status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; - inited = false; - } catch (Exception ex){ - errorString = "Unable to start download -- check url? "; - status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; - s_logger.warn("Exception in constructor -- " + ex.toString()); - } catch (Throwable th) { - s_logger.warn("throwable caught ", th); - } - } - - - @Override - public long download(boolean resume, DownloadCompleteCallback callback) { - switch (status) { - case ABORTED: - case UNRECOVERABLE_ERROR: - case DOWNLOAD_FINISHED: - return 0; - default: - - } + private static final int CHUNK_SIZE = 1024*1024; //1M + private String downloadUrl; + private String toFile; + public TemplateDownloader.Status status= TemplateDownloader.Status.NOT_STARTED; + public String errorString = " "; + private long remoteSize = 0; + public long downloadTime = 0; + public long totalBytes; + private final HttpClient client; + private GetMethod request; + private boolean resume = false; + private DownloadCompleteCallback completionCallback; + StorageLayer _storage; + boolean inited = true; + + private String toDir; + private long MAX_TEMPLATE_SIZE_IN_BYTES; + private ResourceType resourceType = ResourceType.TEMPLATE; + private final HttpMethodRetryHandler myretryhandler; + + + + public HttpTemplateDownloader (StorageLayer storageLayer, String downloadUrl, String toDir, DownloadCompleteCallback callback, long maxTemplateSizeInBytes, String user, String password, Proxy proxy, ResourceType resourceType) { + _storage = storageLayer; + this.downloadUrl = downloadUrl; + setToDir(toDir); + status = TemplateDownloader.Status.NOT_STARTED; + this.resourceType = resourceType; + MAX_TEMPLATE_SIZE_IN_BYTES = maxTemplateSizeInBytes; + + totalBytes = 0; + client = new HttpClient(s_httpClientManager); + + myretryhandler = new HttpMethodRetryHandler() { + @Override + public boolean retryMethod( + final HttpMethod method, + final IOException exception, + int executionCount) { + if (executionCount >= 2) { + // Do not retry if over max retry count + return false; + } + if (exception instanceof NoHttpResponseException) { + // Retry if the server dropped connection on us + return true; + } + if (!method.isRequestSent()) { + // Retry if the request has not been sent fully or + // if it's OK to retry methods that have been sent + return true; + } + // otherwise do not retry + return false; + } + }; + + try { + request = new GetMethod(downloadUrl); + request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler); + completionCallback = callback; + //this.request.setFollowRedirects(false); + + File f = File.createTempFile("dnld", "tmp_", new File(toDir)); + + if (_storage != null) { + _storage.setWorldReadableAndWriteable(f); + } + + toFile = f.getAbsolutePath(); + Pair hostAndPort = UriUtils.validateUrl(downloadUrl); + + if (proxy != null) { + client.getHostConfiguration().setProxy(proxy.getHost(), proxy.getPort()); + if (proxy.getUserName() != null) { + Credentials proxyCreds = new UsernamePasswordCredentials(proxy.getUserName(), proxy.getPassword()); + client.getState().setProxyCredentials(AuthScope.ANY, proxyCreds); + } + } + if ((user != null) && (password != null)) { + client.getParams().setAuthenticationPreemptive(true); + Credentials defaultcreds = new UsernamePasswordCredentials(user, password); + client.getState().setCredentials(new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds); + s_logger.info("Added username=" + user + ", password=" + password + "for host " + hostAndPort.first() + ":" + hostAndPort.second()); + } else { + s_logger.info("No credentials configured for host=" + hostAndPort.first() + ":" + hostAndPort.second()); + } + } catch (IllegalArgumentException iae) { + errorString = iae.getMessage(); + status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; + inited = false; + } catch (Exception ex){ + errorString = "Unable to start download -- check url? "; + status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; + s_logger.warn("Exception in constructor -- " + ex.toString()); + } catch (Throwable th) { + s_logger.warn("throwable caught ", th); + } + } + + + @Override + public long download(boolean resume, DownloadCompleteCallback callback) { + switch (status) { + case ABORTED: + case UNRECOVERABLE_ERROR: + case DOWNLOAD_FINISHED: + return 0; + default: + + } int bytes=0; - File file = new File(toFile); - try { + File file = new File(toFile); + try { - long localFileSize = 0; - if (file.exists() && resume) { - localFileSize = file.length(); - s_logger.info("Resuming download to file (current size)=" + localFileSize); - } + long localFileSize = 0; + if (file.exists() && resume) { + localFileSize = file.length(); + s_logger.info("Resuming download to file (current size)=" + localFileSize); + } Date start = new Date(); - int responseCode=0; - - if (localFileSize > 0 ) { - // require partial content support for resume - request.addRequestHeader("Range", "bytes=" + localFileSize + "-"); - if (client.executeMethod(request) != HttpStatus.SC_PARTIAL_CONTENT) { - errorString = "HTTP Server does not support partial get"; - status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; - return 0; - } - } else if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) { - status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; - errorString = " HTTP Server returned " + responseCode + " (expected 200 OK) "; + int responseCode=0; + + if (localFileSize > 0 ) { + // require partial content support for resume + request.addRequestHeader("Range", "bytes=" + localFileSize + "-"); + if (client.executeMethod(request) != HttpStatus.SC_PARTIAL_CONTENT) { + errorString = "HTTP Server does not support partial get"; + status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; + return 0; + } + } else if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) { + status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; + errorString = " HTTP Server returned " + responseCode + " (expected 200 OK) "; return 0; //FIXME: retry? } @@ -202,16 +199,16 @@ public class HttpTemplateDownloader implements TemplateDownloader { boolean chunked = false; long remoteSize2 = 0; if (contentLengthHeader == null) { - Header chunkedHeader = request.getResponseHeader("Transfer-Encoding"); - if (chunkedHeader == null || !"chunked".equalsIgnoreCase(chunkedHeader.getValue())) { - status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; - errorString=" Failed to receive length of download "; - return 0; //FIXME: what status do we put here? Do we retry? - } else if ("chunked".equalsIgnoreCase(chunkedHeader.getValue())){ - chunked = true; - } + Header chunkedHeader = request.getResponseHeader("Transfer-Encoding"); + if (chunkedHeader == null || !"chunked".equalsIgnoreCase(chunkedHeader.getValue())) { + status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; + errorString=" Failed to receive length of download "; + return 0; //FIXME: what status do we put here? Do we retry? + } else if ("chunked".equalsIgnoreCase(chunkedHeader.getValue())){ + chunked = true; + } } else { - remoteSize2 = Long.parseLong(contentLengthHeader.getValue()); + remoteSize2 = Long.parseLong(contentLengthHeader.getValue()); if ( remoteSize2 == 0 ) { status = TemplateDownloader.Status.DOWNLOAD_FINISHED; String downloaded = "(download complete remote=" + remoteSize + "bytes)"; @@ -222,22 +219,22 @@ public class HttpTemplateDownloader implements TemplateDownloader { } if (remoteSize == 0) { - remoteSize = remoteSize2; + remoteSize = remoteSize2; } if (remoteSize > MAX_TEMPLATE_SIZE_IN_BYTES) { - s_logger.info("Remote size is too large: " + remoteSize + " , max=" + MAX_TEMPLATE_SIZE_IN_BYTES); - status = Status.UNRECOVERABLE_ERROR; - errorString = "Download file size is too large"; - return 0; + s_logger.info("Remote size is too large: " + remoteSize + " , max=" + MAX_TEMPLATE_SIZE_IN_BYTES); + status = Status.UNRECOVERABLE_ERROR; + errorString = "Download file size is too large"; + return 0; } if (remoteSize == 0) { - remoteSize = MAX_TEMPLATE_SIZE_IN_BYTES; + remoteSize = MAX_TEMPLATE_SIZE_IN_BYTES; } - InputStream in = !chunked?new BufferedInputStream(request.getResponseBodyAsStream()) - : new ChunkedInputStream(request.getResponseBodyAsStream()); + InputStream in = !chunked ? new BufferedInputStream(request.getResponseBodyAsStream()) : new ChunkedInputStream( + request.getResponseBodyAsStream()); RandomAccessFile out = new RandomAccessFile(file, "rwd"); out.seek(localFileSize); @@ -249,187 +246,193 @@ public class HttpTemplateDownloader implements TemplateDownloader { boolean done=false; status = TemplateDownloader.Status.IN_PROGRESS; while (!done && status != Status.ABORTED && offset <= remoteSize) { - if ( (bytes = in.read(block, 0, CHUNK_SIZE)) > -1) { - out.write(block, 0, bytes); - offset +=bytes; - out.seek(offset); - totalBytes += bytes; - } else { - done = true; - } + if ( (bytes = in.read(block, 0, CHUNK_SIZE)) > -1) { + out.write(block, 0, bytes); + offset +=bytes; + out.seek(offset); + totalBytes += bytes; + } else { + done = true; + } } Date finish = new Date(); String downloaded = "(incomplete download)"; if (totalBytes >= remoteSize) { - status = TemplateDownloader.Status.DOWNLOAD_FINISHED; - downloaded = "(download complete remote=" + remoteSize + "bytes)"; + status = TemplateDownloader.Status.DOWNLOAD_FINISHED; + downloaded = "(download complete remote=" + remoteSize + "bytes)"; } errorString = "Downloaded " + totalBytes + " bytes " + downloaded; downloadTime += finish.getTime() - start.getTime(); + in.close(); out.close(); return totalBytes; - }catch (HttpException hte) { - status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; - errorString = hte.getMessage(); - } catch (IOException ioe) { - status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; //probably a file write error? - errorString = ioe.getMessage(); - } finally { - if (status == Status.UNRECOVERABLE_ERROR && file.exists() && !file.isDirectory()) { - file.delete(); - } - request.releaseConnection(); + }catch (HttpException hte) { + status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; + errorString = hte.getMessage(); + } catch (IOException ioe) { + status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; //probably a file write error? + errorString = ioe.getMessage(); + } finally { + if (status == Status.UNRECOVERABLE_ERROR && file.exists() && !file.isDirectory()) { + file.delete(); + } + request.releaseConnection(); if (callback != null) { - callback.downloadComplete(status); + callback.downloadComplete(status); } - } - return 0; - } + } + return 0; + } - public String getDownloadUrl() { - return downloadUrl; - } + public String getDownloadUrl() { + return downloadUrl; + } - public String getToFile() { + public String getToFile() { File file = new File(toFile); - return file.getAbsolutePath(); - } - - public TemplateDownloader.Status getStatus() { - return status; - } - - - public long getDownloadTime() { - return downloadTime; - } - - - public long getDownloadedBytes() { - return totalBytes; - } - - @Override - @SuppressWarnings("fallthrough") - public boolean stopDownload() { - switch (getStatus()) { - case IN_PROGRESS: - if (request != null) { - request.abort(); - } - status = TemplateDownloader.Status.ABORTED; - return true; - case UNKNOWN: - case NOT_STARTED: - case RECOVERABLE_ERROR: - case UNRECOVERABLE_ERROR: - case ABORTED: - status = TemplateDownloader.Status.ABORTED; - case DOWNLOAD_FINISHED: - File f = new File(toFile); - if (f.exists()) { - f.delete(); - } - return true; - - default: - return true; - } - } - - @Override - public int getDownloadPercent() { - if (remoteSize == 0) { - return 0; - } - - return (int)(100.0*totalBytes/remoteSize); - } - - @Override - public void run() { - try { - download(resume, completionCallback); - } catch (Throwable t) { - s_logger.warn("Caught exception during download "+ t.getMessage(), t); - errorString = "Failed to install: " + t.getMessage(); - status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; - } - - } - - @Override - public void setStatus(TemplateDownloader.Status status) { - this.status = status; - } - - - - public boolean isResume() { - return resume; - } - - @Override - public String getDownloadError() { - return errorString; - } - - @Override - public String getDownloadLocalPath() { - return getToFile(); - } - - public void setResume(boolean resume) { - this.resume = resume; - } - - public void setToDir(String toDir) { - this.toDir = toDir; - } - - public String getToDir() { - return toDir; - } - - public long getMaxTemplateSizeInBytes() { - return this.MAX_TEMPLATE_SIZE_IN_BYTES; - } - - public static void main(String[] args) { - String url ="http:// dev.mysql.com/get/Downloads/MySQL-5.0/mysql-noinstall-5.0.77-win32.zip/from/http://mirror.services.wisc.edu/mysql/"; - try { - URI uri = new java.net.URI(url); - } catch (URISyntaxException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - TemplateDownloader td = new HttpTemplateDownloader(null, url,"/tmp/mysql", null, TemplateDownloader.DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES, null, null, null, null); - long bytes = td.download(true, null); - if (bytes > 0) { - System.out.println("Downloaded (" + bytes + " bytes)" + " in " + td.getDownloadTime()/1000 + " secs"); - } else { - System.out.println("Failed download"); - } - - } - - @Override - public void setDownloadError(String error) { - errorString = error; - } - - - - @Override - public boolean isInited() { - return inited; - } - - - public ResourceType getResourceType() { - return resourceType; - } + return file.getAbsolutePath(); + } + + @Override + public TemplateDownloader.Status getStatus() { + return status; + } + + + @Override + public long getDownloadTime() { + return downloadTime; + } + + + @Override + public long getDownloadedBytes() { + return totalBytes; + } + + @Override + @SuppressWarnings("fallthrough") + public boolean stopDownload() { + switch (getStatus()) { + case IN_PROGRESS: + if (request != null) { + request.abort(); + } + status = TemplateDownloader.Status.ABORTED; + return true; + case UNKNOWN: + case NOT_STARTED: + case RECOVERABLE_ERROR: + case UNRECOVERABLE_ERROR: + case ABORTED: + status = TemplateDownloader.Status.ABORTED; + case DOWNLOAD_FINISHED: + File f = new File(toFile); + if (f.exists()) { + f.delete(); + } + return true; + + default: + return true; + } + } + + @Override + public int getDownloadPercent() { + if (remoteSize == 0) { + return 0; + } + + return (int)(100.0*totalBytes/remoteSize); + } + + @Override + public void run() { + try { + download(resume, completionCallback); + } catch (Throwable t) { + s_logger.warn("Caught exception during download "+ t.getMessage(), t); + errorString = "Failed to install: " + t.getMessage(); + status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; + } + + } + + @Override + public void setStatus(TemplateDownloader.Status status) { + this.status = status; + } + + + + public boolean isResume() { + return resume; + } + + @Override + public String getDownloadError() { + return errorString; + } + + @Override + public String getDownloadLocalPath() { + return getToFile(); + } + + @Override + public void setResume(boolean resume) { + this.resume = resume; + } + + public void setToDir(String toDir) { + this.toDir = toDir; + } + + public String getToDir() { + return toDir; + } + + @Override + public long getMaxTemplateSizeInBytes() { + return MAX_TEMPLATE_SIZE_IN_BYTES; + } + + public static void main(String[] args) { + String url ="http:// dev.mysql.com/get/Downloads/MySQL-5.0/mysql-noinstall-5.0.77-win32.zip/from/http://mirror.services.wisc.edu/mysql/"; + try { + new java.net.URI(url); + } catch (URISyntaxException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + TemplateDownloader td = new HttpTemplateDownloader(null, url,"/tmp/mysql", null, TemplateDownloader.DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES, null, null, null, null); + long bytes = td.download(true, null); + if (bytes > 0) { + System.out.println("Downloaded (" + bytes + " bytes)" + " in " + td.getDownloadTime()/1000 + " secs"); + } else { + System.out.println("Failed download"); + } + + } + + @Override + public void setDownloadError(String error) { + errorString = error; + } + + + + @Override + public boolean isInited() { + return inited; + } + + + public ResourceType getResourceType() { + return resourceType; + } } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4945d465/core/src/com/cloud/storage/template/LocalTemplateDownloader.java ---------------------------------------------------------------------- diff --git a/core/src/com/cloud/storage/template/LocalTemplateDownloader.java b/core/src/com/cloud/storage/template/LocalTemplateDownloader.java index c8927a1..581524b 100644 --- a/core/src/com/cloud/storage/template/LocalTemplateDownloader.java +++ b/core/src/com/cloud/storage/template/LocalTemplateDownloader.java @@ -34,7 +34,7 @@ import com.cloud.storage.StorageLayer; public class LocalTemplateDownloader extends TemplateDownloaderBase implements TemplateDownloader { public static final Logger s_logger = Logger.getLogger(LocalTemplateDownloader.class); - + public LocalTemplateDownloader(StorageLayer storageLayer, String downloadUrl, String toDir, long maxTemplateSizeInBytes, DownloadCompleteCallback callback) { super(storageLayer, downloadUrl, toDir, maxTemplateSizeInBytes, callback); String filename = downloadUrl.substring(downloadUrl.lastIndexOf(File.separator)); @@ -44,14 +44,14 @@ public class LocalTemplateDownloader extends TemplateDownloaderBase implements T @Override public long download(boolean resume, DownloadCompleteCallback callback) { if (_status == Status.ABORTED || - _status == Status.UNRECOVERABLE_ERROR || - _status == Status.DOWNLOAD_FINISHED) { + _status == Status.UNRECOVERABLE_ERROR || + _status == Status.DOWNLOAD_FINISHED) { return 0; } _start = System.currentTimeMillis(); _resume = resume; - + File src; try { src = new File(new URI(_downloadUrl)); @@ -61,18 +61,20 @@ public class LocalTemplateDownloader extends TemplateDownloaderBase implements T return 0; } File dst = new File(_toFile); - + FileChannel fic = null; FileChannel foc = null; - + FileInputStream fis = null; + FileOutputStream fos = null; + try { - if (_storage != null) { - dst.createNewFile(); - _storage.setWorldReadableAndWriteable(dst); - } - + if (_storage != null) { + dst.createNewFile(); + _storage.setWorldReadableAndWriteable(dst); + } + ByteBuffer buffer = ByteBuffer.allocate(1024 * 512); - FileInputStream fis; + try { fis = new FileInputStream(src); } catch (FileNotFoundException e) { @@ -81,7 +83,6 @@ public class LocalTemplateDownloader extends TemplateDownloaderBase implements T return -1; } fic = fis.getChannel(); - FileOutputStream fos; try { fos = new FileOutputStream(dst); } catch (FileNotFoundException e) { @@ -89,11 +90,11 @@ public class LocalTemplateDownloader extends TemplateDownloaderBase implements T return -1; } foc = fos.getChannel(); - + _remoteSize = src.length(); - this._totalBytes = 0; + _totalBytes = 0; _status = TemplateDownloader.Status.IN_PROGRESS; - + try { while (_status != Status.ABORTED && fic.read(buffer) != -1) { buffer.flip(); @@ -104,13 +105,13 @@ public class LocalTemplateDownloader extends TemplateDownloaderBase implements T } catch (IOException e) { s_logger.warn("Unable to download", e); } - + String downloaded = "(incomplete download)"; if (_totalBytes == _remoteSize) { _status = TemplateDownloader.Status.DOWNLOAD_FINISHED; downloaded = "(download complete)"; } - + _errorString = "Downloaded " + _remoteSize + " bytes " + downloaded; _downloadTime += System.currentTimeMillis() - _start; return _totalBytes; @@ -125,14 +126,28 @@ public class LocalTemplateDownloader extends TemplateDownloaderBase implements T } catch (IOException e) { } } - + if (foc != null) { try { foc.close(); } catch (IOException e) { } } - + + if (fis != null) { + try { + fis.close(); + } catch (IOException e) { + } + } + + if (fos != null) { + try { + fos.close(); + } catch (IOException e) { + } + } + if (_status == Status.UNRECOVERABLE_ERROR && dst.exists()) { dst.delete(); } @@ -141,7 +156,7 @@ public class LocalTemplateDownloader extends TemplateDownloaderBase implements T } } } - + public static void main(String[] args) { String url ="file:///home/ahuang/Download/E3921_P5N7A-VM_manual.zip"; TemplateDownloader td = new LocalTemplateDownloader(null, url,"/tmp/mysql", TemplateDownloader.DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES, null); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4945d465/core/src/com/cloud/storage/template/RawImageProcessor.java ---------------------------------------------------------------------- diff --git a/core/src/com/cloud/storage/template/RawImageProcessor.java b/core/src/com/cloud/storage/template/RawImageProcessor.java index 0e4c8c1..f516d75 100644 --- a/core/src/com/cloud/storage/template/RawImageProcessor.java +++ b/core/src/com/cloud/storage/template/RawImageProcessor.java @@ -25,9 +25,8 @@ import javax.naming.ConfigurationException; import org.apache.log4j.Logger; import com.cloud.exception.InternalErrorException; -import com.cloud.storage.StorageLayer; import com.cloud.storage.Storage.ImageFormat; -import com.cloud.storage.template.Processor.FormatInfo; +import com.cloud.storage.StorageLayer; import com.cloud.utils.component.AdapterBase; @Local(value=Processor.class) http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4945d465/core/src/com/cloud/storage/template/S3TemplateDownloader.java ---------------------------------------------------------------------- diff --git a/core/src/com/cloud/storage/template/S3TemplateDownloader.java b/core/src/com/cloud/storage/template/S3TemplateDownloader.java index 340e0db..05ed64b 100644 --- a/core/src/com/cloud/storage/template/S3TemplateDownloader.java +++ b/core/src/com/cloud/storage/template/S3TemplateDownloader.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.io.InputStream; import java.util.Date; -import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; import org.apache.commons.httpclient.ChunkedInputStream; import org.apache.commons.httpclient.Credentials; import org.apache.commons.httpclient.Header; @@ -43,8 +42,6 @@ import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import com.amazonaws.AmazonClientException; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.ProgressEvent; import com.amazonaws.services.s3.model.ProgressListener; @@ -52,6 +49,9 @@ import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.StorageClass; import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.Upload; + +import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; + import com.cloud.agent.api.storage.Proxy; import com.cloud.agent.api.to.S3TO; import com.cloud.utils.Pair; @@ -89,15 +89,15 @@ public class S3TemplateDownloader implements TemplateDownloader { public S3TemplateDownloader(S3TO storageLayer, String downloadUrl, String installPath, DownloadCompleteCallback callback, long maxTemplateSizeInBytes, String user, String password, Proxy proxy, ResourceType resourceType) { - this.s3 = storageLayer; + s3 = storageLayer; this.downloadUrl = downloadUrl; this.installPath = installPath; - this.status = TemplateDownloader.Status.NOT_STARTED; + status = TemplateDownloader.Status.NOT_STARTED; this.resourceType = resourceType; - this.maxTemplateSizeInByte = maxTemplateSizeInBytes; + maxTemplateSizeInByte = maxTemplateSizeInBytes; - this.totalBytes = 0; - this.client = new HttpClient(s_httpClientManager); + totalBytes = 0; + client = new HttpClient(s_httpClientManager); myretryhandler = new HttpMethodRetryHandler() { @Override @@ -121,12 +121,12 @@ public class S3TemplateDownloader implements TemplateDownloader { }; try { - this.request = new GetMethod(downloadUrl); - this.request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler); - this.completionCallback = callback; + request = new GetMethod(downloadUrl); + request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler); + completionCallback = callback; Pair hostAndPort = UriUtils.validateUrl(downloadUrl); - this.fileName = StringUtils.substringAfterLast(downloadUrl, "/"); + fileName = StringUtils.substringAfterLast(downloadUrl, "/"); if (proxy != null) { client.getHostConfiguration().setProxy(proxy.getHost(), proxy.getPort()); @@ -388,7 +388,7 @@ public class S3TemplateDownloader implements TemplateDownloader { @Override public String getDownloadLocalPath() { - return this.s3Key; + return s3Key; } @Override @@ -398,7 +398,7 @@ public class S3TemplateDownloader implements TemplateDownloader { @Override public long getMaxTemplateSizeInBytes() { - return this.maxTemplateSizeInByte; + return maxTemplateSizeInByte; } @Override http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4945d465/core/src/com/cloud/storage/template/ScpTemplateDownloader.java ---------------------------------------------------------------------- diff --git a/core/src/com/cloud/storage/template/ScpTemplateDownloader.java b/core/src/com/cloud/storage/template/ScpTemplateDownloader.java index 724392f..fbc756f 100644 --- a/core/src/com/cloud/storage/template/ScpTemplateDownloader.java +++ b/core/src/com/cloud/storage/template/ScpTemplateDownloader.java @@ -22,9 +22,10 @@ import java.net.URISyntaxException; import org.apache.log4j.Logger; +import com.trilead.ssh2.SCPClient; + import com.cloud.storage.StorageLayer; import com.cloud.utils.exception.CloudRuntimeException; -import com.trilead.ssh2.SCPClient; public class ScpTemplateDownloader extends TemplateDownloaderBase implements TemplateDownloader { private static final Logger s_logger = Logger.getLogger(ScpTemplateDownloader.class); @@ -83,7 +84,6 @@ public class ScpTemplateDownloader extends TemplateDownloaderBase implements Tem if (port == -1) { port = 22; } - long length = 0; File file = new File(_toFile); com.trilead.ssh2.Connection sshConnection = new com.trilead.ssh2.Connection(uri.getHost(), port); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4945d465/core/src/com/cloud/storage/template/TemplateUploader.java ---------------------------------------------------------------------- diff --git a/core/src/com/cloud/storage/template/TemplateUploader.java b/core/src/com/cloud/storage/template/TemplateUploader.java index 8e0373a..32e8771 100755 --- a/core/src/com/cloud/storage/template/TemplateUploader.java +++ b/core/src/com/cloud/storage/template/TemplateUploader.java @@ -16,9 +16,6 @@ // under the License. package com.cloud.storage.template; -import com.cloud.storage.template.TemplateUploader.UploadCompleteCallback; -import com.cloud.storage.template.TemplateUploader.Status; - public interface TemplateUploader extends Runnable{ /** http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4945d465/core/src/org/apache/cloudstack/storage/command/DownloadCommand.java ---------------------------------------------------------------------- diff --git a/core/src/org/apache/cloudstack/storage/command/DownloadCommand.java b/core/src/org/apache/cloudstack/storage/command/DownloadCommand.java index 84dd59d..9cc3e49 100644 --- a/core/src/org/apache/cloudstack/storage/command/DownloadCommand.java +++ b/core/src/org/apache/cloudstack/storage/command/DownloadCommand.java @@ -26,7 +26,6 @@ import com.cloud.agent.api.storage.Proxy; import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.NfsTO; import com.cloud.storage.Storage.ImageFormat; -import com.cloud.storage.Volume; public class DownloadCommand extends AbstractDownloadCommand implements InternalIdentity { @@ -53,29 +52,29 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal public DownloadCommand(DownloadCommand that) { super(that); - this.hvm = that.hvm; - this.checksum = that.checksum; - this.id = that.id; - this.description = that.description; - this.auth = that.getAuth(); - this.setSecUrl(that.getSecUrl()); - this.maxDownloadSizeInBytes = that.getMaxDownloadSizeInBytes(); - this.resourceType = that.resourceType; - this.installPath = that.installPath; - this._store = that._store; + hvm = that.hvm; + checksum = that.checksum; + id = that.id; + description = that.description; + auth = that.getAuth(); + setSecUrl(that.getSecUrl()); + maxDownloadSizeInBytes = that.getMaxDownloadSizeInBytes(); + resourceType = that.resourceType; + installPath = that.installPath; + _store = that._store; } public DownloadCommand(TemplateObjectTO template, Long maxDownloadSizeInBytes) { super(template.getName(), template.getOrigUrl(), template.getFormat(), template.getAccountId()); - this._store = template.getDataStore(); - this.installPath = template.getPath(); - this.hvm = template.isRequiresHvm(); - this.checksum = template.getChecksum(); - this.id = template.getId(); - this.description = template.getDescription(); + _store = template.getDataStore(); + installPath = template.getPath(); + hvm = template.isRequiresHvm(); + checksum = template.getChecksum(); + id = template.getId(); + description = template.getDescription(); if (_store instanceof NfsTO) { - this.setSecUrl(((NfsTO) _store).getUrl()); + setSecUrl(((NfsTO) _store).getUrl()); } this.maxDownloadSizeInBytes = maxDownloadSizeInBytes; } @@ -87,12 +86,12 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal public DownloadCommand(VolumeObjectTO volume, Long maxDownloadSizeInBytes, String checkSum, String url, ImageFormat format) { super(volume.getName(), url, format, volume.getAccountId()); - this.checksum = checkSum; - this.id = volume.getVolumeId(); - this.installPath = volume.getPath(); - this._store = volume.getDataStore(); + checksum = checkSum; + id = volume.getVolumeId(); + installPath = volume.getPath(); + _store = volume.getDataStore(); this.maxDownloadSizeInBytes = maxDownloadSizeInBytes; - this.resourceType = ResourceType.VOLUME; + resourceType = ResourceType.VOLUME; } @Override public long getId() { @@ -184,6 +183,6 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal } public DataStoreTO getCacheStore() { - return this.cacheStore; + return cacheStore; } } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4945d465/core/test/org/apache/cloudstack/api/agent/test/BackupSnapshotCommandTest.java ---------------------------------------------------------------------- diff --git a/core/test/org/apache/cloudstack/api/agent/test/BackupSnapshotCommandTest.java b/core/test/org/apache/cloudstack/api/agent/test/BackupSnapshotCommandTest.java index 0fee8c6..a7a1fd2 100644 --- a/core/test/org/apache/cloudstack/api/agent/test/BackupSnapshotCommandTest.java +++ b/core/test/org/apache/cloudstack/api/agent/test/BackupSnapshotCommandTest.java @@ -27,7 +27,6 @@ import java.util.Date; import org.junit.Test; import com.cloud.agent.api.BackupSnapshotCommand; -import com.cloud.agent.api.to.StorageFilerTO; import com.cloud.agent.api.to.SwiftTO; import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.StoragePool;