cloudstack-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sw...@apache.org
Subject [20/20] git commit: updated refs/heads/master to 88cd182
Date Thu, 19 May 2016 16:11:47 GMT
Forward merged 4.8 to master and fixed conflicts in #1331


Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/88cd1822
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/88cd1822
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/88cd1822

Branch: refs/heads/master
Commit: 88cd1822729decee1b86076d8494258a41a89522
Parents: b4ad38d 091ca56
Author: Will Stevens <williamstevens@gmail.com>
Authored: Thu May 19 12:08:08 2016 -0400
Committer: Will Stevens <williamstevens@gmail.com>
Committed: Thu May 19 12:08:08 2016 -0400

----------------------------------------------------------------------
 agent/bindir/libvirtqemuhook.in                 |   2 +
 .../datastore/ObjectInDataStoreManagerImpl.java |   1 +
 .../VirtualNetworkApplianceManagerImpl.java     |   3 +
 server/src/com/cloud/test/TestAppender.java     | 178 ++++++++++++
 services/secondary-storage/server/pom.xml       |   3 -
 .../resource/NfsSecondaryStorageResource.java   | 288 +++++++++++++------
 .../LocalNfsSecondaryStorageResourceTest.java   |  44 ++-
 .../NfsSecondaryStorageResourceTest.java        | 118 +++-----
 .../debian/config/opt/cloud/bin/cs/CsDatabag.py |   4 +
 .../debian/config/opt/cloud/bin/cs/CsFile.py    |   1 +
 .../config/opt/cloud/bin/cs/CsRedundant.py      |   3 +
 11 files changed, 454 insertions(+), 191 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cloudstack/blob/88cd1822/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/88cd1822/services/secondary-storage/server/pom.xml
----------------------------------------------------------------------
diff --cc services/secondary-storage/server/pom.xml
index f3d0176,97e68a8..50155cf
--- a/services/secondary-storage/server/pom.xml
+++ b/services/secondary-storage/server/pom.xml
@@@ -23,12 -23,9 +23,9 @@@
    <parent>
      <groupId>org.apache.cloudstack</groupId>
      <artifactId>cloudstack-service-secondary-storage</artifactId>
 -    <version>4.8.1-SNAPSHOT</version>
 +    <version>4.9.0-SNAPSHOT</version>
      <relativePath>../pom.xml</relativePath>
    </parent>
-     <properties>
-     <skipTests>true</skipTests>
-   </properties>
      <dependencies>
      <dependency>
        <groupId>log4j</groupId>

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/88cd1822/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
----------------------------------------------------------------------
diff --cc services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
index 08eab33,f512da0..e7a9076
--- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
+++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
@@@ -16,72 -16,7 +16,91 @@@
  // under the License.
  package org.apache.cloudstack.storage.resource;
  
- import static com.cloud.utils.StringUtils.join;
 +import static com.cloud.utils.storage.S3.S3Utils.putFile;
++import static com.cloud.utils.StringUtils.join;
 +import static java.lang.String.format;
 +import static java.util.Arrays.asList;
 +import static org.apache.commons.lang.StringUtils.substringAfterLast;
 +
 +import java.io.BufferedReader;
 +import java.io.BufferedWriter;
 +import java.io.File;
 +import java.io.FileInputStream;
 +import java.io.FileOutputStream;
 +import java.io.FileReader;
 +import java.io.FileWriter;
 +import java.io.IOException;
 +import java.io.InputStream;
 +import java.io.UnsupportedEncodingException;
 +import java.math.BigInteger;
 +import java.net.InetAddress;
 +import java.net.URI;
 +import java.net.UnknownHostException;
 +import java.security.MessageDigest;
 +import java.security.NoSuchAlgorithmException;
 +import java.util.ArrayList;
 +import java.util.HashMap;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.UUID;
 +
 +import javax.naming.ConfigurationException;
 +
++import com.cloud.exception.InvalidParameterValueException;
++import com.cloud.storage.Storage;
++import com.cloud.storage.template.TemplateConstants;
++import com.cloud.utils.EncryptionUtil;
++import com.google.gson.Gson;
++import com.google.gson.GsonBuilder;
++import io.netty.bootstrap.ServerBootstrap;
++import io.netty.channel.Channel;
++import io.netty.channel.ChannelInitializer;
++import io.netty.channel.ChannelPipeline;
++import io.netty.channel.EventLoopGroup;
++import io.netty.channel.nio.NioEventLoopGroup;
++import io.netty.channel.socket.SocketChannel;
++import io.netty.channel.socket.nio.NioServerSocketChannel;
++import io.netty.handler.codec.http.HttpContentCompressor;
++import io.netty.handler.codec.http.HttpRequestDecoder;
++import io.netty.handler.codec.http.HttpResponseEncoder;
++import io.netty.handler.logging.LogLevel;
++import io.netty.handler.logging.LoggingHandler;
++import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand;
++import org.apache.cloudstack.storage.template.UploadEntity;
++import org.apache.cloudstack.utils.imagestore.ImageStoreUtil;
++import org.apache.commons.codec.digest.DigestUtils;
++import org.apache.commons.io.FileUtils;
++import org.apache.commons.io.FilenameUtils;
++import org.apache.commons.lang.StringUtils;
++import org.apache.http.HttpEntity;
++import org.apache.http.HttpResponse;
++import org.apache.http.NameValuePair;
++import org.apache.http.client.HttpClient;
++import org.apache.http.client.methods.HttpGet;
++import org.apache.http.client.utils.URLEncodedUtils;
++import org.apache.http.impl.client.DefaultHttpClient;
++import org.apache.log4j.Logger;
++
+ import com.amazonaws.services.s3.model.S3ObjectSummary;
++
 +import org.apache.cloudstack.framework.security.keystore.KeystoreManager;
 +import org.apache.cloudstack.storage.command.CopyCmdAnswer;
 +import org.apache.cloudstack.storage.command.CopyCommand;
 +import org.apache.cloudstack.storage.command.DeleteCommand;
 +import org.apache.cloudstack.storage.command.DownloadCommand;
 +import org.apache.cloudstack.storage.command.DownloadProgressCommand;
- import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand;
 +import org.apache.cloudstack.storage.command.UploadStatusAnswer;
 +import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus;
 +import org.apache.cloudstack.storage.command.UploadStatusCommand;
 +import org.apache.cloudstack.storage.template.DownloadManager;
 +import org.apache.cloudstack.storage.template.DownloadManagerImpl;
 +import org.apache.cloudstack.storage.template.DownloadManagerImpl.ZfsPathParser;
- import org.apache.cloudstack.storage.template.UploadEntity;
 +import org.apache.cloudstack.storage.template.UploadManager;
 +import org.apache.cloudstack.storage.template.UploadManagerImpl;
 +import org.apache.cloudstack.storage.to.SnapshotObjectTO;
 +import org.apache.cloudstack.storage.to.TemplateObjectTO;
 +import org.apache.cloudstack.storage.to.VolumeObjectTO;
- import org.apache.cloudstack.utils.imagestore.ImageStoreUtil;
- import org.apache.commons.codec.digest.DigestUtils;
- import org.apache.commons.io.FileUtils;
- import org.apache.commons.io.FilenameUtils;
- import org.apache.commons.lang.StringUtils;
- import org.apache.http.HttpEntity;
- import org.apache.http.HttpResponse;
- import org.apache.http.NameValuePair;
- import org.apache.http.client.HttpClient;
- import org.apache.http.client.methods.HttpGet;
- import org.apache.http.client.utils.URLEncodedUtils;
- import org.apache.http.impl.client.DefaultHttpClient;
- import org.apache.log4j.Logger;
- import org.joda.time.DateTime;
- import org.joda.time.format.ISODateTimeFormat;
 +
- import com.amazonaws.services.s3.model.S3ObjectSummary;
  import com.cloud.agent.api.Answer;
  import com.cloud.agent.api.CheckHealthAnswer;
  import com.cloud.agent.api.CheckHealthCommand;
@@@ -116,13 -51,13 +135,11 @@@ import com.cloud.agent.api.to.NfsTO
  import com.cloud.agent.api.to.S3TO;
  import com.cloud.agent.api.to.SwiftTO;
  import com.cloud.exception.InternalErrorException;
--import com.cloud.exception.InvalidParameterValueException;
  import com.cloud.host.Host;
  import com.cloud.host.Host.Type;
  import com.cloud.hypervisor.Hypervisor.HypervisorType;
  import com.cloud.resource.ServerResourceBase;
  import com.cloud.storage.DataStoreRole;
--import com.cloud.storage.Storage;
  import com.cloud.storage.Storage.ImageFormat;
  import com.cloud.storage.StorageLayer;
  import com.cloud.storage.VMTemplateStorageResourceAssoc;
@@@ -132,40 -67,102 +149,24 @@@ import com.cloud.storage.template.Proce
  import com.cloud.storage.template.QCOW2Processor;
  import com.cloud.storage.template.RawImageProcessor;
  import com.cloud.storage.template.TARProcessor;
--import com.cloud.storage.template.TemplateConstants;
  import com.cloud.storage.template.TemplateLocation;
  import com.cloud.storage.template.TemplateProp;
  import com.cloud.storage.template.VhdProcessor;
  import com.cloud.storage.template.VmdkProcessor;
--import com.cloud.utils.EncryptionUtil;
  import com.cloud.utils.NumbersUtil;
++import com.cloud.utils.storage.S3.S3Utils;
  import com.cloud.utils.SwiftUtil;
  import com.cloud.utils.exception.CloudRuntimeException;
  import com.cloud.utils.net.NetUtils;
  import com.cloud.utils.script.OutputInterpreter;
  import com.cloud.utils.script.Script;
--import com.cloud.utils.storage.S3.S3Utils;
  import com.cloud.vm.SecondaryStorageVm;
--import com.google.gson.Gson;
--import com.google.gson.GsonBuilder;
- 
--import io.netty.bootstrap.ServerBootstrap;
--import io.netty.channel.Channel;
--import io.netty.channel.ChannelInitializer;
--import io.netty.channel.ChannelPipeline;
--import io.netty.channel.EventLoopGroup;
--import io.netty.channel.nio.NioEventLoopGroup;
--import io.netty.channel.socket.SocketChannel;
--import io.netty.channel.socket.nio.NioServerSocketChannel;
--import io.netty.handler.codec.http.HttpContentCompressor;
--import io.netty.handler.codec.http.HttpRequestDecoder;
--import io.netty.handler.codec.http.HttpResponseEncoder;
--import io.netty.handler.logging.LogLevel;
--import io.netty.handler.logging.LoggingHandler;
 -import org.apache.cloudstack.framework.security.keystore.KeystoreManager;
 -import org.apache.cloudstack.storage.command.CopyCmdAnswer;
 -import org.apache.cloudstack.storage.command.CopyCommand;
 -import org.apache.cloudstack.storage.command.DeleteCommand;
 -import org.apache.cloudstack.storage.command.DownloadCommand;
 -import org.apache.cloudstack.storage.command.DownloadProgressCommand;
 -import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand;
 -import org.apache.cloudstack.storage.command.UploadStatusAnswer;
 -import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus;
 -import org.apache.cloudstack.storage.command.UploadStatusCommand;
 -import org.apache.cloudstack.storage.template.DownloadManager;
 -import org.apache.cloudstack.storage.template.DownloadManagerImpl;
 -import org.apache.cloudstack.storage.template.DownloadManagerImpl.ZfsPathParser;
 -import org.apache.cloudstack.storage.template.UploadEntity;
 -import org.apache.cloudstack.storage.template.UploadManager;
 -import org.apache.cloudstack.storage.template.UploadManagerImpl;
 -import org.apache.cloudstack.storage.to.SnapshotObjectTO;
 -import org.apache.cloudstack.storage.to.TemplateObjectTO;
 -import org.apache.cloudstack.storage.to.VolumeObjectTO;
 -import org.apache.cloudstack.utils.imagestore.ImageStoreUtil;
 -import org.apache.commons.codec.digest.DigestUtils;
 -import org.apache.commons.io.FileUtils;
 -import org.apache.commons.io.FilenameUtils;
 -import org.apache.commons.lang.StringUtils;
 -import org.apache.http.HttpEntity;
 -import org.apache.http.HttpResponse;
 -import org.apache.http.NameValuePair;
 -import org.apache.http.client.HttpClient;
 -import org.apache.http.client.methods.HttpGet;
 -import org.apache.http.client.utils.URLEncodedUtils;
 -import org.apache.http.impl.client.DefaultHttpClient;
 -import org.apache.log4j.Logger;
+ import org.joda.time.DateTime;
+ import org.joda.time.format.ISODateTimeFormat;
  
 -import javax.naming.ConfigurationException;
 -import java.io.BufferedReader;
 -import java.io.BufferedWriter;
 -import java.io.File;
 -import java.io.FileInputStream;
 -import java.io.FileOutputStream;
 -import java.io.FileReader;
 -import java.io.FileWriter;
 -import java.io.IOException;
 -import java.io.InputStream;
 -import java.io.UnsupportedEncodingException;
 -import java.math.BigInteger;
 -import java.net.InetAddress;
 -import java.net.URI;
 -import java.net.UnknownHostException;
 -import java.security.MessageDigest;
 -import java.security.NoSuchAlgorithmException;
 -import java.util.ArrayList;
 -import java.util.HashMap;
 -import java.util.List;
 -import java.util.Map;
 -import java.util.UUID;
 -
 -import static com.cloud.utils.StringUtils.join;
 -import static com.cloud.utils.storage.S3.S3Utils.putFile;
 -import static java.lang.String.format;
 -import static java.util.Arrays.asList;
 -import static org.apache.commons.lang.StringUtils.substringAfterLast;
 -
  public class NfsSecondaryStorageResource extends ServerResourceBase implements SecondaryStorageResource
{
  
-     private static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResource.class);
+     public static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResource.class);
  
      private static final String TEMPLATE_ROOT_DIR = "template/tmpl";
      private static final String VOLUME_ROOT_DIR = "volumes";
@@@ -933,6 -925,118 +921,118 @@@
          }
      }
  
+     /***
+      *This method will create a file using the filenName and metaFileName.
+      *That file will contain the given attributes (unique name, file name, size, and virtualSize).
+      *
+      * @param metaFileName : The path of the metadata file
+      * @param filename      :attribute:  Filename of the template
+      * @param uniqueName    :attribute:  Unique name of the template
+      * @param size          :attribute:  physical size of the template
+      * @param virtualSize   :attribute:  virtual size of the template
+      * @return File representing the metadata file
+      * @throws IOException
+      */
+ 
+     protected File swiftWriteMetadataFile(String metaFileName, String uniqueName, String
filename, long size, long virtualSize) throws IOException {
+         File metaFile = new File(metaFileName);
+         FileWriter writer = new FileWriter(metaFile);
+         BufferedWriter bufferWriter = new BufferedWriter(writer);
+         bufferWriter.write("uniquename=" + uniqueName);
+         bufferWriter.write("\n");
+         bufferWriter.write("filename=" + filename);
+         bufferWriter.write("\n");
+         bufferWriter.write("size=" + size);
+         bufferWriter.write("\n");
+         bufferWriter.write("virtualsize=" + virtualSize);
+         bufferWriter.close();
+         writer.close();
+         return metaFile;
+     }
+ 
+     /**
+      * Creates a template.properties for Swift with its correct unique name
+      *
+      * @param swift  The swift object
+      * @param srcFile Source file on the staging NFS
+      * @param containerName Destination container
+      * @return true on successful write
+      */
+     protected boolean swiftUploadMetadataFile(SwiftTO swift, File srcFile, String containerName)
throws IOException {
+ 
+         String uniqueName = FilenameUtils.getBaseName(srcFile.getName());
+ 
+         File uniqDir = _storage.createUniqDir();
+         String metaFileName = uniqDir.getAbsolutePath() + File.separator + _tmpltpp;
+         _storage.create(uniqDir.getAbsolutePath(), _tmpltpp);
+ 
+         long virtualSize = getVirtualSize(srcFile, getTemplateFormat(srcFile.getName()));
+ 
+         File metaFile = swiftWriteMetadataFile(metaFileName,
+                                                 uniqueName,
+                                                 srcFile.getName(),
+                                                 srcFile.length(),
+                                                 virtualSize);
+ 
+         SwiftUtil.putObject(swift, metaFile, containerName, _tmpltpp);
+         metaFile.delete();
+         uniqDir.delete();
+ 
+         return true;
+     }
+ 
+     /**
+      * Copies data from NFS and uploads it into a Swift container
+      *
+      * @param cmd CopyComand
+      * @return CopyCmdAnswer
+      */
+     protected Answer copyFromNfsToSwift(CopyCommand cmd) {
+ 
+         final DataTO srcData = cmd.getSrcTO();
+         final DataTO destData = cmd.getDestTO();
+ 
+         DataStoreTO srcDataStore = srcData.getDataStore();
+         NfsTO srcStore = (NfsTO)srcDataStore;
+         DataStoreTO destDataStore = destData.getDataStore();
 -        File srcFile = getFile(srcData.getPath(), srcStore.getUrl());
++        File srcFile = getFile(srcData.getPath(), srcStore.getUrl(), _nfsVersion);
+ 
+         SwiftTO swift = (SwiftTO)destDataStore;
+ 
+         try {
+ 
+             String containerName = SwiftUtil.getContainerName(destData.getObjectType().toString(),
destData.getId());
+             String swiftPath = SwiftUtil.putObject(swift, srcFile, containerName, srcFile.getName());
+ 
+ 
+             DataTO retObj = null;
+             if (destData.getObjectType() == DataObjectType.TEMPLATE) {
+                 swiftUploadMetadataFile(swift, srcFile, containerName);
+                 TemplateObjectTO newTemplate = new TemplateObjectTO();
+                 newTemplate.setPath(swiftPath);
+                 newTemplate.setSize(getVirtualSize(srcFile, getTemplateFormat(srcFile.getName())));
+                 newTemplate.setPhysicalSize(srcFile.length());
+                 newTemplate.setFormat(getTemplateFormat(srcFile.getName()));
+                 retObj = newTemplate;
+             } else if (destData.getObjectType() == DataObjectType.VOLUME) {
+                 VolumeObjectTO newVol = new VolumeObjectTO();
+                 newVol.setPath(containerName);
+                 newVol.setSize(getVirtualSize(srcFile, getTemplateFormat(srcFile.getName())));
+                 retObj = newVol;
+             } else if (destData.getObjectType() == DataObjectType.SNAPSHOT) {
+                 SnapshotObjectTO newSnapshot = new SnapshotObjectTO();
+                 newSnapshot.setPath(containerName);
+                 retObj = newSnapshot;
+             }
+ 
+             return new CopyCmdAnswer(retObj);
+ 
+         } catch (Exception e) {
+             s_logger.error("failed to upload " + srcData.getPath(), e);
+             return new CopyCmdAnswer("failed to upload " + srcData.getPath() + e.toString());
+         }
+     }
+ 
      String swiftDownload(SwiftTO swift, String container, String rfilename, String lFullPath)
{
          Script command = new Script("/bin/bash", s_logger);
          command.add("-c");


Mime
View raw message