cloudstack-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject [20/48] git commit: updated refs/heads/4.9-bountycastle-daan to 98bf0ca
Date Tue, 24 May 2016 09:49:25 GMT
Fix Sync of template.properties in Swift


Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/f5ac8ddd
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/f5ac8ddd
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/f5ac8ddd

Branch: refs/heads/4.9-bountycastle-daan
Commit: f5ac8dddedaec6ed5e21069f150f86229b43f02c
Parents: 7a0b37a
Author: Syed <syed1.mushtaq@gmail.com>
Authored: Mon Dec 14 17:37:28 2015 -0500
Committer: Syed <syed1.mushtaq@gmail.com>
Committed: Fri May 13 22:08:16 2016 +0000

----------------------------------------------------------------------
 .../datastore/ObjectInDataStoreManagerImpl.java |   1 +
 server/src/com/cloud/test/TestAppender.java     | 178 +++++++++
 services/secondary-storage/server/pom.xml       |   3 -
 .../resource/NfsSecondaryStorageResource.java   | 373 ++++++++++++-------
 .../LocalNfsSecondaryStorageResourceTest.java   |  44 ++-
 .../NfsSecondaryStorageResourceTest.java        | 118 +++---
 6 files changed, 481 insertions(+), 236 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cloudstack/blob/f5ac8ddd/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java
b/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java
index b1d6127..5117b7c 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java
@@ -156,6 +156,7 @@ public class ObjectInDataStoreManagerImpl implements ObjectInDataStoreManager
{
                         // template.properties
                         // there
                     }
+
                     ts.setInstallPath(installPath);
                     ts.setState(ObjectInDataStoreStateMachine.State.Allocated);
                     ts = templateDataStoreDao.persist(ts);

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/f5ac8ddd/server/src/com/cloud/test/TestAppender.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/test/TestAppender.java b/server/src/com/cloud/test/TestAppender.java
new file mode 100644
index 0000000..539cac1
--- /dev/null
+++ b/server/src/com/cloud/test/TestAppender.java
@@ -0,0 +1,178 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+package com.cloud.test;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Objects;
+import com.google.common.collect.ImmutableMap;
+import org.apache.log4j.AppenderSkeleton;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.spi.LoggingEvent;
+import org.springframework.util.Assert;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.base.Strings.isNullOrEmpty;
+import static java.lang.String.format;
+import static org.apache.log4j.Level.ALL;
+import static org.apache.log4j.Level.DEBUG;
+import static org.apache.log4j.Level.ERROR;
+import static org.apache.log4j.Level.FATAL;
+import static org.apache.log4j.Level.INFO;
+import static org.apache.log4j.Level.OFF;
+
+/**
+*
+* Tracks one or more patterns to determine whether or not they have been
+* logged. It uses a streaming approach to determine whether or not a message
+* has a occurred to prevent unnecessary memory consumption. Instances of this
+* of this class are created using the {@link TestAppenderBuilder}.
+*
+* To use this class, register a one or more expected patterns by level as part
+* of the test setup and retain an reference to the appender instance. After the
+* expected logging events have occurred in the test case, call
+* {@link TestAppender#assertMessagesLogged()} which will fail the test if any of the
+* expected patterns were not logged.
+*
+*/
+public final class TestAppender extends AppenderSkeleton {
+    private final static String APPENDER_NAME = "test_appender";
+    private final ImmutableMap<Level, Set<PatternResult>> expectedPatternResults;
+    private TestAppender(final Map<Level, Set<PatternResult>> expectedPatterns)
{
+        super();
+        expectedPatternResults = ImmutableMap.copyOf(expectedPatterns);
+    }
+    protected void append(LoggingEvent loggingEvent) {
+        checkArgument(loggingEvent != null, "append requires a non-null loggingEvent");
+        final Level level = loggingEvent.getLevel();
+        checkState(expectedPatternResults.containsKey(level), "level " + level + " not supported
by append");
+        for (final PatternResult patternResult : expectedPatternResults.get(level)) {
+            if (patternResult.getPattern().matcher(loggingEvent.getRenderedMessage()).matches())
{
+                patternResult.markFound();
+            }
+        }
+    }
+
+    public void close() {
+// Do nothing ...
+    }
+    public boolean requiresLayout() {
+        return false;
+    }
+    public void assertMessagesLogged() {
+        final List<String> unloggedPatterns = new ArrayList<>();
+        for (final Map.Entry<Level, Set<PatternResult>> expectedPatternResult
: expectedPatternResults.entrySet()) {
+            for (final PatternResult patternResults : expectedPatternResult.getValue()) {
+                if (!patternResults.isFound()) {
+                    unloggedPatterns.add(format("%1$s was not logged for level %2$s",
+                            patternResults.getPattern().toString(), expectedPatternResult.getKey()));
+                }
+            }
+        }
+        if (!unloggedPatterns.isEmpty()) {
+            //Raise an assert
+            Assert.isTrue(false, Joiner.on(",").join(unloggedPatterns));
+        }
+    }
+
+    private static final class PatternResult {
+        private final Pattern pattern;
+        private boolean foundFlag = false;
+        private PatternResult(Pattern pattern) {
+            super();
+            this.pattern = pattern;
+        }
+        public Pattern getPattern() {
+            return pattern;
+        }
+        public void markFound() {
+        // This operation is thread-safe because the value will only ever be switched from
false to true. Therefore,
+        // multiple threads mutating the value for a pattern will not corrupt the value ...
+            foundFlag = true;
+        }
+        public boolean isFound() {
+            return foundFlag;
+        }
+        @Override
+        public boolean equals(Object thatObject) {
+            if (this == thatObject) {
+                return true;
+            }
+            if (thatObject == null || getClass() != thatObject.getClass()) {
+                return false;
+            }
+            PatternResult thatPatternResult = (PatternResult) thatObject;
+            return foundFlag == thatPatternResult.foundFlag &&
+                    Objects.equal(pattern, thatPatternResult.pattern);
+        }
+        @Override
+        public int hashCode() {
+            return Objects.hashCode(pattern, foundFlag);
+        }
+        @Override
+        public String toString() {
+            return format("Pattern Result [ pattern: %1$s, markFound: %2$s ]", pattern.toString(),
foundFlag);
+        }
+    }
+
+    public static final class TestAppenderBuilder {
+        private final Map<Level, Set<PatternResult>> expectedPatterns;
+        public TestAppenderBuilder() {
+            super();
+            expectedPatterns = new HashMap<>();
+            expectedPatterns.put(ALL, new HashSet<PatternResult>());
+            expectedPatterns.put(DEBUG, new HashSet<PatternResult>());
+            expectedPatterns.put(ERROR, new HashSet<PatternResult>());
+            expectedPatterns.put(FATAL, new HashSet<PatternResult>());
+            expectedPatterns.put(INFO, new HashSet<PatternResult>());
+            expectedPatterns.put(OFF, new HashSet<PatternResult>());
+        }
+        public TestAppenderBuilder addExpectedPattern(final Level level, final String pattern)
{
+            checkArgument(level != null, "addExpectedPattern requires a non-null level");
+            checkArgument(!isNullOrEmpty(pattern), "addExpectedPattern requires a non-blank
pattern");
+            checkState(expectedPatterns.containsKey(level), "level " + level + " is not supported
by " + getClass().getName());
+            expectedPatterns.get(level).add(new PatternResult(Pattern.compile(pattern)));
+            return this;
+        }
+        public TestAppender build() {
+            return new TestAppender(expectedPatterns);
+        }
+    }
+    /**
+     *
+     * Attaches a {@link TestAppender} to a {@link Logger} and ensures that it is the only
+     * test appender attached to the logger.
+     *
+     * @param logger The logger which will be monitored by the test
+     * @param testAppender The test appender to attach to {@code logger}
+     */
+    public static void safeAddAppender(Logger logger, TestAppender testAppender) {
+        logger.removeAppender(APPENDER_NAME);
+        logger.addAppender(testAppender);
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/f5ac8ddd/services/secondary-storage/server/pom.xml
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/pom.xml b/services/secondary-storage/server/pom.xml
index 417e0a6..63aef27 100644
--- a/services/secondary-storage/server/pom.xml
+++ b/services/secondary-storage/server/pom.xml
@@ -26,9 +26,6 @@
     <version>4.7.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
-    <properties>
-    <skipTests>true</skipTests>
-  </properties>
     <dependencies>
     <dependency>
       <groupId>log4j</groupId>

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/f5ac8ddd/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
index 768a177..f512da0 100644
--- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
+++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
@@ -16,91 +16,7 @@
 // under the License.
 package org.apache.cloudstack.storage.resource;
 
-import static com.cloud.utils.storage.S3.S3Utils.putFile;
-import static com.cloud.utils.StringUtils.join;
-import static java.lang.String.format;
-import static java.util.Arrays.asList;
-import static org.apache.commons.lang.StringUtils.substringAfterLast;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.UnsupportedEncodingException;
-import java.math.BigInteger;
-import java.net.InetAddress;
-import java.net.URI;
-import java.net.UnknownHostException;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-
-import javax.naming.ConfigurationException;
-
-import com.cloud.exception.InvalidParameterValueException;
-import com.cloud.storage.Storage;
-import com.cloud.storage.template.TemplateConstants;
-import com.cloud.utils.EncryptionUtil;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import io.netty.bootstrap.ServerBootstrap;
-import io.netty.channel.Channel;
-import io.netty.channel.ChannelInitializer;
-import io.netty.channel.ChannelPipeline;
-import io.netty.channel.EventLoopGroup;
-import io.netty.channel.nio.NioEventLoopGroup;
-import io.netty.channel.socket.SocketChannel;
-import io.netty.channel.socket.nio.NioServerSocketChannel;
-import io.netty.handler.codec.http.HttpContentCompressor;
-import io.netty.handler.codec.http.HttpRequestDecoder;
-import io.netty.handler.codec.http.HttpResponseEncoder;
-import io.netty.handler.logging.LogLevel;
-import io.netty.handler.logging.LoggingHandler;
-import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand;
-import org.apache.cloudstack.storage.template.UploadEntity;
-import org.apache.cloudstack.utils.imagestore.ImageStoreUtil;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.FilenameUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
-import org.apache.http.NameValuePair;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.utils.URLEncodedUtils;
-import org.apache.http.impl.client.DefaultHttpClient;
-import org.apache.log4j.Logger;
-
 import com.amazonaws.services.s3.model.S3ObjectSummary;
-
-import org.apache.cloudstack.framework.security.keystore.KeystoreManager;
-import org.apache.cloudstack.storage.command.CopyCmdAnswer;
-import org.apache.cloudstack.storage.command.CopyCommand;
-import org.apache.cloudstack.storage.command.DeleteCommand;
-import org.apache.cloudstack.storage.command.DownloadCommand;
-import org.apache.cloudstack.storage.command.DownloadProgressCommand;
-import org.apache.cloudstack.storage.command.UploadStatusAnswer;
-import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus;
-import org.apache.cloudstack.storage.command.UploadStatusCommand;
-import org.apache.cloudstack.storage.template.DownloadManager;
-import org.apache.cloudstack.storage.template.DownloadManagerImpl;
-import org.apache.cloudstack.storage.template.DownloadManagerImpl.ZfsPathParser;
-import org.apache.cloudstack.storage.template.UploadManager;
-import org.apache.cloudstack.storage.template.UploadManagerImpl;
-import org.apache.cloudstack.storage.to.SnapshotObjectTO;
-import org.apache.cloudstack.storage.to.TemplateObjectTO;
-import org.apache.cloudstack.storage.to.VolumeObjectTO;
-
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.CheckHealthAnswer;
 import com.cloud.agent.api.CheckHealthCommand;
@@ -135,11 +51,13 @@ import com.cloud.agent.api.to.NfsTO;
 import com.cloud.agent.api.to.S3TO;
 import com.cloud.agent.api.to.SwiftTO;
 import com.cloud.exception.InternalErrorException;
+import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.host.Host;
 import com.cloud.host.Host.Type;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.resource.ServerResourceBase;
 import com.cloud.storage.DataStoreRole;
+import com.cloud.storage.Storage;
 import com.cloud.storage.Storage.ImageFormat;
 import com.cloud.storage.StorageLayer;
 import com.cloud.storage.VMTemplateStorageResourceAssoc;
@@ -149,24 +67,102 @@ import com.cloud.storage.template.Processor.FormatInfo;
 import com.cloud.storage.template.QCOW2Processor;
 import com.cloud.storage.template.RawImageProcessor;
 import com.cloud.storage.template.TARProcessor;
+import com.cloud.storage.template.TemplateConstants;
 import com.cloud.storage.template.TemplateLocation;
 import com.cloud.storage.template.TemplateProp;
 import com.cloud.storage.template.VhdProcessor;
 import com.cloud.storage.template.VmdkProcessor;
+import com.cloud.utils.EncryptionUtil;
 import com.cloud.utils.NumbersUtil;
-import com.cloud.utils.storage.S3.S3Utils;
 import com.cloud.utils.SwiftUtil;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.net.NetUtils;
 import com.cloud.utils.script.OutputInterpreter;
 import com.cloud.utils.script.Script;
+import com.cloud.utils.storage.S3.S3Utils;
 import com.cloud.vm.SecondaryStorageVm;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import io.netty.bootstrap.ServerBootstrap;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelPipeline;
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.channel.socket.SocketChannel;
+import io.netty.channel.socket.nio.NioServerSocketChannel;
+import io.netty.handler.codec.http.HttpContentCompressor;
+import io.netty.handler.codec.http.HttpRequestDecoder;
+import io.netty.handler.codec.http.HttpResponseEncoder;
+import io.netty.handler.logging.LogLevel;
+import io.netty.handler.logging.LoggingHandler;
+import org.apache.cloudstack.framework.security.keystore.KeystoreManager;
+import org.apache.cloudstack.storage.command.CopyCmdAnswer;
+import org.apache.cloudstack.storage.command.CopyCommand;
+import org.apache.cloudstack.storage.command.DeleteCommand;
+import org.apache.cloudstack.storage.command.DownloadCommand;
+import org.apache.cloudstack.storage.command.DownloadProgressCommand;
+import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand;
+import org.apache.cloudstack.storage.command.UploadStatusAnswer;
+import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus;
+import org.apache.cloudstack.storage.command.UploadStatusCommand;
+import org.apache.cloudstack.storage.template.DownloadManager;
+import org.apache.cloudstack.storage.template.DownloadManagerImpl;
+import org.apache.cloudstack.storage.template.DownloadManagerImpl.ZfsPathParser;
+import org.apache.cloudstack.storage.template.UploadEntity;
+import org.apache.cloudstack.storage.template.UploadManager;
+import org.apache.cloudstack.storage.template.UploadManagerImpl;
+import org.apache.cloudstack.storage.to.SnapshotObjectTO;
+import org.apache.cloudstack.storage.to.TemplateObjectTO;
+import org.apache.cloudstack.storage.to.VolumeObjectTO;
+import org.apache.cloudstack.utils.imagestore.ImageStoreUtil;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.utils.URLEncodedUtils;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.log4j.Logger;
 import org.joda.time.DateTime;
 import org.joda.time.format.ISODateTimeFormat;
 
+import javax.naming.ConfigurationException;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.math.BigInteger;
+import java.net.InetAddress;
+import java.net.URI;
+import java.net.UnknownHostException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import static com.cloud.utils.StringUtils.join;
+import static com.cloud.utils.storage.S3.S3Utils.putFile;
+import static java.lang.String.format;
+import static java.util.Arrays.asList;
+import static org.apache.commons.lang.StringUtils.substringAfterLast;
+
 public class NfsSecondaryStorageResource extends ServerResourceBase implements SecondaryStorageResource
{
 
-    private static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResource.class);
+    public static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResource.class);
 
     private static final String TEMPLATE_ROOT_DIR = "template/tmpl";
     private static final String VOLUME_ROOT_DIR = "volumes";
@@ -499,10 +495,10 @@ public class NfsSecondaryStorageResource extends ServerResourceBase
implements S
             String destFileFullPath = destFile.getAbsolutePath() + File.separator + fileName;
             s_logger.debug("copy snapshot " + srcFile.getAbsolutePath() + " to template "
+ destFileFullPath);
             Script.runSimpleBashScript("cp " + srcFile.getAbsolutePath() + " " + destFileFullPath);
-            String metaFileName = destFile.getAbsolutePath() + File.separator + "template.properties";
+            String metaFileName = destFile.getAbsolutePath() + File.separator + _tmpltpp;
             File metaFile = new File(metaFileName);
             try {
-                _storage.create(destFile.getAbsolutePath(), "template.properties");
+                _storage.create(destFile.getAbsolutePath(), _tmpltpp);
                 try ( // generate template.properties file
                      FileWriter writer = new FileWriter(metaFile);
                      BufferedWriter bufferWriter = new BufferedWriter(writer);
@@ -597,32 +593,14 @@ public class NfsSecondaryStorageResource extends ServerResourceBase
implements S
                     return answer;
                 }
                 s_logger.debug("starting copy template to swift");
-                DataTO newTemplate = answer.getNewData();
-                File templateFile = getFile(newTemplate.getPath(), ((NfsTO)srcDataStore).getUrl());
-                SwiftTO swift = (SwiftTO)destDataStore;
-                String containterName = SwiftUtil.getContainerName(destData.getObjectType().toString(),
destData.getId());
-                String swiftPath = SwiftUtil.putObject(swift, templateFile, containterName,
templateFile.getName());
-                //upload template.properties
-                File properties = new File(templateFile.getParent() + File.separator + _tmpltpp);
-                if (properties.exists()) {
-                    SwiftUtil.putObject(swift, properties, containterName, _tmpltpp);
-                }
+                TemplateObjectTO newTemplate = (TemplateObjectTO)answer.getNewData();
+                newTemplate.setDataStore(srcDataStore);
+                CopyCommand newCpyCmd = new CopyCommand(newTemplate, destData, cmd.getWait(),
cmd.executeInSequence());
+                Answer result = copyFromNfsToSwift(newCpyCmd);
 
-                //clean up template data on staging area
-                try {
-                    DeleteCommand deleteCommand = new DeleteCommand(newTemplate);
-                    execute(deleteCommand);
-                } catch (Exception e) {
-                    s_logger.debug("Failed to clean up staging area:", e);
-                }
+                cleanupStagingNfs(newTemplate);
+                return result;
 
-                TemplateObjectTO template = new TemplateObjectTO();
-                template.setPath(swiftPath);
-                template.setSize(templateFile.length());
-                template.setPhysicalSize(template.getSize());
-                SnapshotObjectTO snapshot = (SnapshotObjectTO)srcData;
-                template.setFormat(snapshot.getVolume().getFormat());
-                return new CopyCmdAnswer(template);
             } else if (destDataStore instanceof S3TO) {
                 //create template on the same data store
                 CopyCmdAnswer answer =
@@ -635,18 +613,27 @@ public class NfsSecondaryStorageResource extends ServerResourceBase
implements S
                 newTemplate.setDataStore(srcDataStore);
                 CopyCommand newCpyCmd = new CopyCommand(newTemplate, destData, cmd.getWait(),
cmd.executeInSequence());
                 Answer result = copyFromNfsToS3(newCpyCmd);
-                //clean up template data on staging area
-                try {
-                    DeleteCommand deleteCommand = new DeleteCommand(newTemplate);
-                    execute(deleteCommand);
-                } catch (Exception e) {
-                    s_logger.debug("Failed to clean up staging area:", e);
-                }
+
+                cleanupStagingNfs(newTemplate);
+
                 return result;
             }
         }
-        s_logger.debug("Failed to create templat from snapshot");
-        return new CopyCmdAnswer("Unsupported prototcol");
+        s_logger.debug("Failed to create template from snapshot");
+        return new CopyCmdAnswer("Unsupported protocol");
+    }
+
+    /**
+     * clean up template data on staging area
+     * @param newTemplate: The template on the secondary storage that needs to be cleaned
up
+     */
+    protected void cleanupStagingNfs(TemplateObjectTO newTemplate) {
+        try {
+            DeleteCommand deleteCommand = new DeleteCommand(newTemplate);
+            execute(deleteCommand);
+        } catch (Exception e) {
+            s_logger.debug("Failed to clean up staging area:", e);
+        }
     }
 
     protected Answer copyFromNfsToImage(CopyCommand cmd) {
@@ -759,22 +746,18 @@ public class NfsSecondaryStorageResource extends ServerResourceBase
implements S
             String container = "T-" + cmd.getId();
             String swiftPath = SwiftUtil.putObject(swiftTO, file, container, null);
 
+            long virtualSize = getVirtualSize(file, getTemplateFormat(file.getName()));
+            long size = file.length();
+            String uniqueName = cmd.getName();
+
             //put metda file
             File uniqDir = _storage.createUniqDir();
-            String metaFileName = uniqDir.getAbsolutePath() + File.separator + "template.properties";
-            _storage.create(uniqDir.getAbsolutePath(), "template.properties");
-            File metaFile = new File(metaFileName);
-            FileWriter writer = new FileWriter(metaFile);
-            BufferedWriter bufferWriter = new BufferedWriter(writer);
-            bufferWriter.write("uniquename=" + cmd.getName());
-            bufferWriter.write("\n");
-            bufferWriter.write("filename=" + fileName);
-            bufferWriter.write("\n");
-            bufferWriter.write("size=" + file.length());
-            bufferWriter.close();
-            writer.close();
-
-            SwiftUtil.putObject(swiftTO, metaFile, container, "template.properties");
+            String metaFileName = uniqDir.getAbsolutePath() + File.separator + _tmpltpp;
+            _storage.create(uniqDir.getAbsolutePath(), _tmpltpp);
+
+            File metaFile = swiftWriteMetadataFile(metaFileName, uniqueName, fileName, size,
virtualSize);
+
+            SwiftUtil.putObject(swiftTO, metaFile, container, _tmpltpp);
             metaFile.delete();
             uniqDir.delete();
             String md5sum = null;
@@ -785,7 +768,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements
S
             }
 
             DownloadAnswer answer =
-                    new DownloadAnswer(null, 100, null, VMTemplateStorageResourceAssoc.Status.DOWNLOADED,
swiftPath, swiftPath, file.length(), file.length(), md5sum);
+                    new DownloadAnswer(null, 100, null, VMTemplateStorageResourceAssoc.Status.DOWNLOADED,
swiftPath, swiftPath, virtualSize, file.length(), md5sum);
             return answer;
         } catch (IOException e) {
             s_logger.debug("Failed to register template into swift", e);
@@ -942,6 +925,118 @@ public class NfsSecondaryStorageResource extends ServerResourceBase
implements S
         }
     }
 
+    /***
+     *This method will create a file using the filenName and metaFileName.
+     *That file will contain the given attributes (unique name, file name, size, and virtualSize).
+     *
+     * @param metaFileName : The path of the metadata file
+     * @param filename      :attribute:  Filename of the template
+     * @param uniqueName    :attribute:  Unique name of the template
+     * @param size          :attribute:  physical size of the template
+     * @param virtualSize   :attribute:  virtual size of the template
+     * @return File representing the metadata file
+     * @throws IOException
+     */
+
+    protected File swiftWriteMetadataFile(String metaFileName, String uniqueName, String
filename, long size, long virtualSize) throws IOException {
+        File metaFile = new File(metaFileName);
+        FileWriter writer = new FileWriter(metaFile);
+        BufferedWriter bufferWriter = new BufferedWriter(writer);
+        bufferWriter.write("uniquename=" + uniqueName);
+        bufferWriter.write("\n");
+        bufferWriter.write("filename=" + filename);
+        bufferWriter.write("\n");
+        bufferWriter.write("size=" + size);
+        bufferWriter.write("\n");
+        bufferWriter.write("virtualsize=" + virtualSize);
+        bufferWriter.close();
+        writer.close();
+        return metaFile;
+    }
+
+    /**
+     * Creates a template.properties for Swift with its correct unique name
+     *
+     * @param swift  The swift object
+     * @param srcFile Source file on the staging NFS
+     * @param containerName Destination container
+     * @return true on successful write
+     */
+    protected boolean swiftUploadMetadataFile(SwiftTO swift, File srcFile, String containerName)
throws IOException {
+
+        String uniqueName = FilenameUtils.getBaseName(srcFile.getName());
+
+        File uniqDir = _storage.createUniqDir();
+        String metaFileName = uniqDir.getAbsolutePath() + File.separator + _tmpltpp;
+        _storage.create(uniqDir.getAbsolutePath(), _tmpltpp);
+
+        long virtualSize = getVirtualSize(srcFile, getTemplateFormat(srcFile.getName()));
+
+        File metaFile = swiftWriteMetadataFile(metaFileName,
+                                                uniqueName,
+                                                srcFile.getName(),
+                                                srcFile.length(),
+                                                virtualSize);
+
+        SwiftUtil.putObject(swift, metaFile, containerName, _tmpltpp);
+        metaFile.delete();
+        uniqDir.delete();
+
+        return true;
+    }
+
+    /**
+     * Copies data from NFS and uploads it into a Swift container
+     *
+     * @param cmd CopyComand
+     * @return CopyCmdAnswer
+     */
+    protected Answer copyFromNfsToSwift(CopyCommand cmd) {
+
+        final DataTO srcData = cmd.getSrcTO();
+        final DataTO destData = cmd.getDestTO();
+
+        DataStoreTO srcDataStore = srcData.getDataStore();
+        NfsTO srcStore = (NfsTO)srcDataStore;
+        DataStoreTO destDataStore = destData.getDataStore();
+        File srcFile = getFile(srcData.getPath(), srcStore.getUrl());
+
+        SwiftTO swift = (SwiftTO)destDataStore;
+
+        try {
+
+            String containerName = SwiftUtil.getContainerName(destData.getObjectType().toString(),
destData.getId());
+            String swiftPath = SwiftUtil.putObject(swift, srcFile, containerName, srcFile.getName());
+
+
+            DataTO retObj = null;
+            if (destData.getObjectType() == DataObjectType.TEMPLATE) {
+                swiftUploadMetadataFile(swift, srcFile, containerName);
+                TemplateObjectTO newTemplate = new TemplateObjectTO();
+                newTemplate.setPath(swiftPath);
+                newTemplate.setSize(getVirtualSize(srcFile, getTemplateFormat(srcFile.getName())));
+                newTemplate.setPhysicalSize(srcFile.length());
+                newTemplate.setFormat(getTemplateFormat(srcFile.getName()));
+                retObj = newTemplate;
+            } else if (destData.getObjectType() == DataObjectType.VOLUME) {
+                VolumeObjectTO newVol = new VolumeObjectTO();
+                newVol.setPath(containerName);
+                newVol.setSize(getVirtualSize(srcFile, getTemplateFormat(srcFile.getName())));
+                retObj = newVol;
+            } else if (destData.getObjectType() == DataObjectType.SNAPSHOT) {
+                SnapshotObjectTO newSnapshot = new SnapshotObjectTO();
+                newSnapshot.setPath(containerName);
+                retObj = newSnapshot;
+            }
+
+            return new CopyCmdAnswer(retObj);
+
+        } catch (Exception e) {
+            s_logger.error("failed to upload " + srcData.getPath(), e);
+            return new CopyCmdAnswer("failed to upload " + srcData.getPath() + e.toString());
+        }
+    }
+
     String swiftDownload(SwiftTO swift, String container, String rfilename, String lFullPath)
{
         Script command = new Script("/bin/bash", s_logger);
         command.add("-c");
@@ -1458,13 +1553,13 @@ public class NfsSecondaryStorageResource extends ServerResourceBase
implements S
         Map<String, TemplateProp> tmpltInfos = new HashMap<String, TemplateProp>();
         for (String container : containers) {
             if (container.startsWith("T-")) {
-                String[] files = SwiftUtil.list(swift, container, "template.properties");
+                String[] files = SwiftUtil.list(swift, container, _tmpltpp);
                 if (files.length != 1) {
                     continue;
                 }
                 try {
                     File tempFile = File.createTempFile("template", ".tmp");
-                    File tmpFile = SwiftUtil.getObject(swift, tempFile, container + File.separator
+ "template.properties");
+                    File tmpFile = SwiftUtil.getObject(swift, tempFile, container + File.separator
+ _tmpltpp);
                     if (tmpFile == null) {
                         continue;
                     }
@@ -1779,7 +1874,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase
implements S
             } else {
                 boolean found = false;
                 for (File f : tmpltFiles) {
-                    if (!found && f.getName().equals("template.properties")) {
+                    if (!found && f.getName().equals(_tmpltpp)) {
                         found = true;
                     }
 

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/f5ac8ddd/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java
b/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java
index 52bde6a..b33ce3b 100644
--- a/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java
+++ b/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java
@@ -18,29 +18,6 @@
  */
 package org.apache.cloudstack.storage.resource;
 
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.Map;
-import java.util.Properties;
-import java.util.UUID;
-
-import javax.naming.ConfigurationException;
-
-import junit.framework.Assert;
-import junit.framework.TestCase;
-
-import org.apache.log4j.Logger;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-import org.apache.cloudstack.storage.command.CopyCmdAnswer;
-import org.apache.cloudstack.storage.command.CopyCommand;
-import org.apache.cloudstack.storage.command.DownloadCommand;
-import org.apache.cloudstack.storage.to.TemplateObjectTO;
-
 import com.cloud.agent.api.storage.DownloadAnswer;
 import com.cloud.agent.api.storage.ListTemplateAnswer;
 import com.cloud.agent.api.storage.ListTemplateCommand;
@@ -51,7 +28,28 @@ import com.cloud.storage.DataStoreRole;
 import com.cloud.storage.Storage;
 import com.cloud.utils.PropertiesUtil;
 import com.cloud.utils.exception.CloudRuntimeException;
+import junit.framework.Assert;
+import junit.framework.TestCase;
+import org.apache.cloudstack.storage.command.CopyCmdAnswer;
+import org.apache.cloudstack.storage.command.CopyCommand;
+import org.apache.cloudstack.storage.command.DownloadCommand;
+import org.apache.cloudstack.storage.to.TemplateObjectTO;
+import org.apache.log4j.Logger;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import javax.naming.ConfigurationException;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Properties;
+import java.util.UUID;
 
+@Ignore
 public class LocalNfsSecondaryStorageResourceTest extends TestCase {
     private static Map<String, Object> testParams;
 

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/f5ac8ddd/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java
b/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java
index 13ddb35..e437a0f 100644
--- a/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java
+++ b/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java
@@ -18,91 +18,67 @@
  */
 package org.apache.cloudstack.storage.resource;
 
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URI;
-import java.util.Map;
-import java.util.Properties;
-
-import javax.naming.ConfigurationException;
-
-import junit.framework.Assert;
-import junit.framework.TestCase;
-
+import com.cloud.test.TestAppender;
+import org.apache.cloudstack.storage.command.DeleteCommand;
+import org.apache.cloudstack.storage.to.TemplateObjectTO;
 import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
 
-import com.cloud.utils.PropertiesUtil;
-import com.cloud.utils.exception.CloudRuntimeException;
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.StringWriter;
 
-public class NfsSecondaryStorageResourceTest extends TestCase {
-    private static Map<String, Object> testParams;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.spy;
 
-    private static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResourceTest.class.getName());
+@RunWith(PowerMockRunner.class)
+public class NfsSecondaryStorageResourceTest {
 
-    NfsSecondaryStorageResource resource;
+    private NfsSecondaryStorageResource resource;
 
     @Before
-    @Override
-    public void setUp() throws ConfigurationException {
-        s_logger.setLevel(Level.ALL);
+    public void setUp() {
         resource = new NfsSecondaryStorageResource();
-        resource.setInSystemVM(true);
-        testParams = PropertiesUtil.toMap(loadProperties());
-        resource.configureStorageLayerClass(testParams);
-        Object testLocalRoot = testParams.get("testLocalRoot");
-        if (testLocalRoot != null) {
-            resource.setParentPath((String)testLocalRoot);
-        }
     }
 
     @Test
-    public void testMount() throws Exception {
-        String sampleUriStr = "cifs://192.168.1.128/CSHV3?user=administrator&password=1pass%40word1&foo=bar";
-        URI sampleUri = new URI(sampleUriStr);
-
-        s_logger.info("Check HostIp parsing");
-        String hostIpStr = resource.getUriHostIp(sampleUri);
-        Assert.assertEquals("Expected host IP " + sampleUri.getHost() + " and actual host
IP " + hostIpStr + " differ.", sampleUri.getHost(), hostIpStr);
-
-        s_logger.info("Check option parsing");
-        String expected = "user=administrator,password=1pass@word1,foo=bar,";
-        String actualOpts = resource.parseCifsMountOptions(sampleUri);
-        Assert.assertEquals("Options should be " + expected + " and not " + actualOpts, expected,
actualOpts);
-
-        // attempt a configured mount
-        final Map<String, Object> params = PropertiesUtil.toMap(loadProperties());
-        String sampleMount = (String)params.get("testCifsMount");
-        if (!sampleMount.isEmpty()) {
-            s_logger.info("functional test, mount " + sampleMount);
-            URI realMntUri = new URI(sampleMount);
-            String mntSubDir = resource.mountUri(realMntUri);
-            s_logger.info("functional test, umount " + mntSubDir);
-            resource.umount(resource.getMountingRoot() + mntSubDir, realMntUri);
-        } else {
-            s_logger.info("no entry for testCifsMount in " + "./conf/agent.properties - skip
functional test");
-        }
-    }
+    @PrepareForTest(NfsSecondaryStorageResource.class)
+    public void testSwiftWriteMetadataFile() throws Exception {
+        String expected = "uniquename=test\nfilename=testfile\nsize=100\nvirtualsize=1000";
 
-    public static Properties loadProperties() throws ConfigurationException {
-        Properties properties = new Properties();
-        final File file = PropertiesUtil.findConfigFile("agent.properties");
-        if (file == null) {
-            throw new ConfigurationException("Unable to find agent.properties.");
-        }
-        s_logger.info("agent.properties found at " + file.getAbsolutePath());
-        try(FileInputStream fs = new FileInputStream(file);) {
-            properties.load(fs);
-        } catch (final FileNotFoundException ex) {
-            throw new CloudRuntimeException("Cannot find the file: " + file.getAbsolutePath(),
ex);
-        } catch (final IOException ex) {
-            throw new CloudRuntimeException("IOException in reading " + file.getAbsolutePath(),
ex);
-        }
-        return properties;
+        StringWriter stringWriter = new StringWriter();
+        BufferedWriter bufferWriter = new BufferedWriter(stringWriter);
+        PowerMockito.whenNew(BufferedWriter.class).withArguments(any(FileWriter.class)).thenReturn(bufferWriter);
+
+        resource.swiftWriteMetadataFile("testfile", "test", "testfile", 100, 1000);
+
+        Assert.assertEquals(expected, stringWriter.toString());
     }
 
+    @Test
+    public void testCleanupStagingNfs() throws Exception{
+
+        NfsSecondaryStorageResource spyResource = spy(resource);
+        RuntimeException exception = new RuntimeException();
+        doThrow(exception).when(spyResource).execute(any(DeleteCommand.class));
+        TemplateObjectTO mockTemplate = Mockito.mock(TemplateObjectTO.class);
+
+        TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder();
+        appenderBuilder.addExpectedPattern(Level.DEBUG, "Failed to clean up staging area:");
+        TestAppender testLogAppender = appenderBuilder.build();
+        TestAppender.safeAddAppender(NfsSecondaryStorageResource.s_logger, testLogAppender);
+
+        spyResource.cleanupStagingNfs(mockTemplate);
+
+        testLogAppender.assertMessagesLogged();
+
+    }
 }


Mime
View raw message