Mailing-List: contact commits-help@cloudstack.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@cloudstack.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: ahuang@apache.org
To: commits@cloudstack.apache.org
Date: Wed, 05 Feb 2014 01:39:42 -0000
Message-Id: <da4d83b09b114a12acfd32b3f1b0c1da@git.apache.org>
In-Reply-To: <0b93ed374eba4a6490c753e7790ce970@git.apache.org>
References: <0b93ed374eba4a6490c753e7790ce970@git.apache.org>
Subject: [04/11] Moved the secondary storage service into its own server
 directory

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4be3b993/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadManagerImpl.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadManagerImpl.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadManagerImpl.java
new file mode 100755
index 0000000..cdbc52d
--- /dev/null
+++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadManagerImpl.java
@@ -0,0 +1,550 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.storage.template;
+
+import java.io.File;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import javax.naming.ConfigurationException;
+
+import org.apache.log4j.Logger;
+
+import org.apache.cloudstack.storage.resource.SecondaryStorageResource;
+
+import com.cloud.agent.api.storage.CreateEntityDownloadURLAnswer;
+import com.cloud.agent.api.storage.CreateEntityDownloadURLCommand;
+import com.cloud.agent.api.storage.DeleteEntityDownloadURLAnswer;
+import com.cloud.agent.api.storage.DeleteEntityDownloadURLCommand;
+import com.cloud.agent.api.storage.UploadAnswer;
+import com.cloud.agent.api.storage.UploadCommand;
+import com.cloud.agent.api.storage.UploadProgressCommand;
+import com.cloud.storage.Storage.ImageFormat;
+import com.cloud.storage.StorageLayer;
+import com.cloud.storage.Upload;
+import com.cloud.storage.UploadVO;
+import com.cloud.storage.template.FtpTemplateUploader;
+import com.cloud.storage.template.TemplateUploader;
+import com.cloud.storage.template.TemplateUploader.Status;
+import com.cloud.storage.template.TemplateUploader.UploadCompleteCallback;
+import com.cloud.utils.NumbersUtil;
+import com.cloud.utils.component.ManagerBase;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.script.Script;
+
+public class UploadManagerImpl extends ManagerBase implements UploadManager {
+
+    public class Completion implements UploadCompleteCallback {
+        private final String jobId;
+
+        public Completion(String jobId) {
+            this.jobId = jobId;
+        }
+
+        @Override
+        public void uploadComplete(Status status) {
+            setUploadStatus(jobId, status);
+        }
+    }
+
+    private static class UploadJob {
+        private final TemplateUploader tu;
+
+        public UploadJob(TemplateUploader tu, String jobId, long id, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
+                String installPathPrefix) {
+            super();
+            this.tu = tu;
+        }
+
+        public TemplateUploader getTemplateUploader() {
+            return tu;
+        }
+
+        public void cleanup() {
+            if (tu != null) {
+                String upldPath = tu.getUploadLocalPath();
+                if (upldPath != null) {
+                    File f = new File(upldPath);
+                    f.delete();
+                }
+            }
+        }
+
+    }
+
+    public static final Logger s_logger = Logger.getLogger(UploadManagerImpl.class);
+    private ExecutorService threadPool;
+    private final Map<String, UploadJob> jobs = new ConcurrentHashMap<String, UploadJob>();
+    private String parentDir;
+    private final String extractMountPoint = "/mnt/SecStorage/extractmnt";
+    private StorageLayer _storage;
+    private boolean hvm;
+
+    @Override
+    public String uploadPublicTemplate(long id, String url, String name, ImageFormat format, Long accountId, String descr, String cksum, String installPathPrefix,
+            String userName, String passwd, long templateSizeInBytes) {
+
+        UUID uuid = UUID.randomUUID();
+        String jobId = uuid.toString();
+
+        String completePath = parentDir + File.separator + installPathPrefix;
+        s_logger.debug("Starting upload from " + completePath);
+
+        URI uri;
+        try {
+            uri = new URI(url);
+        } catch (URISyntaxException e) {
+            s_logger.error("URI is incorrect: " + url);
+            throw new CloudRuntimeException("URI is incorrect: " + url);
+        }
+        TemplateUploader tu;
+        if ((uri != null) && (uri.getScheme() != null)) {
+            if (uri.getScheme().equalsIgnoreCase("ftp")) {
+                tu = new FtpTemplateUploader(completePath, url, new Completion(jobId), templateSizeInBytes);
+            } else {
+                s_logger.error("Scheme is not supported " + url);
+                throw new CloudRuntimeException("Scheme is not supported " + url);
+            }
+        } else {
+            s_logger.error("Unable to download from URL: " + url);
+            throw new CloudRuntimeException("Unable to download from URL: " + url);
+        }
+        UploadJob uj = new UploadJob(tu, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix);
+        jobs.put(jobId, uj);
+        threadPool.execute(tu);
+
+        return jobId;
+
+    }
+
+    @Override
+    public String getUploadError(String jobId) {
+        UploadJob uj = jobs.get(jobId);
+        if (uj != null) {
+            return uj.getTemplateUploader().getUploadError();
+        }
+        return null;
+    }
+
+    @Override
+    public int getUploadPct(String jobId) {
+        UploadJob uj = jobs.get(jobId);
+        if (uj != null) {
+            return uj.getTemplateUploader().getUploadPercent();
+        }
+        return 0;
+    }
+
+    @Override
+    public Status getUploadStatus(String jobId) {
+        UploadJob job = jobs.get(jobId);
+        if (job != null) {
+            TemplateUploader tu = job.getTemplateUploader();
+            if (tu != null) {
+                return tu.getStatus();
+            }
+        }
+        return Status.UNKNOWN;
+    }
+
+    public static UploadVO.Status convertStatus(Status tds) {
+        switch (tds) {
+        case ABORTED:
+            return UploadVO.Status.NOT_UPLOADED;
+        case UPLOAD_FINISHED:
+            return UploadVO.Status.UPLOAD_IN_PROGRESS;
+        case IN_PROGRESS:
+            return UploadVO.Status.UPLOAD_IN_PROGRESS;
+        case NOT_STARTED:
+            return UploadVO.Status.NOT_UPLOADED;
+        case RECOVERABLE_ERROR:
+            return UploadVO.Status.NOT_UPLOADED;
+        case UNKNOWN:
+            return UploadVO.Status.UNKNOWN;
+        case UNRECOVERABLE_ERROR:
+            return UploadVO.Status.UPLOAD_ERROR;
+        case POST_UPLOAD_FINISHED:
+            return UploadVO.Status.UPLOADED;
+        default:
+            return UploadVO.Status.UNKNOWN;
+        }
+    }
+
+    @Override
+    public com.cloud.storage.UploadVO.Status getUploadStatus2(String jobId) {
+        return convertStatus(getUploadStatus(jobId));
+    }
+
+    @Override
+    public String getPublicTemplateRepo() {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    private UploadAnswer handleUploadProgressCmd(UploadProgressCommand cmd) {
+        String jobId = cmd.getJobId();
+        UploadAnswer answer;
+        UploadJob uj = null;
+        if (jobId != null)
+            uj = jobs.get(jobId);
+        if (uj == null) {
+            return new UploadAnswer(null, 0, "Cannot find job", com.cloud.storage.UploadVO.Status.UNKNOWN, "", "", 0);
+        }
+        TemplateUploader td = uj.getTemplateUploader();
+        switch (cmd.getRequest()) {
+        case GET_STATUS:
+            break;
+        case ABORT:
+            td.stopUpload();
+            sleep();
+            break;
+            /*case RESTART:
+            td.stopUpload();
+            sleep();
+            threadPool.execute(td);
+            break;*/
+        case PURGE:
+            td.stopUpload();
+            answer =
+                    new UploadAnswer(jobId, getUploadPct(jobId), getUploadError(jobId), getUploadStatus2(jobId), getUploadLocalPath(jobId), getInstallPath(jobId),
+                            getUploadTemplateSize(jobId));
+            jobs.remove(jobId);
+            return answer;
+        default:
+            break; // TODO
+        }
+        return new UploadAnswer(jobId, getUploadPct(jobId), getUploadError(jobId), getUploadStatus2(jobId), getUploadLocalPath(jobId), getInstallPath(jobId),
+                getUploadTemplateSize(jobId));
+    }
+
+    @Override
+    public UploadAnswer handleUploadCommand(SecondaryStorageResource resource, UploadCommand cmd) {
+        s_logger.warn("Handling the upload " + cmd.getInstallPath() + " " + cmd.getId());
+        if (cmd instanceof UploadProgressCommand) {
+            return handleUploadProgressCmd((UploadProgressCommand)cmd);
+        }
+
+        String user = null;
+        String password = null;
+        String jobId =
+                uploadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(),
+                        cmd.getInstallPath(), user, password, cmd.getTemplateSizeInBytes());
+        sleep();
+        return new UploadAnswer(jobId, getUploadPct(jobId), getUploadError(jobId), getUploadStatus2(jobId), getUploadLocalPath(jobId), getInstallPath(jobId),
+                getUploadTemplateSize(jobId));
+    }
+
+    @Override
+    public CreateEntityDownloadURLAnswer handleCreateEntityURLCommand(CreateEntityDownloadURLCommand cmd) {
+
+        boolean isApacheUp = checkAndStartApache();
+        if (!isApacheUp) {
+            String errorString = "Error in starting Apache server ";
+            s_logger.error(errorString);
+            return new CreateEntityDownloadURLAnswer(errorString, CreateEntityDownloadURLAnswer.RESULT_FAILURE);
+        }
+        // Create the directory structure so that its visible under apache server root
+        String extractDir = "/var/www/html/userdata/";
+        Script command = new Script("mkdir", s_logger);
+        command.add("-p");
+        command.add(extractDir);
+        String result = command.execute();
+        if (result != null) {
+            String errorString = "Error in creating directory =" + result;
+            s_logger.error(errorString);
+            return new CreateEntityDownloadURLAnswer(errorString, CreateEntityDownloadURLAnswer.RESULT_FAILURE);
+        }
+
+        // Create a random file under the directory for security reasons.
+        String uuid = cmd.getExtractLinkUUID();
+        command = new Script("touch", s_logger);
+        command.add(extractDir + uuid);
+        result = command.execute();
+        if (result != null) {
+            String errorString = "Error in creating file " + uuid + " ,error: " + result;
+            s_logger.warn(errorString);
+            return new CreateEntityDownloadURLAnswer(errorString, CreateEntityDownloadURLAnswer.RESULT_FAILURE);
+        }
+
+        // Create a symbolic link from the actual directory to the template location. The entity would be directly visible under /var/www/html/userdata/cmd.getInstallPath();
+        command = new Script("/bin/bash", s_logger);
+        command.add("-c");
+        command.add("ln -sf /mnt/SecStorage/" + cmd.getParent() + File.separator + cmd.getInstallPath() + " " + extractDir + uuid);
+        result = command.execute();
+        if (result != null) {
+            String errorString = "Error in linking  err=" + result;
+            s_logger.error(errorString);
+            return new CreateEntityDownloadURLAnswer(errorString, CreateEntityDownloadURLAnswer.RESULT_FAILURE);
+        }
+
+        return new CreateEntityDownloadURLAnswer("", CreateEntityDownloadURLAnswer.RESULT_SUCCESS);
+
+    }
+
+    @Override
+    public DeleteEntityDownloadURLAnswer handleDeleteEntityDownloadURLCommand(DeleteEntityDownloadURLCommand cmd) {
+
+        //Delete the soft link. Example path = volumes/8/74eeb2c6-8ab1-4357-841f-2e9d06d1f360.vhd
+        s_logger.warn("handleDeleteEntityDownloadURLCommand Path:" + cmd.getPath() + " Type:" + cmd.getType().toString());
+        String path = cmd.getPath();
+        Script command = new Script("/bin/bash", s_logger);
+        command.add("-c");
+
+        //We just need to remove the UUID.vhd
+        String extractUrl = cmd.getExtractUrl();
+        command.add("unlink /var/www/html/userdata/" + extractUrl.substring(extractUrl.lastIndexOf(File.separator) + 1));
+        String result = command.execute();
+        if (result != null) {
+            String errorString = "Error in deleting =" + result;
+            s_logger.warn(errorString);
+            return new DeleteEntityDownloadURLAnswer(errorString, CreateEntityDownloadURLAnswer.RESULT_FAILURE);
+        }
+
+        // If its a volume also delete the Hard link since it was created only for the purpose of download.
+        if (cmd.getType() == Upload.Type.VOLUME) {
+            command = new Script("/bin/bash", s_logger);
+            command.add("-c");
+            command.add("rm -f /mnt/SecStorage/" + cmd.getParentPath() + File.separator + path);
+            s_logger.warn(" " + parentDir + File.separator + path);
+            result = command.execute();
+            if (result != null) {
+                String errorString = "Error in linking  err=" + result;
+                s_logger.warn(errorString);
+                return new DeleteEntityDownloadURLAnswer(errorString, CreateEntityDownloadURLAnswer.RESULT_FAILURE);
+            }
+        }
+
+        return new DeleteEntityDownloadURLAnswer("", CreateEntityDownloadURLAnswer.RESULT_SUCCESS);
+    }
+
+    private String getInstallPath(String jobId) {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    private String getUploadLocalPath(String jobId) {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    private long getUploadTemplateSize(String jobId) {
+        return 0;
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
+
+        String value = null;
+
+        _storage = (StorageLayer)params.get(StorageLayer.InstanceConfigKey);
+        if (_storage == null) {
+            value = (String)params.get(StorageLayer.ClassConfigKey);
+            if (value == null) {
+                throw new ConfigurationException("Unable to find the storage layer");
+            }
+
+            Class<StorageLayer> clazz;
+            try {
+                clazz = (Class<StorageLayer>)Class.forName(value);
+                _storage = clazz.newInstance();
+            } catch (ClassNotFoundException e) {
+                throw new ConfigurationException("Unable to instantiate " + value);
+            } catch (InstantiationException e) {
+                throw new ConfigurationException("Unable to instantiate " + value);
+            } catch (IllegalAccessException e) {
+                throw new ConfigurationException("Unable to instantiate " + value);
+            }
+        }
+
+        String inSystemVM = (String)params.get("secondary.storage.vm");
+        if (inSystemVM != null && "true".equalsIgnoreCase(inSystemVM)) {
+            s_logger.info("UploadManager: starting additional services since we are inside system vm");
+            startAdditionalServices();
+            //blockOutgoingOnPrivate();
+        }
+
+        value = (String)params.get("install.numthreads");
+        final int numInstallThreads = NumbersUtil.parseInt(value, 10);
+
+        String scriptsDir = (String)params.get("template.scripts.dir");
+        if (scriptsDir == null) {
+            scriptsDir = "scripts/storage/secondary";
+        }
+
+        // Add more processors here.
+        threadPool = Executors.newFixedThreadPool(numInstallThreads);
+
+        return true;
+    }
+
+    private void startAdditionalServices() {
+
+        Script command = new Script("rm", s_logger);
+        command.add("-rf");
+        command.add(extractMountPoint);
+        String result = command.execute();
+        if (result != null) {
+            s_logger.warn("Error in creating file " + extractMountPoint + " ,error: " + result);
+            return;
+        }
+
+        command = new Script("touch", s_logger);
+        command.add(extractMountPoint);
+        result = command.execute();
+        if (result != null) {
+            s_logger.warn("Error in creating file " + extractMountPoint + " ,error: " + result);
+            return;
+        }
+
+        command = new Script("/bin/bash", s_logger);
+        command.add("-c");
+        command.add("ln -sf " + parentDir + " " + extractMountPoint);
+        result = command.execute();
+        if (result != null) {
+            s_logger.warn("Error in linking  err=" + result);
+            return;
+        }
+
+    }
+
+    /**
+     * Get notified of change of job status. Executed in context of uploader thread
+     *
+     * @param jobId
+     *            the id of the job
+     * @param status
+     *            the status of the job
+     */
+    public void setUploadStatus(String jobId, Status status) {
+        UploadJob uj = jobs.get(jobId);
+        if (uj == null) {
+            s_logger.warn("setUploadStatus for jobId: " + jobId + ", status=" + status + " no job found");
+            return;
+        }
+        TemplateUploader tu = uj.getTemplateUploader();
+        s_logger.warn("Upload Completion for jobId: " + jobId + ", status=" + status);
+        s_logger.warn("UploadedBytes=" + tu.getUploadedBytes() + ", error=" + tu.getUploadError() + ", pct=" + tu.getUploadPercent());
+
+        switch (status) {
+        case ABORTED:
+        case NOT_STARTED:
+        case UNRECOVERABLE_ERROR:
+            // Delete the entity only if its a volume. TO DO - find a better way of finding it a volume.
+            if (uj.getTemplateUploader().getUploadLocalPath().indexOf("volume") > -1) {
+                uj.cleanup();
+            }
+            break;
+        case UNKNOWN:
+            return;
+        case IN_PROGRESS:
+            s_logger.info("Resuming jobId: " + jobId + ", status=" + status);
+            tu.setResume(true);
+            threadPool.execute(tu);
+            break;
+        case RECOVERABLE_ERROR:
+            threadPool.execute(tu);
+            break;
+        case UPLOAD_FINISHED:
+            tu.setUploadError("Upload success, starting install ");
+            String result = postUpload(jobId);
+            if (result != null) {
+                s_logger.error("Failed post upload script: " + result);
+                tu.setStatus(Status.UNRECOVERABLE_ERROR);
+                tu.setUploadError("Failed post upload script: " + result);
+            } else {
+                s_logger.warn("Upload completed successfully at " + new SimpleDateFormat().format(new Date()));
+                tu.setStatus(Status.POST_UPLOAD_FINISHED);
+                tu.setUploadError("Upload completed successfully at " + new SimpleDateFormat().format(new Date()));
+            }
+            // Delete the entity only if its a volume. TO DO - find a better way of finding it a volume.
+            if (uj.getTemplateUploader().getUploadLocalPath().indexOf("volume") > -1) {
+                uj.cleanup();
+            }
+            break;
+        default:
+            break;
+        }
+    }
+
+    private String postUpload(String jobId) {
+        return null;
+    }
+
+    private void sleep() {
+        try {
+            Thread.sleep(3000);
+        } catch (InterruptedException e) {
+            // ignore
+        }
+    }
+
+    private boolean checkAndStartApache() {
+
+        //Check whether the Apache server is running
+        Script command = new Script("/bin/bash", s_logger);
+        command.add("-c");
+        command.add("if [ -d /etc/apache2 ] ; then service apache2 status | grep pid; else service httpd status | grep pid; fi ");
+        String result = command.execute();
+
+        //Apache Server is not running. Try to start it.
+        if (result != null) {
+
+            /*s_logger.warn("Apache server not running, trying to start it");
+            String port = Integer.toString(TemplateConstants.DEFAULT_TMPLT_COPY_PORT);
+            String intf = TemplateConstants.DEFAULT_TMPLT_COPY_INTF;
+
+            command = new Script("/bin/bash", s_logger);
+            command.add("-c");
+            command.add("iptables -D INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + port + " -j DROP;" +
+                        "iptables -D INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + port + " -j HTTP;" +
+                        "iptables -D INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j DROP;" +
+                        "iptables -D INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j HTTP;" +
+                        "iptables -F HTTP;" +
+                        "iptables -X HTTP;" +
+                        "iptables -N HTTP;" +
+                        "iptables -I INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + port + " -j DROP;" +
+                        "iptables -I INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j DROP;" +
+                        "iptables -I INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + port + " -j HTTP;" +
+                        "iptables -I INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j HTTP;");
+
+            result = command.execute();
+            if (result != null) {
+                s_logger.warn("Error in opening up httpd port err=" + result );
+                return false;
+            }*/
+
+            command = new Script("/bin/bash", s_logger);
+            command.add("-c");
+            command.add("if [ -d /etc/apache2 ] ; then service apache2 start; else service httpd start; fi ");
+            result = command.execute();
+            if (result != null) {
+                s_logger.warn("Error in starting httpd service err=" + result);
+                return false;
+            }
+        }
+
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4be3b993/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java b/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java
new file mode 100644
index 0000000..e0fcbae
--- /dev/null
+++ b/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cloudstack.storage.resource;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Properties;
+import java.util.UUID;
+
+import javax.naming.ConfigurationException;
+
+import junit.framework.Assert;
+import junit.framework.TestCase;
+
+import org.apache.log4j.Logger;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import org.apache.cloudstack.storage.command.CopyCmdAnswer;
+import org.apache.cloudstack.storage.command.CopyCommand;
+import org.apache.cloudstack.storage.command.DownloadCommand;
+import org.apache.cloudstack.storage.to.TemplateObjectTO;
+
+import com.cloud.agent.api.storage.DownloadAnswer;
+import com.cloud.agent.api.storage.ListTemplateAnswer;
+import com.cloud.agent.api.storage.ListTemplateCommand;
+import com.cloud.agent.api.to.DataObjectType;
+import com.cloud.agent.api.to.NfsTO;
+import com.cloud.agent.api.to.SwiftTO;
+import com.cloud.storage.DataStoreRole;
+import com.cloud.storage.Storage;
+import com.cloud.utils.PropertiesUtil;
+import com.cloud.utils.exception.CloudRuntimeException;
+
+public class LocalNfsSecondaryStorageResourceTest extends TestCase {
+    private static Map<String, Object> testParams;
+
+    private static final Logger s_logger = Logger.getLogger(LocalNfsSecondaryStorageResourceTest.class.getName());
+
+    LocalNfsSecondaryStorageResource resource;
+
+    @Before
+    @Override
+    public void setUp() throws ConfigurationException {
+        resource = new LocalNfsSecondaryStorageResource();
+        resource.setInSystemVM(true);
+
+        testParams = PropertiesUtil.toMap(loadProperties());
+        resource.configureStorageLayerClass(testParams);
+        Object testLocalRoot = testParams.get("testLocalRoot");
+        resource.setParentPath("/mnt");
+
+        if (testLocalRoot != null) {
+            resource.setParentPath((String)testLocalRoot);
+        }
+
+        System.setProperty("paths.script", "/Users/edison/develop/asf-master/script");
+        //resource.configure("test", new HashMap<String, Object>());
+    }
+
+    @Test
+    public void testExecuteRequest() throws Exception {
+        TemplateObjectTO template = Mockito.mock(TemplateObjectTO.class);
+        NfsTO cacheStore = Mockito.mock(NfsTO.class);
+        Mockito.when(cacheStore.getUrl()).thenReturn("nfs://nfs2.lab.vmops.com/export/home/edison/");
+        SwiftTO swift = Mockito.mock(SwiftTO.class);
+        Mockito.when(swift.getEndPoint()).thenReturn("https://objects.dreamhost.com/auth");
+        Mockito.when(swift.getAccount()).thenReturn("cloudstack");
+        Mockito.when(swift.getUserName()).thenReturn("images");
+        Mockito.when(swift.getKey()).thenReturn("oxvELQaOD1U5_VyosGfA-wpZ7uBWEff-CUBGCM0u");
+
+        Mockito.when(template.getDataStore()).thenReturn(swift);
+        Mockito.when(template.getPath()).thenReturn("template/1/1/");
+        Mockito.when(template.isRequiresHvm()).thenReturn(true);
+        Mockito.when(template.getId()).thenReturn(1L);
+        Mockito.when(template.getFormat()).thenReturn(Storage.ImageFormat.VHD);
+        Mockito.when(template.getOrigUrl()).thenReturn("http://nfs1.lab.vmops.com/templates/test.bz2");
+        Mockito.when(template.getName()).thenReturn(UUID.randomUUID().toString());
+        Mockito.when(template.getObjectType()).thenReturn(DataObjectType.TEMPLATE);
+
+        DownloadCommand cmd = new DownloadCommand(template, 100000L);
+        cmd.setCacheStore(cacheStore);
+        DownloadAnswer answer = (DownloadAnswer)resource.executeRequest(cmd);
+        Assert.assertTrue(answer.getResult());
+
+        Mockito.when(template.getPath()).thenReturn(answer.getInstallPath());
+        Mockito.when(template.getDataStore()).thenReturn(swift);
+        //download swift:
+        Mockito.when(cacheStore.getRole()).thenReturn(DataStoreRole.ImageCache);
+        TemplateObjectTO destTemplate = Mockito.mock(TemplateObjectTO.class);
+        Mockito.when(destTemplate.getPath()).thenReturn("template/1/2");
+        Mockito.when(destTemplate.getDataStore()).thenReturn(cacheStore);
+        Mockito.when(destTemplate.getObjectType()).thenReturn(DataObjectType.TEMPLATE);
+        CopyCommand cpyCmd = new CopyCommand(template, destTemplate, 10000, true);
+        CopyCmdAnswer copyCmdAnswer = (CopyCmdAnswer)resource.executeRequest(cpyCmd);
+        Assert.assertTrue(copyCmdAnswer.getResult());
+
+        //list template
+        ListTemplateCommand listCmd = new ListTemplateCommand(swift);
+        ListTemplateAnswer listAnswer = (ListTemplateAnswer)resource.executeRequest(listCmd);
+
+        Assert.assertTrue(listAnswer.getTemplateInfo().size() > 0);
+    }
+
+    public static Properties loadProperties() throws ConfigurationException {
+        Properties properties = new Properties();
+        final File file = PropertiesUtil.findConfigFile("agent.properties");
+        if (file == null) {
+            throw new ConfigurationException("Unable to find agent.properties.");
+        }
+
+        s_logger.info("agent.properties found at " + file.getAbsolutePath());
+
+        try {
+            properties.load(new FileInputStream(file));
+        } catch (final FileNotFoundException ex) {
+            throw new CloudRuntimeException("Cannot find the file: " + file.getAbsolutePath(), ex);
+        } catch (final IOException ex) {
+            throw new CloudRuntimeException("IOException in reading " + file.getAbsolutePath(), ex);
+        }
+        return properties;
+    }
+}

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4be3b993/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java b/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java
new file mode 100644
index 0000000..e0ae4c5
--- /dev/null
+++ b/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cloudstack.storage.resource;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.util.Map;
+import java.util.Properties;
+
+import javax.naming.ConfigurationException;
+
+import junit.framework.Assert;
+import junit.framework.TestCase;
+
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.cloud.utils.PropertiesUtil;
+import com.cloud.utils.exception.CloudRuntimeException;
+
+public class NfsSecondaryStorageResourceTest extends TestCase {
+    private static Map<String, Object> testParams;
+
+    private static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResourceTest.class.getName());
+
+    NfsSecondaryStorageResource resource;
+
+    @Before
+    @Override
+    public void setUp() throws ConfigurationException {
+        s_logger.setLevel(Level.ALL);
+        resource = new NfsSecondaryStorageResource();
+        resource.setInSystemVM(true);
+        testParams = PropertiesUtil.toMap(loadProperties());
+        resource.configureStorageLayerClass(testParams);
+        Object testLocalRoot = testParams.get("testLocalRoot");
+        if (testLocalRoot != null) {
+            resource.setParentPath((String)testLocalRoot);
+        }
+    }
+
+    @Test
+    public void testMount() throws Exception {
+        String sampleUriStr = "cifs://192.168.1.128/CSHV3?user=administrator&password=1pass%40word1&foo=bar";
+        URI sampleUri = new URI(sampleUriStr);
+
+        s_logger.info("Check HostIp parsing");
+        String hostIpStr = resource.getUriHostIp(sampleUri);
+        Assert.assertEquals("Expected host IP " + sampleUri.getHost() + " and actual host IP " + hostIpStr + " differ.", sampleUri.getHost(), hostIpStr);
+
+        s_logger.info("Check option parsing");
+        String expected = "user=administrator,password=1pass@word1,foo=bar,";
+        String actualOpts = resource.parseCifsMountOptions(sampleUri);
+        Assert.assertEquals("Options should be " + expected + " and not " + actualOpts, expected, actualOpts);
+
+        // attempt a configured mount
+        final Map<String, Object> params = PropertiesUtil.toMap(loadProperties());
+        String sampleMount = (String)params.get("testCifsMount");
+        if (!sampleMount.isEmpty()) {
+            s_logger.info("functional test, mount " + sampleMount);
+            URI realMntUri = new URI(sampleMount);
+            String mntSubDir = resource.mountUri(realMntUri);
+            s_logger.info("functional test, umount " + mntSubDir);
+            resource.umount(resource.getMountingRoot() + mntSubDir, realMntUri);
+        } else {
+            s_logger.info("no entry for testCifsMount in " + "./conf/agent.properties - skip functional test");
+        }
+    }
+
+    public static Properties loadProperties() throws ConfigurationException {
+        Properties properties = new Properties();
+        final File file = PropertiesUtil.findConfigFile("agent.properties");
+        if (file == null) {
+            throw new ConfigurationException("Unable to find agent.properties.");
+        }
+
+        s_logger.info("agent.properties found at " + file.getAbsolutePath());
+
+        try {
+            properties.load(new FileInputStream(file));
+        } catch (final FileNotFoundException ex) {
+            throw new CloudRuntimeException("Cannot find the file: " + file.getAbsolutePath(), ex);
+        } catch (final IOException ex) {
+            throw new CloudRuntimeException("IOException in reading " + file.getAbsolutePath(), ex);
+        }
+        return properties;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4be3b993/services/secondary-storage/src/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResource.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResource.java b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResource.java
deleted file mode 100644
index 9393ee2..0000000
--- a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResource.java
+++ /dev/null
@@ -1,95 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.apache.cloudstack.storage.resource;
-
-import java.net.URI;
-import java.util.concurrent.Executors;
-
-import org.apache.log4j.Logger;
-import org.springframework.stereotype.Component;
-
-import org.apache.cloudstack.storage.template.DownloadManagerImpl;
-
-import com.cloud.agent.api.Answer;
-import com.cloud.agent.api.Command;
-import com.cloud.storage.JavaStorageLayer;
-import com.cloud.utils.exception.CloudRuntimeException;
-import com.cloud.utils.script.Script;
-
-@Component
-public class LocalNfsSecondaryStorageResource extends NfsSecondaryStorageResource {
-
-    private static final Logger s_logger = Logger.getLogger(LocalNfsSecondaryStorageResource.class);
-
-    public LocalNfsSecondaryStorageResource() {
-        this._dlMgr = new DownloadManagerImpl();
-        ((DownloadManagerImpl)_dlMgr).setThreadPool(Executors.newFixedThreadPool(10));
-        _storage = new JavaStorageLayer();
-        this._inSystemVM = false;
-    }
-
-    @Override
-    public void setParentPath(String path) {
-        this._parent = path;
-    }
-
-    @Override
-    public Answer executeRequest(Command cmd) {
-        return super.executeRequest(cmd);
-    }
-
-    @Override
-    synchronized public String getRootDir(String secUrl) {
-        try {
-            URI uri = new URI(secUrl);
-            String dir = mountUri(uri);
-            return _parent + "/" + dir;
-        } catch (Exception e) {
-            String msg = "GetRootDir for " + secUrl + " failed due to " + e.toString();
-            s_logger.error(msg, e);
-            throw new CloudRuntimeException(msg);
-        }
-    }
-
-    @Override
-    protected void mount(String localRootPath, String remoteDevice, URI uri) {
-        ensureLocalRootPathExists(localRootPath, uri);
-
-        if (mountExists(localRootPath, uri)) {
-            return;
-        }
-
-        attemptMount(localRootPath, remoteDevice, uri);
-
-        // Change permissions for the mountpoint - seems to bypass authentication
-        Script script = new Script(true, "chmod", _timeout, s_logger);
-        script.add("777", localRootPath);
-        String result = script.execute();
-        if (result != null) {
-            String errMsg = "Unable to set permissions for " + localRootPath + " due to " + result;
-            s_logger.error(errMsg);
-            throw new CloudRuntimeException(errMsg);
-        }
-        s_logger.debug("Successfully set 777 permission for " + localRootPath);
-
-        // XXX: Adding the check for creation of snapshots dir here. Might have
-        // to move it somewhere more logical later.
-        checkForSnapshotsDir(localRootPath);
-        checkForVolumesDir(localRootPath);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4be3b993/services/secondary-storage/src/org/apache/cloudstack/storage/resource/LocalSecondaryStorageResource.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/LocalSecondaryStorageResource.java b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/LocalSecondaryStorageResource.java
deleted file mode 100644
index bdfe7e8..0000000
--- a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/LocalSecondaryStorageResource.java
+++ /dev/null
@@ -1,240 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.apache.cloudstack.storage.resource;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import javax.naming.ConfigurationException;
-
-import org.apache.log4j.Logger;
-
-import org.apache.cloudstack.storage.command.DownloadCommand;
-import org.apache.cloudstack.storage.command.DownloadProgressCommand;
-import org.apache.cloudstack.storage.template.DownloadManager;
-import org.apache.cloudstack.storage.template.DownloadManagerImpl;
-
-import com.cloud.agent.api.Answer;
-import com.cloud.agent.api.CheckHealthAnswer;
-import com.cloud.agent.api.CheckHealthCommand;
-import com.cloud.agent.api.Command;
-import com.cloud.agent.api.ComputeChecksumCommand;
-import com.cloud.agent.api.PingCommand;
-import com.cloud.agent.api.PingStorageCommand;
-import com.cloud.agent.api.ReadyAnswer;
-import com.cloud.agent.api.ReadyCommand;
-import com.cloud.agent.api.SecStorageSetupCommand;
-import com.cloud.agent.api.StartupCommand;
-import com.cloud.agent.api.StartupStorageCommand;
-import com.cloud.agent.api.storage.ListTemplateAnswer;
-import com.cloud.agent.api.storage.ListTemplateCommand;
-import com.cloud.agent.api.to.NfsTO;
-import com.cloud.host.Host;
-import com.cloud.host.Host.Type;
-import com.cloud.resource.ServerResourceBase;
-import com.cloud.storage.Storage;
-import com.cloud.storage.Storage.StoragePoolType;
-import com.cloud.storage.StorageLayer;
-import com.cloud.storage.template.TemplateProp;
-import com.cloud.utils.component.ComponentContext;
-
-public class LocalSecondaryStorageResource extends ServerResourceBase implements SecondaryStorageResource {
-    private static final Logger s_logger = Logger.getLogger(LocalSecondaryStorageResource.class);
-    int _timeout;
-
-    String _instance;
-    String _parent;
-
-    String _dc;
-    String _pod;
-    String _guid;
-
-    StorageLayer _storage;
-
-    DownloadManager _dlMgr;
-
-    @Override
-    public void disconnected() {
-    }
-
-    @Override
-    public String getRootDir(String url) {
-        return getRootDir();
-
-    }
-
-    public String getRootDir() {
-        return _parent;
-    }
-
-    @Override
-    public Answer executeRequest(Command cmd) {
-        if (cmd instanceof DownloadProgressCommand) {
-            return _dlMgr.handleDownloadCommand(this, (DownloadProgressCommand)cmd);
-        } else if (cmd instanceof DownloadCommand) {
-            return _dlMgr.handleDownloadCommand(this, (DownloadCommand)cmd);
-        } else if (cmd instanceof CheckHealthCommand) {
-            return new CheckHealthAnswer((CheckHealthCommand)cmd, true);
-        } else if (cmd instanceof SecStorageSetupCommand) {
-            return new Answer(cmd, true, "success");
-        } else if (cmd instanceof ReadyCommand) {
-            return new ReadyAnswer((ReadyCommand)cmd);
-        } else if (cmd instanceof ListTemplateCommand) {
-            return execute((ListTemplateCommand)cmd);
-        } else if (cmd instanceof ComputeChecksumCommand) {
-            return execute((ComputeChecksumCommand)cmd);
-        } else {
-            return Answer.createUnsupportedCommandAnswer(cmd);
-        }
-    }
-
-    private Answer execute(ComputeChecksumCommand cmd) {
-        return new Answer(cmd, false, null);
-    }
-
-    private Answer execute(ListTemplateCommand cmd) {
-        String root = getRootDir();
-        Map<String, TemplateProp> templateInfos = _dlMgr.gatherTemplateInfo(root);
-        return new ListTemplateAnswer(((NfsTO)cmd.getDataStore()).getUrl(), templateInfos);
-    }
-
-    @Override
-    public Type getType() {
-        return Host.Type.LocalSecondaryStorage;
-    }
-
-    @Override
-    public PingCommand getCurrentStatus(final long id) {
-        return new PingStorageCommand(Host.Type.Storage, id, new HashMap<String, Boolean>());
-    }
-
-    @Override
-    @SuppressWarnings("unchecked")
-    public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
-        super.configure(name, params);
-
-        _guid = (String)params.get("guid");
-        if (_guid == null) {
-            throw new ConfigurationException("Unable to find the guid");
-        }
-
-        _dc = (String)params.get("zone");
-        if (_dc == null) {
-            throw new ConfigurationException("Unable to find the zone");
-        }
-        _pod = (String)params.get("pod");
-
-        _instance = (String)params.get("instance");
-
-        _parent = (String)params.get("mount.path");
-        if (_parent == null) {
-            throw new ConfigurationException("No directory specified.");
-        }
-
-        _storage = (StorageLayer)params.get(StorageLayer.InstanceConfigKey);
-        if (_storage == null) {
-            String value = (String)params.get(StorageLayer.ClassConfigKey);
-            if (value == null) {
-                value = "com.cloud.storage.JavaStorageLayer";
-            }
-
-            try {
-                Class<StorageLayer> clazz = (Class<StorageLayer>)Class.forName(value);
-                _storage = ComponentContext.inject(clazz);
-            } catch (ClassNotFoundException e) {
-                throw new ConfigurationException("Unable to find class " + value);
-            }
-        }
-
-        if (!_storage.mkdirs(_parent)) {
-            s_logger.warn("Unable to create the directory " + _parent);
-            throw new ConfigurationException("Unable to create the directory " + _parent);
-        }
-
-        s_logger.info("Mount point established at " + _parent);
-
-        params.put("template.parent", _parent);
-        params.put(StorageLayer.InstanceConfigKey, _storage);
-
-        _dlMgr = new DownloadManagerImpl();
-        _dlMgr.configure("DownloadManager", params);
-
-        return true;
-    }
-
-    @Override
-    public boolean start() {
-        return true;
-    }
-
-    @Override
-    public boolean stop() {
-        return true;
-    }
-
-    @Override
-    public StartupCommand[] initialize() {
-
-        final StartupStorageCommand cmd =
-            new StartupStorageCommand(_parent, StoragePoolType.Filesystem, 1024l * 1024l * 1024l * 1024l, _dlMgr.gatherTemplateInfo(_parent));
-        cmd.setResourceType(Storage.StorageResourceType.LOCAL_SECONDARY_STORAGE);
-        cmd.setIqn("local://");
-        fillNetworkInformation(cmd);
-        cmd.setDataCenter(_dc);
-        cmd.setPod(_pod);
-        cmd.setGuid(_guid);
-        cmd.setName(_guid);
-        cmd.setVersion(LocalSecondaryStorageResource.class.getPackage().getImplementationVersion());
-
-        return new StartupCommand[] {cmd};
-    }
-
-    @Override
-    protected String getDefaultScriptsDir() {
-        return "scripts/storage/secondary";
-    }
-
-    @Override
-    public void setName(String name) {
-        // TODO Auto-generated method stub
-
-    }
-
-    @Override
-    public void setConfigParams(Map<String, Object> params) {
-        // TODO Auto-generated method stub
-
-    }
-
-    @Override
-    public Map<String, Object> getConfigParams() {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public int getRunLevel() {
-        // TODO Auto-generated method stub
-        return 0;
-    }
-
-    @Override
-    public void setRunLevel(int level) {
-        // TODO Auto-generated method stub
-
-    }
-}