cloudstack-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From raj...@apache.org
Subject git commit: updated refs/heads/volume-upload to 2a2d436
Date Tue, 20 Jan 2015 10:33:36 GMT
Repository: cloudstack
Updated Branches:
  refs/heads/volume-upload a31e1bff3 -> 2a2d4364e


made changes to move a file to the specified localtion

Signed-off-by: Rajani Karuturi <rajanikaruturi@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/2a2d4364
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/2a2d4364
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/2a2d4364

Branch: refs/heads/volume-upload
Commit: 2a2d4364e14903d51cdc3d3a7b4e2eb979a60445
Parents: a31e1bf
Author: Bharat Kumar <bharat.kumar@citrix.com>
Authored: Mon Jan 19 10:13:03 2015 +0530
Committer: Rajani Karuturi <rajanikaruturi@gmail.com>
Committed: Tue Jan 20 16:02:33 2015 +0530

----------------------------------------------------------------------
 .../resource/NfsSecondaryStorageResource.java   | 214 +++++++++++++++++--
 .../storage/template/DownloadManager.java       |   2 +
 .../storage/template/DownloadManagerImpl.java   |   7 +-
 .../storage/template/UploadEntity.java          |  76 ++++++-
 4 files changed, 274 insertions(+), 25 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2a2d4364/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
index 246138d..6ad7df5 100755
--- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
+++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
@@ -45,6 +45,7 @@ import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
@@ -225,7 +226,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements
S
     protected String _parent = "/mnt/SecStorage";
     final private String _tmpltpp = "template.properties";
     protected String createTemplateFromSnapshotXenScript;
-    private HashMap<Long,UploadEntity> uploadEntityStateMap = new HashMap<Long,UploadEntity>();
+    private HashMap<String,UploadEntity> uploadEntityStateMap = new HashMap<String,UploadEntity>();
 
     public void setParentPath(String path) {
         _parent = path;
@@ -1361,7 +1362,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase
implements S
         // Create request handler registry
         UriHttpAsyncRequestHandlerMapper reqistry = new UriHttpAsyncRequestHandlerMapper();
         // Register the default handler for all URIs
-        reqistry.register("/upload*", new PostUploadRequestHandler());
+        reqistry.register("/upload*", new PostUploadRequestHandler(_params,this,_dlMgr));
         // Create server-side HTTP protocol handler
         HttpAsyncService protocolHandler = new HttpAsyncService(httpproc, reqistry) {
 
@@ -2628,32 +2629,100 @@ public class NfsSecondaryStorageResource extends ServerResourceBase
implements S
     private class PostUploadRequestHandler implements HttpAsyncRequestHandler<HttpRequest>
{
         //private  static final Logger s_logger = Logger.getLogger(PostUploadRequestHandler.class);
 
-        public void handleuplod(long entityId, String absolutePath, String entityName, long
entitySize, byte[] data) {
+        private String createTmpltScr;
+        private String createVolScr;
+        private String scriptsDir;
+        private int installTimeoutPerGig = 180 * 60 * 1000;
+        private Map<String,ImageFormat> formatMap = new HashMap<>();
+        private DownloadManager _downloadMgr;
+        private SecondaryStorageResource _resource;
+
+        public PostUploadRequestHandler(Map<String, Object> params, SecondaryStorageResource
resource, DownloadManager dlmgr) {
+                  configure(params, dlmgr);
+                  this._resource=resource;
+        }
+
+        public void configure(Map<String, Object> params, DownloadManager dlmgr) {
+
+            _downloadMgr=dlmgr;
+            String scriptsDir = (String)params.get("template.scripts.dir");
+            if (scriptsDir == null) {
+                scriptsDir = "scripts/storage/secondary";
+            }
+           try {
+               createTmpltScr = Script.findScript(scriptsDir, "createtmplt.sh");
+               if (createTmpltScr == null) {
+                   throw new ConfigurationException("Unable to find createtmplt.sh");
+               }
+               s_logger.info("createtmplt.sh found in " + createTmpltScr);
+               createVolScr = Script.findScript(scriptsDir, "createvolume.sh");
+               if (createVolScr == null) {
+                   throw new ConfigurationException("Unable to find createvolume.sh");
+               }
+               s_logger.info("createvolume.sh found in " + createVolScr);
+
+           }catch (Exception e) {
+               s_logger.debug("failed to configure the postupload handler "+e);
+           }
+
+            formatMap.put("qcow2",ImageFormat.QCOW2);
+            formatMap.put("vhd",ImageFormat.VHD);
+            formatMap.put("iso",ImageFormat.ISO);
+            formatMap.put("vmdk",ImageFormat.VMDK);
+            formatMap.put("tar",ImageFormat.TAR);
+            formatMap.put("ova",ImageFormat.OVA);
+
+        }
+
+        public void handleuplod(long entityId, String installPathPrefix, String uuid, String
localTemppath, String entityName, long entitySize, String resourceType, String imageFormat,
byte[] data, boolean isHvm, String chksum, String dataStoreUrl) {
             UploadEntity uploadEntity=null;
             try {
 
-                if (uploadEntityStateMap.containsKey(entityId)) {
+                if (uploadEntityStateMap.containsKey(uuid)) {
                     //the file upload entity has been created.
-                    uploadEntity = uploadEntityStateMap.get(entityId);
-                    uploadEntity.getFilewriter().write(data);
-                    uploadEntity.incremetByteCount(data.length);
-                    if (uploadEntity.getDownloadedsize() == uploadEntity.getEntitysize())
{
-                        uploadEntity.setStatus(UploadEntity.Status.COMPLETED);
-                        uploadEntity.getFile().renameTo(new File(uploadEntity.getAbsoluteFilePath()));
-                        uploadEntity.getFilewriter().close();
-                    } else {
-                        uploadEntity.setStatus(UploadEntity.Status.IN_PROGRESS);
+                    uploadEntity = uploadEntityStateMap.get(uuid);
+                    if (uploadEntity.getUploadState() == UploadEntity.Status.ERROR) {
+                        s_logger.debug("received data write requres for a uplod entity in
error state. Ignoring.");
+                    }else if (uploadEntity.getUploadState()== UploadEntity.Status.COMPLETED)
{
+                        s_logger.debug("received a write request for entity which is alredy
downloaded. Ignoring.");
+                    }else if (uploadEntity.getUploadState()== UploadEntity.Status.IN_PROGRESS)
{
+                        uploadEntity.getFilewriter().write(data);
+                        uploadEntity.incremetByteCount(data.length);
+                        if (uploadEntity.getDownloadedsize() == uploadEntity.getEntitysize())
{
+                            postUpload(uploadEntity);
+                        } else {
+                            uploadEntity.setStatus(UploadEntity.Status.IN_PROGRESS);
+                        }
                     }
                 }else{
                     //this is a new upload.
-                    uploadEntity = new UploadEntity(entitySize, UploadEntity.Status.IN_PROGRESS,
entityName, absolutePath);
-                    uploadEntityStateMap.put(entityId, uploadEntity);
-                    File tempFile = File.createTempFile("dnld_" + entityId,absolutePath);
+                    uploadEntity = new UploadEntity(uuid, entityId, entitySize, UploadEntity.Status.IN_PROGRESS,
entityName, installPathPrefix);
+                    uploadEntity.setResourceType("volume".equalsIgnoreCase(resourceType)
? UploadEntity.ResourceType.VOLUME : UploadEntity.ResourceType.TEMPLATE);
+                    uploadEntity.setFormat(formatMap.get(imageFormat.toLowerCase()));
+                    //relative path wihour ssvm mount info.
+                    uploadEntity.setTemplatePath(installPathPrefix);
+                    installPathPrefix = _resource.getRootDir(dataStoreUrl) + File.separator
+ installPathPrefix;
+                    uploadEntity.setInstallPathPrefix(installPathPrefix);
+                    uploadEntity.setEntitysize(entitySize);
+                    uploadEntity.setHvm(isHvm);
+                    uploadEntity.setFilename(entityName);
+                    if (!_storage.exists(installPathPrefix)) {
+                        _storage.mkdir(installPathPrefix);
+                    }
+                    File tempFile = File.createTempFile("dnld_", entityName,new File(installPathPrefix));
+                    uploadEntity.setFile(tempFile);
                     FileOutputStream filewriter = new FileOutputStream(tempFile.getAbsolutePath());
                     uploadEntity.setFilewriter(filewriter);
                     filewriter.write(data);
                     uploadEntity.setFile(tempFile);
                     uploadEntity.incremetByteCount(data.length);
+                    uploadEntityStateMap.put(uuid, uploadEntity);
+                    if (uploadEntity.getDownloadedsize() == uploadEntity.getEntitysize())
{
+                        postUpload(uploadEntity);
+
+                    } else {
+                        uploadEntity.setStatus(UploadEntity.Status.IN_PROGRESS);
+                    }
                 }
             } catch (Exception e) {
                 uploadEntity.setErrorMessage(e.getMessage());
@@ -2667,6 +2736,117 @@ public class NfsSecondaryStorageResource extends ServerResourceBase
implements S
                 s_logger.debug("exception occured while writing to a file " + e);
             }
         }
+        public String postUpload(UploadEntity uploadEntity) {
+
+            String resourcePath = uploadEntity.getInstallPathPrefix(); // install path prefix
// path with mount
+            // directory
+            String finalResourcePath = uploadEntity.getTmpltPath(); // template download
+            // path on secondary
+            // storage
+            UploadEntity.ResourceType resourceType = uploadEntity.getResourceType();
+
+            File originalTemplate = uploadEntity.getFile();
+            //String checkSum = computeCheckSum(originalTemplate);
+            //if (checkSum == null) {
+            //  s_logger.warn("Something wrong happened when try to calculate the checksum
of downloaded template!");
+            //}
+            //dnld.setCheckSum(checkSum);
+
+            int imgSizeGigs = (int)Math.ceil(_storage.getSize(originalTemplate.getPath())
* 1.0d / (1024 * 1024 * 1024));
+            imgSizeGigs++; // add one just in case
+            long timeout = (long)imgSizeGigs * installTimeoutPerGig;
+            Script scr = null;
+            String script = resourceType == UploadEntity.ResourceType.TEMPLATE ? createTmpltScr
: createVolScr;
+            scr = new Script(script, timeout, s_logger);
+            scr.add("-s", Integer.toString(imgSizeGigs));
+            scr.add("-S", Long.toString(UploadEntity.s_maxTemplateSize));
+            //if (uploadEntity.getDescription() != null && dnld.getDescription().length()
> 1) {
+            //    scr.add("-d", dnld.getDescription());
+            //}
+            if (uploadEntity.isHvm()) {
+                scr.add("-h");
+            }
+
+            // add options common to ISO and template
+            String extension = uploadEntity.getFormat().getFileExtension();
+            String templateName = "";
+            if (extension.equals("iso")) {
+                templateName = uploadEntity.getUuid().trim().replace(" ", "_");
+            } else {
+                templateName = java.util.UUID.nameUUIDFromBytes((uploadEntity.getFilename()
+ System.currentTimeMillis()).getBytes()).toString();
+            }
+
+            // run script to mv the temporary template file to the final template
+            // file
+            String templateFilename = templateName + "." + extension;
+            uploadEntity.setTemplatePath(finalResourcePath + "/" + templateFilename);
+            scr.add("-n", templateFilename);
+
+            scr.add("-t", resourcePath);
+            scr.add("-f", uploadEntity.getFile().getAbsolutePath()); // this is the temporary
+            // template file downloaded
+            if (uploadEntity.getChksum() != null && uploadEntity.getChksum().length()
> 1) {
+                scr.add("-c", uploadEntity.getChksum());
+            }
+            scr.add("-u"); // cleanup
+            String result;
+            result = scr.execute();
+
+            if (result != null) {
+                return result;
+            }
+
+            // Set permissions for the downloaded template
+            File downloadedTemplate = new File(resourcePath + "/" + templateFilename);
+            _storage.setWorldReadableAndWriteable(downloadedTemplate);
+
+            // Set permissions for template/volume.properties
+            String propertiesFile = resourcePath;
+            if (resourceType == UploadEntity.ResourceType.TEMPLATE) {
+                propertiesFile += "/template.properties";
+            } else {
+                propertiesFile += "/volume.properties";
+            }
+            File templateProperties = new File(propertiesFile);
+            _storage.setWorldReadableAndWriteable(templateProperties);
+
+            TemplateLocation loc = new TemplateLocation(_storage, resourcePath);
+            try {
+                loc.create(uploadEntity.getEntityId(), true, uploadEntity.getFilename());
+            } catch (IOException e) {
+                s_logger.warn("Something is wrong with template location " + resourcePath,
e);
+                loc.purge();
+                return "Unable to download due to " + e.getMessage();
+            }
+
+            Iterator<Processor> en = _downloadMgr.getProcessesors().values().iterator();
+            while (en.hasNext()) {
+                Processor processor = en.next();
+
+                FormatInfo info = null;
+                try {
+                    info = processor.process(resourcePath, null, templateName);
+                } catch (InternalErrorException e) {
+                    s_logger.error("Template process exception ", e);
+                    return e.toString();
+                }
+                if (info != null) {
+                    loc.addFormat(info);
+                    //dnld.setTemplatesize(info.virtualSize);
+                    //dnld.setTemplatePhysicalSize(info.size);
+                    break;
+                }
+            }
+
+            if (!loc.save()) {
+                s_logger.warn("Cleaning up because we're unable to save the formats");
+                loc.purge();
+            }
+            uploadEntity.setStatus(UploadEntity.Status.COMPLETED);
+            uploadEntityStateMap.put(uploadEntity.getUuid(), uploadEntity);
+
+            return null;
+        }
 
         private void parsePostBody(InputStream input, OutputStream output, Map<String,
String> params) throws IOException {
             BufferedReader reader = new BufferedReader(new InputStreamReader(input));
@@ -2745,7 +2925,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase
implements S
             TemplateOrVolumePostUploadCommand cmd = gson.fromJson(metadata, TemplateOrVolumePostUploadCommand.class);
 
             //call handle upload method.
-            handleuplod(cmd.getEntityId(), cmd.getAbsolutePath(), cmd.getName(), output.size(),
output.toByteArray());
+            handleuplod(cmd.getEntityId(),cmd.getAbsolutePath(), cmd.getEntityUUID(),cmd.getLocalPath(),cmd.getName(),output.size(),
cmd.getType(), cmd.getImageFormat(), output.toByteArray(), false, cmd.getChecksum(), cmd.getDataTo());
 
             s_logger.error(new String(data));
 

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2a2d4364/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManager.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManager.java
b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManager.java
index d0abe2c..9003aec 100644
--- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManager.java
+++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManager.java
@@ -52,6 +52,8 @@ public interface DownloadManager extends Manager {
     public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat
format, boolean hvm, Long accountId, String descr, String cksum,
         String installPathPrefix, String user, String password, long maxTemplateSizeInBytes,
Proxy proxy, ResourceType resourceType);
 
+    Map getProcessesors();
+
     /**
      * Get the status of a download job
      * @param jobId job Id

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2a2d4364/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
index 9678b6b..f9b2ff1 100755
--- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
+++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
@@ -88,7 +88,7 @@ import com.cloud.utils.script.Script;
 public class DownloadManagerImpl extends ManagerBase implements DownloadManager {
     private String _name;
     StorageLayer _storage;
-    Map<String, Processor> _processors;
+    public Map<String, Processor> _processors;
 
     public class Completion implements DownloadCompleteCallback {
         private final String jobId;
@@ -103,6 +103,11 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
         }
     }
 
+    @Override
+    public Map getProcessesors() {
+        return _processors;
+    }
+
     private static class DownloadJob {
         private final TemplateDownloader td;
         private final String tmpltName;

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2a2d4364/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadEntity.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadEntity.java
b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadEntity.java
index 7741ca5..3066c6b 100644
--- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadEntity.java
+++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadEntity.java
@@ -18,6 +18,7 @@
 package org.apache.cloudstack.storage.template;
 
 
+import com.cloud.storage.Storage;
 import java.io.File;
 import java.io.FileOutputStream;
 
@@ -25,10 +26,19 @@ public class UploadEntity {
     private long filesize;
     private long downloadedsize;
     private String filename;
-    private String absoluteFilePath;
+    private String installPathPrefix;
+    private String templatePath;
+    private boolean isHvm;
+    private Storage.ImageFormat format;
+    private String uuid;
+    private long entityId;
+    private String chksum;
 
 
 
+    public static enum ResourceType {
+        VOLUME, TEMPLATE
+    }
 
     public static enum Status {
         UNKNOWN, IN_PROGRESS, COMPLETED, ERROR
@@ -38,13 +48,18 @@ public class UploadEntity {
     private FileOutputStream filewriter = null;
     private String errorMessage=null;
     private File file;
+    private ResourceType resourceType;
+
+    public static long s_maxTemplateSize = 50L * 1024L * 1024L * 1024L;
 
-    public UploadEntity(long filesize, Status status, String filename, String absoluteFilePath)
{
+    public UploadEntity(String uuid,long entityId,long filesize, Status status, String filename,
String installPathPrefix){
+        this.uuid=uuid;
         this.filesize=filesize;
         this.uploadState=status;
         this.downloadedsize=0l;
         this.filename=filename;
-        this.absoluteFilePath=absoluteFilePath;
+        this.installPathPrefix = installPathPrefix;
+        this.entityId=entityId;
     }
 
     public void setEntitysize(long filesize) {
@@ -102,13 +117,60 @@ public class UploadEntity {
     public void setFilename(String filename) {
         this.filename = filename;
     }
-    public String getAbsoluteFilePath() {
-        return absoluteFilePath;
+    public String getInstallPathPrefix() {
+        return installPathPrefix;
+    }
+
+    public void setInstallPathPrefix(String absoluteFilePath) {
+        this.installPathPrefix = absoluteFilePath;
     }
 
-    public void setAbsoluteFilePath(String absoluteFilePath) {
-        this.absoluteFilePath = absoluteFilePath;
+    public String getTmpltPath() {
+        return templatePath;
     }
 
+    public void setTemplatePath(String templatePath) {
+        this.templatePath=templatePath;
+    }
+
+    public ResourceType getResourceType() {
+        return resourceType;
+    }
+
+    public void setResourceType(ResourceType resourceType) {
+        this.resourceType = resourceType;
+    }
+
+    public boolean isHvm() {
+        return isHvm;
+    }
+
+    public void setHvm(boolean isHvm) {
+        this.isHvm = isHvm;
+    }
+
+    public Storage.ImageFormat getFormat() {
+        return format;
+    }
+
+    public void setFormat(Storage.ImageFormat format) {
+        this.format = format;
+    }
+
+    public String getUuid() {
+        return uuid;
+    }
+
+    public long getEntityId() {
+        return entityId;
+    }
+
+    public String getChksum() {
+        return chksum;
+    }
+
+    public void setChksum(String chksum) {
+        this.chksum = chksum;
+    }
 
 }


Mime
View raw message