cloudstack-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (CLOUDSTACK-10146) Bypass Secondary Storage for KVM templates
Date Thu, 04 Jan 2018 08:22:00 GMT

    [ https://issues.apache.org/jira/browse/CLOUDSTACK-10146?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16310973#comment-16310973
] 

ASF GitHub Bot commented on CLOUDSTACK-10146:
---------------------------------------------

marcaurele commented on a change in pull request #2379: CLOUDSTACK-10146: Bypass Secondary
Storage for KVM templates
URL: https://github.com/apache/cloudstack/pull/2379#discussion_r159597834
 
 

 ##########
 File path: server/src/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java
 ##########
 @@ -0,0 +1,225 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package org.apache.cloudstack.direct.download;
+
+import com.cloud.agent.AgentManager;
+import com.cloud.agent.api.Answer;
+import com.cloud.host.Host;
+import com.cloud.host.HostVO;
+import com.cloud.host.Status;
+import com.cloud.host.dao.HostDao;
+import com.cloud.hypervisor.Hypervisor;
+import com.cloud.storage.DataStoreRole;
+import com.cloud.storage.VMTemplateStoragePoolVO;
+import com.cloud.storage.VMTemplateStorageResourceAssoc;
+import com.cloud.storage.VMTemplateVO;
+import com.cloud.storage.dao.VMTemplateDao;
+import com.cloud.storage.dao.VMTemplatePoolDao;
+import com.cloud.utils.component.ManagerBase;
+import com.cloud.utils.exception.CloudRuntimeException;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import javax.inject.Inject;
+
+import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
+import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer;
+import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand.DownloadProtocol;
+import org.apache.cloudstack.agent.directdownload.HttpDirectDownloadCommand;
+import org.apache.cloudstack.agent.directdownload.MetalinkDirectDownloadCommand;
+import org.apache.cloudstack.agent.directdownload.NfsDirectDownloadCommand;
+import org.apache.cloudstack.agent.directdownload.SetupDirectDownloadCertificate;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
+import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
+import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore;
+import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
+import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
+import org.apache.log4j.Logger;
+
+public class DirectDownloadManagerImpl extends ManagerBase implements DirectDownloadManager
{
+
+    private static final Logger s_logger = Logger.getLogger(DirectDownloadManagerImpl.class);
+    protected static final String httpHeaderDetailKey = "HTTP_HEADER";
+
+    @Inject
+    VMTemplateDao vmTemplateDao;
+    @Inject
+    PrimaryDataStoreDao primaryDataStoreDao;
+    @Inject
+    HostDao hostDao;
+    @Inject
+    AgentManager agentManager;
+    @Inject
+    VMTemplatePoolDao vmTemplatePoolDao;
+    @Inject
+    DataStoreManager dataStoreManager;
+
+    @Override
+    public List<Class<?>> getCommands() {
+        final List<Class<?>> cmdList = new ArrayList<Class<?>>();
+        return cmdList;
+    }
+
+    /**
+     * Return protocol to use from provided URL
+     * @param url
+     * @return
+     */
+    public static DownloadProtocol getProtocolFromUrl(String url) {
+        URI uri;
+        try {
+            uri = new URI(url);
+        } catch (URISyntaxException e) {
+            throw new CloudRuntimeException("URI is incorrect: " + url);
+        }
+        if ((uri != null) && (uri.getScheme() != null)) {
+            if (uri.getPath().endsWith(".metalink")) {
+                return DownloadProtocol.METALINK;
+            } else if (uri.getScheme().equalsIgnoreCase("http")) {
+                return DownloadProtocol.HTTP;
+            } else if (uri.getScheme().equalsIgnoreCase("https")) {
+                return DownloadProtocol.HTTPS;
+            } else if (uri.getScheme().equalsIgnoreCase("nfs")) {
+                return DownloadProtocol.NFS;
+            } else {
+                throw new CloudRuntimeException("Scheme is not supported " + url);
+            }
+        } else {
+            throw new CloudRuntimeException("URI is incorrect: " + url);
+        }
+    }
+
+    /**
+     * Return HTTP headers from template details
+     * @param templateDetails
+     * @return
+     */
+    protected Map<String, String> getHeadersFromDetails(Map<String, String> templateDetails)
{
+        Map<String, String> headers = new HashMap<>();
+        for (String key : templateDetails.keySet()) {
+            if (key.startsWith(httpHeaderDetailKey)) {
+                String header = key.split(":")[1];
+                String value = templateDetails.get(key);
+                headers.put(header, value);
+            }
+        }
+        return headers;
+    }
+
+    @Override
+    public void downloadTemplate(long templateId, long poolId, long hostId) {
+        VMTemplateVO template = vmTemplateDao.findById(templateId);
+        StoragePoolVO pool = primaryDataStoreDao.findById(poolId);
+        HostVO host = hostDao.findById(hostId);
+        if (pool == null) {
+            throw new CloudRuntimeException("Storage pool " + poolId + " could not be found");
+        }
+        if (template == null) {
+            throw new CloudRuntimeException("Template " + templateId + " could not be found");
+        }
+        if (host == null) {
+            throw new CloudRuntimeException("Host " + hostId + " could not be found");
+        }
+        if (!template.isDirectDownload()) {
+            throw new CloudRuntimeException("Template " + templateId + " is not marked for
direct download");
+        }
+        Map<String, String> details = template.getDetails();
+        String url = template.getUrl();
+        String checksum = template.getChecksum();
+        Map<String, String> headers = getHeadersFromDetails(details);
+        DataStore store = dataStoreManager.getDataStore(poolId, DataStoreRole.Primary);
+        if (store == null) {
+            throw new CloudRuntimeException("Data store " + poolId + " could not be found");
+        }
+        PrimaryDataStore primaryDataStore = (PrimaryDataStore) store;
+        PrimaryDataStoreTO to = (PrimaryDataStoreTO) primaryDataStore.getTO();
+
+        DownloadProtocol protocol = getProtocolFromUrl(url);
+        DirectDownloadCommand cmd = getDirectDownloadCommandFromProtocol(protocol, url, templateId,
to, checksum, headers);
+        Answer answer = agentManager.easySend(hostId, cmd);
+        if (answer == null || !answer.getResult()) {
+            throw new CloudRuntimeException("Host " + hostId + " could not download template
" +
+                    templateId + " on pool " + poolId);
+        }
+
+        VMTemplateStoragePoolVO sPoolRef = vmTemplatePoolDao.findByPoolTemplate(poolId, templateId);
+        if (sPoolRef == null) {
+            if (s_logger.isDebugEnabled()) {
+                s_logger.debug("Not found (templateId:" + templateId + " poolId: " + poolId
+ ") in template_spool_ref, persisting it");
+            }
+            DirectDownloadAnswer ans = (DirectDownloadAnswer) answer;
+            sPoolRef = new VMTemplateStoragePoolVO(poolId, templateId);
+            sPoolRef.setDownloadPercent(100);
+            sPoolRef.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED);
+            sPoolRef.setState(ObjectInDataStoreStateMachine.State.Ready);
+            sPoolRef.setTemplateSize(ans.getTemplateSize());
+            sPoolRef.setLocalDownloadPath(ans.getInstallPath());
+            sPoolRef.setInstallPath(ans.getInstallPath());
+            vmTemplatePoolDao.persist(sPoolRef);
+        }
+    }
+
+    /**
+     * Return DirectDownloadCommand according to the protocol
+     * @param protocol
+     * @param url
+     * @param templateId
+     * @param destPool
+     * @return
+     */
+    private DirectDownloadCommand getDirectDownloadCommandFromProtocol(DownloadProtocol protocol,
String url, Long templateId, PrimaryDataStoreTO destPool,
+                                                                       String checksum, Map<String,
String> httpHeaders) {
+        if (protocol.equals(DownloadProtocol.HTTP) || protocol.equals(DownloadProtocol.HTTPS))
{
+            return new HttpDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders);
+        } else if (protocol.equals(DownloadProtocol.NFS)) {
+            return new NfsDirectDownloadCommand(url, templateId, destPool, checksum);
+        } else if (protocol.equals(DownloadProtocol.METALINK)) {
+            return new MetalinkDirectDownloadCommand(url, templateId, destPool, checksum);
+        } else {
+            return null;
+        }
+    }
+
+    @Override
+    public void uploadCertificateToHosts(String certificateCer, String certificateName) {
+        List<HostVO> hosts = hostDao.listAllHostsByType(Host.Type.Routing)
+                .stream()
+                .filter(x -> x.getStatus().equals(Status.Up) &&
+                            x.getHypervisorType().equals(Hypervisor.HypervisorType.KVM))
+                .collect(Collectors.toList());
+        for (HostVO host : hosts) {
+            uploadCertificate(certificateCer, certificateName, host.getId());
+        }
+    }
+
+    private void uploadCertificate(String certificate, String certificateName, long hostId)
{
+        SetupDirectDownloadCertificate cmd = new SetupDirectDownloadCertificate(certificate,
certificateName);
+        Answer answer = agentManager.easySend(hostId, cmd);
+        if (answer != null && answer.getResult()) {
+            s_logger.debug("Certificate " + certificateName + "successfully uploaded to host:
" + hostId);
 
 Review comment:
   would be good to have it at the `info`level

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


> Bypass Secondary Storage for KVM templates
> ------------------------------------------
>
>                 Key: CLOUDSTACK-10146
>                 URL: https://issues.apache.org/jira/browse/CLOUDSTACK-10146
>             Project: CloudStack
>          Issue Type: Bug
>      Security Level: Public(Anyone can view this level - this is the default.) 
>    Affects Versions: 4.11.0.0
>            Reporter: Nicolas Vazquez
>            Assignee: Nicolas Vazquez
>




--
This message was sent by Atlassian JIRA
(v6.4.14#64029)

Mime
View raw message