From commits-return-87959-archive-asf-public=cust-asf.ponee.io@cloudstack.apache.org Fri May 24 12:36:50 2019 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [207.244.88.153]) by mx-eu-01.ponee.io (Postfix) with SMTP id A9CB318077A for ; Fri, 24 May 2019 14:36:49 +0200 (CEST) Received: (qmail 76242 invoked by uid 500); 24 May 2019 12:36:47 -0000 Mailing-List: contact commits-help@cloudstack.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@cloudstack.apache.org Delivered-To: mailing list commits@cloudstack.apache.org Received: (qmail 76105 invoked by uid 99); 24 May 2019 12:36:47 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 24 May 2019 12:36:47 +0000 From: GitBox To: commits@cloudstack.apache.org Subject: [GitHub] [cloudstack] DaanHoogland commented on a change in pull request #3350: Retrieve diagnostics data rebase Message-ID: <155870140758.13723.16551728996710933601.gitbox@gitbox.apache.org> Date: Fri, 24 May 2019 12:36:47 -0000 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit DaanHoogland commented on a change in pull request #3350: Retrieve diagnostics data rebase URL: https://github.com/apache/cloudstack/pull/3350#discussion_r287330713 ########## File path: server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java ########## @@ -126,10 +191,333 @@ protected String prepareShellCmd(String cmdType, String ipAddress, String option } } + @Override + @ActionEvent(eventType = EventTypes.EVENT_SYSTEM_VM_DIAGNOSTICS, eventDescription = "getting diagnostics files on system vm", async = true) + public String getDiagnosticsDataCommand(GetDiagnosticsDataCmd cmd) { + Long vmId = cmd.getId(); + List optionalFilesList = cmd.getFilesList(); + VMInstanceVO vmInstance = getSystemVMInstance(vmId); + long zoneId = vmInstance.getDataCenterId(); + + List fileList = getFileListToBeRetrieved(optionalFilesList, vmInstance); + + if (CollectionUtils.isEmpty(fileList)) { + throw new CloudRuntimeException("Failed to generate diagnostics file list for retrieval."); + } + + Long vmHostId = vmInstance.getHostId(); + + // Find Secondary Storage with enough Disk Quota in the current Zone + final DataStore store = getImageStore(vmInstance.getDataCenterId()); + + Answer zipFilesAnswer = zipDiagnosticsFilesInSystemVm(vmInstance, fileList); + + if (zipFilesAnswer == null) { + throw new CloudRuntimeException(String.format("Failed to generate diagnostics zip file in VM %s", vmInstance.getUuid())); + } + + if (!zipFilesAnswer.getResult()) { + throw new CloudRuntimeException(String.format("Failed to generate diagnostics zip file file in VM %s due to %s", vmInstance.getUuid(), zipFilesAnswer.getDetails())); + } + + // Copy zip file from system VM to secondary storage + String zipFileInSystemVm = zipFilesAnswer.getDetails().replace("\n", ""); + Pair copyToSecondaryStorageResults = copyZipFileToSecondaryStorage(vmInstance, vmHostId, zipFileInSystemVm, store); + + // Send cleanup zip file cleanup command to system VM + Answer fileCleanupAnswer = deleteDiagnosticsZipFileInsystemVm(vmInstance, zipFileInSystemVm); + if (fileCleanupAnswer == null) { + LOGGER.error(String.format("Failed to cleanup diagnostics zip file on vm: %s", vmInstance.getUuid())); + } else { + if (!fileCleanupAnswer.getResult()) { + LOGGER.error(String.format("Zip file cleanup for vm %s has failed with: %s", vmInstance.getUuid(), fileCleanupAnswer.getDetails())); + } + } + + if (!copyToSecondaryStorageResults.first()) { + throw new CloudRuntimeException(String.format("Failed to copy %s to secondary storage %s due to %s.", zipFileInSystemVm, store.getUri(), copyToSecondaryStorageResults.second())); + } + + // Now we need to create the file download URL + // Find ssvm of store + VMInstanceVO ssvm = getSecondaryStorageVmInZone(zoneId); + if (ssvm == null) { + throw new CloudRuntimeException("No ssvm found in Zone with ID: " + zoneId); + } + // Secondary Storage install path = "diagnostics_data/diagnostics_files_xxxx.tar + String installPath = DIAGNOSTICS_DATA_DIRECTORY + "/" + zipFileInSystemVm.replace("/root", ""); + return createFileDownloadUrl(store, ssvm.getHypervisorType(), installPath); + } + + /** + * Copy retrieved diagnostics zip file from system vm to secondary storage + * For VMware use the mgmt server, and for Xen/KVM use the hyperhost of the target VM + * The strategy is to mount secondary storage on mgmt server or host and scp directly to /mnt/SecStorage/diagnostics_data + * + * @param fileToCopy zip file in system vm to be copied + * @param store secondary storage to copy zip file to + */ + private Pair copyZipFileToSecondaryStorage(VMInstanceVO vmInstance, Long vmHostId, String fileToCopy, DataStore store) { + String vmControlIp = getVMSshIp(vmInstance); + if (StringUtils.isBlank(vmControlIp)) { + return new Pair<>(false, "Unable to find system vm ssh/control IP for vm with ID: " + vmInstance.getId()); + } + Pair copyResult; + if (vmInstance.getHypervisorType() == Hypervisor.HypervisorType.VMware) { + copyResult = orchestrateCopyToSecondaryStorageVMware(store, vmControlIp, fileToCopy); + } else { + copyResult = orchestrateCopyToSecondaryStorageNonVMware(store, vmControlIp, fileToCopy, vmHostId); + } + return copyResult; + } + + private void configureNetworkElementCommand(NetworkElementCommand cmd, VMInstanceVO vmInstance) { + Map accessDetails = networkManager.getSystemVMAccessDetails(vmInstance); + if (StringUtils.isBlank(accessDetails.get(NetworkElementCommand.ROUTER_IP))) { + throw new CloudRuntimeException("Unable to set system vm ControlIP for system vm with ID: " + vmInstance.getId()); + } + cmd.setAccessDetail(accessDetails); + } + + private Answer zipDiagnosticsFilesInSystemVm(VMInstanceVO vmInstance, List fileList) { + final PrepareFilesCommand cmd = new PrepareFilesCommand(fileList, DataRetrievalTimeout.value()); + configureNetworkElementCommand(cmd, vmInstance); + Answer answer = agentManager.easySend(vmInstance.getHostId(), cmd); + return answer; + } + + private Answer deleteDiagnosticsZipFileInsystemVm(VMInstanceVO vmInstance, String zipFileName) { + final DeleteFileInVrCommand cmd = new DeleteFileInVrCommand(zipFileName); + configureNetworkElementCommand(cmd, vmInstance); + return agentManager.easySend(vmInstance.getHostId(), cmd); + } + + /** + * Generate a list of diagnostics file to be retrieved depending on the system VM type + * + * @param optionalFileList Optional list of files that user may want to retrieve, empty by default + * @param vmInstance system VM instance, either SSVM, CPVM or VR + * @return a list of files to be retrieved for system VM, either generated from defaults depending on the VM type, or specified + * by the optional list param + */ + private List getFileListToBeRetrieved(List optionalFileList, VMInstanceVO vmInstance) { + DiagnosticsFilesList fileListObject = DiagnosticsFilesListFactory.getDiagnosticsFilesList(optionalFileList, vmInstance); + List fileList = new ArrayList<>(); + + if (fileListObject != null) { + fileList = fileListObject.generateFileList(); + } + return fileList; + } + + private Pair orchestrateCopyToSecondaryStorageNonVMware(final DataStore store, final String vmControlIp, String fileToCopy, Long vmHostId) { + CopyToSecondaryStorageCommand toSecondaryStorageCommand = new CopyToSecondaryStorageCommand(store.getUri(), vmControlIp, fileToCopy); + Answer copyToSecondaryAnswer = agentManager.easySend(vmHostId, toSecondaryStorageCommand); + Pair copyAnswer; + if (copyToSecondaryAnswer != null) { + copyAnswer = new Pair<>(copyToSecondaryAnswer.getResult(), copyToSecondaryAnswer.getDetails()); + } else { + copyAnswer = new Pair<>(false, "Diagnostics Zip file to secondary storage failed"); + } + return copyAnswer; + } + + private Pair orchestrateCopyToSecondaryStorageVMware(final DataStore store, final String vmSshIp, String diagnosticsFile) { + String mountPoint; + boolean success; + + Integer nfsVersion = imageStoreDetailsUtil.getNfsVersion(store.getId()); + mountPoint = mountManager.getMountPoint(store.getUri(), nfsVersion); + if (StringUtils.isNotBlank(mountPoint)) { + LOGGER.info(String.format("Copying %s from %s to secondary store %s", diagnosticsFile, vmSshIp, store.getUri())); + + // dirIn/mnt/SecStorage/uuid/diagnostics_data + String dataDirectoryInSecondaryStore = String.format("%s/%s", mountPoint, DIAGNOSTICS_DATA_DIRECTORY); + try { + File dataDirectory = new File(dataDirectoryInSecondaryStore); + boolean existsInSecondaryStore = dataDirectory.exists() || dataDirectory.mkdir(); + + // Modify directory file permissions + Path path = Paths.get(dataDirectory.getAbsolutePath()); + setDirFilePermissions(path); + + if (existsInSecondaryStore) { + // scp from system VM to mounted sec storage directory + int port = 3922; + File permKey = new File("/var/cloudstack/management/.ssh/id_rsa"); + SshHelper.scpFrom(vmSshIp, port, "root", permKey, dataDirectoryInSecondaryStore, diagnosticsFile); + } + // Verify File copy to Secondary Storage + File fileInSecondaryStore = new File(dataDirectoryInSecondaryStore + diagnosticsFile.replace("/root", "")); + success = fileInSecondaryStore.exists(); + } catch (Exception e) { + String msg = String.format("Exception caught during scp from %s to secondary store %s: ", vmSshIp, dataDirectoryInSecondaryStore); + LOGGER.error(msg); + return new Pair<>(false, msg); + } finally { + // umount secondary storage + umountSecondaryStorage(mountPoint); + } + } else { + return new Pair<>(false, "Failed to mount secondary storage:" + store.getName()); + } + return new Pair<>(success, "File copied to secondary storage successfully"); + } + + // Get ssvm from the zone to use for creating entity download URL + private VMInstanceVO getSecondaryStorageVmInZone(Long zoneId) { + List ssvm = instanceDao.listByZoneIdAndType(zoneId, VirtualMachine.Type.SecondaryStorageVm); + return (CollectionUtils.isEmpty(ssvm)) ? null : ssvm.get(0); + } + + /** + * Iterate through all Image stores in the current running zone and select any that has less than 95% disk usage Review comment: comment is off: ```95%``` should be ```DiskQuotaPercentageThreshold .value()%``` ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: users@infra.apache.org With regards, Apache Git Services