Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 94563200C4C for ; Tue, 4 Apr 2017 23:48:38 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 92E3C160B77; Tue, 4 Apr 2017 21:48:38 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 1FD9A160BA2 for ; Tue, 4 Apr 2017 23:48:35 +0200 (CEST) Received: (qmail 82069 invoked by uid 500); 4 Apr 2017 21:48:35 -0000 Mailing-List: contact commits-help@geode.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@geode.apache.org Delivered-To: mailing list commits@geode.apache.org Received: (qmail 81904 invoked by uid 99); 4 Apr 2017 21:48:35 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 04 Apr 2017 21:48:35 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 144CDDFF0F; Tue, 4 Apr 2017 21:48:35 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: khowe@apache.org To: commits@geode.apache.org Date: Tue, 04 Apr 2017 21:48:38 -0000 Message-Id: <5b686fc95fe542188bef0c7d51ebee5a@git.apache.org> In-Reply-To: <570700c45527470fa68b4ec13366fd6e@git.apache.org> References: <570700c45527470fa68b4ec13366fd6e@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [4/5] geode git commit: GEODE-2420: Add classes to estimated the size of exported logs archived-at: Tue, 04 Apr 2017 21:48:38 -0000 GEODE-2420: Add classes to estimated the size of exported logs - New Function class added with tests - Code to invoke new function added, but disabled, in ExportLogsCommand - New option added, but disabled, for specifying size limit for exported logs - Refactoring: extracted the interceptor from ExportLogsCommand - Refactoring: renamed ExportLogCommand to ExportLogsCommand - New Serializable class to return the log size estimate to the command - Tests for the log sizing related classes Project: http://git-wip-us.apache.org/repos/asf/geode/repo Commit: http://git-wip-us.apache.org/repos/asf/geode/commit/0c15c6e0 Tree: http://git-wip-us.apache.org/repos/asf/geode/tree/0c15c6e0 Diff: http://git-wip-us.apache.org/repos/asf/geode/diff/0c15c6e0 Branch: refs/heads/develop Commit: 0c15c6e01e7095f61e0e2df9866534f1cf6489cc Parents: 4a09e88 Author: Ken Howe Authored: Wed Mar 29 11:10:20 2017 -0700 Committer: Ken Howe Committed: Tue Apr 4 13:51:07 2017 -0700 ---------------------------------------------------------------------- .../internal/cli/commands/ExportLogCommand.java | 236 ---- .../cli/commands/ExportLogsCommand.java | 278 +++++ .../cli/commands/ExportLogsInterceptor.java | 109 ++ .../cli/functions/ExportLogsFunction.java | 7 +- .../cli/functions/ExportedLogsSizeInfo.java | 107 ++ .../cli/functions/SizeExportLogsFunction.java | 87 ++ .../internal/cli/i18n/CliStrings.java | 6 + .../management/internal/cli/util/LogSizer.java | 116 ++ .../cli/commands/ExportLogsCommandTest.java | 129 +++ .../cli/commands/ExportLogsDUnitTest.java | 4 +- .../commands/ExportLogsFileSizeLimitTest.java | 88 ++ .../ExportLogsInterceptorJUnitTest.java | 4 +- .../cli/commands/ExportLogsStatsDUnitTest.java | 2 +- .../cli/commands/ExportLogsTestSuite.java | 27 + .../cli/commands/LogLevelInterceptorTest.java | 2 +- .../cli/functions/ExportedLogsSizeInfoTest.java | 145 +++ .../SizeExportLogsFunctionCacheTest.java | 183 +++ .../SizeExportLogsFunctionFileTest.java | 183 +++ .../cli/functions/SizeExportLogsTestSuite.java | 29 + .../internal/cli/util/LogSizerTest.java | 99 ++ .../dunit/rules/GfshShellConnectionRule.java | 4 +- .../sanctionedDataSerializables.txt | 1066 +++++++++--------- .../codeAnalysis/sanctionedSerializables.txt | 1 + 23 files changed, 2134 insertions(+), 778 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/geode/blob/0c15c6e0/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ExportLogCommand.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ExportLogCommand.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ExportLogCommand.java deleted file mode 100644 index 3f147c1..0000000 --- a/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ExportLogCommand.java +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.geode.management.internal.cli.commands; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.geode.cache.Region; -import org.apache.geode.distributed.DistributedMember; -import org.apache.geode.internal.cache.GemFireCacheImpl; -import org.apache.geode.internal.logging.LogService; -import org.apache.geode.management.cli.CliMetaData; -import org.apache.geode.management.cli.ConverterHint; -import org.apache.geode.management.cli.Result; -import org.apache.geode.management.internal.cli.AbstractCliAroundInterceptor; -import org.apache.geode.management.internal.cli.CliUtil; -import org.apache.geode.management.internal.cli.GfshParseResult; -import org.apache.geode.management.internal.cli.functions.ExportLogsFunction; -import org.apache.geode.management.internal.cli.i18n.CliStrings; -import org.apache.geode.management.internal.cli.result.ResultBuilder; -import org.apache.geode.management.internal.cli.util.ExportLogsCacheWriter; -import org.apache.geode.internal.logging.log4j.LogLevel; -import org.apache.geode.management.internal.configuration.utils.ZipUtils; -import org.apache.geode.management.internal.security.ResourceOperation; -import org.apache.geode.security.ResourcePermission; -import org.apache.logging.log4j.Logger; -import org.springframework.shell.core.CommandMarker; -import org.springframework.shell.core.annotation.CliCommand; -import org.springframework.shell.core.annotation.CliOption; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.LocalDateTime; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -public class ExportLogCommand implements CommandMarker { - public final static String FORMAT = "yyyy/MM/dd/HH/mm/ss/SSS/z"; - public final static String ONLY_DATE_FORMAT = "yyyy/MM/dd"; - private final static Logger logger = LogService.getLogger(); - - @CliCommand(value = CliStrings.EXPORT_LOGS, help = CliStrings.EXPORT_LOGS__HELP) - @CliMetaData(shellOnly = false, isFileDownloadOverHttp = true, - interceptor = "org.apache.geode.management.internal.cli.commands.ExportLogCommand$ExportLogsInterceptor", - relatedTopic = {CliStrings.TOPIC_GEODE_SERVER, CliStrings.TOPIC_GEODE_DEBUG_UTIL}) - @ResourceOperation(resource = ResourcePermission.Resource.CLUSTER, - operation = ResourcePermission.Operation.READ) - public Result exportLogs( - @CliOption(key = CliStrings.EXPORT_LOGS__DIR, help = CliStrings.EXPORT_LOGS__DIR__HELP, - mandatory = false) String dirName, - @CliOption(key = CliStrings.EXPORT_LOGS__GROUP, - unspecifiedDefaultValue = CliMetaData.ANNOTATION_NULL_VALUE, - optionContext = ConverterHint.MEMBERGROUP, - help = CliStrings.EXPORT_LOGS__GROUP__HELP) String[] groups, - @CliOption(key = CliStrings.EXPORT_LOGS__MEMBER, - unspecifiedDefaultValue = CliMetaData.ANNOTATION_NULL_VALUE, - optionContext = ConverterHint.ALL_MEMBER_IDNAME, - help = CliStrings.EXPORT_LOGS__MEMBER__HELP) String[] memberIds, - @CliOption(key = CliStrings.EXPORT_LOGS__LOGLEVEL, - unspecifiedDefaultValue = LogService.DEFAULT_LOG_LEVEL, - optionContext = ConverterHint.LOG_LEVEL, - help = CliStrings.EXPORT_LOGS__LOGLEVEL__HELP) String logLevel, - @CliOption(key = CliStrings.EXPORT_LOGS__UPTO_LOGLEVEL, unspecifiedDefaultValue = "false", - help = CliStrings.EXPORT_LOGS__UPTO_LOGLEVEL__HELP) boolean onlyLogLevel, - @CliOption(key = CliStrings.EXPORT_LOGS__MERGELOG, unspecifiedDefaultValue = "false", - help = CliStrings.EXPORT_LOGS__MERGELOG__HELP) boolean mergeLog, - @CliOption(key = CliStrings.EXPORT_LOGS__STARTTIME, - unspecifiedDefaultValue = CliMetaData.ANNOTATION_NULL_VALUE, - help = CliStrings.EXPORT_LOGS__STARTTIME__HELP) String start, - @CliOption(key = CliStrings.EXPORT_LOGS__ENDTIME, - unspecifiedDefaultValue = CliMetaData.ANNOTATION_NULL_VALUE, - help = CliStrings.EXPORT_LOGS__ENDTIME__HELP) String end, - @CliOption(key = CliStrings.EXPORT_LOGS__LOGSONLY, unspecifiedDefaultValue = "false", - specifiedDefaultValue = "true", - help = CliStrings.EXPORT_LOGS__LOGSONLY__HELP) boolean logsOnly, - @CliOption(key = CliStrings.EXPORT_LOGS__STATSONLY, unspecifiedDefaultValue = "false", - specifiedDefaultValue = "true", - help = CliStrings.EXPORT_LOGS__STATSONLY__HELP) boolean statsOnly) { - Result result = null; - GemFireCacheImpl cache = GemFireCacheImpl.getInstance(); - try { - Set targetMembers = - CliUtil.findMembersIncludingLocators(groups, memberIds); - - if (targetMembers.isEmpty()) { - return ResultBuilder.createUserErrorResult(CliStrings.NO_MEMBERS_FOUND_MESSAGE); - } - - Map zipFilesFromMembers = new HashMap<>(); - for (DistributedMember server : targetMembers) { - Region region = ExportLogsFunction.createOrGetExistingExportLogsRegion(true, cache); - - ExportLogsCacheWriter cacheWriter = - (ExportLogsCacheWriter) region.getAttributes().getCacheWriter(); - - cacheWriter.startFile(server.getName()); - - CliUtil.executeFunction(new ExportLogsFunction(), - new ExportLogsFunction.Args(start, end, logLevel, onlyLogLevel, logsOnly, statsOnly), - server).getResult(); - Path zipFile = cacheWriter.endFile(); - ExportLogsFunction.destroyExportLogsRegion(cache); - - // only put the zipfile in the map if it is not null - if (zipFile != null) { - logger.info("Received zip file from member {}: {}", server.getId(), zipFile); - zipFilesFromMembers.put(server.getId(), zipFile); - } - } - - if (zipFilesFromMembers.isEmpty()) { - return ResultBuilder.createUserErrorResult("No files to be exported."); - } - - Path tempDir = Files.createTempDirectory("exportedLogs"); - // make sure the directory is created, so that even if there is no files unzipped to this dir, - // we can - // still zip it and send an empty zip file back to the client - Path exportedLogsDir = tempDir.resolve("exportedLogs"); - FileUtils.forceMkdir(exportedLogsDir.toFile()); - - for (Path zipFile : zipFilesFromMembers.values()) { - Path unzippedMemberDir = - exportedLogsDir.resolve(zipFile.getFileName().toString().replace(".zip", "")); - ZipUtils.unzip(zipFile.toAbsolutePath().toString(), unzippedMemberDir.toString()); - FileUtils.deleteQuietly(zipFile.toFile()); - } - - Path workingDir = Paths.get(System.getProperty("user.dir")); - Path exportedLogsZipFile = workingDir - .resolve("exportedLogs_" + System.currentTimeMillis() + ".zip").toAbsolutePath(); - - logger.info("Zipping into: " + exportedLogsZipFile.toString()); - ZipUtils.zipDirectory(exportedLogsDir, exportedLogsZipFile); - FileUtils.deleteDirectory(tempDir.toFile()); - result = ResultBuilder.createInfoResult(exportedLogsZipFile.toString()); - } catch (Exception ex) { - logger.error(ex, ex); - result = ResultBuilder.createGemFireErrorResult(ex.getMessage()); - } finally { - ExportLogsFunction.destroyExportLogsRegion(cache); - } - logger.debug("Exporting logs returning = {}", result); - return result; - } - - /** - * after the export logs, will need to copy the tempFile to the desired location and delete the - * temp file. - */ - public static class ExportLogsInterceptor extends AbstractCliAroundInterceptor { - @Override - public Result preExecution(GfshParseResult parseResult) { - // the arguments are in the order of it's being declared - Map arguments = parseResult.getParamValueStrings(); - - // validates groupId and memberIds not both set - if (arguments.get("group") != null && arguments.get("member") != null) { - return ResultBuilder.createUserErrorResult("Can't specify both group and member."); - } - - // validate log level - String logLevel = arguments.get("log-level"); - if (StringUtils.isBlank(logLevel) || LogLevel.getLevel(logLevel) == null) { - return ResultBuilder.createUserErrorResult("Invalid log level: " + logLevel); - } - - // validate start date and end date - String start = arguments.get("start-time"); - String end = arguments.get("end-time"); - if (start != null && end != null) { - // need to make sure end is later than start - LocalDateTime startTime = ExportLogsFunction.parseTime(start); - LocalDateTime endTime = ExportLogsFunction.parseTime(end); - if (startTime.isAfter(endTime)) { - return ResultBuilder.createUserErrorResult("start-time has to be earlier than end-time."); - } - } - - // validate onlyLogs and onlyStats - boolean onlyLogs = Boolean.parseBoolean(arguments.get("logs-only")); - boolean onlyStats = Boolean.parseBoolean(arguments.get("stats-only")); - if (onlyLogs && onlyStats) { - return ResultBuilder.createUserErrorResult("logs-only and stats-only can't both be true"); - } - - return ResultBuilder.createInfoResult(""); - } - - @Override - public Result postExecution(GfshParseResult parseResult, Result commandResult, Path tempFile) { - // in the command over http case, the command result is in the downloaded temp file - if (tempFile != null) { - Path dirPath; - String dirName = parseResult.getParamValueStrings().get("dir"); - if (StringUtils.isBlank(dirName)) { - dirPath = Paths.get(System.getProperty("user.dir")); - } else { - dirPath = Paths.get(dirName); - } - String fileName = "exportedLogs_" + System.currentTimeMillis() + ".zip"; - File exportedLogFile = dirPath.resolve(fileName).toFile(); - try { - FileUtils.copyFile(tempFile.toFile(), exportedLogFile); - FileUtils.deleteQuietly(tempFile.toFile()); - commandResult = ResultBuilder - .createInfoResult("Logs exported to: " + exportedLogFile.getAbsolutePath()); - } catch (IOException e) { - logger.error(e.getMessage(), e); - commandResult = ResultBuilder.createGemFireErrorResult(e.getMessage()); - } - } else if (commandResult.getStatus() == Result.Status.OK) { - commandResult = ResultBuilder.createInfoResult( - "Logs exported to the connected member's file system: " + commandResult.nextLine()); - } - return commandResult; - } - } - -} http://git-wip-us.apache.org/repos/asf/geode/blob/0c15c6e0/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ExportLogsCommand.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ExportLogsCommand.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ExportLogsCommand.java new file mode 100644 index 0000000..37cbd71 --- /dev/null +++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ExportLogsCommand.java @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.geode.management.internal.cli.commands; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.geode.cache.Region; +import org.apache.geode.distributed.DistributedMember; +import org.apache.geode.internal.cache.GemFireCacheImpl; +import org.apache.geode.internal.logging.LogService; +import org.apache.geode.management.cli.CliMetaData; +import org.apache.geode.management.cli.ConverterHint; +import org.apache.geode.management.cli.Result; +import org.apache.geode.management.internal.cli.CliUtil; +import org.apache.geode.management.internal.cli.functions.ExportLogsFunction; +import org.apache.geode.management.internal.cli.functions.ExportedLogsSizeInfo; +import org.apache.geode.management.internal.cli.functions.SizeExportLogsFunction; +import org.apache.geode.management.internal.cli.i18n.CliStrings; +import org.apache.geode.management.internal.cli.result.ResultBuilder; +import org.apache.geode.management.internal.cli.util.ExportLogsCacheWriter; +import org.apache.geode.management.internal.configuration.utils.ZipUtils; +import org.apache.geode.management.internal.security.ResourceOperation; +import org.apache.geode.security.ResourcePermission; +import org.apache.logging.log4j.Logger; +import org.springframework.shell.core.CommandMarker; +import org.springframework.shell.core.annotation.CliCommand; +import org.springframework.shell.core.annotation.CliOption; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class ExportLogsCommand implements CommandMarker { + + private static final Logger logger = LogService.getLogger(); + + public static final String FORMAT = "yyyy/MM/dd/HH/mm/ss/SSS/z"; + public static final String ONLY_DATE_FORMAT = "yyyy/MM/dd"; + + private static final Pattern DISK_SPACE_LIMIT_PATTERN = Pattern.compile("(\\d+)([mgtMGT]?)"); + + @CliCommand(value = CliStrings.EXPORT_LOGS, help = CliStrings.EXPORT_LOGS__HELP) + @CliMetaData(shellOnly = false, isFileDownloadOverHttp = true, + interceptor = "org.apache.geode.management.internal.cli.commands.ExportLogsInterceptor", + relatedTopic = {CliStrings.TOPIC_GEODE_SERVER, CliStrings.TOPIC_GEODE_DEBUG_UTIL}) + @ResourceOperation(resource = ResourcePermission.Resource.CLUSTER, + operation = ResourcePermission.Operation.READ) + public Result exportLogs( + @CliOption(key = CliStrings.EXPORT_LOGS__DIR, help = CliStrings.EXPORT_LOGS__DIR__HELP, + mandatory = false) String dirName, + @CliOption(key = CliStrings.EXPORT_LOGS__GROUP, + unspecifiedDefaultValue = CliMetaData.ANNOTATION_NULL_VALUE, + optionContext = ConverterHint.MEMBERGROUP, + help = CliStrings.EXPORT_LOGS__GROUP__HELP) String[] groups, + @CliOption(key = CliStrings.EXPORT_LOGS__MEMBER, + unspecifiedDefaultValue = CliMetaData.ANNOTATION_NULL_VALUE, + optionContext = ConverterHint.ALL_MEMBER_IDNAME, + help = CliStrings.EXPORT_LOGS__MEMBER__HELP) String[] memberIds, + @CliOption(key = CliStrings.EXPORT_LOGS__LOGLEVEL, + unspecifiedDefaultValue = LogService.DEFAULT_LOG_LEVEL, + optionContext = ConverterHint.LOG_LEVEL, + help = CliStrings.EXPORT_LOGS__LOGLEVEL__HELP) String logLevel, + @CliOption(key = CliStrings.EXPORT_LOGS__UPTO_LOGLEVEL, unspecifiedDefaultValue = "false", + help = CliStrings.EXPORT_LOGS__UPTO_LOGLEVEL__HELP) boolean onlyLogLevel, + @CliOption(key = CliStrings.EXPORT_LOGS__MERGELOG, unspecifiedDefaultValue = "false", + help = CliStrings.EXPORT_LOGS__MERGELOG__HELP) boolean mergeLog, + @CliOption(key = CliStrings.EXPORT_LOGS__STARTTIME, + unspecifiedDefaultValue = CliMetaData.ANNOTATION_NULL_VALUE, + help = CliStrings.EXPORT_LOGS__STARTTIME__HELP) String start, + @CliOption(key = CliStrings.EXPORT_LOGS__ENDTIME, + unspecifiedDefaultValue = CliMetaData.ANNOTATION_NULL_VALUE, + help = CliStrings.EXPORT_LOGS__ENDTIME__HELP) String end, + @CliOption(key = CliStrings.EXPORT_LOGS__LOGSONLY, unspecifiedDefaultValue = "false", + specifiedDefaultValue = "true", + help = CliStrings.EXPORT_LOGS__LOGSONLY__HELP) boolean logsOnly, + @CliOption(key = CliStrings.EXPORT_LOGS__STATSONLY, unspecifiedDefaultValue = "false", + specifiedDefaultValue = "true", + help = CliStrings.EXPORT_LOGS__STATSONLY__HELP) boolean statsOnly) { + // @CliOption(key = CliStrings.EXPORT_LOGS__FILESIZELIMIT, + // unspecifiedDefaultValue = CliStrings.EXPORT_LOGS__FILESIZELIMIT__UNSPECIFIED_DEFAULT, + // specifiedDefaultValue = CliStrings.EXPORT_LOGS__FILESIZELIMIT__SPECIFIED_DEFAULT, + // help = CliStrings.EXPORT_LOGS__FILESIZELIMIT__HELP) String fileSizeLimit) { + Result result = null; + GemFireCacheImpl cache = GemFireCacheImpl.getInstance(); + try { + Set targetMembers = + CliUtil.findMembersIncludingLocators(groups, memberIds); + + if (targetMembers.isEmpty()) { + return ResultBuilder.createUserErrorResult(CliStrings.NO_MEMBERS_FOUND_MESSAGE); + } + + if (false) { + // TODO: get estimated size of exported logs from all servers first + Map fileSizesFromMembers = new HashMap<>(); + for (DistributedMember server : targetMembers) { + SizeExportLogsFunction.Args args = new SizeExportLogsFunction.Args(start, end, logLevel, + onlyLogLevel, logsOnly, statsOnly); + + List results = (List) CliUtil + .executeFunction(new SizeExportLogsFunction(), args, server).getResult(); + long estimatedSize = 0; + long diskAvailable = 0; + long diskSize = 0; + List res = (List) results.get(0); + if (res.get(0) instanceof ExportedLogsSizeInfo) { + ExportedLogsSizeInfo sizeInfo = (ExportedLogsSizeInfo) res.get(0); + estimatedSize = sizeInfo.getLogsSize(); + diskAvailable = sizeInfo.getDiskAvailable(); + diskSize = sizeInfo.getDiskSize(); + } else { + estimatedSize = 0; + } + + + logger.info("Received file size from member {}: {}", server.getId(), estimatedSize); + } + + // TODO: Check log size limits on the locator + } + + // get zipped files from all servers next + Map zipFilesFromMembers = new HashMap<>(); + for (DistributedMember server : targetMembers) { + Region region = ExportLogsFunction.createOrGetExistingExportLogsRegion(true, cache); + + ExportLogsCacheWriter cacheWriter = + (ExportLogsCacheWriter) region.getAttributes().getCacheWriter(); + + cacheWriter.startFile(server.getName()); + + CliUtil.executeFunction(new ExportLogsFunction(), + new ExportLogsFunction.Args(start, end, logLevel, onlyLogLevel, logsOnly, statsOnly), + server).getResult(); + Path zipFile = cacheWriter.endFile(); + ExportLogsFunction.destroyExportLogsRegion(cache); + + // only put the zipfile in the map if it is not null + if (zipFile != null) { + logger.info("Received zip file from member {}: {}", server.getId(), zipFile); + zipFilesFromMembers.put(server.getId(), zipFile); + } + } + + if (zipFilesFromMembers.isEmpty()) { + return ResultBuilder.createUserErrorResult("No files to be exported."); + } + + Path tempDir = Files.createTempDirectory("exportedLogs"); + // make sure the directory is created, so that even if there is no files unzipped to this dir, + // we can + // still zip it and send an empty zip file back to the client + Path exportedLogsDir = tempDir.resolve("exportedLogs"); + FileUtils.forceMkdir(exportedLogsDir.toFile()); + + for (Path zipFile : zipFilesFromMembers.values()) { + Path unzippedMemberDir = + exportedLogsDir.resolve(zipFile.getFileName().toString().replace(".zip", "")); + ZipUtils.unzip(zipFile.toAbsolutePath().toString(), unzippedMemberDir.toString()); + FileUtils.deleteQuietly(zipFile.toFile()); + } + + Path workingDir = Paths.get(System.getProperty("user.dir")); + Path exportedLogsZipFile = workingDir + .resolve("exportedLogs_" + System.currentTimeMillis() + ".zip").toAbsolutePath(); + + logger.info("Zipping into: " + exportedLogsZipFile.toString()); + ZipUtils.zipDirectory(exportedLogsDir, exportedLogsZipFile); + FileUtils.deleteDirectory(tempDir.toFile()); + + // try { + // isFileSizeCheckEnabledAndWithinLimit(parseFileSizeLimit(fileSizeLimit), + // exportedLogsZipFile.toFile()); + // } catch (IllegalArgumentException e) { + // return ResultBuilder.createUserErrorResult("TOO BIG: fileSizeLimit = " + fileSizeLimit); + // } + + result = ResultBuilder.createInfoResult(exportedLogsZipFile.toString()); + } catch (Exception ex) { + logger.error(ex.getMessage(), ex); + result = ResultBuilder.createGemFireErrorResult(ex.getMessage()); + } finally { + ExportLogsFunction.destroyExportLogsRegion(cache); + } + logger.debug("Exporting logs returning = {}", result); + return result; + } + + /** + * Returns file size limit in bytes + */ + int parseFileSizeLimit(String fileSizeLimit) { + if (StringUtils.isEmpty(fileSizeLimit)) { + return 0; + } + + int sizeLimit = parseSize(fileSizeLimit); + int byteMultiplier = parseByteMultiplier(fileSizeLimit); + + return sizeLimit * byteMultiplier; + } + + /** + * Throws IllegalArgumentException if file size is over fileSizeLimitBytes + */ + void checkOverDiskSpaceThreshold(int fileSizeLimitBytes, File file) { + // TODO:GEODE-2420: warn user if exportedLogsZipFile size > threshold + if (FileUtils.sizeOf(file) > fileSizeLimitBytes) { + throw new IllegalArgumentException("TOO BIG"); // FileTooBigException + } + } + + /** + * Throws IllegalArgumentException if file size is over fileSizeLimitBytes false == limit is zero + * true == file size is less than limit exception == file size is over limit + */ + boolean isFileSizeCheckEnabledAndWithinLimit(int fileSizeLimitBytes, File file) { + // TODO:GEODE-2420: warn user if exportedLogsZipFile size > threshold + if (fileSizeLimitBytes < 1) { + return false; + } + if (FileUtils.sizeOf(file) < fileSizeLimitBytes) { + return true; + } + throw new IllegalArgumentException("TOO BIG: fileSizeLimit = " + fileSizeLimitBytes + + ", fileSize = " + FileUtils.sizeOf(file)); // FileTooBigException + } + + static int parseSize(String diskSpaceLimit) { + Matcher matcher = DISK_SPACE_LIMIT_PATTERN.matcher(diskSpaceLimit); + if (matcher.matches()) { + return Integer.parseInt(matcher.group(1)); + } else { + throw new IllegalArgumentException(); + } + } + + static int parseByteMultiplier(String diskSpaceLimit) { + Matcher matcher = DISK_SPACE_LIMIT_PATTERN.matcher(diskSpaceLimit); + if (!matcher.matches()) { + throw new IllegalArgumentException(); + } + switch (matcher.group(2).toLowerCase()) { + case "t": + return (int) Math.pow(1024, 4); + case "g": + return (int) Math.pow(1024, 3); + case "m": + default: + return (int) Math.pow(1024, 2); + } + } + + static final int MEGABYTE = (int) Math.pow(1024, 2); + static final int GIGABYTE = (int) Math.pow(1024, 3); + static final int TERABYTE = (int) Math.pow(1024, 4); + +} http://git-wip-us.apache.org/repos/asf/geode/blob/0c15c6e0/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ExportLogsInterceptor.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ExportLogsInterceptor.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ExportLogsInterceptor.java new file mode 100644 index 0000000..dc20da3 --- /dev/null +++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/commands/ExportLogsInterceptor.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.geode.management.internal.cli.commands; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.geode.internal.logging.LogService; +import org.apache.geode.internal.logging.log4j.LogLevel; +import org.apache.geode.management.cli.Result; +import org.apache.geode.management.internal.cli.AbstractCliAroundInterceptor; +import org.apache.geode.management.internal.cli.GfshParseResult; +import org.apache.geode.management.internal.cli.functions.ExportLogsFunction; +import org.apache.geode.management.internal.cli.result.ResultBuilder; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.Logger; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.LocalDateTime; +import java.util.Map; + +/** + * after the export logs, will need to copy the tempFile to the desired location and delete the temp + * file. + */ +public class ExportLogsInterceptor extends AbstractCliAroundInterceptor { + private static final Logger logger = LogService.getLogger(); + + @Override + public Result preExecution(GfshParseResult parseResult) { + // the arguments are in the order of it's being declared + Map arguments = parseResult.getParamValueStrings(); + + // validates groupId and memberIds not both set + if (arguments.get("group") != null && arguments.get("member") != null) { + return ResultBuilder.createUserErrorResult("Can't specify both group and member."); + } + + // validate log level + String logLevel = arguments.get("log-level"); + if (StringUtils.isBlank(logLevel) || LogLevel.getLevel(logLevel) == null) { + return ResultBuilder.createUserErrorResult("Invalid log level: " + logLevel); + } + + // validate start date and end date + String start = arguments.get("start-time"); + String end = arguments.get("end-time"); + if (start != null && end != null) { + // need to make sure end is later than start + LocalDateTime startTime = ExportLogsFunction.parseTime(start); + LocalDateTime endTime = ExportLogsFunction.parseTime(end); + if (startTime.isAfter(endTime)) { + return ResultBuilder.createUserErrorResult("start-time has to be earlier than end-time."); + } + } + + // validate onlyLogs and onlyStats + boolean onlyLogs = Boolean.parseBoolean(arguments.get("logs-only")); + boolean onlyStats = Boolean.parseBoolean(arguments.get("stats-only")); + if (onlyLogs && onlyStats) { + return ResultBuilder.createUserErrorResult("logs-only and stats-only can't both be true"); + } + + return ResultBuilder.createInfoResult(""); + } + + @Override + public Result postExecution(GfshParseResult parseResult, Result commandResult, Path tempFile) { + // in the command over http case, the command result is in the downloaded temp file + if (tempFile != null) { + Path dirPath; + String dirName = parseResult.getParamValueStrings().get("dir"); + if (StringUtils.isBlank(dirName)) { + dirPath = Paths.get(System.getProperty("user.dir")); + } else { + dirPath = Paths.get(dirName); + } + String fileName = "exportedLogs_" + System.currentTimeMillis() + ".zip"; + File exportedLogFile = dirPath.resolve(fileName).toFile(); + try { + FileUtils.copyFile(tempFile.toFile(), exportedLogFile); + FileUtils.deleteQuietly(tempFile.toFile()); + commandResult = ResultBuilder + .createInfoResult("Logs exported to: " + exportedLogFile.getAbsolutePath()); + } catch (IOException e) { + logger.error(e.getMessage(), e); + commandResult = ResultBuilder.createGemFireErrorResult(e.getMessage()); + } + } else if (commandResult.getStatus() == Result.Status.OK) { + commandResult = ResultBuilder.createInfoResult( + "Logs exported to the connected member's file system: " + commandResult.nextLine()); + } + return commandResult; + } +} http://git-wip-us.apache.org/repos/asf/geode/blob/0c15c6e0/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/ExportLogsFunction.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/ExportLogsFunction.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/ExportLogsFunction.java index cbdf1c4..13124c5 100644 --- a/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/ExportLogsFunction.java +++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/ExportLogsFunction.java @@ -28,7 +28,7 @@ import org.apache.geode.internal.InternalEntity; import org.apache.geode.internal.cache.GemFireCacheImpl; import org.apache.geode.internal.cache.InternalRegionArguments; import org.apache.geode.internal.logging.LogService; -import org.apache.geode.management.internal.cli.commands.ExportLogCommand; +import org.apache.geode.management.internal.cli.commands.ExportLogsCommand; import org.apache.geode.management.internal.cli.util.ExportLogsCacheWriter; import org.apache.geode.management.internal.cli.util.LogExporter; import org.apache.geode.management.internal.cli.util.LogFilter; @@ -76,6 +76,7 @@ public class ExportLogsFunction implements Function, InternalEntity { Args args = (Args) context.getArguments(); File baseLogFile = null; File baseStatsFile = null; + if (args.isIncludeLogs() && !config.getLogFile().toString().isEmpty()) { baseLogFile = config.getLogFile().getAbsoluteFile(); } @@ -208,11 +209,11 @@ public class ExportLogsFunction implements Function, InternalEntity { } try { - SimpleDateFormat df = new SimpleDateFormat(ExportLogCommand.FORMAT); + SimpleDateFormat df = new SimpleDateFormat(ExportLogsCommand.FORMAT); return df.parse(dateString).toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); } catch (ParseException e) { try { - SimpleDateFormat df = new SimpleDateFormat(ExportLogCommand.ONLY_DATE_FORMAT); + SimpleDateFormat df = new SimpleDateFormat(ExportLogsCommand.ONLY_DATE_FORMAT); return df.parse(dateString).toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); } catch (ParseException e1) { return null; http://git-wip-us.apache.org/repos/asf/geode/blob/0c15c6e0/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/ExportedLogsSizeInfo.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/ExportedLogsSizeInfo.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/ExportedLogsSizeInfo.java new file mode 100644 index 0000000..6d56f12 --- /dev/null +++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/ExportedLogsSizeInfo.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.geode.management.internal.cli.functions; + +import org.apache.geode.DataSerializable; +import org.apache.geode.DataSerializer; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +public class ExportedLogsSizeInfo implements DataSerializable { + private long logsSize; + private long diskAvailable; + private long diskSize; + + // Used for deserialization only + public ExportedLogsSizeInfo() { + logsSize = 0; + diskAvailable = 0; + diskSize = 0; + } + + public ExportedLogsSizeInfo(long logsSize, long diskAvailable, long diskSize) { + this.logsSize = logsSize; + this.diskAvailable = diskAvailable; + this.diskSize = diskSize; + } + + public long getDiskSize() { + return diskSize; + } + + public long getDiskAvailable() { + return diskAvailable; + } + + public long getLogsSize() { + + return logsSize; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + ExportedLogsSizeInfo that = (ExportedLogsSizeInfo) o; + + if (logsSize != that.logsSize) { + return false; + } + if (diskAvailable != that.diskAvailable) { + return false; + } + return diskSize == that.diskSize; + } + + @Override + public int hashCode() { + int result = (int) (logsSize ^ (logsSize >>> 32)); + result = 31 * result + (int) (diskAvailable ^ (diskAvailable >>> 32)); + result = 31 * result + (int) (diskSize ^ (diskSize >>> 32)); + return result; + } + + @Override + public void toData(DataOutput out) throws IOException { + DataSerializer.writeLong(logsSize, out); + DataSerializer.writeLong(diskAvailable, out); + DataSerializer.writeLong(diskSize, out); + } + + @Override + public void fromData(DataInput in) throws IOException, ClassNotFoundException { + logsSize = DataSerializer.readLong(in); + diskAvailable = DataSerializer.readLong(in); + diskSize = DataSerializer.readLong(in); + } + + @Override + public String toString() { + StringBuilder result = new StringBuilder(); + result.append("["); + result.append("logsSize: " + logsSize); + result.append(", diskAvailable: " + diskAvailable); + result.append(", diskSize: " + diskSize); + result.append("]"); + return result.toString(); + } +} http://git-wip-us.apache.org/repos/asf/geode/blob/0c15c6e0/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunction.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunction.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunction.java new file mode 100644 index 0000000..1718898 --- /dev/null +++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunction.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.geode.management.internal.cli.functions; + +import org.apache.geode.cache.Cache; +import org.apache.geode.cache.CacheFactory; +import org.apache.geode.cache.execute.Function; +import org.apache.geode.cache.execute.FunctionContext; +import org.apache.geode.distributed.DistributedMember; +import org.apache.geode.distributed.internal.DistributionConfig; +import org.apache.geode.internal.InternalEntity; +import org.apache.geode.internal.cache.GemFireCacheImpl; +import org.apache.geode.internal.logging.LogService; +import org.apache.geode.management.internal.cli.util.LogFilter; +import org.apache.geode.management.internal.cli.util.LogSizer; +import org.apache.logging.log4j.Logger; + +import java.io.File; +import java.io.IOException; +import java.text.ParseException; +import java.util.Arrays; + +public class SizeExportLogsFunction extends ExportLogsFunction implements Function, InternalEntity { + private static final Logger LOGGER = LogService.getLogger(); + private static final long serialVersionUID = 1L; + + protected Cache getCache() { + return CacheFactory.getAnyInstance(); + } + + @Override + public void execute(final FunctionContext context) { + try { + GemFireCacheImpl cache = GemFireCacheImpl.getInstance(); + DistributionConfig config = cache.getDistributedSystem().getConfig(); + Args args = (Args) context.getArguments(); + long diskAvailable = config.getLogFile().getUsableSpace(); + long diskSize = config.getLogFile().getTotalSpace(); + long estimatedSize = estimateLogFileSize(cache.getMyId(), config.getLogFile(), + config.getStatisticArchiveFile(), args); + + context.getResultSender().lastResult( + Arrays.asList(new ExportedLogsSizeInfo(estimatedSize, diskAvailable, diskSize))); + + } catch (Exception e) { + e.printStackTrace(); + LOGGER.error(e.getMessage()); + context.getResultSender().sendException(e); + } + } + + long estimateLogFileSize(final DistributedMember member, final File logFile, + final File statArchive, final Args args) throws ParseException, IOException { + LOGGER.info("SizeExportLogsFunction started for member {}", member); + + File baseLogFile = null; + File baseStatsFile = null; + + if (args.isIncludeLogs() && !logFile.toString().isEmpty()) { + baseLogFile = logFile.getAbsoluteFile(); + } + if (args.isIncludeStats() && !statArchive.toString().isEmpty()) { + baseStatsFile = statArchive.getAbsoluteFile(); + } + + LogFilter logFilter = new LogFilter(args.getLogLevel(), args.isThisLogLevelOnly(), + args.getStartTime(), args.getEndTime()); + + long estimatedSize = new LogSizer(logFilter, baseLogFile, baseStatsFile).getFilteredSize(); + + LOGGER.info("Estimated log file size: " + estimatedSize); + + return estimatedSize; + } +} http://git-wip-us.apache.org/repos/asf/geode/blob/0c15c6e0/geode-core/src/main/java/org/apache/geode/management/internal/cli/i18n/CliStrings.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/i18n/CliStrings.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/i18n/CliStrings.java index 5b1f089..dfc4cf1 100644 --- a/geode-core/src/main/java/org/apache/geode/management/internal/cli/i18n/CliStrings.java +++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/i18n/CliStrings.java @@ -1434,6 +1434,12 @@ public class CliStrings { public static final String EXPORT_LOGS__STATSONLY = "stats-only"; public static final String EXPORT_LOGS__LOGSONLY__HELP = "Whether to only export logs"; public static final String EXPORT_LOGS__STATSONLY__HELP = "Whether to only export statistics"; + public static final String EXPORT_LOGS__FILESIZELIMIT = "file-size-limit"; + public static final String EXPORT_LOGS__FILESIZELIMIT__HELP = + "Limits size of the file that can be exported. Specify zero for no limit. Value is in megabytes by default or [m|g|t] may be specified."; + public static final String EXPORT_LOGS__FILESIZELIMIT__SPECIFIED_DEFAULT = "0"; + public static final String EXPORT_LOGS__FILESIZELIMIT__UNSPECIFIED_DEFAULT = "100m"; + /* export stack-trace command */ public static final String EXPORT_STACKTRACE = "export stack-traces"; http://git-wip-us.apache.org/repos/asf/geode/blob/0c15c6e0/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/LogSizer.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/LogSizer.java b/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/LogSizer.java new file mode 100644 index 0000000..0a799f6 --- /dev/null +++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/LogSizer.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.geode.management.internal.cli.util; + +import static java.util.stream.Collectors.toList; + +import org.apache.geode.internal.logging.LogService; +import org.apache.logging.log4j.Logger; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.text.ParseException; +import java.util.Collections; +import java.util.List; +import java.util.Scanner; +import java.util.function.Predicate; +import java.util.stream.Stream; + +public class LogSizer { + private static final Logger LOGGER = LogService.getLogger(); + + private final LogFilter logFilter; + private final File baseLogFile; + private final File baseStatsFile; + + private long filteredSize; + + /** + * @param logFilter the filter that's used to check if we need to accept the file or the logLine + * @param baseLogFile if not null, we will export the logs in that directory + * @param baseStatsFile if not null, we will export stats in that directory + */ + public LogSizer(LogFilter logFilter, File baseLogFile, File baseStatsFile) throws ParseException { + assert logFilter != null; + this.logFilter = logFilter; + this.baseLogFile = baseLogFile; + this.baseStatsFile = baseStatsFile; + filteredSize = 0; + } + + /** + * @return combined size of stat archives and filtered log files in bytes + */ + public long getFilteredSize() throws IOException { + + if (baseLogFile != null) { + for (Path logFile : findLogFiles(baseLogFile.toPath().getParent())) { + filteredSize += filterAndSize(logFile); + } + } + + if (baseStatsFile != null) { + for (Path statFile : findStatFiles(baseStatsFile.toPath().getParent())) { + filteredSize += statFile.toFile().length(); + } + } + + return filteredSize; + } + + /** + * @return size of file in bytes + */ + protected long filterAndSize(Path originalLogFile) throws FileNotFoundException { + long size = 0; + Scanner in = new Scanner(originalLogFile.toFile()); + while (in.hasNextLine()) { + String line = in.nextLine(); + + LogFilter.LineFilterResult result = this.logFilter.acceptsLine(line); + + if (result == LogFilter.LineFilterResult.REMAINDER_OF_FILE_REJECTED) { + break; + } + size += line.length() + File.separator.length(); + } + return size; + } + + protected List findLogFiles(Path workingDir) throws IOException { + Predicate logFileSelector = (Path file) -> file.toString().toLowerCase().endsWith(".log"); + return findFiles(workingDir, logFileSelector); + } + + + protected List findStatFiles(Path workingDir) throws IOException { + Predicate statFileSelector = + (Path file) -> file.toString().toLowerCase().endsWith(".gfs"); + return findFiles(workingDir, statFileSelector); + } + + private List findFiles(Path workingDir, Predicate fileSelector) throws IOException { + Stream selectedFiles = null; + if (!workingDir.toFile().isDirectory()) { + return Collections.emptyList(); + } + selectedFiles = Files.list(workingDir).filter(fileSelector).filter(this.logFilter::acceptsFile); + + return selectedFiles.collect(toList()); + } +} http://git-wip-us.apache.org/repos/asf/geode/blob/0c15c6e0/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsCommandTest.java ---------------------------------------------------------------------- diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsCommandTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsCommandTest.java new file mode 100644 index 0000000..a02c07f --- /dev/null +++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsCommandTest.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.geode.management.internal.cli.commands; + +import static org.apache.geode.management.internal.cli.commands.ExportLogsCommand.*; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; + +import org.apache.geode.cache.Cache; +import org.apache.geode.cache.execute.Execution; +import org.apache.geode.cache.execute.ResultCollector; +import org.apache.geode.distributed.DistributedMember; +import org.apache.geode.test.junit.categories.UnitTest; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(UnitTest.class) +public class ExportLogsCommandTest { + + private ExportLogsCommand createExportLogsCommand(final Cache cache, + final DistributedMember distributedMember, final Execution functionExecutor) { + return new TestExportLogsCommand(cache, distributedMember, functionExecutor); + } + + @Test + public void parseSize_sizeWithUnit_shouldReturnSize() throws Exception { + assertThat(ExportLogsCommand.parseSize("1000m")).isEqualTo(1000); + } + + @Test + public void parseSize_sizeWithoutUnit_shouldReturnSize() throws Exception { + assertThat(ExportLogsCommand.parseSize("1000")).isEqualTo(1000); + } + + @Test + public void parseByteMultiplier_sizeWithoutUnit_shouldReturnDefaultUnit() throws Exception { + assertThat(ExportLogsCommand.parseByteMultiplier("1000")).isEqualTo(MEGABYTE); + } + + @Test + public void parseByteMultiplier_sizeWith_m_shouldReturnUnit() throws Exception { + assertThat(ExportLogsCommand.parseByteMultiplier("1000m")).isEqualTo(MEGABYTE); + } + + @Test + public void parseByteMultiplier_sizeWith_g_shouldReturnUnit() throws Exception { + assertThat(ExportLogsCommand.parseByteMultiplier("1000g")).isEqualTo(GIGABYTE); + } + + @Test + public void parseByteMultiplier_sizeWith_t_shouldReturnUnit() throws Exception { + assertThat(ExportLogsCommand.parseByteMultiplier("1000t")).isEqualTo(TERABYTE); + } + + @Test + public void parseByteMultiplier_sizeWith_M_shouldReturnUnit() throws Exception { + assertThat(ExportLogsCommand.parseByteMultiplier("1000M")).isEqualTo(MEGABYTE); + } + + @Test + public void parseByteMultiplier_sizeWith_G_shouldReturnUnit() throws Exception { + assertThat(ExportLogsCommand.parseByteMultiplier("1000G")).isEqualTo(GIGABYTE); + } + + @Test + public void parseByteMultiplier_sizeWith_T_shouldReturnUnit() throws Exception { + assertThat(ExportLogsCommand.parseByteMultiplier("1000T")).isEqualTo(TERABYTE); + } + + @Test + public void parseByteMultiplier_illegalUnit_shouldThrow() throws Exception { + assertThatThrownBy(() -> ExportLogsCommand.parseByteMultiplier("1000q")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void parseSize_garbage_shouldThrow() throws Exception { + assertThatThrownBy(() -> ExportLogsCommand.parseSize("bizbap")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void parseByteMultiplier_garbage_shouldThrow() throws Exception { + assertThatThrownBy(() -> ExportLogsCommand.parseByteMultiplier("bizbap")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Ignore + @Test + public void sizeFromMember_withinLimits() throws Exception { + final Cache mockCache = mock(Cache.class); + final DistributedMember mockDistributedMember = mock(DistributedMember.class); + final Execution mockFuntionExecutor = mock(Execution.class); + final ResultCollector mockResultCollector = mock(ResultCollector.class); + + final ExportLogsCommand cmd = + createExportLogsCommand(mockCache, mockDistributedMember, mockFuntionExecutor); + // cmd.exportLogs(); + } + + private static class TestExportLogsCommand extends ExportLogsCommand { + + private final Cache cache; + private final DistributedMember distributedMember; + private final Execution functionExecutor; + + public TestExportLogsCommand(final Cache cache, final DistributedMember distributedMember, + final Execution functionExecutor) { + assert cache != null; + this.cache = cache; + this.distributedMember = distributedMember; + this.functionExecutor = functionExecutor; + } + } +} http://git-wip-us.apache.org/repos/asf/geode/blob/0c15c6e0/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsDUnitTest.java ---------------------------------------------------------------------- diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsDUnitTest.java index 95edd42..d0180d0 100644 --- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsDUnitTest.java +++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsDUnitTest.java @@ -19,8 +19,8 @@ package org.apache.geode.management.internal.cli.commands; import static java.util.stream.Collectors.joining; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toSet; -import static org.apache.geode.management.internal.cli.commands.ExportLogCommand.FORMAT; -import static org.apache.geode.management.internal.cli.commands.ExportLogCommand.ONLY_DATE_FORMAT; +import static org.apache.geode.management.internal.cli.commands.ExportLogsCommand.FORMAT; +import static org.apache.geode.management.internal.cli.commands.ExportLogsCommand.ONLY_DATE_FORMAT; import static org.assertj.core.api.Assertions.assertThat; import org.apache.commons.io.FileUtils; http://git-wip-us.apache.org/repos/asf/geode/blob/0c15c6e0/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsFileSizeLimitTest.java ---------------------------------------------------------------------- diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsFileSizeLimitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsFileSizeLimitTest.java new file mode 100644 index 0000000..ec2bcfe --- /dev/null +++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsFileSizeLimitTest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.geode.management.internal.cli.commands; + +import static org.apache.geode.management.internal.cli.commands.ExportLogsCommand.MEGABYTE; +import static org.assertj.core.api.Assertions.*; + +import org.apache.commons.io.FileUtils; +import org.apache.geode.test.junit.categories.IntegrationTest; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.TestName; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; + +@Category(IntegrationTest.class) +public class ExportLogsFileSizeLimitTest { + + private File dir; + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Rule + public TestName testName = new TestName(); + + @Before + public void before() throws Exception { + this.dir = this.temporaryFolder.getRoot(); + } + + @Test + public void sizeOverLimitThrows() throws Exception { + File file = new File(this.dir, this.testName.getMethodName()); + fillUpFile(file, MEGABYTE * 2); + + ExportLogsCommand exportLogsCommand = new ExportLogsCommand(); + + assertThatThrownBy( + () -> exportLogsCommand.isFileSizeCheckEnabledAndWithinLimit(MEGABYTE, file)); + } + + @Test + public void sizeLessThanLimitIsOk() throws Exception { + File file = new File(this.dir, this.testName.getMethodName()); + fillUpFile(file, MEGABYTE / 2); + + ExportLogsCommand exportLogsCommand = new ExportLogsCommand(); + + assertThat(exportLogsCommand.isFileSizeCheckEnabledAndWithinLimit(MEGABYTE, file)).isTrue(); + } + + @Test + public void sizeZeroIsUnlimitedSize() throws Exception { + File file = new File(this.dir, this.testName.getMethodName()); + fillUpFile(file, MEGABYTE * 2); + + ExportLogsCommand exportLogsCommand = new ExportLogsCommand(); + + assertThat(exportLogsCommand.isFileSizeCheckEnabledAndWithinLimit(0, file)).isFalse(); + } + + private void fillUpFile(File file, int sizeInBytes) throws IOException { + PrintWriter writer = new PrintWriter(file, "UTF-8"); + while (FileUtils.sizeOf(file) < sizeInBytes) { + writer.println("this is a line of data in the file"); + } + writer.close(); + } + +} http://git-wip-us.apache.org/repos/asf/geode/blob/0c15c6e0/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsInterceptorJUnitTest.java ---------------------------------------------------------------------- diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsInterceptorJUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsInterceptorJUnitTest.java index 573701f..ec8fdb6 100644 --- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsInterceptorJUnitTest.java +++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsInterceptorJUnitTest.java @@ -31,14 +31,14 @@ import java.util.Map; @Category(UnitTest.class) public class ExportLogsInterceptorJUnitTest { - private ExportLogCommand.ExportLogsInterceptor interceptor; + private ExportLogsInterceptor interceptor; private GfshParseResult parseResult; private Map arguments; private Result result; @Before public void before() { - interceptor = new ExportLogCommand.ExportLogsInterceptor(); + interceptor = new ExportLogsInterceptor(); parseResult = Mockito.mock(GfshParseResult.class); arguments = new HashMap<>(); arguments.put("log-level", "info"); http://git-wip-us.apache.org/repos/asf/geode/blob/0c15c6e0/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsStatsDUnitTest.java ---------------------------------------------------------------------- diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsStatsDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsStatsDUnitTest.java index 5a4d274..44a0362 100644 --- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsStatsDUnitTest.java +++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsStatsDUnitTest.java @@ -19,7 +19,7 @@ import static org.apache.geode.distributed.ConfigurationProperties.HTTP_SERVICE_ import static org.apache.geode.distributed.ConfigurationProperties.HTTP_SERVICE_PORT; import static org.apache.geode.distributed.ConfigurationProperties.JMX_MANAGER_PORT; import static org.apache.geode.distributed.ConfigurationProperties.STATISTIC_ARCHIVE_FILE; -import static org.apache.geode.management.internal.cli.commands.ExportLogCommand.ONLY_DATE_FORMAT; +import static org.apache.geode.management.internal.cli.commands.ExportLogsCommand.ONLY_DATE_FORMAT; import static org.assertj.core.api.Assertions.assertThat; import com.google.common.collect.Sets; http://git-wip-us.apache.org/repos/asf/geode/blob/0c15c6e0/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsTestSuite.java ---------------------------------------------------------------------- diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsTestSuite.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsTestSuite.java new file mode 100644 index 0000000..90a92f3 --- /dev/null +++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExportLogsTestSuite.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.geode.management.internal.cli.commands; + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +/** + * All of the JUnit, DUnit and Integration tests for gfsh command export logs. + */ +@Suite.SuiteClasses({ExportLogsCommandTest.class, ExportLogsFileSizeLimitTest.class, + ExportLogsIntegrationTest.class, ExportLogsDUnitTest.class,}) +@RunWith(Suite.class) +public class ExportLogsTestSuite { +} http://git-wip-us.apache.org/repos/asf/geode/blob/0c15c6e0/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/LogLevelInterceptorTest.java ---------------------------------------------------------------------- diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/LogLevelInterceptorTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/LogLevelInterceptorTest.java index 41b9715..bcbe07c 100644 --- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/LogLevelInterceptorTest.java +++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/LogLevelInterceptorTest.java @@ -41,7 +41,7 @@ public class LogLevelInterceptorTest { @Before public void before() { - interceptors.add(new ExportLogCommand.ExportLogsInterceptor()); + interceptors.add(new ExportLogsInterceptor()); interceptors.add(new ConfigCommands.AlterRuntimeInterceptor()); interceptors.add(new MiscellaneousCommands.ChangeLogLevelInterceptor()); http://git-wip-us.apache.org/repos/asf/geode/blob/0c15c6e0/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/ExportedLogsSizeInfoTest.java ---------------------------------------------------------------------- diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/ExportedLogsSizeInfoTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/ExportedLogsSizeInfoTest.java new file mode 100644 index 0000000..017d70d --- /dev/null +++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/ExportedLogsSizeInfoTest.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.geode.management.internal.cli.functions; + +import static org.assertj.core.api.Assertions.*; + +import org.apache.geode.test.junit.categories.UnitTest; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; + +@Category(UnitTest.class) +public class ExportedLogsSizeInfoTest { + @Before + public void setUp() throws Exception { + + } + + @After + public void tearDown() throws Exception { + + } + + @Test + public final void testExportedLogsSizeInfoConstructor() { + ExportedLogsSizeInfo sizeDetail = new ExportedLogsSizeInfo(1L, 11L, 111L); + assertThat(sizeDetail).isNotNull(); + assertThat(sizeDetail.getLogsSize()).isEqualTo(1L); + assertThat(sizeDetail.getDiskAvailable()).isEqualTo(11L); + assertThat(sizeDetail.getDiskSize()).isEqualTo(111L); + } + + @Test + public final void testExportedLogsSizeInfoZeroArgConstructor() { + ExportedLogsSizeInfo sizeDetail = new ExportedLogsSizeInfo(); + assertThat(sizeDetail).isNotNull(); + assertThat(sizeDetail.getLogsSize()).isEqualTo(0L); + assertThat(sizeDetail.getDiskAvailable()).isEqualTo(0L); + assertThat(sizeDetail.getDiskSize()).isEqualTo(0L); + } + + @Test + public void equals_returnsTrueForTwoInstancesWithTheSameFieldValues() throws Exception { + ExportedLogsSizeInfo sizeDetail1 = new ExportedLogsSizeInfo(2L, 22L, 222L); + ExportedLogsSizeInfo sizeDetail2 = new ExportedLogsSizeInfo(2L, 22L, 222L); + assertThat(sizeDetail1.equals(sizeDetail1)).isTrue(); + assertThat(sizeDetail1.equals(sizeDetail2)).isTrue(); + assertThat(sizeDetail2.equals(sizeDetail1)).isTrue(); + } + + @Test + public void equals_returnsFalseWhenLogsSizeDiffers() throws Exception { + ExportedLogsSizeInfo sizeDetail1 = new ExportedLogsSizeInfo(3L, 33L, 333L); + ExportedLogsSizeInfo sizeDetail2 = new ExportedLogsSizeInfo(33L, 33L, 333L); + assertThat(sizeDetail1.equals(sizeDetail2)).isFalse(); + assertThat(sizeDetail2.equals(sizeDetail1)).isFalse(); + } + + @Test + public void equals_returnsFalseWhenAvailableDiskDiffers() throws Exception { + ExportedLogsSizeInfo sizeDetail1 = new ExportedLogsSizeInfo(4L, 44L, 444L); + ExportedLogsSizeInfo sizeDetail2 = new ExportedLogsSizeInfo(4L, 4L, 444L); + assertThat(sizeDetail1.equals(sizeDetail2)).isFalse(); + assertThat(sizeDetail2.equals(sizeDetail1)).isFalse(); + } + + @Test + public void equals_returnsFalseWheneDiskSizeDiffers() throws Exception { + ExportedLogsSizeInfo sizeDetail1 = new ExportedLogsSizeInfo(5L, 55L, 555L); + ExportedLogsSizeInfo sizeDetail2 = new ExportedLogsSizeInfo(5L, 55L, 55L); + assertThat(sizeDetail1.equals(sizeDetail2)).isFalse(); + assertThat(sizeDetail2.equals(sizeDetail1)).isFalse(); + } + + @Test + public void equals_returnsFalseForComparisonWithNullObject() throws Exception { + ExportedLogsSizeInfo sizeDetail1 = new ExportedLogsSizeInfo(6L, 66L, 666L); + ExportedLogsSizeInfo sizeDetail2 = null; + assertThat(sizeDetail1.equals(sizeDetail2)).isFalse(); + } + + @Test + public final void testClassInequality() { + ExportedLogsSizeInfo sizeDeatai1 = new ExportedLogsSizeInfo(7L, 77L, 777L); + String sizeDetail2 = sizeDeatai1.toString(); + assertThat(sizeDeatai1.equals(sizeDetail2)).isFalse(); + assertThat(sizeDetail2.equals(sizeDeatai1)).isFalse(); + } + + @Test + public void testHashCode() throws Exception { + ExportedLogsSizeInfo sizeDetail1 = new ExportedLogsSizeInfo(); + ExportedLogsSizeInfo sizeDetail2 = new ExportedLogsSizeInfo(8L, 88L, 888L); + ExportedLogsSizeInfo sizeDetail3 = new ExportedLogsSizeInfo(88L, 8L, 888L); + + assertThat(sizeDetail1.hashCode()).isNotEqualTo(sizeDetail2.hashCode()); + assertThat(sizeDetail1.hashCode()).isNotEqualTo(sizeDetail3.hashCode()); + assertThat(sizeDetail2.hashCode()).isNotEqualTo(sizeDetail3.hashCode()); + + assertThat(sizeDetail1.hashCode()).isEqualTo(0); + assertThat(sizeDetail2.hashCode()).isEqualTo(11304); + assertThat(sizeDetail3.hashCode()).isEqualTo(85704); + + } + + @Test + public void deserialization_setsFieldsToOriginalUnserializedValues() throws Exception { + ExportedLogsSizeInfo sizeDetail = new ExportedLogsSizeInfo(9L, 99L, 999L); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutput out = new DataOutputStream(baos); + sizeDetail.toData(out); + ExportedLogsSizeInfo sizeDetailIn = new ExportedLogsSizeInfo(); + sizeDetailIn.fromData(new DataInputStream(new ByteArrayInputStream(baos.toByteArray()))); + + assertThat(sizeDetailIn).isEqualTo(sizeDetail); + } + + @Test + public void testToString() throws Exception { + ExportedLogsSizeInfo sizeDetail = new ExportedLogsSizeInfo(10L, 100L, 1000L); + assertThat(sizeDetail.toString()) + .isEqualTo("[logsSize: 10, diskAvailable: 100, diskSize: 1000]"); + + } + +} http://git-wip-us.apache.org/repos/asf/geode/blob/0c15c6e0/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunctionCacheTest.java ---------------------------------------------------------------------- diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunctionCacheTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunctionCacheTest.java new file mode 100644 index 0000000..8de605f --- /dev/null +++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunctionCacheTest.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.geode.management.internal.cli.functions; + +import static org.assertj.core.api.Assertions.*; +import static org.apache.geode.distributed.ConfigurationProperties.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.apache.commons.io.FileUtils; +import org.apache.geode.cache.Cache; +import org.apache.geode.cache.CacheFactory; +import org.apache.geode.cache.execute.FunctionContext; +import org.apache.geode.cache.execute.ResultSender; +import org.apache.geode.distributed.DistributedMember; +import org.apache.geode.internal.cache.execute.FunctionContextImpl; +import org.apache.geode.test.junit.categories.IntegrationTest; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.TestName; + +import java.io.File; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; + +@Category(IntegrationTest.class) +public class SizeExportLogsFunctionCacheTest { + + private Cache cache; + + private SizeExportLogsFunction.Args nonFilteringArgs; + private TestResultSender resultSender; + private FunctionContext functionContext; + private File dir; + private DistributedMember member; + File logFile; + File statFile; + String name; + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Rule + public TestName testName = new TestName(); + + @Before + public void before() throws Throwable { + name = testName.getMethodName(); + + this.dir = this.temporaryFolder.getRoot(); + logFile = new File(dir, name + ".log"); + statFile = new File(dir, name + ".gfs"); + + this.nonFilteringArgs = new ExportLogsFunction.Args(null, null, null, false, false, false); + functionContext = new FunctionContextImpl("functionId", nonFilteringArgs, resultSender); + + } + + @After + public void after() throws Exception { + if (this.cache != null) { + this.cache.close(); + } + FileUtils.deleteDirectory(dir); + } + + @Test + public void withFiles_returnsCombinedSizeResult() throws Throwable { + Properties config = new Properties(); + config.setProperty(NAME, name); + config.setProperty(LOCATORS, ""); + config.setProperty(MCAST_PORT, "0"); + config.setProperty(LOG_FILE, logFile.getAbsolutePath()); + config.setProperty(STATISTIC_ARCHIVE_FILE, statFile.getAbsolutePath()); + + this.cache = new CacheFactory(config).create(); + TestResultSender resultSender = new TestResultSender(); + FunctionContext context = new FunctionContextImpl("functionId", nonFilteringArgs, resultSender); + + // log and stat files sizes are not constant with a real cache running, so check for the sizer + // estimate within a range + long initalFileSizes = FileUtils.sizeOf(logFile) + FileUtils.sizeOf(statFile); + new SizeExportLogsFunction().execute(context); + long finalFileSizes = FileUtils.sizeOf(logFile) + FileUtils.sizeOf(statFile); + getAndVerifySizeEstimate(resultSender, initalFileSizes, finalFileSizes); + } + + @Test + public void noFiles_returnsZeroResult() throws Throwable { + Properties config = new Properties(); + config.setProperty(NAME, name); + config.setProperty(LOCATORS, ""); + config.setProperty(MCAST_PORT, "0"); + + this.cache = new CacheFactory(config).create(); + + TestResultSender resultSender = new TestResultSender(); + FunctionContext context = new FunctionContextImpl("functionId", nonFilteringArgs, resultSender); + + new SizeExportLogsFunction().execute(context); + getAndVerifySizeEstimate(resultSender, 0L); + } + + private void getAndVerifySizeEstimate(TestResultSender resultSender, long expectedSize) + throws Throwable { + getAndVerifySizeEstimate(resultSender, expectedSize, expectedSize); + } + + private void getAndVerifySizeEstimate(TestResultSender resultSender, long minExpected, + long maxExpected) throws Throwable { + List results = resultSender.getResults(); + + assertThat(results).isNotNull(); + assertThat(results.size()).isEqualTo(1); + List result = (List) results.get(0); + assertThat(result).isNotNull(); + assertThat(((ExportedLogsSizeInfo) result.get(0)).getLogsSize()) + .isGreaterThanOrEqualTo(minExpected).isLessThanOrEqualTo(maxExpected); + } + + @Test + public void withFunctionError_shouldThrow() throws Throwable { + Properties config = new Properties(); + config.setProperty(NAME, name); + config.setProperty(LOCATORS, ""); + config.setProperty(MCAST_PORT, "0"); + + this.cache = new CacheFactory().create(); + + TestResultSender resultSender = new TestResultSender(); + FunctionContext context = new FunctionContextImpl("functionId", null, resultSender); + + new SizeExportLogsFunction().execute(context); + assertThatThrownBy(() -> resultSender.getResults()).isInstanceOf(NullPointerException.class); + } + + private static class TestResultSender implements ResultSender { + + private final List results = new LinkedList(); + + private Throwable t; + + protected List getResults() throws Throwable { + if (t != null) { + throw t; + } + return Collections.unmodifiableList(results); + } + + @Override + public void lastResult(final Object lastResult) { + results.add(lastResult); + } + + @Override + public void sendResult(final Object oneResult) { + results.add(oneResult); + } + + @Override + public void sendException(final Throwable t) { + this.t = t; + } + } +} http://git-wip-us.apache.org/repos/asf/geode/blob/0c15c6e0/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunctionFileTest.java ---------------------------------------------------------------------- diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunctionFileTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunctionFileTest.java new file mode 100644 index 0000000..87ad7d2 --- /dev/null +++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunctionFileTest.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.geode.management.internal.cli.functions; + +import static org.assertj.core.api.Assertions.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +import org.apache.geode.cache.execute.FunctionContext; +import org.apache.geode.management.internal.cli.functions.ExportLogsFunction.Args; +import org.apache.commons.io.FileUtils; +import org.apache.geode.distributed.DistributedMember; +import org.apache.geode.test.junit.categories.IntegrationTest; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.TestName; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; + +@Category(IntegrationTest.class) +public class SizeExportLogsFunctionFileTest { + + private File dir; + private DistributedMember member; + private SizeExportLogsFunction.Args nonFilteringArgs; + private FunctionContext functionContext; + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Rule + public TestName testName = new TestName(); + + @Before + public void before() throws Exception { + this.dir = this.temporaryFolder.getRoot(); + this.member = mock(DistributedMember.class); + this.nonFilteringArgs = new Args(null, null, null, false, false, false); + } + + @After + public void after() throws Exception { + FileUtils.deleteDirectory(dir); + } + + @Test + public void bothFiles_returnsCombinedSize() throws Exception { + List logFiles = + createLogFiles(new File(dir.getName(), testName.getMethodName()), 1, 1, FileUtils.ONE_KB); + File logFile = logFiles.get(0); + long logFileSize = FileUtils.sizeOf(logFiles.get(0)); + + List statFiles = + createStatFiles(new File(dir.getName(), testName.getMethodName()), 1, 1, FileUtils.ONE_KB); + File statArchive = statFiles.get(0); + long statFileSize = FileUtils.sizeOf(statArchive); + + SizeExportLogsFunction function = new SizeExportLogsFunction(); + assertThat(function.estimateLogFileSize(this.member, logFile, statArchive, nonFilteringArgs)) + .isEqualTo(logFileSize + statFileSize); + } + + private long expectedSize; + + @Test + public void manyFiles_returnsCombinedSize() throws Exception { + expectedSize = 0; + List logFiles = + createLogFiles(new File(dir.getName(), testName.getMethodName()), 1, 3, FileUtils.ONE_KB); + logFiles.forEach((file) -> { + expectedSize += FileUtils.sizeOf(file); + }); + + List statFiles = createStatFiles(new File(dir.getName(), testName.getMethodName()), 1, 2, + FileUtils.ONE_KB * 2); + statFiles.forEach((file) -> { + expectedSize += FileUtils.sizeOf(file); + }); + + SizeExportLogsFunction function = new SizeExportLogsFunction(); + assertThat(function.estimateLogFileSize(this.member, logFiles.get(0), statFiles.get(0), + nonFilteringArgs)).isEqualTo(expectedSize); + } + + @Test + public void emptyFiles_returnsZeroSize() throws Exception { + List logFiles = + createLogFiles(new File(dir.getName(), testName.getMethodName()), 1, 3, 0); + + List statFiles = + createStatFiles(new File(dir.getName(), testName.getMethodName()), 1, 2, 0); + SizeExportLogsFunction function = new SizeExportLogsFunction(); + assertThat(function.estimateLogFileSize(this.member, logFiles.get(0), statFiles.get(0), + nonFilteringArgs)).isEqualTo(0); + } + + @Test + public void nullFiles_returnsZeroSize() throws Exception { + File nullLogFile = new File(dir.getPath(), "nullLogFile"); + File nullStatFile = new File(dir.getPath(), "nullStatFile"); + SizeExportLogsFunction function = new SizeExportLogsFunction(); + assertThat( + function.estimateLogFileSize(this.member, nullLogFile, nullStatFile, nonFilteringArgs)) + .isEqualTo(0); + } + + private List createLogFiles(File logFile, int mainId, int numberOfFiles, long sizeOfFile) + throws IOException { + List files = new ArrayList<>(); + for (int i = 0; i < numberOfFiles; i++) { + String name = + baseName(logFile.getName()) + "-" + formatId(mainId) + "-" + formatId(i + 1) + ".log"; + File file = createFile(name, sizeOfFile, true); + files.add(file); + } + return files; + } + + private List createStatFiles(File logFile, int mainId, int numberOfFiles, long sizeOfFile) + throws IOException { + List files = new ArrayList<>(); + for (int i = 0; i < numberOfFiles; i++) { + String name = + baseName(logFile.getName()) + "-" + formatId(mainId) + "-" + formatId(i + 1) + ".gfs"; + File file = createFile(name, sizeOfFile, false); + files.add(file); + } + return files; + } + + private String baseName(String logFileName) { + // base log file: myfile.log + // mainId childId for rolling + // myfile-01-01.log + // myfile-01-02.log + // pass in myfile.log + // return myfile + return null; + } + + private String formatId(final int id) { + return String.format("%02d", id); + } + + private File createFile(String name, long sizeInBytes, boolean lineFeed) throws IOException { + File file = new File(this.dir, name); + fillUpFile(file, sizeInBytes, lineFeed); + return file; + } + + private void fillUpFile(File file, long sizeInBytes, boolean lineFeed) throws IOException { + PrintWriter writer = new PrintWriter(file, "UTF-8"); + while (FileUtils.sizeOf(file) < sizeInBytes) { + writer.print("this is a line of data in the file"); + if (lineFeed) { + writer.println(); + } + } + writer.close(); + } + +} http://git-wip-us.apache.org/repos/asf/geode/blob/0c15c6e0/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsTestSuite.java ---------------------------------------------------------------------- diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsTestSuite.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsTestSuite.java new file mode 100644 index 0000000..e70a750 --- /dev/null +++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsTestSuite.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.geode.management.internal.cli.functions; + +import org.apache.geode.management.internal.cli.util.LogSizerTest; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +/** + * JUnit, DUnit and Integration tests for SizeExportLogsFunction. + */ +@Suite.SuiteClasses({SizeExportLogsFunctionCacheTest.class, SizeExportLogsFunctionFileTest.class, + LogSizerTest.class, ExportedLogsSizeInfoTest.class}) +@RunWith(Suite.class) +public class SizeExportLogsTestSuite { + +}