geode-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kh...@apache.org
Subject geode git commit: Refined log size estimation.
Date Tue, 21 Mar 2017 23:42:37 GMT
Repository: geode
Updated Branches:
  refs/heads/feature/GEODE-2420 8215ae0a4 -> fb3526ae5


Refined log size estimation.

Added tests of the Function class


Project: http://git-wip-us.apache.org/repos/asf/geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/geode/commit/fb3526ae
Tree: http://git-wip-us.apache.org/repos/asf/geode/tree/fb3526ae
Diff: http://git-wip-us.apache.org/repos/asf/geode/diff/fb3526ae

Branch: refs/heads/feature/GEODE-2420
Commit: fb3526ae5a839417364798e0e61fc949c26f1430
Parents: 8215ae0
Author: Ken Howe <khowe@pivotal.io>
Authored: Tue Mar 21 16:41:10 2017 -0700
Committer: Ken Howe <khowe@pivotal.io>
Committed: Tue Mar 21 16:41:10 2017 -0700

----------------------------------------------------------------------
 .../cli/functions/SizeExportLogsFunction.java   | 176 ++++++++-----------
 .../management/internal/cli/util/LogSizer.java  | 129 ++++++++++++++
 .../SizeExportLogsFunctionIntegrationTest.java  | 164 +++++++++++++++--
 3 files changed, 356 insertions(+), 113 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/geode/blob/fb3526ae/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunction.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunction.java
b/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunction.java
index 4b1d954..dfd8f3c 100644
--- a/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunction.java
+++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunction.java
@@ -14,46 +14,27 @@
  */
 package org.apache.geode.management.internal.cli.functions;
 
-import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang.StringUtils;
-import org.apache.geode.cache.AttributesFactory;
-import org.apache.geode.cache.Cache;
-import org.apache.geode.cache.DataPolicy;
-import org.apache.geode.cache.Region;
-import org.apache.geode.cache.Scope;
 import org.apache.geode.cache.execute.Function;
 import org.apache.geode.cache.execute.FunctionContext;
 import org.apache.geode.distributed.DistributedMember;
 import org.apache.geode.distributed.internal.DistributionConfig;
 import org.apache.geode.internal.InternalEntity;
 import org.apache.geode.internal.cache.GemFireCacheImpl;
-import org.apache.geode.internal.cache.InternalRegionArguments;
 import org.apache.geode.internal.logging.LogService;
-import org.apache.geode.management.internal.cli.commands.ExportLogsCommand;
-import org.apache.geode.management.internal.cli.util.ExportLogsCacheWriter;
-import org.apache.geode.management.internal.cli.util.LogExporter;
 import org.apache.geode.management.internal.cli.util.LogFilter;
-import org.apache.geode.management.internal.configuration.domain.Configuration;
-import org.apache.logging.log4j.Level;
+import org.apache.geode.management.internal.cli.util.LogSizer;
 import org.apache.logging.log4j.Logger;
 
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.IOException;
-import java.io.Serializable;
-import java.nio.file.Path;
 import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.time.LocalDateTime;
-import java.time.ZoneId;
 import java.util.Arrays;
 
-public class SizeExportLogsFunction implements Function, InternalEntity {
+public class SizeExportLogsFunction extends ExportLogsFunction implements Function, InternalEntity
{
   private static final Logger LOGGER = LogService.getLogger();
   private static final long serialVersionUID = 1L;
 
-  private static final int BUFFER_SIZE = 1024;
-
   @Override
   public void execute(final FunctionContext context) {
     try {
@@ -92,91 +73,82 @@ public class SizeExportLogsFunction implements Function, InternalEntity
{
     LogFilter logFilter = new LogFilter(args.getLogLevel(), args.isThisLogLevelOnly(),
         args.getStartTime(), args.getEndTime());
 
-    // TODO: int estimatedSize = new LogSizer(logFilter, baseLogFile, baseStatsFile).export();
-
-    Path exportedZipFile = new LogExporter(logFilter, baseLogFile, baseStatsFile).export();
-
-    // nothing to return back
-    if (exportedZipFile == null) {
-      return -1;
-    }
-
-    long estimatedSize = FileUtils.sizeOf(exportedZipFile.toFile());
+    long estimatedSize = new LogSizer(logFilter, baseLogFile, baseStatsFile).getFilteredSize();
 
     LOGGER.info("Estimated log file size: " + estimatedSize);
 
     return estimatedSize;
   }
 
-  @Override
-  public boolean isHA() {
-    return false;
-  }
-
-  public static class Args implements Serializable {
-    private LocalDateTime startTime;
-    private LocalDateTime endTime;
-    private Level logLevel;
-    private boolean thisLogLevelOnly;
-    private boolean includeLogs;
-    private boolean includeStats;
-
-    public Args(String startTime, String endTime, String logLevel, boolean logLevelOnly,
-                boolean logsOnly, boolean statsOnly) {
-      this.startTime = parseTime(startTime);
-      this.endTime = parseTime(endTime);
-
-      if (StringUtils.isBlank(logLevel)) {
-        this.logLevel = Level.INFO;
-      } else {
-        this.logLevel = Level.getLevel(logLevel.toUpperCase());
-      }
-      this.thisLogLevelOnly = logLevelOnly;
-
-      this.includeLogs = !statsOnly;
-      this.includeStats = !logsOnly;
-    }
-
-    public LocalDateTime getStartTime() {
-      return startTime;
-    }
-
-    public LocalDateTime getEndTime() {
-      return endTime;
-    }
-
-    public Level getLogLevel() {
-      return logLevel;
-    }
-
-    public boolean isThisLogLevelOnly() {
-      return thisLogLevelOnly;
-    }
-
-    public boolean isIncludeLogs() {
-      return includeLogs;
-    }
-
-    public boolean isIncludeStats() {
-      return includeStats;
-    }
-  }
-
-  public static LocalDateTime parseTime(String dateString) {
-    if (dateString == null) {
-      return null;
-    }
-
-    try {
-      SimpleDateFormat df = new SimpleDateFormat(ExportLogsCommand.FORMAT);
-      return df.parse(dateString).toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime();
-    } catch (ParseException e) {
-      try {
-        SimpleDateFormat df = new SimpleDateFormat(ExportLogsCommand.ONLY_DATE_FORMAT);
-        return df.parse(dateString).toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime();
-      } catch (ParseException e1) {
-        return null;
-      }
-    }
-  }
+//  @Override
+//  public boolean isHA() {
+//    return false;
+//  }
+//
+//  public static class Args implements Serializable {
+//    private LocalDateTime startTime;
+//    private LocalDateTime endTime;
+//    private Level logLevel;
+//    private boolean thisLogLevelOnly;
+//    private boolean includeLogs;
+//    private boolean includeStats;
+//
+//    public Args(String startTime, String endTime, String logLevel, boolean logLevelOnly,
+//                boolean logsOnly, boolean statsOnly) {
+//      this.startTime = parseTime(startTime);
+//      this.endTime = parseTime(endTime);
+//
+//      if (StringUtils.isBlank(logLevel)) {
+//        this.logLevel = Level.INFO;
+//      } else {
+//        this.logLevel = Level.getLevel(logLevel.toUpperCase());
+//      }
+//      this.thisLogLevelOnly = logLevelOnly;
+//
+//      this.includeLogs = !statsOnly;
+//      this.includeStats = !logsOnly;
+//    }
+//
+//    public LocalDateTime getStartTime() {
+//      return startTime;
+//    }
+//
+//    public LocalDateTime getEndTime() {
+//      return endTime;
+//    }
+//
+//    public Level getLogLevel() {
+//      return logLevel;
+//    }
+//
+//    public boolean isThisLogLevelOnly() {
+//      return thisLogLevelOnly;
+//    }
+//
+//    public boolean isIncludeLogs() {
+//      return includeLogs;
+//    }
+//
+//    public boolean isIncludeStats() {
+//      return includeStats;
+//    }
+//  }
+//
+//  public static LocalDateTime parseTime(String dateString) {
+//    if (dateString == null) {
+//      return null;
+//    }
+//
+//    try {
+//      SimpleDateFormat df = new SimpleDateFormat(ExportLogsCommand.FORMAT);
+//      return df.parse(dateString).toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime();
+//    } catch (ParseException e) {
+//      try {
+//        SimpleDateFormat df = new SimpleDateFormat(ExportLogsCommand.ONLY_DATE_FORMAT);
+//        return df.parse(dateString).toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime();
+//      } catch (ParseException e1) {
+//        return null;
+//      }
+//    }
+//  }
 }

http://git-wip-us.apache.org/repos/asf/geode/blob/fb3526ae/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/LogSizer.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/LogSizer.java
b/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/LogSizer.java
new file mode 100644
index 0000000..4a6ea60
--- /dev/null
+++ b/geode-core/src/main/java/org/apache/geode/management/internal/cli/util/LogSizer.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information
regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version
2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain
a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
express
+ * or implied. See the License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.geode.management.internal.cli.util;
+
+import static java.util.stream.Collectors.toList;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.geode.internal.logging.LogService;
+import org.apache.logging.log4j.Logger;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.text.ParseException;
+import java.util.List;
+import java.util.Scanner;
+import java.util.function.Predicate;
+import java.util.stream.Stream;
+
+public class LogSizer {
+  private static final Logger LOGGER = LogService.getLogger();
+
+  private final LogFilter logFilter;
+  private final File baseLogFile;
+  private final File baseStatsFile;
+
+  private long filteredSize;
+
+  /**
+   * @param logFilter the filter that's used to check if we need to accept the file or the
logLine
+   * @param baseLogFile if not null, we will export the logs in that directory
+   * @param baseStatsFile if not null, we will export stats in that directory
+   */
+  public LogSizer(LogFilter logFilter, File baseLogFile, File baseStatsFile)
+      throws ParseException {
+    assert logFilter != null;
+    this.logFilter = logFilter;
+    this.baseLogFile = baseLogFile;
+    this.baseStatsFile = baseStatsFile;
+    filteredSize = 0;
+  }
+
+  /**
+   * @return Path to the zip file that has all the filtered files, null if no files are selected
to
+   *         export.
+   */
+  public long getFilteredSize() throws IOException {
+    Path tempDirectory = Files.createTempDirectory("exportLogs");
+
+    if (baseLogFile != null) {
+      for (Path logFile : findLogFiles(baseLogFile.toPath().getParent())) {
+        filteredSize += filterAndSize(logFile);
+      }
+    }
+
+    if (baseStatsFile != null) {
+      for (Path statFile : findStatFiles(baseStatsFile.toPath().getParent())) {
+        filteredSize += FileUtils.sizeOf(statFile.toFile());
+      }
+    }
+
+    FileUtils.deleteDirectory(tempDirectory.toFile());
+
+    return filteredSize;
+  }
+
+  /**
+   * @return size of file in bytes
+   */
+  protected long filterAndSize(Path originalLogFile) throws FileNotFoundException {
+    long size = 0;
+    Scanner in = new Scanner(originalLogFile.toFile());
+    while (in.hasNextLine()) {
+      String line = in.nextLine();
+
+      LogFilter.LineFilterResult result = this.logFilter.acceptsLine(line);
+
+      if (result == LogFilter.LineFilterResult.REMAINDER_OF_FILE_REJECTED) {
+        break;
+      }
+      size += line.length() + File.separator.length();
+    }
+    return size;
+  }
+
+
+  private void writeLine(String line, BufferedWriter writer) {
+    try {
+      writer.write(line);
+      writer.newLine();
+    } catch (IOException e) {
+      throw new RuntimeException("Unable to write to log file", e);
+    }
+  }
+
+  protected List<Path> findLogFiles(Path workingDir) throws IOException {
+    Predicate<Path> logFileSelector = (Path file) -> file.toString().toLowerCase().endsWith(".log");
+    return findFiles(workingDir, logFileSelector);
+  }
+
+
+  protected List<Path> findStatFiles(Path workingDir) throws IOException {
+    Predicate<Path> statFileSelector =
+        (Path file) -> file.toString().toLowerCase().endsWith(".gfs");
+    return findFiles(workingDir, statFileSelector);
+  }
+
+  private List<Path> findFiles(Path workingDir, Predicate<Path> fileSelector)
throws IOException {
+    Stream<Path> selectedFiles =
+        Files.list(workingDir).filter(fileSelector).filter(this.logFilter::acceptsFile);
+
+    return selectedFiles.collect(toList());
+  }
+}

http://git-wip-us.apache.org/repos/asf/geode/blob/fb3526ae/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunctionIntegrationTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunctionIntegrationTest.java
b/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunctionIntegrationTest.java
index 7903165..4bba6f2 100644
--- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunctionIntegrationTest.java
+++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/functions/SizeExportLogsFunctionIntegrationTest.java
@@ -15,9 +15,15 @@
 package org.apache.geode.management.internal.cli.functions;
 
 import static org.assertj.core.api.Assertions.*;
+import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
 
-import org.apache.geode.management.internal.cli.functions.SizeExportLogsFunction.Args;
+import org.apache.geode.cache.execute.FunctionContext;
+import org.apache.geode.cache.execute.ResultSender;
+import org.apache.geode.internal.cache.execute.FunctionContextImpl;
+import org.apache.geode.management.internal.cli.functions.ExportLogsFunction.Args;
 import org.apache.commons.io.FileUtils;
 import org.apache.geode.distributed.DistributedMember;
 import org.apache.geode.test.junit.categories.IntegrationTest;
@@ -31,12 +37,19 @@ import org.junit.rules.TestName;
 import java.io.File;
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
 
 @Category(IntegrationTest.class)
 public class SizeExportLogsFunctionIntegrationTest {
 
   private File dir;
   private DistributedMember member;
+  private SizeExportLogsFunction.Args nonFilteringArgs;
+  private TestResultSender resultSender;
+  private FunctionContext functionContext;
 
   @Rule
   public TemporaryFolder temporaryFolder = new TemporaryFolder();
@@ -48,29 +61,158 @@ public class SizeExportLogsFunctionIntegrationTest {
   public void before() throws Exception {
     this.dir = this.temporaryFolder.getRoot();
     this.member = mock(DistributedMember.class);
+
+    this.nonFilteringArgs = new Args(null, null, null, false,false, false);
+    resultSender = new TestResultSender();
+    functionContext = new FunctionContextImpl("functionId", nonFilteringArgs, resultSender);
+  }
+
+  @Test
+  public void bothFiles_returnsCombinedSize() throws Exception {
+    List<File> logFiles = createLogFiles(new File(dir.getName(), testName.getMethodName()),
1, 1,
+        FileUtils.ONE_KB);
+    File logFile = logFiles.get(0);
+    long logFileSize = FileUtils.sizeOf(logFiles.get(0));
+
+    List<File> statFiles = createStatFiles(new File(dir.getName(), testName.getMethodName()),
1, 1, FileUtils.ONE_KB);
+    File statArchive = statFiles.get(0);
+    long statFileSize = FileUtils.sizeOf(statArchive);
+
+    SizeExportLogsFunction function = new SizeExportLogsFunction();
+    assertThat(function.estimateLogFileSize(this.member, logFile, statArchive, nonFilteringArgs)).isEqualTo(logFileSize
+ statFileSize);
   }
 
+  private long expectedSize;
   @Test
-  public void functionReturnsSize() throws Exception {
-    File logFile = new File(this.dir, this.testName.getMethodName() + ".log");
-    fillUpFile(logFile, (int) Math.pow(1024, 2));
+  public void manyFiles_returnsCombinedSize() throws Exception {
+    expectedSize = 0;
+    List<File> logFiles = createLogFiles(new File(dir.getName(), testName.getMethodName()),
1, 3, FileUtils.ONE_KB);
+    logFiles.forEach((file) -> {expectedSize += FileUtils.sizeOf(file);});
+//    long logFileSize = FileUtils.sizeOf(logFile);
+
+    List<File> statFiles = createStatFiles(new File(dir.getName(), testName.getMethodName()),
1, 2, FileUtils.ONE_KB * 2);
+    statFiles.forEach((file) -> {expectedSize += FileUtils.sizeOf(file);});
+//    long statFileSize = FileUtils.sizeOf(statArchive);
+
+    SizeExportLogsFunction function = new SizeExportLogsFunction();
+    assertThat(function.estimateLogFileSize(this.member, logFiles.get(0), statFiles.get(0),
nonFilteringArgs)).isEqualTo(expectedSize);
+  }
 
-    File statArchive = new File(this.dir, this.testName.getMethodName() + ".gfs");
-    fillUpFile(statArchive, (int) Math.pow(1024, 2));
+  @Test
+  public void emptyFiles_returnsZeroSize() throws Exception {
+    List<File> logFiles = createLogFiles(new File(dir.getName(), testName.getMethodName()),
1, 3, 0);
+//    logFiles.forEach((file) -> {expectedSize += FileUtils.sizeOf(file);});
+//    long logFileSize = FileUtils.sizeOf(logFile);
 
-    SizeExportLogsFunction.Args args = new Args(null, null, null, false,
-    false,  false);
+    List<File> statFiles = createStatFiles(new File(dir.getName(), testName.getMethodName()),
1, 2, 0);
+//    statFiles.forEach((file) -> {expectedSize += FileUtils.sizeOf(file);});
+//    long statFileSize = FileUtils.sizeOf(statArchive);
+    SizeExportLogsFunction function = new SizeExportLogsFunction();
+    assertThat(function.estimateLogFileSize(this.member, logFiles.get(0), statFiles.get(0),
nonFilteringArgs)).isEqualTo(0);
+  }
+
+  @Test
+  public void nullFiles_returnsZeroSize() throws Exception {
+    File nullLogFile = new File(dir.getPath(), "nullLogFile");
+    File nullStatFile = new File(dir.getPath(), "nullStatFile");
+    SizeExportLogsFunction function = new SizeExportLogsFunction();
+    assertThat(function.estimateLogFileSize(this.member, nullLogFile, nullStatFile, nonFilteringArgs)).isEqualTo(0);
+  }
+
+  private List<File> createLogFiles(File logFile, int mainId, int numberOfFiles, long
sizeOfFile)
+      throws IOException {
+    List<File> files = new ArrayList<>();
+    for (int i = 0; i < numberOfFiles; i++) {
+      String name = baseName(logFile.getName()) + "-" + formatId(mainId) + "-" + formatId(i+1)
+ ".log";
+      File file = createFile(name, sizeOfFile, true);
+      files.add(file);
+    }
+    return files;
+  }
+
+  private List<File> createStatFiles(File logFile, int mainId, int numberOfFiles, long
sizeOfFile)
+      throws IOException {
+    List<File> files = new ArrayList<>();
+    for (int i = 0; i < numberOfFiles; i++) {
+      String name = baseName(logFile.getName()) + "-" + formatId(mainId) + "-" + formatId(i+1)
+ ".gfs";
+      File file = createFile(name, sizeOfFile, false);
+      files.add(file);
+    }
+    return files;
+  }
 
+  @Test
+  public void negativeEstimatedSize() throws Exception {
+    File nullLogFile = new File(dir.getPath(), "nullLogFile");
+    File nullStatFile = new File(dir.getPath(), "nullStatFile");
     SizeExportLogsFunction function = new SizeExportLogsFunction();
-    assertThat(function.estimateLogFileSize(this.member, logFile, statArchive, args)).isEqualTo(0);
+    SizeExportLogsFunction spyFunction = spy(function);
+
+    doReturn(-1L).when(spyFunction).estimateLogFileSize(this.member, nullLogFile, nullStatFile,
nonFilteringArgs);
+//    when(spyFunction.estimateLogFileSize(this.member, nullLogFile, nullStatFile, nonFilteringArgs)).thenReturn(-1L);
+
+    spyFunction.execute(functionContext);
+
+
+  }
+
+  private String baseName(String logFileName) {
+    // base log file: myfile.log
+    // mainId childId for rolling
+    // myfile-01-01.log
+    // myfile-01-02.log
+    // pass in myfile.log
+    // return myfile
+    return null;
+  }
+
+  private String formatId(final int id) {
+    return String.format("%02d", id);
+  }
+
+  private File createFile(String name, long sizeInBytes, boolean lineFeed) throws IOException
{
+    File file = new File(this.dir, name);
+    fillUpFile(file, sizeInBytes, lineFeed);
+    return file;
   }
 
-  private void fillUpFile(File file, int sizeInBytes) throws IOException {
+  private void fillUpFile(File file, long sizeInBytes, boolean lineFeed) throws IOException
{
     PrintWriter writer = new PrintWriter(file, "UTF-8");
     while (FileUtils.sizeOf(file) < sizeInBytes) {
-      writer.println("this is a line of data in the file");
+      writer.print("this is a line of data in the file");
+      if (lineFeed) {
+        writer.println();
+      }
     }
     writer.close();
   }
 
+  private static class TestResultSender implements ResultSender {
+
+    private final List<Object> results = new LinkedList<Object>();
+
+    private Throwable t;
+
+    protected List<Object> getResults() throws Throwable {
+      if (t != null) {
+        throw t;
+      }
+      return Collections.unmodifiableList(results);
+    }
+
+    @Override
+    public void lastResult(final Object lastResult) {
+      results.add(lastResult);
+    }
+
+    @Override
+    public void sendResult(final Object oneResult) {
+      results.add(oneResult);
+    }
+
+    @Override
+    public void sendException(final Throwable t) {
+      this.t = t;
+    }
+  }
 }
\ No newline at end of file


Mime
View raw message