ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ira...@apache.org
Subject [ignite-teamcity-bot] 02/04: Increased the number of files to save logs in 2 weeks #168
Date Thu, 09 Jul 2020 20:19:03 GMT
This is an automated email from the ASF dual-hosted git repository.

irakov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git

commit b745f638c715d232b1c40c10647e285ec2f912e8
Author: sergeyuttsel <uttsel@gmail.com>
AuthorDate: Thu Jul 9 23:13:47 2020 +0300

    Increased the number of files to save logs in 2 weeks #168
    
    Signed-off-by: Ivan Rakov <ivan.glukos@gmail.com>
---
 .../org/apache/ignite/ci/db/Ignite2Configurer.java |  2 +-
 .../apache/ignite/ci/tcbot/TcBotWebAppModule.java  |  2 +
 .../ignite/tcbot/engine/cleaner/Cleaner.java       | 90 ++++++++++++++++------
 .../ignite/tcbot/engine/defect/DefectsStorage.java |  9 ++-
 .../ignite/tcbot/engine/issue/IssuesStorage.java   |  9 ++-
 .../apache/ignite/tcignited/build/FatBuildDao.java | 24 +++++-
 6 files changed, 103 insertions(+), 33 deletions(-)

diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/Ignite2Configurer.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/Ignite2Configurer.java
index 6b0257c..69ddad8 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/Ignite2Configurer.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/Ignite2Configurer.java
@@ -56,7 +56,7 @@ public class Ignite2Configurer {
         logFilePolicy.setContext(logCtx);
         logFilePolicy.setParent(rollingFa);
         logFilePolicy.setFileNamePattern(new File(logs, "logfile-%d{yyyy-MM-dd_HH}.log").getAbsolutePath());
-        logFilePolicy.setMaxHistory(7);
+        logFilePolicy.setMaxHistory(24*7*2);
         logFilePolicy.start();
 
         final String activeFileName = logFilePolicy.getActiveFileName();
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/TcBotWebAppModule.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/TcBotWebAppModule.java
index 969e6b6..29ef04b 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/TcBotWebAppModule.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/TcBotWebAppModule.java
@@ -38,6 +38,7 @@ import org.apache.ignite.tcbot.common.conf.IDataSourcesConfigSupplier;
 import org.apache.ignite.tcbot.common.exeption.ExceptionUtil;
 import org.apache.ignite.tcbot.common.exeption.ServicesStartingException;
 import org.apache.ignite.tcbot.engine.TcBotEngineModule;
+import org.apache.ignite.tcbot.engine.cleaner.Cleaner;
 import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
 import org.apache.ignite.tcbot.engine.pool.TcUpdatePool;
 import org.apache.ignite.tcbot.notify.TcBotNotificationsModule;
@@ -78,6 +79,7 @@ public class TcBotWebAppModule extends AbstractModule {
         bind(ObserverTask.class).in(new SingletonScope());
         bind(BuildObserver.class).in(new SingletonScope());
         bind(VisasHistoryStorage.class).in(new SingletonScope());
+        bind(Cleaner.class).in(new SingletonScope());
 
         install(new TcBotPersistenceModule());
         install(new TeamcityIgnitedModule());
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/cleaner/Cleaner.java
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/cleaner/Cleaner.java
index 4c5ac1f..6d14a26 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/cleaner/Cleaner.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/cleaner/Cleaner.java
@@ -18,12 +18,16 @@ package org.apache.ignite.tcbot.engine.cleaner;
 
 import java.io.File;
 import java.time.ZonedDateTime;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
 import javax.inject.Inject;
 import org.apache.ignite.ci.teamcity.ignited.buildcondition.BuildConditionDao;
+import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.tcbot.common.conf.TcBotWorkDir;
 import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
 import org.apache.ignite.tcbot.common.interceptor.MonitoredTask;
@@ -39,6 +43,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class Cleaner {
+    private final AtomicBoolean init = new AtomicBoolean();
+
     @Inject IIssuesStorage issuesStorage;
     @Inject FatBuildDao fatBuildDao;
     @Inject SuiteInvocationHistoryDao suiteInvocationHistoryDao;
@@ -55,41 +61,62 @@ public class Cleaner {
     private ScheduledExecutorService executorService;
 
     @AutoProfiling
-    @MonitoredTask(name = "Clean ignite cache data and logs")
+    @MonitoredTask(name = "Clean old cache data and log files")
     public void clean() {
-        if (cfg.getCleanerConfig().enabled()) {
-            try {
+        try {
+            if (cfg.getCleanerConfig().enabled()) {
                 long safeDays = cfg.getCleanerConfig().safeDays();
+
                 int numOfItemsToDel = cfg.getCleanerConfig().numOfItemsToDel();
-                long thresholdDate = ZonedDateTime.now().minusDays(safeDays).toInstant().toEpochMilli();
 
-                removeCacheEntries(thresholdDate, numOfItemsToDel);
-                removeLogFiles(thresholdDate, numOfItemsToDel);
-            }
-            catch (Exception e) {
-                e.printStackTrace();
+                ZonedDateTime thresholdDate = ZonedDateTime.now().minusDays(safeDays);
+
+                logger.info("Some data (numOfItemsToDel=" + numOfItemsToDel + ") older than
" + thresholdDate + " will be removed.");
 
-                logger.error("Periodic cache clean failed: " + e.getMessage(), e);
+                long thresholdEpochMilli = thresholdDate.toInstant().toEpochMilli();
+
+                removeCacheEntries(thresholdEpochMilli, numOfItemsToDel);
+
+                removeLogFiles(thresholdEpochMilli, numOfItemsToDel);
             }
+            else
+                logger.info("Periodic cache clean disabled.");
+        }
+        catch (Throwable e) {
+            logger.error("Periodic cache clean failed: " + e.getMessage(), e);
+
+            e.printStackTrace();
         }
-        else
-            logger.info("Periodic cache clean disabled.");
 
     }
 
     private void removeCacheEntries(long thresholdDate, int numOfItemsToDel) {
         List<Long> oldBuildsKeys = fatBuildDao.getOldBuilds(thresholdDate, numOfItemsToDel);
 
+        List<String> strOldBuildsKeys = oldBuildsKeys.stream().map(compositeId ->
{
+                IgniteBiTuple<Integer, Integer> idTuple = FatBuildDao.cacheKeyToSrvIdAndBuildId(compositeId);
+                return "TeamCity id: " + idTuple.get1() + " build id: " + idTuple.get2();
+            })
+            .collect(Collectors.toList());
+
+        logger.info("Builds will be removed (" + strOldBuildsKeys.size() + "): " + strOldBuildsKeys);
+
         for (Long buildCacheKey : oldBuildsKeys) {
             suiteInvocationHistoryDao.remove(buildCacheKey);
+
             buildLogCheckResultDao.remove(buildCacheKey);
+
             buildRefDao.remove(buildCacheKey);
+
             buildStartTimeStorage.remove(buildCacheKey);
+
             buildConditionDao.remove(buildCacheKey);
+
             fatBuildDao.remove(buildCacheKey);
         }
 
         defectsStorage.removeOldDefects(thresholdDate, numOfItemsToDel);
+
         issuesStorage.removeOldIssues(thresholdDate, numOfItemsToDel);
     }
 
@@ -98,34 +125,47 @@ public class Cleaner {
 
         for (String srvId : cfg.getServerIds()) {
             File srvIdLogDir = new File(workDir, cfg.getTeamcityConfig(srvId).logsDirectory());
+
             removeFiles(srvIdLogDir, thresholdDate, numOfItemsToDel);
         }
 
         File tcBotLogDir = new File(workDir, "tcbot_logs");
+
         removeFiles(tcBotLogDir, thresholdDate, numOfItemsToDel);
     }
 
     private void removeFiles(File dir, long thresholdDate, int numOfItemsToDel) {
         File[] logFiles = dir.listFiles();
+
+        List<File> filesToRmv = new ArrayList<>(numOfItemsToDel);
+
         if (logFiles != null)
-            for (File file : logFiles) {
+            for (File file : logFiles)
                 if (file.lastModified() < thresholdDate && numOfItemsToDel-- >
0)
-                    file.delete();
-            }
+                    filesToRmv.add(file);
+
+        logger.info("In the directory " + dir + " files will be removed (" +
+            filesToRmv.size() + "): " + filesToRmv.stream().map(File::getName).collect(Collectors.toList())
+        );
+
+        for (File file : filesToRmv) {
+            file.delete();
+        }
+
     }
 
     public void startBackgroundClean() {
-        suiteInvocationHistoryDao.init();
-        buildLogCheckResultDao.init();
-        buildRefDao.init();
-        buildStartTimeStorage.init();
-        buildConditionDao.init();
-        fatBuildDao.init();
-
-        executorService = Executors.newSingleThreadScheduledExecutor();
+        if (init.compareAndSet(false, true)) {
+            suiteInvocationHistoryDao.init();
+            buildLogCheckResultDao.init();
+            buildRefDao.init();
+            buildStartTimeStorage.init();
+            buildConditionDao.init();
+            fatBuildDao.init();
 
-        executorService.scheduleAtFixedRate(this::clean, 30, 30, TimeUnit.MINUTES);
-//        executorService.scheduleAtFixedRate(this::clean, 0, 10, TimeUnit.SECONDS);
+            executorService = Executors.newSingleThreadScheduledExecutor();
 
+            executorService.scheduleAtFixedRate(this::clean, 5, 30, TimeUnit.MINUTES);
+        }
     }
 }
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/defect/DefectsStorage.java
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/defect/DefectsStorage.java
index 23008bf..407925a 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/defect/DefectsStorage.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/defect/DefectsStorage.java
@@ -180,8 +180,13 @@ public class DefectsStorage {
 
         ScanQuery<Integer, BinaryObject> scan =
             new ScanQuery<>((key, defect) -> {
-                long resolvedTs = defect.<Long>field("resolvedTs");
-                return resolvedTs > 0 && resolvedTs < thresholdDate;
+                Long resolvedTs = 0L;
+
+                if (defect.hasField("resolvedTs"))
+                    resolvedTs = defect.<Long>field("resolvedTs");
+
+                return (resolvedTs > 0 && resolvedTs < thresholdDate) ||
+                    !defect.hasField("resolvedTs");
             });
 
         for (Cache.Entry<Integer, BinaryObject> entry : cacheWithBinary.query(scan))
{
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/issue/IssuesStorage.java
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/issue/IssuesStorage.java
index 1e93c2b..554691f 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/issue/IssuesStorage.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/issue/IssuesStorage.java
@@ -133,8 +133,13 @@ public class IssuesStorage implements IIssuesStorage {
 
         ScanQuery<BinaryObject, BinaryObject> scan =
             new ScanQuery<>((issueKey, issue) -> {
-                Long detectedTs = issue.<Long>field("detectedTs");
-                return detectedTs != null && detectedTs < thresholdDate &&
detectedTs > 0;
+                Long detectedTs = null;
+
+                if (issue.hasField("detectedTs"))
+                    detectedTs = issue.<Long>field("detectedTs");
+
+                return (detectedTs != null && detectedTs < thresholdDate &&
detectedTs > 0) ||
+                    !issue.hasField("detectedTs");
             });
 
         for (Cache.Entry<BinaryObject, BinaryObject> entry : cacheWithBinary.query(scan))
{
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
index 889597e..425e31b 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
@@ -19,6 +19,7 @@ package org.apache.ignite.tcignited.build;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Iterables;
+import java.time.ZonedDateTime;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Date;
@@ -47,6 +48,7 @@ import org.apache.ignite.cache.affinity.Affinity;
 import org.apache.ignite.cache.query.ScanQuery;
 import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
 import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
 import org.apache.ignite.tcbot.persistence.CacheConfigs;
 import org.apache.ignite.tcbot.persistence.IStringCompactor;
@@ -177,6 +179,17 @@ public class FatBuildDao {
     }
 
     /**
+     * @param cacheKey cache key.
+     */
+    public static IgniteBiTuple<Integer, Integer> cacheKeyToSrvIdAndBuildId(long cacheKey)
{
+        IgniteBiTuple<Integer, Integer> srvIdAndBuildId = new IgniteBiTuple<>();
+
+        srvIdAndBuildId.set((int)(cacheKey >> 32), (int)cacheKey);
+
+        return srvIdAndBuildId;
+    }
+
+    /**
      * @param srvIdMaskHigh Server id mask high.
      * @param buildId Build id.
      */
@@ -410,12 +423,18 @@ public class FatBuildDao {
         IgniteCache<Long, BinaryObject> cacheWithBinary = buildsCache.withKeepBinary();
 
         ScanQuery<Long, BinaryObject> scan = new ScanQuery<>((key, fatBuild)
-> {
-                long startDate = fatBuild.<Long>field("startDate");
-                return startDate > 0 && startDate < thresholdDate;
+                Long startDate = 0L;
+
+                if (fatBuild.hasField("startDate"))
+                    startDate = fatBuild.<Long>field("startDate");
+
+                return (startDate > 0 && startDate < thresholdDate) ||
+                    !fatBuild.hasField("startDate");
             }
         );
 
         List<Long> oldBuildsKeys = new ArrayList<>(numOfItemsToDel);
+
         for (Cache.Entry<Long, BinaryObject> entry : cacheWithBinary.query(scan)) {
             if (numOfItemsToDel > 0) {
                 numOfItemsToDel--;
@@ -445,5 +464,4 @@ public class FatBuildDao {
             return entry.getValue() == null ? BuildRefDao.cacheKeyToBuildId(key) : null;
         }
     }
-
 }


Mime
View raw message