ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dpav...@apache.org
Subject [ignite-teamcity-bot] branch master updated: Time consuming builds detection, including timed out suites - Fixes #132.
Date Sat, 06 Jul 2019 12:30:47 GMT
This is an automated email from the ASF dual-hosted git repository.

dpavlov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git


The following commit(s) were added to refs/heads/master by this push:
     new 5c2d80c  Time consuming builds detection, including timed out suites - Fixes #132.
5c2d80c is described below

commit 5c2d80c3ae7eb0153b2131abc9729b8d89b0a1b6
Author: Dmitriy Pavlov <dpavlov@apache.org>
AuthorDate: Sat Jul 6 15:30:32 2019 +0300

    Time consuming builds detection, including timed out suites - Fixes #132.
    
    Signed-off-by: Dmitriy Pavlov <dpavlov@apache.org>
---
 .../ci/tcbot/TcBotBusinessServicesModule.java      |   4 +
 .../org/apache/ignite/ci/web/model/Version.java    |   2 +-
 .../web/rest/buildtime/BuildTimeRestService.java   |  57 ++++++++
 .../src/main/webapp/buildtime.html                 | 139 +++++++++++++++++++
 ignite-tc-helper-web/src/main/webapp/index.html    |   5 +
 .../org/apache/ignite/tcbot/common/TcBotConst.java |   3 +
 .../tcbot/engine/buildtime/BuildTimeService.java   | 115 ++++++++++++++++
 .../{DsProblemRef.java => BuildTimeRecordUi.java}  |  17 +--
 .../{DsProblemRef.java => BuildTimeResultUi.java}  |  16 +--
 .../ignite/tcbot/engine/ui/DsHistoryStatUi.java    |   2 +-
 .../ignite/tcbot/engine/ui/DsProblemRef.java       |   1 +
 .../ignited/fatbuild/FatBuildCompacted.java        |   8 ++
 .../ignited/fatbuild/StatisticsCompacted.java      |  40 ++----
 .../ignite/tcignited/TeamcityIgnitedImpl.java      |  11 +-
 .../apache/ignite/tcignited/build/FatBuildDao.java | 153 ++++++++++++++++++---
 .../ignite/tcignited/buildref/BuildRefDao.java     |  13 +-
 .../tcignited/buildtime/BuildTimeRecord.java       |  27 ++--
 .../tcignited/buildtime/BuildTimeResult.java       |  81 +++++++++++
 .../ignite/tcignited/history/HistoryCollector.java | 138 ++++++++++++++++---
 .../tcignited/history/RunHistCompactedDao.java     |  78 ++++++++++-
 20 files changed, 804 insertions(+), 106 deletions(-)

diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/TcBotBusinessServicesModule.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/TcBotBusinessServicesModule.java
index 2134a3f..f0d636f 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/TcBotBusinessServicesModule.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/TcBotBusinessServicesModule.java
@@ -19,6 +19,7 @@ package org.apache.ignite.ci.tcbot;
 import com.google.inject.AbstractModule;
 import com.google.inject.internal.SingletonScope;
 import org.apache.ignite.ci.issue.IssuesStorage;
+import org.apache.ignite.tcbot.engine.buildtime.BuildTimeService;
 import org.apache.ignite.tcbot.engine.chain.BuildChainProcessor;
 import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
 import org.apache.ignite.ci.tcbot.conf.LocalFilesBasedConfig;
@@ -42,5 +43,8 @@ public class TcBotBusinessServicesModule extends AbstractModule {
         bind(MasterTrendsService.class).in(new SingletonScope());
         bind(ITcBotBgAuth.class).to(TcBotBgAuthImpl.class).in(new SingletonScope());
         bind(BuildChainProcessor.class).in(new SingletonScope());
+
+        //todo move to bot engine module
+        bind(BuildTimeService.class).in(new SingletonScope());
     }
 }
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
index d284a46..27b0990 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
@@ -28,7 +28,7 @@ package org.apache.ignite.ci.web.model;
     public static final String GITHUB_REF = "https://github.com/apache/ignite-teamcity-bot";
 
     /** TC Bot Version. */
-    public static final String VERSION = "20190704";
+    public static final String VERSION = "20190706";
 
     /** Java version, where Web App is running. */
     public String javaVer;
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/buildtime/BuildTimeRestService.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/buildtime/BuildTimeRestService.java
new file mode 100644
index 0000000..43c39de
--- /dev/null
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/buildtime/BuildTimeRestService.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ci.web.rest.buildtime;
+
+import org.apache.ignite.ci.user.ITcBotUserCreds;
+import org.apache.ignite.ci.web.CtxListener;
+import org.apache.ignite.tcbot.engine.buildtime.BuildTimeService;
+import org.apache.ignite.tcbot.engine.ui.BuildTimeResultUi;
+import org.jetbrains.annotations.Nullable;
+
+import javax.servlet.ServletContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+
+@Path(BuildTimeRestService.LONG_RUNNING_SUMMARY)
+@Produces(MediaType.APPLICATION_JSON)
+public class BuildTimeRestService {
+    public static final String LONG_RUNNING_SUMMARY = "buildtime";
+
+    /** Servlet Context. */
+    @Context
+    private ServletContext ctx;
+
+    /** Current Request. */
+    @Context
+    private HttpServletRequest req;
+
+    @GET
+    @Path("analytics")
+    public BuildTimeResultUi loadAnalytics(@Nullable @QueryParam("branch") String branchOrNull) {
+        final ITcBotUserCreds creds = ITcBotUserCreds.get(req);
+
+        final BuildTimeService tbProc = CtxListener.getInjector(ctx).getInstance(BuildTimeService.class);
+
+        return tbProc.analytics(creds);
+    }
+}
diff --git a/ignite-tc-helper-web/src/main/webapp/buildtime.html b/ignite-tc-helper-web/src/main/webapp/buildtime.html
new file mode 100644
index 0000000..7090beb
--- /dev/null
+++ b/ignite-tc-helper-web/src/main/webapp/buildtime.html
@@ -0,0 +1,139 @@
+<html>
+<head>
+    <title>Apache Ignite Teamcity Bot - Build Time Analytics</title>
+    <link rel="icon" href="img/leaf-icon-png-7066.png">
+
+    <link rel="stylesheet" href="https://code.jquery.com/ui/1.12.1/themes/base/jquery-ui.css">
+
+
+    <script src="https://code.jquery.com/jquery-1.12.4.js"></script>
+    <script src="https://code.jquery.com/ui/1.12.1/jquery-ui.js"></script>
+
+
+    <script src="https://cdn.jsdelivr.net/npm/vue/dist/vue.js"></script>
+    <!-- production version, optimized for size and speed -->
+    <!--<script src="https://cdn.jsdelivr.net/npm/vue"></script>-->
+
+    <script src="https://cdn.jsdelivr.net/npm/vuetify/dist/vuetify.js"></script>
+    <link href="https://fonts.googleapis.com/css?family=Roboto:100,300,400,500,700,900|Material+Icons" rel="stylesheet">
+    <link href="https://cdn.jsdelivr.net/npm/vuetify/dist/vuetify.min.css" rel="stylesheet">
+
+    <link rel="stylesheet" href="css/style-1.5.css">
+
+    <script src="js/common-1.6.js"></script>
+    <script>
+$(document).ready(function() {
+    $.getScript("js/common-1.6.js", function(data, textStatus, jqxhr){ });
+
+    $( document ).tooltip();
+    loadData();
+});
+
+function loadAnalytics() {
+    let app = new Vue({
+        el: '#app',
+        data () {
+            return {
+                headers: [
+                    { text: 'Build Type', value: 'buildType' },
+                    { text: 'Average Duration', value: 'averageDuration' },
+                    { text: 'Duration', value: 'totalDuration' }
+                ],
+                byBuildType: [
+                ]
+            }
+        },
+        created () {
+            this.initialize()
+        },
+
+        methods: {
+            setBuildTimeStat(data) {
+                this.byBuildType = data.byBuildType;
+                this.timedOutByBuildType = data.timedOutByBuildType;
+
+                $("#loadStatus").html("");
+            },
+            initialize() {
+                $.ajax({ url: "/rest/buildtime/analytics",
+                    success: this.setBuildTimeStat,
+                    error: showErrInLoadStatus });
+            }
+        }
+    })
+}
+
+function loadData() {
+    $("#loadStatus").html("<img src='https://www.wallies.com/filebin/images/loading_apple.gif' width=20px height=20px> Please wait");
+
+    $("#version").html(" " + "<a href=\"monitoring.html\">TC Bot Moniroting Page</a> <br>");
+    $.ajax({
+        url: "rest/branches/version",
+        success: showVersionInfo,
+        error: showErrInLoadStatus
+    });
+
+
+    loadAnalytics();
+}
+
+
+</script>
+</head>
+<body>
+
+<div id="loadStatus"></div>
+
+
+<div class="formgroup"  id="app">
+
+    <v-app id="readyForReview">
+        <!--<v-expansion-panel>-->
+            <!--<v-expansion-panel-content-->
+                    <!--v-for="(item,i) in 1"-->
+                    <!--:key="i"-->
+            <!--&gt;-->
+                <template v-slot:header>
+                    <div>Build types longest avg.duration</div>
+                </template>
+                <v-card>
+                    <div>Longest average duration (more than 90 minutes)</div>
+                    <v-data-table
+                            :headers="headers"
+                            :items="byBuildType"
+                            class="elevation-1"
+                    >
+                        <template v-slot:items="props">
+                            <!--<td>-->
+                                <!--<a :href="props.item.prHtmlUrl">{{ props.item.prNumber }} </a> {{ props.item.prTitle }}</td>-->
+                            <!--<td class="text-xs-right">{{ props.item.b }}</td>-->
+                            <!--<td class="text-xs-right">-->
+                                <!--<img :src="props.item.prAuthorAvatarUrl" width='20px' height='20px'> {{ props.item.prAuthor }}</td>-->
+                            <td class="text-xs-right">{{ props.item.buildType }}</td>
+                            <td class="text-xs-right">{{ props.item.averageDuration }}</td>
+                            <td class="text-xs-right">{{ props.item.totalDuration }}</td>
+                        </template>
+                    </v-data-table>
+
+                    <div>Timed out suites average duration (more than 60 minutes)</div>
+                    <v-data-table
+                            :headers="headers"
+                            :items="timedOutByBuildType"
+                            class="elevation-1"
+                    >
+                        <template v-slot:items="props">
+                            <td class="text-xs-right">{{ props.item.buildType }}</td>
+                            <td class="text-xs-right">{{ props.item.averageDuration }}</td>
+                            <td class="text-xs-right">{{ props.item.totalDuration }}</td>
+                        </template>
+                    </v-data-table>
+                </v-card>
+            <!--</v-expansion-panel-content>-->
+        <!--</v-expansion-panel>-->
+
+    </v-app>
+</div>
+
+<div id="version"></div>
+</body>
+</html>
\ No newline at end of file
diff --git a/ignite-tc-helper-web/src/main/webapp/index.html b/ignite-tc-helper-web/src/main/webapp/index.html
index e60ab52..86f12f3 100644
--- a/ignite-tc-helper-web/src/main/webapp/index.html
+++ b/ignite-tc-helper-web/src/main/webapp/index.html
@@ -37,6 +37,11 @@ function loadData() {
         },
         error: showErrInLoadStatus
     });
+
+    $.ajax({ url: "/rest/buildtime/analytics",
+        success: function (data) {
+            $("#loadStatus").html("");
+        }, error: showErrInLoadStatus });
 }
 
 
diff --git a/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/TcBotConst.java b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/TcBotConst.java
index 3dd56ee..44433db 100644
--- a/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/TcBotConst.java
+++ b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/TcBotConst.java
@@ -26,6 +26,9 @@ public class TcBotConst {
     /** Max days to keep test invocatoin data in run statistics: affects Bot Visa. */
     public static final int HISTORY_MAX_DAYS = 21;
 
+    /** Absulte Max days to keep track about build existence, 42 */
+    public static final int BUILD_MAX_DAYS = HISTORY_MAX_DAYS * 2;
+
     /** History collection process: build id per server ID border days. */
     public static final int HISTORY_BUILD_ID_BORDER_DAYS = HISTORY_MAX_DAYS + 2;
 
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/buildtime/BuildTimeService.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/buildtime/BuildTimeService.java
new file mode 100644
index 0000000..09dfaa9
--- /dev/null
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/buildtime/BuildTimeService.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.tcbot.engine.buildtime;
+
+import org.apache.ignite.tcbot.common.interceptor.MonitoredTask;
+import org.apache.ignite.tcbot.common.util.TimeUtil;
+import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
+import org.apache.ignite.tcbot.engine.ui.BuildTimeRecordUi;
+import org.apache.ignite.tcbot.engine.ui.BuildTimeResultUi;
+import org.apache.ignite.tcbot.persistence.IStringCompactor;
+import org.apache.ignite.tcbot.persistence.scheduler.IScheduler;
+import org.apache.ignite.tcignited.ITeamcityIgnited;
+import org.apache.ignite.tcignited.ITeamcityIgnitedProvider;
+import org.apache.ignite.tcignited.build.FatBuildDao;
+import org.apache.ignite.tcignited.buildref.BuildRefDao;
+import org.apache.ignite.tcignited.buildtime.BuildTimeRecord;
+import org.apache.ignite.tcignited.buildtime.BuildTimeResult;
+import org.apache.ignite.tcignited.creds.ICredentialsProv;
+import org.apache.ignite.tcignited.history.HistoryCollector;
+
+import javax.inject.Inject;
+import java.time.Duration;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+public class BuildTimeService {
+    @Inject private ITeamcityIgnitedProvider tcProv;
+
+    /** Config. */
+    @Inject private ITcBotConfig cfg;
+
+    @Inject private FatBuildDao fatBuildDao;
+
+    @Inject private IStringCompactor compactor;
+
+    @Inject private HistoryCollector historyCollector;
+
+    @Inject private IScheduler scheduler;
+
+    private volatile BuildTimeResult lastRes1d = new BuildTimeResult();
+
+    @Inject private BuildRefDao buildRefDao;
+
+    public BuildTimeResultUi analytics(ICredentialsProv prov) {
+        if (buildRefDao.buildRefsCache() == null)
+            return new BuildTimeResultUi();
+
+        Collection<String> allServers = cfg.getServerIds();
+
+        scheduler.sheduleNamed("BuildTimeService.loadAnalytics",
+                this::loadAnalytics, 15, TimeUnit.MINUTES);
+
+        Set<Integer> availableServers = allServers.stream()
+                .filter(prov::hasAccess)
+                .map(ITeamcityIgnited::serverIdToInt)
+                .collect(Collectors.toSet());
+
+        BuildTimeResultUi resultUi = new BuildTimeResultUi();
+
+        long minDuration = Duration.ofMinutes(90).toMillis();
+        long minDurationTimeout = Duration.ofMinutes(60).toMillis();
+        int cntToInclude = 50;
+        BuildTimeResult res = lastRes1d;
+
+        res.topByBuildTypes(availableServers, minDuration, cntToInclude)
+                .stream().map(this::convertToUi).forEach(e -> resultUi.byBuildType.add(e));
+
+        res.topTimeoutsByBuildTypes(availableServers, minDurationTimeout, cntToInclude)
+                .stream().map(this::convertToUi).forEach(e -> resultUi.timedOutByBuildType.add(e));
+
+        return resultUi;
+    }
+
+    public BuildTimeRecordUi convertToUi(Map.Entry<Long, BuildTimeRecord> e) {
+        BuildTimeRecordUi buildTimeRecordUi = new BuildTimeRecordUi();
+        Long key = e.getKey();
+        int btId = BuildTimeResult.cacheKeyToBuildType(key);
+        buildTimeRecordUi.buildType = compactor.getStringFromId(btId);
+
+        buildTimeRecordUi.averageDuration = TimeUtil.millisToDurationPrintable(e.getValue().avgDuration());
+        buildTimeRecordUi.totalDuration =  TimeUtil.millisToDurationPrintable(e.getValue().totalDuration());
+        return buildTimeRecordUi;
+    }
+
+    @SuppressWarnings("WeakerAccess")
+    @MonitoredTask(name = "Load Build Time Analytics")
+    protected void loadAnalytics() {
+        int days = 1;
+
+        List<Long> idsToCheck = historyCollector.findAllRecentBuilds(days, cfg.getServerIds());
+
+        BuildTimeResult res = fatBuildDao.loadBuildTimeResult(days, idsToCheck);
+
+        lastRes1d = res;
+    }
+}
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/BuildTimeRecordUi.java
similarity index 73%
copy from tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java
copy to tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/BuildTimeRecordUi.java
index 23fe3bd..f411d83 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/BuildTimeRecordUi.java
@@ -14,18 +14,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.apache.ignite.tcbot.engine.ui;
 
-/**
- * Detailed status of failures: Reference to some Issue with current suite or test detected by the Bot.
- * Currently contains only display name.
- */
-public class DsProblemRef {
-    public String name;
-    public String webUrl;
-
-    public DsProblemRef(String name) {
-        this.name = name;
-    }
+@SuppressWarnings({"WeakerAccess", "PublicField"})
+public class BuildTimeRecordUi {
+    public String buildType;
+    public String averageDuration;
+    public String totalDuration;
 }
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/BuildTimeResultUi.java
similarity index 73%
copy from tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java
copy to tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/BuildTimeResultUi.java
index 23fe3bd..37e9a21 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/BuildTimeResultUi.java
@@ -17,15 +17,11 @@
 
 package org.apache.ignite.tcbot.engine.ui;
 
-/**
- * Detailed status of failures: Reference to some Issue with current suite or test detected by the Bot.
- * Currently contains only display name.
- */
-public class DsProblemRef {
-    public String name;
-    public String webUrl;
+import java.util.ArrayList;
+import java.util.List;
 
-    public DsProblemRef(String name) {
-        this.name = name;
-    }
+@SuppressWarnings({"WeakerAccess", "PublicField"})
+public class BuildTimeResultUi {
+    public List<BuildTimeRecordUi> byBuildType = new ArrayList<>();
+    public List<BuildTimeRecordUi> timedOutByBuildType = new ArrayList<>();
 }
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsHistoryStatUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsHistoryStatUi.java
index a718e47..6107c70 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsHistoryStatUi.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsHistoryStatUi.java
@@ -23,7 +23,7 @@ import javax.annotation.Nullable;
 /**
  * Detailed status of failures: Test/suite failure summary: contains statistic of failures and total runs for suite or for test.
  */
-public class DsHistoryStatUi {
+@SuppressWarnings("PublicField") public class DsHistoryStatUi {
     /** Registered number of failures from TC helper DB */
     @Nullable public Integer failures;
 
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java
index 23fe3bd..3c98b34 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java
+++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java
@@ -21,6 +21,7 @@ package org.apache.ignite.tcbot.engine.ui;
  * Detailed status of failures: Reference to some Issue with current suite or test detected by the Bot.
  * Currently contains only display name.
  */
+@SuppressWarnings({"WeakerAccess", "PublicField"})
 public class DsProblemRef {
     public String name;
     public String webUrl;
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
index eaeeb54..c47d8e9 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
@@ -554,6 +554,10 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
                 .forEach(this.problems::add);
     }
 
+    public Long statisticValue(Integer propCode) {
+        return statistics == null ? null : statistics.statisticValue(propCode);
+    }
+
     public Long buildDuration(IStringCompactor compactor) {
         return statistics == null ? null : statistics.buildDuration(compactor);
     }
@@ -711,4 +715,8 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
 
         return cnt;
     }
+
+    public long getFinishDateTs() {
+        return finishDate;
+    }
 }
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/StatisticsCompacted.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/StatisticsCompacted.java
index 12a0f67..514542b 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/StatisticsCompacted.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/StatisticsCompacted.java
@@ -29,6 +29,8 @@ import org.apache.ignite.internal.util.GridLongList;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nullable;
+
 /**
  * Statistics values to be saved in compacted form.
  */
@@ -85,10 +87,14 @@ public class StatisticsCompacted {
     public Long buildDuration(IStringCompactor compactor) {
         Integer buildDurationId = compactor.getStringIdIfPresent(Statistics.BUILD_DURATION);
 
-        if (buildDurationId == null)
+        return statisticValue(buildDurationId);
+    }
+
+    @Nullable public Long statisticValue(@Nullable Integer propCode) {
+        if (propCode == null)
             return null;
 
-        long val = findPropertyValue(buildDurationId);
+        long val = findPropertyValue(propCode);
 
         return val >= 0 ? val : null;
     }
@@ -100,12 +106,7 @@ public class StatisticsCompacted {
     public Long buildDurationNetTime(IStringCompactor compactor) {
         Integer buildDurationNetId = compactor.getStringIdIfPresent(Statistics.BUILD_DURATION_NET_TIME);
 
-        if (buildDurationNetId == null)
-            return null;
-
-        long val = findPropertyValue(buildDurationNetId);
-
-        return val >= 0 ? val : null;
+        return statisticValue(buildDurationNetId);
     }
 
     /**
@@ -115,12 +116,7 @@ public class StatisticsCompacted {
     public Long artifcactPublishingDuration(IStringCompactor compactor) {
         Integer buildDurationNetId = compactor.getStringIdIfPresent(Statistics.ARTIFACTS_PUBLISHING_DURATION);
 
-        if (buildDurationNetId == null)
-            return null;
-
-        long val = findPropertyValue(buildDurationNetId);
-
-        return val >= 0 ? val : null;
+        return statisticValue(buildDurationNetId);
     }
 
     /**
@@ -130,12 +126,7 @@ public class StatisticsCompacted {
     public Long dependeciesResolvingDuration(IStringCompactor compactor) {
         Integer buildDurationNetId = compactor.getStringIdIfPresent(Statistics.DEPENDECIES_RESOLVING_DURATION);
 
-        if (buildDurationNetId == null)
-            return null;
-
-        long val = findPropertyValue(buildDurationNetId);
-
-        return val >= 0 ? val : null;
+        return statisticValue(buildDurationNetId);
     }
 
     /**
@@ -145,15 +136,10 @@ public class StatisticsCompacted {
     public Long sourceUpdateDuration(IStringCompactor compactor) {
         Integer buildDurationNetId = compactor.getStringIdIfPresent(Statistics.SOURCES_UPDATE_DURATION);
 
-        if (buildDurationNetId == null)
-            return null;
-
-        long val = findPropertyValue(buildDurationNetId);
-
-        return val >= 0 ? val : null;
+        return statisticValue(buildDurationNetId);
     }
 
-    private long findPropertyValue(int propCode) {
+    public long findPropertyValue(int propCode) {
         if (keys == null)
             return -1L;
 
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
index 892b2e2..ce10502 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java
@@ -569,12 +569,17 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
 
         long ts = highBuild.getStartDateTs();
 
-        return ts > 0 ? ts : null;
+        if (ts > 0) {
+            runHistCompactedDao.setBuildStartTime(srvIdMaskHigh, buildId, ts);
+
+            return ts;
+        } else
+            return null;
     }
 
-    @GuavaCached(maximumSize = 100000, expireAfterAccessSecs = 90, softValues = true)
+    //@GuavaCached(maximumSize = 100000, expireAfterAccessSecs = 90, softValues = true)
     public Long getBuildStartTime(int buildId) {
-        return runHistCompactedDao.getBuildStartTime(srvIdMaskHigh, buildId);
+        return histCollector.getBuildStartTime(srvIdMaskHigh, buildId);
     }
 
     /** {@inheritDoc} */
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
index 82da2c8..5b4740d 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/FatBuildDao.java
@@ -19,23 +19,6 @@ package org.apache.ignite.tcignited.build;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Iterables;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import java.util.stream.StreamSupport;
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
-import javax.cache.Cache;
-import javax.cache.processor.EntryProcessorException;
-import javax.cache.processor.EntryProcessorResult;
-import javax.cache.processor.MutableEntry;
-import javax.inject.Inject;
-import javax.inject.Provider;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.binary.BinaryObject;
@@ -45,8 +28,10 @@ import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
 import org.apache.ignite.tcbot.persistence.CacheConfigs;
 import org.apache.ignite.tcbot.persistence.IStringCompactor;
 import org.apache.ignite.tcignited.buildref.BuildRefDao;
+import org.apache.ignite.tcignited.buildtime.BuildTimeResult;
 import org.apache.ignite.tcignited.history.HistoryCollector;
 import org.apache.ignite.tcservice.model.changes.ChangesList;
+import org.apache.ignite.tcservice.model.hist.BuildRef;
 import org.apache.ignite.tcservice.model.result.Build;
 import org.apache.ignite.tcservice.model.result.problems.ProblemOccurrence;
 import org.apache.ignite.tcservice.model.result.stat.Statistics;
@@ -54,6 +39,19 @@ import org.apache.ignite.tcservice.model.result.tests.TestOccurrencesFull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import javax.cache.Cache;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.EntryProcessorResult;
+import javax.cache.processor.MutableEntry;
+import javax.inject.Inject;
+import javax.inject.Provider;
+import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
 /**
  *
  */
@@ -63,6 +61,7 @@ public class FatBuildDao {
 
     /** Cache name */
     public static final String TEAMCITY_FAT_BUILD_CACHE_NAME = "teamcityFatBuild";
+    public static final int MAX_FAT_BUILD_CHUNK = 32 * 10;
 
     /** Ignite provider. */
     @Inject private Provider<Ignite> igniteProvider;
@@ -206,6 +205,17 @@ public class FatBuildDao {
 
     /**
      * @param srvId Server id.
+     * @param buildId Build Id.
+     */
+    @Nullable public Long getBuildStartTime(int srvId, Integer buildId) {
+        IgniteCache<Long, BinaryObject> cacheBin = buildsCache.withKeepBinary();
+        long key = buildIdToCacheKey(srvId, buildId);
+
+        return cacheBin.invoke(key, new GetStartTimeProc());
+    }
+
+    /**
+     * @param srvId Server id.
      * @param ids Ids.
      */
     public Map<Integer, Long> getBuildStartTime(int srvId, Set<Integer> ids) {
@@ -213,7 +223,7 @@ public class FatBuildDao {
         Set<Long> keys = buildsIdsToCacheKeys(srvId, ids);
         HashMap<Integer, Long> res = new HashMap<>();
 
-        Iterables.partition(keys, 32 * 10).forEach(
+        Iterables.partition(keys, MAX_FAT_BUILD_CHUNK).forEach(
             chunk -> {
                 Map<Long, EntryProcessorResult<Long>> map = cacheBin.invokeAll(keys, new GetStartTimeProc());
                 map.forEach((k, r) -> {
@@ -227,6 +237,113 @@ public class FatBuildDao {
         return res;
     }
 
+    public BuildTimeResult loadBuildTimeResult(int ageDays, List<Long> idsToCheck) {
+        int stateRunning = compactor.getStringId(BuildRef.STATE_RUNNING);
+        Integer buildDurationId = compactor.getStringIdIfPresent(Statistics.BUILD_DURATION);
+        int timeoutProblemCode = compactor.getStringId(ProblemOccurrence.TC_EXECUTION_TIMEOUT);
+
+        BuildTimeResult res = new BuildTimeResult();
+
+        // also may take affinity into account
+        Iterables.partition(idsToCheck, MAX_FAT_BUILD_CHUNK).forEach(
+                chunk -> {
+                    HashSet<Long> keys = new HashSet<>(chunk);
+                    Map<Long, FatBuildCompacted> all = buildsCache.getAll(keys);
+                    all.forEach((key, build) -> {
+                        if (build.isComposite())
+                            return;
+
+                        long runningTime = getBuildRunningTime(stateRunning, buildDurationId, build);
+                        if (runningTime > 0) {
+                            int buildTypeId = build.buildTypeId();
+                            System.err.println("Running " + runningTime + " BT: " + buildTypeId);
+
+                            int srvId = BuildRefDao.cacheKeyToSrvId(key);
+                            boolean hasTimeout = build.hasBuildProblemType(timeoutProblemCode);
+
+                            res.add(srvId, buildTypeId, runningTime, hasTimeout);
+                        }
+                    });
+                }
+        );
+
+        return res;
+
+    }
+
+    public static long getBuildRunningTime(int stateRunning, Integer buildDurationId,
+                                           FatBuildCompacted buildBinary) {
+        long startTs = buildBinary.getStartDateTs();
+
+        if (startTs <= 0)
+            return -1;
+
+        int state = buildBinary.state();
+
+        long runningTime = -1;
+        if (stateRunning == state)
+            runningTime = System.currentTimeMillis() - startTs;
+
+        if (runningTime < 0) {
+            if (buildDurationId != null) {
+                Long val = buildBinary.statisticValue(buildDurationId);
+
+                runningTime = (val != null && val >= 0) ? val : -1;
+            }
+
+
+        }
+
+        if (runningTime < 0) {
+            long finishTs = buildBinary.getFinishDateTs();
+
+            if (finishTs > 0)
+                runningTime = finishTs - startTs;
+        }
+
+        return runningTime;
+    }
+
+    public static long getBuildRunningTime(int stateRunning, Integer buildDurationId,
+        BinaryObject buildBinary) {
+        Long startTs = buildBinary.field("startDate");
+
+        if (startTs == null || startTs <= 0)
+            return -1;
+
+
+        int status = buildBinary.field("status");
+        int state = buildBinary.field("state");
+
+        long runningTime = -1;
+        if(stateRunning == state)
+            runningTime = System.currentTimeMillis() - startTs;
+
+        if(runningTime<0){
+
+            if (buildDurationId != null) {
+                BinaryObject statistics = buildBinary.field("statistics");
+                if(statistics!=null) {
+                    // statistics.field()
+                }
+                long val = -1; //statistics.findPropertyValue(buildDurationId);
+
+                runningTime = val >= 0 ? val : -1;
+            }
+
+
+        }
+
+        if(runningTime<0) {
+            Long finishTs= buildBinary.field("finishDate");
+
+            if(finishTs!=null)
+                runningTime = finishTs - startTs;
+        }
+
+        return runningTime;
+    }
+
     private static class GetStartTimeProc implements CacheEntryProcessor<Long, BinaryObject, Long> {
         public GetStartTimeProc() {
         }
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
index e8c8ae0..72ad31a 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
@@ -104,7 +104,7 @@ public class BuildRefDao {
      * @param srvId Server id.
      */
     public static boolean isKeyForServer(Long key, int srvId) {
-        return key!=null && key >> 32 == srvId;
+        return key!=null && cacheKeyToSrvId(key) == srvId;
     }
 
     /**
@@ -169,6 +169,13 @@ public class BuildRefDao {
     }
 
     /**
+     * @param cacheKey Cache key.
+     */
+    public static int cacheKeyToSrvId(long cacheKey) {
+        return (int)(cacheKey >> 32);
+    }
+
+    /**
      * @param srvId Server id mask high.
      * @param buildTypeId Build type (suite) id.
      * @param bracnhNameQry Bracnh name query.
@@ -307,4 +314,8 @@ public class BuildRefDao {
         return StreamSupport.stream(buildRefsCache.spliterator(), false)
                 .filter(entry -> isKeyForServer(entry.getKey(), srvId));
     }
+
+    public IgniteCache<Long, BuildRefCompacted> buildRefsCache() {
+        return buildRefsCache;
+    }
 }
diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildtime/BuildTimeRecord.java
similarity index 66%
copy from tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java
copy to tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildtime/BuildTimeRecord.java
index 23fe3bd..63e8026 100644
--- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/DsProblemRef.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildtime/BuildTimeRecord.java
@@ -14,18 +14,25 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+package org.apache.ignite.tcignited.buildtime;
 
-package org.apache.ignite.tcbot.engine.ui;
+public class BuildTimeRecord {
+    private long totaltime;
+    private int cnt;
 
-/**
- * Detailed status of failures: Reference to some Issue with current suite or test detected by the Bot.
- * Currently contains only display name.
- */
-public class DsProblemRef {
-    public String name;
-    public String webUrl;
+    public void addInvocation(long runningTimeMs) {
+        totaltime += runningTimeMs;
+        cnt++;
+    }
+
+    public long avgDuration() {
+        if (cnt == 0)
+            return 0;
+
+        return totaltime / cnt;
+    }
 
-    public DsProblemRef(String name) {
-        this.name = name;
+    public long totalDuration() {
+        return totaltime;
     }
 }
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildtime/BuildTimeResult.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildtime/BuildTimeResult.java
new file mode 100644
index 0000000..0721fd9
--- /dev/null
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildtime/BuildTimeResult.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.tcignited.buildtime;
+
+import java.util.*;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class BuildTimeResult {
+    private Map<Long, BuildTimeRecord> btByBuildType = new HashMap<>();
+    private Map<Long, BuildTimeRecord> timedOutByBuildType = new HashMap<>();
+
+    public void add(int srvId, int buildTypeId, long runningTime, boolean hasTimeout) {
+        long cacheKey = buildTypeToCacheKey(srvId, buildTypeId);
+        btByBuildType.computeIfAbsent(cacheKey, k -> new BuildTimeRecord()).addInvocation(runningTime);
+
+        if (hasTimeout)
+            timedOutByBuildType.computeIfAbsent(cacheKey, k -> new BuildTimeRecord()).addInvocation(runningTime);
+    }
+
+    public static long buildTypeToCacheKey(long srvId, int btId) {
+        return (long)btId | srvId << 32;
+    }
+
+    public static int cacheKeyToSrvId(long cacheKey) {
+        return (int)(cacheKey >> 32);
+    }
+
+    public static int cacheKeyToBuildType(Long cacheKey) {
+        long l = cacheKey << 32;
+        return (int)(l >> 32);
+    }
+
+
+    public List<Map.Entry<Long, BuildTimeRecord>> topByBuildTypes(Set<Integer> availableServers,
+                                                                  long minAvgDurationMs,
+                                                                  int maxCnt) {
+        return filtered(btByBuildType, availableServers, minAvgDurationMs)
+                .sorted(Comparator.comparing(
+                        (Function<Map.Entry<Long, BuildTimeRecord>, Long>) entry -> entry.getValue().avgDuration())
+                        .reversed())
+                .limit(maxCnt)
+                .collect(Collectors.toList());
+    }
+
+    public List<Map.Entry<Long, BuildTimeRecord>> topTimeoutsByBuildTypes(Set<Integer> availableServers,
+                                                                  long minAvgDurationMs,
+                                                                  int maxCnt) {
+        return filtered(timedOutByBuildType, availableServers, minAvgDurationMs)
+                .sorted(Comparator.comparing(
+                        (Function<Map.Entry<Long, BuildTimeRecord>, Long>) entry -> entry.getValue().avgDuration())
+                        .reversed())
+                .limit(maxCnt)
+                .collect(Collectors.toList());
+    }
+
+    private Stream<Map.Entry<Long, BuildTimeRecord>> filtered(Map<Long, BuildTimeRecord> map, Set<Integer> availableServers, long minAvgDurationMs) {
+        return map.entrySet().stream()
+                .filter(e -> {
+                    Long key = e.getKey();
+                    int srvId = cacheKeyToSrvId(key);
+                    return availableServers.contains(srvId);
+                })
+                .filter(e -> e.getValue().avgDuration() > minAvgDurationMs);
+    }
+}
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
index e9ca205..1246a3e 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java
@@ -16,42 +16,46 @@
  */
 package org.apache.ignite.tcignited.history;
 
-import com.google.common.base.Preconditions;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.collect.Iterables;
-import java.time.Duration;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.BiPredicate;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import javax.inject.Inject;
-import javax.inject.Provider;
 import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.ScanQuery;
 import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
 import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
 import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
 import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey;
 import org.apache.ignite.tcbot.common.TcBotConst;
 import org.apache.ignite.tcbot.common.exeption.ExceptionUtil;
+import org.apache.ignite.tcbot.common.exeption.ServicesStartingException;
 import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
 import org.apache.ignite.tcbot.persistence.IStringCompactor;
+import org.apache.ignite.tcignited.ITeamcityIgnited;
 import org.apache.ignite.tcignited.build.FatBuildDao;
 import org.apache.ignite.tcignited.build.SuiteHistory;
 import org.apache.ignite.tcignited.buildref.BranchEquivalence;
 import org.apache.ignite.tcignited.buildref.BuildRefDao;
+import org.apache.ignite.tcservice.model.hist.BuildRef;
 import org.apache.ignite.tcservice.model.result.tests.TestOccurrence;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.cache.Cache;
+import javax.inject.Inject;
+import javax.inject.Provider;
+import java.time.Duration;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BiPredicate;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
 /**
  *
  */
@@ -143,9 +147,18 @@ public class HistoryCollector {
         long curTs = System.currentTimeMillis();
         Set<Integer> buildIds = bRefsList.stream()
             .filter(b -> {
-                AtomicInteger biggestIdOutOfScope = biggestBuildIdOutOfHistoryScope.get(srvId);
+               /* AtomicInteger biggestIdOutOfScope = biggestBuildIdOutOfHistoryScope.get(srvId);
                 int outOfScopeBuildId = biggestIdOutOfScope == null ? -1 : biggestIdOutOfScope.get();
                 return b.id() > outOfScopeBuildId;
+                */
+
+                Integer maxBuildIdForDay = runHistCompactedDao.getBorderForAgeForBuildId(srvId, TcBotConst.HISTORY_BUILD_ID_BORDER_DAYS);
+
+                if (maxBuildIdForDay == null)
+                    return true;
+
+                return b.id()>maxBuildIdForDay;
+
             })
             .filter(this::applicableForHistory)
             .map(BuildRefCompacted::id)
@@ -225,7 +238,7 @@ public class HistoryCollector {
         int normalizedBaseBranch) {
         Map<Integer, SuiteInvocation> suiteRunHist = histDao.getSuiteRunHist(srvId, buildTypeId, normalizedBaseBranch);
 
-        System.out.println("***** Found history for suite "
+        logger.info("***** Found history for suite "
             + compactor.getStringFromId(buildTypeId)
             + " branch " + compactor.getStringFromId(normalizedBaseBranch) + ": " + suiteRunHist.size() );
 
@@ -325,4 +338,91 @@ public class HistoryCollector {
         return getSuiteHist(srvId, buildTypeId, normalizedBaseBranch);
     }
 
+
+    /**
+     * @param srvId Server id.
+     * @param buildId Build id.
+     */
+    public Long getBuildStartTime(int srvId, int buildId) {
+        Long time = runHistCompactedDao.getBuildStartTime(srvId, buildId);
+
+        if (time != null)
+            return time;
+
+        time = fatBuildDao.getBuildStartTime(srvId, buildId);
+
+        if (time != null)
+            runHistCompactedDao.setBuildStartTime(srvId, buildId, time);
+
+        return time;
+    }
+
+    public List<Long> findAllRecentBuilds(int days, Collection<String> allServers) {
+        IgniteCache<Long, BuildRefCompacted> cache = buildRefDao.buildRefsCache();
+        if (cache == null)
+            throw new ServicesStartingException(new RuntimeException("Ignite is not yet available"));
+
+        IgniteCache<Long, BinaryObject> cacheBin = cache.withKeepBinary();
+
+        final Map<Integer, Integer> preBorder = new HashMap<>();
+
+        allServers.stream()
+                .map(ITeamcityIgnited::serverIdToInt)
+                .forEach(srvId -> {
+                    Integer borderForAgeForBuildId = runHistCompactedDao.getBorderForAgeForBuildId(srvId, days);
+                    if (borderForAgeForBuildId != null)
+                        preBorder.put(srvId, borderForAgeForBuildId);
+                });
+
+        final int stateQueued = compactor.getStringId(BuildRef.STATE_QUEUED);
+
+        long minTs = System.currentTimeMillis() - Duration.ofDays(days).toMillis();
+        QueryCursor<Cache.Entry<Long, BinaryObject>> query = cacheBin.query(
+                new ScanQuery<Long, BinaryObject>()
+                        .setFilter((key, v) -> {
+                            int srvId = BuildRefDao.cacheKeyToSrvId(key);
+                            Integer buildIdBorder = preBorder.get(srvId);
+                            if (buildIdBorder != null) {
+                                int id = v.field("id");
+                                if (id < buildIdBorder)
+                                    return false;// pre-filtered build out of scope
+                            }
+                            int state = v.field("state");
+
+                            return stateQueued != state;
+                        }));
+
+        int cnt = 0;
+        List<Long> idsToCheck = new ArrayList<>();
+
+        try (QueryCursor<Cache.Entry<Long, BinaryObject>> cursor = query) {
+            for (Cache.Entry<Long, BinaryObject> next : cursor) {
+                Long key = next.getKey();
+                int srvId = BuildRefDao.cacheKeyToSrvId(key);
+
+                int buildId = BuildRefDao.cacheKeyToBuildId(key);
+
+                Integer borderBuildId = runHistCompactedDao.getBorderForAgeForBuildId(srvId, days);
+
+                boolean passesDate = borderBuildId == null || buildId >= borderBuildId;
+
+                if (!passesDate)
+                    continue;
+
+                Long startTs = getBuildStartTime(srvId, buildId);
+                if (startTs == null || startTs < minTs)
+                    continue; //time not saved in the DB, skip
+
+                System.err.println("Found build at srv [" + srvId + "]: [" + buildId + "] to analyze, ts=" + startTs);
+
+                cnt++;
+
+                idsToCheck.add(key);
+            }
+        }
+
+        System.err.println("Total builds to load " + cnt);
+
+        return idsToCheck;
+    }
 }
diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
index 5a18465..aa49966 100644
--- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
+++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/RunHistCompactedDao.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.tcignited.history;
 
+import java.time.Duration;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -24,7 +25,10 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicIntegerArray;
 import java.util.stream.Collectors;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -42,6 +46,7 @@ import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
 import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistCompacted;
 import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.tcbot.common.TcBotConst;
 import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
 import org.apache.ignite.tcbot.common.interceptor.GuavaCached;
 import org.apache.ignite.tcbot.persistence.CacheConfigs;
@@ -51,7 +56,7 @@ import org.apache.ignite.tcignited.buildref.BuildRefDao;
 import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch;
 
 /**
- *
+ * TODO: rename to build start time storage
  */
 public class RunHistCompactedDao {
     /** Cache name. */
@@ -78,6 +83,17 @@ public class RunHistCompactedDao {
     /** Build start time. */
     private IgniteCache<Long, Long> buildStartTime;
 
+    /**
+     * Biggest build ID, which is older than particular days count.
+     * Map: server ID-> Array of build Ids
+     * Array[0] = max build ID
+     * Array[1] = max build ID older than 1 day
+     */
+    private final ConcurrentMap<Integer, AtomicIntegerArray> maxBuildIdOlderThanDays = new ConcurrentHashMap<>();
+
+    /** Millis in day. */
+    private static final long MILLIS_IN_DAY = Duration.ofDays(1).toMillis();
+
     /** Compactor. */
     @Inject private IStringCompactor compactor;
 
@@ -146,14 +162,27 @@ public class RunHistCompactedDao {
         if (ts == null || ts <= 0)
             return null;
 
+        processBuildForBorder(srvId, buildId, ts);
+
         return ts;
     }
 
+    public boolean setBuildStartTime(int srvId, int buildId, long ts) {
+        if (ts <= 0)
+            return false;
+
+        processBuildForBorder(srvId, buildId, ts);
+
+        return buildStartTime.putIfAbsent(buildIdToCacheKey(srvId, buildId), ts);
+    }
+
     @AutoProfiling
     public boolean setBuildProcessed(int srvId, int buildId, long ts) {
         if (ts <= 0)
             return false;
 
+        processBuildForBorder(srvId, buildId, ts);
+
         return buildStartTime.putIfAbsent(buildIdToCacheKey(srvId, buildId), ts);
     }
 
@@ -263,8 +292,13 @@ public class RunHistCompactedDao {
         Map<Integer, Long> res = new HashMap<>();
 
         buildStartTime.getAll(cacheKeys).forEach((k, ts) -> {
-            if (ts != null && ts > 0)
-                res.put(BuildRefDao.cacheKeyToBuildId(k), ts);
+            if (ts != null && ts > 0) {
+                int buildId = BuildRefDao.cacheKeyToBuildId(k);
+
+                res.put(buildId, ts);
+
+                processBuildForBorder(srvId, buildId, ts);
+            }
         });
 
         return res;
@@ -274,10 +308,46 @@ public class RunHistCompactedDao {
         Map<Long, Long> res = new HashMap<>();
 
         builds.forEach((buildId, ts) -> {
-            if (ts != null && ts > 0)
+            if (ts != null && ts > 0) {
                 res.put(buildIdToCacheKey(srvId, buildId), ts);
+
+                processBuildForBorder(srvId, buildId, ts);
+            }
         });
 
         buildStartTime.putAll(res);
     }
+
+    private void processBuildForBorder(int srvId, Integer buildId, Long ts) {
+        if (ts == null || ts <= 0)
+            return;
+
+        AtomicIntegerArray arr = maxBuildIdOlderThanDays.computeIfAbsent(srvId,
+            k -> new AtomicIntegerArray(TcBotConst.BUILD_MAX_DAYS + 1));
+
+        long ageMs = System.currentTimeMillis() - ts;
+        if (ageMs < 0)
+            return;
+
+        long days = ageMs / MILLIS_IN_DAY;
+        if (days > TcBotConst.BUILD_MAX_DAYS)
+            days = TcBotConst.BUILD_MAX_DAYS;
+
+        arr.accumulateAndGet((int)days, buildId, Math::max);
+    }
+
+    @Nullable public Integer getBorderForAgeForBuildId(int srvId, int ageDays) {
+        AtomicIntegerArray arr = maxBuildIdOlderThanDays.get(srvId);
+        if (arr == null)
+            return null;
+
+        for (int i = ageDays; i < TcBotConst.BUILD_MAX_DAYS; i++) {
+            int buildId = arr.get(i);
+            if (buildId != 0)
+                return buildId;
+        }
+
+        return null;
+    }
+
 }


Mime
View raw message