ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dpav...@apache.org
Subject [ignite-teamcity-bot] branch incremental-build-upload updated: Incremental builds implemented
Date Wed, 19 Sep 2018 17:08:47 GMT
This is an automated email from the ASF dual-hosted git repository.

dpavlov pushed a commit to branch incremental-build-upload
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git


The following commit(s) were added to refs/heads/incremental-build-upload by this push:
     new 4ac2525  Incremental builds implemented
4ac2525 is described below

commit 4ac252579d1be88a2f89cac3416071bf840f1976
Author: Dmitriy Pavlov <dpavlov@apache.org>
AuthorDate: Wed Sep 19 20:08:42 2018 +0300

    Incremental builds implemented
---
 .../main/java/org/apache/ignite/ci/ITeamcity.java  | 16 +++----
 .../apache/ignite/ci/IgnitePersistentTeamcity.java | 51 +++++++++++++++++-----
 .../org/apache/ignite/ci/IgniteTeamcityHelper.java | 48 ++++++++++----------
 .../runners/IgniteTeamcityHelperRunnerExample.java |  4 +-
 4 files changed, 72 insertions(+), 47 deletions(-)

diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
index 4e9f9d6..ccc3fea 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
@@ -66,16 +66,17 @@ public interface ITeamcity extends AutoCloseable {
      * @return list of builds in historical order, recent builds coming last
      */
     default List<BuildRef> getFinishedBuilds(String projectId, String branch) {
-        return getFinishedBuilds(projectId, branch, null);
+        return getFinishedBuilds(projectId, branch, null, null);
     };
 
     /**
      * @param projectId suite ID (string without spaces)
      * @param branch
      * @param cnt builds count
+     * @param sinceBuildNumber
      * @return list of builds in historical order, recent builds coming last
      */
-    List<BuildRef> getFinishedBuilds(String projectId, String branch, Long cnt);
+    List<BuildRef> getFinishedBuilds(String projectId, String branch, Long cnt, Integer
sinceBuildNumber);
 
     /**
      * Includes snapshot dependencies failed builds into list
@@ -85,7 +86,7 @@ public interface ITeamcity extends AutoCloseable {
      * @return list of builds in historical order, recent builds coming last
      */
     default List<BuildRef> getFinishedBuildsIncludeSnDepFailed(String projectId, String
branch){
-        return getFinishedBuildsIncludeSnDepFailed(projectId, branch, null);
+        return getFinishedBuildsIncludeSnDepFailed(projectId, branch, null, null);
     };
 
     /**
@@ -94,9 +95,10 @@ public interface ITeamcity extends AutoCloseable {
      * @param projectId suite ID (string without spaces)
      * @param branch branch in TC identification
      * @param cnt builds count
+     * @param sinceBuildNumber limit builds export with some build number, not operational
for Persistent connection
      * @return list of builds in historical order, recent builds coming last
      */
-    List<BuildRef> getFinishedBuildsIncludeSnDepFailed(String projectId, String branch,
Long cnt);
+    List<BuildRef> getFinishedBuildsIncludeSnDepFailed(String projectId, String branch,
Long cnt, Integer sinceBuildNumber);
 
     /**   */
     CompletableFuture<List<BuildRef>> getRunningBuilds(@Nullable String branch);
@@ -104,12 +106,8 @@ public interface ITeamcity extends AutoCloseable {
     /**   */
     CompletableFuture<List<BuildRef>> getQueuedBuilds(@Nullable String branch);
 
-    default int[] getBuildNumbersFromHistory(String projectId, String branchNameForHist)
{
-        return getBuildNumbersFromHistory(projectId, branchNameForHist, null);
-    }
-
     default int[] getBuildNumbersFromHistory(String projectId, String branchNameForHist,
Long cnt) {
-        return getFinishedBuilds(projectId, branchNameForHist, cnt).stream().mapToInt(BuildRef::getId).toArray();
+        return getFinishedBuilds(projectId, branchNameForHist, cnt, null).stream().mapToInt(BuildRef::getId).toArray();
     }
 
     Build getBuild(String href);
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
index d3b038d..e9d1150 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
@@ -33,6 +33,7 @@ import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.Lock;
+import java.util.function.BiFunction;
 import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
@@ -71,6 +72,8 @@ import org.apache.ignite.ci.util.CacheUpdateUtil;
 import org.apache.ignite.ci.util.CollectionUtil;
 import org.apache.ignite.ci.util.ObjectInterner;
 import org.jetbrains.annotations.NotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.xml.sax.SAXParseException;
 
 import static org.apache.ignite.ci.BuildChainProcessor.normalizeBranch;
@@ -79,6 +82,9 @@ import static org.apache.ignite.ci.BuildChainProcessor.normalizeBranch;
  *
  */
 public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity, ITeamcity, ITcAnalytics
{
+    /** Logger. */
+    private static final Logger logger = LoggerFactory.getLogger(IgniteTeamcityHelper.class);
+
     //V1 caches, 1024 parts
 
     //V2 caches, 32 parts
@@ -267,12 +273,12 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity,
ITea
         return loaded;
     }
 
-    protected  <K> List<BuildRef> loadBuildHistory(IgniteCache<K, Expirable<List<BuildRef>>>
cache,
-                                                                int seconds,
-                                                                Long cnt,
-                                                                K key,
-                                                                Function<K, List<BuildRef>>
realLoad) {
-        @Nullable Expirable<List<BuildRef>> persistedBuilds = readBuildHistEntry(
 cache, (K) key);
+    protected <K> List<BuildRef> loadBuildHistory(IgniteCache<K, Expirable<List<BuildRef>>>
cache,
+                                                  int seconds,
+                                                  Long cnt,
+                                                  K key,
+                                                  BiFunction<K, Integer, List<BuildRef>>
realLoad) {
+        @Nullable Expirable<List<BuildRef>> persistedBuilds = readBuildHistEntry(cache,
(K) key);
 
         if (persistedBuilds != null
                 && (persistedBuilds.isAgeLessThanSecs(seconds)
@@ -297,9 +303,25 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity,
ITea
 
             //todo sinceBuild:(number:) // --todo -10 build numbers
 
+            Integer sinceBuildNumber = null;
+            if(persistedBuilds!=null) {
+                List<BuildRef> prevData = persistedBuilds.getData();
+                if (prevData.size() > 10) {
+                    BuildRef buildRef = prevData.get(prevData.size() - 10);
+
+                    if (!Strings.isNullOrEmpty(buildRef.buildNumber)) {
+                        try {
+                            sinceBuildNumber = Integer.valueOf(buildRef.buildNumber);
+                        } catch (NumberFormatException e) {
+                            logger.info("", e);
+                        }
+                    }
+                }
+            }
             List<BuildRef> dataFromRest;
             try {
-                dataFromRest = realLoad.apply(key);
+                dataFromRest = realLoad.apply(key, sinceBuildNumber);
+
             }
             catch (Exception e) {
                 if (Throwables.getRootCause(e) instanceof FileNotFoundException) {
@@ -353,11 +375,15 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity,
ITea
 
     /** {@inheritDoc} */
     @AutoProfiling
-    @Override public List<BuildRef> getFinishedBuilds(String projectId, String branch,
Long cnt) {
+    @Override public List<BuildRef> getFinishedBuilds(String projectId,
+                                                      String branch,
+                                                      Long cnt,
+                                                      Integer ignored) {
+        //todo may be support sinceBuildNo
         final SuiteInBranch suiteInBranch = new SuiteInBranch(projectId, branch);
 
         List<BuildRef> buildRefs = loadBuildHistory(buildHistCache(), 60, cnt, suiteInBranch,
-            (key) -> teamcity.getFinishedBuilds(projectId, branch, cnt));
+            (key, sinceBuildNumber) -> teamcity.getFinishedBuilds(projectId, branch, cnt,
sinceBuildNumber));
 
         if (cnt == null)
             return buildRefs;
@@ -385,11 +411,14 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity,
ITea
 
     /** {@inheritDoc} */
     @AutoProfiling
-    @Override public List<BuildRef> getFinishedBuildsIncludeSnDepFailed(String projectId,
String branch, Long cnt) {
+    @Override public List<BuildRef> getFinishedBuildsIncludeSnDepFailed(String projectId,
+                                                                        String branch,
+                                                                        Long cnt,
+                                                                        Integer ignored)
{
         final SuiteInBranch suiteInBranch = new SuiteInBranch(projectId, branch);
 
         return loadBuildHistory(buildHistIncFailedCache(), 60, cnt, suiteInBranch,
-            (key) -> teamcity.getFinishedBuildsIncludeSnDepFailed(projectId, branch, cnt));
+            (key, sinceBuildNumber) -> teamcity.getFinishedBuildsIncludeSnDepFailed(projectId,
branch, cnt, sinceBuildNumber));
     }
 
 
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityHelper.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityHelper.java
index ba8a7d6..871d9e9 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityHelper.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityHelper.java
@@ -468,30 +468,25 @@ public class IgniteTeamcityHelper implements ITeamcity {
     }
 
     private List<BuildRef> getBuildHistory(@Nullable String buildTypeId,
-        @Nullable String branchName,
-        boolean dfltFilter,
-        @Nullable String state){
-
-        return getBuildHistory(buildTypeId, branchName, dfltFilter, state, null);
-    }
-
-    private List<BuildRef> getBuildHistory(@Nullable String buildTypeId,
-        @Nullable String branchName,
-        boolean dfltFilter,
-        @Nullable String state,
-        @Nullable Long cnt) {
-        String btFilter = isNullOrEmpty(buildTypeId) ? "" : ",buildType:" + buildTypeId +
"";
+                                           @Nullable String branchName,
+                                           boolean dfltFilter,
+                                           @Nullable String state,
+                                           @Nullable Long cnt,
+                                           @Nullable Integer sinceBuildNumber) {
+        String btFilter = isNullOrEmpty(buildTypeId) ? "" : ",buildType:" + buildTypeId;
         String stateFilter = isNullOrEmpty(state) ? "" : (",state:" + state);
-        String branchFilter = isNullOrEmpty(branchName) ? "" :",branch:" + branchName;
+        String branchFilter = isNullOrEmpty(branchName) ? "" : ",branch:" + branchName;
+        String buildNoFilter = sinceBuildNumber == null ? "" : ",sinceBuild:(number:" + sinceBuildNumber
+ ")";
         long cntFilter = cnt == null ? DEFAULT_BUILDS_COUNT : cnt;
 
         return sendGetXmlParseJaxb(host + "app/rest/latest/builds"
-            + "?locator="
-            + "defaultFilter:" + dfltFilter
-            + btFilter
-            + stateFilter
-            + branchFilter
-            + ",count:" + cntFilter, Builds.class).getBuildsNonNull();
+                + "?locator="
+                + "defaultFilter:" + dfltFilter
+                + btFilter
+                + stateFilter
+                + branchFilter
+                + buildNoFilter
+                + ",count:" + cntFilter, Builds.class).getBuildsNonNull();
     }
 
     @AutoProfiling
@@ -566,18 +561,21 @@ public class IgniteTeamcityHelper implements ITeamcity {
     @Override public List<BuildRef> getFinishedBuilds(String projectId,
         String branch) {
 
-        return getFinishedBuilds(projectId, branch, null);
+        return getFinishedBuilds(projectId, branch, null, null);
     }
 
     /** {@inheritDoc} */
     @Override
     @AutoProfiling
     public List<BuildRef> getFinishedBuilds(String projectId,
-        String branch, Long cnt) {
+                                            String branch, Long cnt,
+                                            @Nullable Integer sinceBuildNumber) {
         List<BuildRef> finished = getBuildHistory(projectId,
             UrlUtil.escape(branch),
             true,
-            null, cnt);
+            null,
+                cnt,
+                sinceBuildNumber);
 
         return finished.stream().filter(BuildRef::isNotCancelled).collect(Collectors.toList());
     }
@@ -590,7 +588,7 @@ public class IgniteTeamcityHelper implements ITeamcity {
 
     /** {@inheritDoc} */
     @Override
-    @AutoProfiling public List<BuildRef> getFinishedBuildsIncludeSnDepFailed(String
projectId, String branch, Long cnt) {
+    @AutoProfiling public List<BuildRef> getFinishedBuildsIncludeSnDepFailed(String
projectId, String branch, Long cnt, Integer sinceBuildNumber) {
         return getBuildsInState(projectId, branch, BuildRef.STATE_FINISHED, cnt);
     }
 
@@ -614,7 +612,7 @@ public class IgniteTeamcityHelper implements ITeamcity {
         List<BuildRef> finished = getBuildHistory(projectId,
             UrlUtil.escape(branch),
             false,
-            state, cnt);
+            state, cnt, null);
         return finished.stream().filter(BuildRef::isNotCancelled).collect(Collectors.toList());
     }
 
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/IgniteTeamcityHelperRunnerExample.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/IgniteTeamcityHelperRunnerExample.java
index 92ff0c8..bdfc34a 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/IgniteTeamcityHelperRunnerExample.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/IgniteTeamcityHelperRunnerExample.java
@@ -19,7 +19,6 @@ package  org.apache.ignite.ci.runners;
 
 import com.google.common.base.Throwables;
 import java.io.File;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -30,6 +29,7 @@ import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.stream.Collectors;
+
 import org.apache.ignite.ci.IgniteTeamcityHelper;
 import org.apache.ignite.ci.tcmodel.conf.BuildType;
 import org.apache.ignite.ci.tcmodel.conf.bt.BuildTypeFull;
@@ -57,7 +57,7 @@ public class IgniteTeamcityHelperRunnerExample {
                 if (bt.getName().toLowerCase().contains("pds")
                     // || bt.getName().toLowerCase().contains("cache")
                     ) {
-                    int[] ints = helper.getBuildNumbersFromHistory(bt.getName(), branchNameForHist);
+                    int[] ints = helper.getBuildNumbersFromHistory(bt.getName(), branchNameForHist,
null);
 
                     List<CompletableFuture<File>> fileFutList = helper.standardProcessLogs(ints);
                     List<File> collect = getFuturesResults(fileFutList);


Mime
View raw message