ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dpav...@apache.org
Subject [ignite-teamcity-bot] branch ignite-10562 updated: IGNITE-10562 Change logic of build diverged handling: TC Bot DB contains invalid values for build references ID maps to value with other ID.
Date Thu, 06 Dec 2018 17:04:55 GMT
This is an automated email from the ASF dual-hosted git repository.

dpavlov pushed a commit to branch ignite-10562
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git


The following commit(s) were added to refs/heads/ignite-10562 by this push:
     new 68c0b05  IGNITE-10562 Change logic of build diverged handling: TC Bot DB contains
invalid values for build references ID maps to value with other ID.
68c0b05 is described below

commit 68c0b05bda5a13809b56d32aa180ac351d45f92d
Author: Dmitriy Pavlov <dpavlov@apache.org>
AuthorDate: Thu Dec 6 20:04:49 2018 +0300

    IGNITE-10562 Change logic of build diverged handling: TC Bot DB contains invalid values
for build references ID maps to value with other ID.
---
 .../ci/teamcity/ignited/BuildRefCompacted.java     | 12 +++-
 .../ci/teamcity/ignited/TeamcityIgnitedImpl.java   | 21 +++++--
 .../ignited/fatbuild/FatBuildCompacted.java        | 12 +++-
 .../ci/teamcity/ignited/fatbuild/FatBuildDao.java  | 30 ++-------
 .../ignited/fatbuild/ProactiveFatBuildSync.java    | 73 ++++++++++++++--------
 5 files changed, 91 insertions(+), 57 deletions(-)

diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java
index ada9317..599a4cb 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java
@@ -66,7 +66,7 @@ public class BuildRefCompacted {
         state = compactor.getStringId(ref.state());
     }
 
-    public void setId(Integer buildId) {
+    public void setId(@Nullable Integer buildId) {
         this.id = buildId == null ? -1 : buildId;
     }
 
@@ -158,11 +158,21 @@ public class BuildRefCompacted {
     }
 
     /** */
+    protected void status(int status) {
+        this.status = status;
+    }
+
+    /** */
     public int state() {
         return state;
     }
 
     /** */
+    public void state(int state) {
+        this.state = state;
+    }
+
+    /** */
     public boolean isFakeStub() {
         return id() < 0;
     }
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
index 8112ff5..f3125cb 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
@@ -386,7 +386,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
      * Enables scheduleing for build refs/builds/history sync
      */
     public void ensureActualizeRequested() {
-        scheduler.sheduleNamed(taskName("actualizeRecentBuildRefs"), this::actualizeRecentBuildRefs,
2, TimeUnit.MINUTES);
+        scheduler.sheduleNamed(taskName("actualizeRecentBuildRefs"), () -> actualizeRecentBuildRefs(srvNme),
2, TimeUnit.MINUTES);
 
         // schedule find missing later
         buildSync.ensureActualizationRequested(srvNme, conn);
@@ -488,11 +488,16 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
         return changes.values();
     }
 
+    String actualizeRecentBuildRefs() {
+        return actualizeRecentBuildRefs(srvNme);
+    }
 
     /**
      *
+     * @param srvNme TC service name
      */
-    void actualizeRecentBuildRefs() {
+    @MonitoredTask(name = "Prepare Actualize BuildRefs(srv, full resync)", nameExtArgsIndexes
= {0})
+    String actualizeRecentBuildRefs(String srvNme) {
         List<BuildRefCompacted> running = buildRefDao.getQueuedAndRunning(srvIdMaskHigh);
 
         Set<Integer> paginateUntil = new HashSet<>();
@@ -501,18 +506,22 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
         List<Integer> runningIds = running.stream().map(BuildRefCompacted::id).collect(Collectors.toList());
         OptionalInt max = runningIds.stream().mapToInt(i -> i).max();
         if (max.isPresent()) {
-            runningIds.forEach(id->{
-                if(id > (max.getAsInt() - MAX_ID_DIFF_TO_ENFORCE_CONTINUE_SCAN))
+            runningIds.forEach(id -> {
+                if (id > (max.getAsInt() - MAX_ID_DIFF_TO_ENFORCE_CONTINUE_SCAN))
                     paginateUntil.add(id);
                 else
                     directUpload.add(id);
             });
         }
+
+        int cntFreshBuilds = paginateUntil.size();
+
         //schedule direct reload for Fat Builds for all queued too-old builds
         buildSync.scheduleBuildsLoad(conn, directUpload);
 
         runActualizeBuildRefs(srvNme, false, paginateUntil);
 
+        int freshButNotFoundByBuildsRefsScan = paginateUntil.size();
         if (!paginateUntil.isEmpty()) {
             //some builds may stuck in the queued or running, enforce loading now
             buildSync.doLoadBuilds(-1, srvNme, conn, paginateUntil);
@@ -520,6 +529,10 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
 
         // schedule full resync later
         scheduler.invokeLater(this::sheduleResyncBuildRefs, 15, TimeUnit.MINUTES);
+
+        return "Build queue " + running.size() + ", relatively fresh: " + cntFreshBuilds
+
+             ", fresh but not found by scan: " + freshButNotFoundByBuildsRefsScan +
+            ", old builds sheduled " + directUpload.size();
     }
 
     /**
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
index 3f4ff2b..31cba27 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
@@ -506,7 +506,15 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
         return invocation;
     }
 
-    public void setVersion(short version) {
-        this._ver = version;
+    public void setVersion(short ver) {
+        this._ver = ver;
     }
+
+    public FatBuildCompacted setCancelled(IStringCompactor compactor) {
+        status(compactor.getStringId(BuildRef.STATUS_UNKNOWN));
+        state(compactor.getStringId(BuildRef.STATE_FINISHED));
+
+        return this;
+    }
+
 }
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
index b7257e1..6a3381d 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
@@ -23,8 +23,9 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
-import java.util.function.BiConsumer;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
 import javax.annotation.Nullable;
 import javax.cache.Cache;
 import javax.inject.Inject;
@@ -32,8 +33,6 @@ import javax.inject.Provider;
 import javax.validation.constraints.NotNull;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.query.QueryCursor;
-import org.apache.ignite.cache.query.SqlQuery;
 import org.apache.ignite.ci.db.TcHelperDb;
 import org.apache.ignite.ci.di.AutoProfiling;
 import org.apache.ignite.ci.tcmodel.changes.ChangesList;
@@ -123,12 +122,6 @@ public class FatBuildDao {
         buildsCache.put(buildIdToCacheKey(srvIdMaskHigh, buildId), newBuild);
     }
 
-
-    @AutoProfiling
-    public boolean removeFatBuild(int srvIdMaskHigh, int buildId) {
-       return  buildsCache.remove(buildIdToCacheKey(srvIdMaskHigh, buildId));
-    }
-
     public static int[] extractChangeIds(@NotNull ChangesList changesList) {
         return changesList.changes().stream().mapToInt(
                         ch -> {
@@ -188,20 +181,9 @@ public class FatBuildDao {
         return buildsCache.containsKey(buildIdToCacheKey(srvIdMaskHigh, buildId));
     }
 
-    public void forEachBuildsV5(int srvId, BiConsumer<Long, FatBuildCompacted> processor)
{
-        try (QueryCursor<Cache.Entry<Long, FatBuildCompacted>> qryCursor  = buildsCache.query(
-            new SqlQuery<Long, FatBuildCompacted>(FatBuildCompacted.class, "_ver <
?")
-                .setArgs(FatBuildCompacted.LATEST_VERSION))) {
-
-            for (Cache.Entry<Long, FatBuildCompacted> next : qryCursor) {
-                Long key = next.getKey();
-
-                if (!isKeyForServer(key, srvId))
-                    continue;
-
-                processor.accept(next.getKey(), next.getValue());
-            }
-        }
-
+    public Stream<Cache.Entry<Long, FatBuildCompacted>> outdatedVersionEntries(int
srvId) {
+        return StreamSupport.stream(buildsCache.spliterator(), false)
+            .filter(entry -> entry.getValue().isOutdatedEntityVersion())
+            .filter(entry -> isKeyForServer(entry.getKey(), srvId));
     }
 }
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
index a77b899..b41ed9b 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
@@ -16,11 +16,9 @@
  */
 package org.apache.ignite.ci.teamcity.ignited.fatbuild;
 
-import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
 import com.google.common.base.Throwables;
 import java.util.stream.Stream;
-import org.apache.ignite.ci.ITeamcity;
 import org.apache.ignite.ci.di.AutoProfiling;
 import org.apache.ignite.ci.di.MonitoredTask;
 import org.apache.ignite.ci.di.scheduler.IScheduler;
@@ -245,22 +243,36 @@ public class ProactiveFatBuildSync {
      */
     public void ensureActualizationRequested(String srvName, ITeamcityConn conn) {
         scheduler.sheduleNamed(taskName("findMissingBuildsFromBuildRef", srvName),
-                () -> findMissingBuildsFromBuildRef(srvName, conn), 360, TimeUnit.MINUTES);
+            () -> findMissingBuildsFromBuildRef(srvName, conn), 360, TimeUnit.MINUTES);
 
         scheduler.sheduleNamed(taskName("migrateBuildsToV6", srvName),
             () -> migrateBuildsToV6(srvName, conn), 8, TimeUnit.HOURS);
     }
 
+    /**
+     * @param srvName Server name.
+     * @param conn Connection.
+     */
     @MonitoredTask(name = "Migrate Builds to V6", nameExtArgsIndexes = {0})
     public String migrateBuildsToV6(String srvName, ITeamcityConn conn) {
         int srvId = ITeamcityIgnited.serverIdToInt(srvName);
 
         AtomicInteger cnt = new AtomicInteger();
-        fatBuildDao.forEachBuildsV5(srvId, (id, fatBuild) -> {
+        AtomicInteger divergedIds = new AtomicInteger();
+        fatBuildDao.outdatedVersionEntries(srvId).forEach(entry -> {
             cnt.incrementAndGet();
+            int buildId = BuildRefDao.cacheKeyToBuildId(entry.getKey());
+            FatBuildCompacted transformed = transformV5Build(
+                srvId,
+                buildId,
+                entry.getValue());
+
+            if (transformed != null)
+                divergedIds.incrementAndGet();
         });
 
-        return cnt.get() + " builds found having version < 6";
+        return "Found: " + cnt.get() + " builds found having version < 6 and "
+            + divergedIds.get() + " with ID divergence.";
     }
 
     /**
@@ -320,23 +332,8 @@ public class ProactiveFatBuildSync {
         final int srvIdMask = ITeamcityIgnited.serverIdToInt(srvName);
 
         if (existingBuild != null && existingBuild.isOutdatedEntityVersion()) {
-            int ver = existingBuild.version();
-            if (ver == FatBuildCompacted.VER_FULL_DATA_BUT_ID_CONFLICTS_POSSIBLE) {
-                if (Objects.equals(buildId, existingBuild.id())) {
-                    existingBuild.setVersion(FatBuildCompacted.LATEST_VERSION);
-
-                    fatBuildDao.putFatBuild(srvIdMask, buildId, existingBuild);
-
-                    return null;
-                }
-                else {
-                    logger.warn("Build inconsistency found in the DB, removing build " +
existingBuild.getId());
-
-                    existingBuild = null;
-
-                    fatBuildDao.removeFatBuild(srvIdMask, buildId);
-                }
-            }
+            if (existingBuild.version() == FatBuildCompacted.VER_FULL_DATA_BUT_ID_CONFLICTS_POSSIBLE)
+                return transformV5Build(srvIdMask, buildId, existingBuild);
         }
 
         Build build;
@@ -349,9 +346,11 @@ public class ProactiveFatBuildSync {
 
             if (build.isFakeStub())
                 build.setCancelled(); // probably now it will not happen because of direct
connection to TC.
-            else
-                Preconditions.checkState(Objects.equals(build.getId(), buildId),
-                    "Build IDs are not consistent: returned " + build.getId() + " queued
is " + buildId);
+            else {
+                if(!Objects.equals(build.getId(), buildId))
+                    throw new FileNotFoundException(
+                        "Build IDs are not consistent: returned " + build.getId() + " queued
is " + buildId);
+            }
 
             if (build.testOccurrences != null && !build.isComposite()) { // don't
query tests for compoite
                 String nextHref = null;
@@ -397,7 +396,7 @@ public class ProactiveFatBuildSync {
 
                     problems = existingBuild.problems(compactor);
 
-                    //todo extract new parameters or save fat build without XML
+                    //todo extract new parameters or save fat build without XML convertions
                     // - existingBuild.statistics();
                     // - int[] changes = existingBuild.changes();
                 }
@@ -419,4 +418,26 @@ public class ProactiveFatBuildSync {
         // new save will be performed with new entity version for compacted build
         return fatBuildDao.saveBuild(srvIdMask, buildId, build, tests, problems, statistics,
changesList, existingBuild);
     }
+
+    @Nullable
+    public FatBuildCompacted transformV5Build(int srvIdMask, int buildId, @NotNull FatBuildCompacted
existingBuild) {
+        if (Objects.equals(buildId, existingBuild.id())) {
+            existingBuild.setVersion(FatBuildCompacted.LATEST_VERSION);
+
+            fatBuildDao.putFatBuild(srvIdMask, buildId, existingBuild);
+
+            return null;
+        }
+        else {
+            logger.warn("Build inconsistency found in the DB, removing build " + existingBuild.getId());
+
+            FatBuildCompacted buildCompacted = new FatBuildCompacted()
+                .setFakeStub(true)
+                .setCancelled(compactor);
+
+            fatBuildDao.putFatBuild(srvIdMask, buildId, buildCompacted);
+
+            return buildCompacted;
+        }
+    }
 }


Mime
View raw message