ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dpav...@apache.org
Subject [ignite-teamcity-bot] 01/01: IGNITE-10562 TC Bot DB contains invalid values for build references ID maps to value with other ID.
Date Thu, 06 Dec 2018 10:49:10 GMT
This is an automated email from the ASF dual-hosted git repository.

dpavlov pushed a commit to branch ignite-10562
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git

commit fa585895216942a865f286fb826a5a0866782dec
Author: Dmitriy Pavlov <dpavlov@apache.org>
AuthorDate: Thu Dec 6 13:48:49 2018 +0300

    IGNITE-10562 TC Bot DB contains invalid values for build references ID maps to value with
other ID.
---
 .../main/java/org/apache/ignite/ci/ITeamcity.java  | 10 +---------
 .../apache/ignite/ci/IgnitePersistentTeamcity.java |  6 ++++++
 .../apache/ignite/ci/IgniteTeamcityConnection.java |  6 ++++++
 .../apache/ignite/ci/runners/ClientTmpHelper.java  | 14 +++++++++++--
 .../ignite/ci/runners/RemoteClientTmpHelper.java   | 21 ++++++++++++++------
 .../ci/tcbot/builds/CompareBuildsService.java      |  2 +-
 .../ci/teamcity/ignited/TeamcityIgnitedImpl.java   |  6 ++----
 .../ignited/fatbuild/FatBuildCompacted.java        | 18 +++++++++++++++--
 .../ci/teamcity/ignited/fatbuild/FatBuildDao.java  |  6 ++++++
 .../ignited/fatbuild/ProactiveFatBuildSync.java    | 23 ++++++++++++++++++++--
 10 files changed, 86 insertions(+), 26 deletions(-)

diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
index 62332e1..80ed20d 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/ITeamcity.java
@@ -68,15 +68,7 @@ public interface ITeamcity extends ITeamcityConn {
     @Deprecated
     Build getBuild(String href);
 
-    default Build getBuild(int buildId) {
-        return getBuild(getBuildHrefById(buildId));
-    }
-
-    @NotNull default String getBuildHrefById(int id) {
-        return buildHref(id);
-    }
-
-    @NotNull static String buildHref(int id) {
+    @NotNull public static String buildHref(int id) {
         return "app/rest/latest/builds/id:" + id;
     }
 
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
index 4b256ce..88f486c 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgnitePersistentTeamcity.java
@@ -325,6 +325,7 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity,
ITea
         return new SuiteInBranch(loaded.suiteId(), normalizeBranch(loaded.branchName));
     }
 
+    @Deprecated
     private Build realLoadBuild(String href1) {
         try {
             return teamcity.getBuild(href1);
@@ -352,6 +353,11 @@ public class IgnitePersistentTeamcity implements IAnalyticsEnabledTeamcity,
ITea
         return teamcity.host();
     }
 
+    /** {@inheritDoc} */
+    @Override public Build getBuild(int buildId) {
+        return teamcity.getBuild(buildId);
+    }
+
     @Deprecated
     private void registerCriticalBuildProblemInStat(BuildRef build, ProblemOccurrences problems)
{
         boolean criticalFail = problems.getProblemsNonNull().stream().anyMatch(occurrence
->
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
index ce5a319..40e5c3a 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/IgniteTeamcityConnection.java
@@ -446,6 +446,12 @@ public class IgniteTeamcityConnection implements ITeamcity {
         return getJaxbUsingHref(href, Build.class);
     }
 
+    /** {@inheritDoc} */
+    @AutoProfiling
+    @Override public Build getBuild(int buildId) {
+        return getJaxbUsingHref(ITeamcity.buildHref(buildId), Build.class);
+    }
+
     /**
      * @param href Href.
      * @param elem Element class.
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/ClientTmpHelper.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/ClientTmpHelper.java
index 14207a0..f05804f 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/ClientTmpHelper.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/ClientTmpHelper.java
@@ -28,14 +28,24 @@ import org.apache.ignite.ci.user.UserAndSessionsStorage;
  * Utility class for local connection to TC helper DB (server) and any manipulations with
data needed.
  */
 public class ClientTmpHelper {
+    /**
+     * @param args Args.
+     */
     public static void main(String[] args) {
-        try (Ignite ignite = TcHelperDb.startClient()){
+        int inconsistent;
+        try (Ignite ignite = TcHelperDb.startClient()) {
             RemoteClientTmpHelper.DUMPS = "dumpsLocal";
 
-            RemoteClientTmpHelper.validateBuildIdConsistency(ignite);
+            inconsistent = RemoteClientTmpHelper.validateBuildIdConsistency(ignite);
         }
+
+        System.err.println("Inconsistent builds in queue found [" +
+            +inconsistent + "]");
     }
 
+    /**
+     * @param args Args.
+     */
     public static void main0(String[] args) {
         Ignite ignite = TcHelperDb.startClient();
 
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/RemoteClientTmpHelper.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/RemoteClientTmpHelper.java
index 07c7ef9..772c62f 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/RemoteClientTmpHelper.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/runners/RemoteClientTmpHelper.java
@@ -23,6 +23,7 @@ import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.util.concurrent.atomic.AtomicInteger;
 import javax.cache.Cache;
 import javax.xml.bind.JAXBException;
 import org.apache.ignite.Ignite;
@@ -55,7 +56,8 @@ public class RemoteClientTmpHelper {
         }
     }
 
-    public static void validateBuildIdConsistency(Ignite ignite) {
+    public static int validateBuildIdConsistency(Ignite ignite) {
+        AtomicInteger inconsistent = new AtomicInteger();
         String apacheSrvName = "apache";
         int apache = ITeamcityIgnited.serverIdToInt(apacheSrvName);
 
@@ -65,8 +67,8 @@ public class RemoteClientTmpHelper {
         IgniteCache<Long, BuildRefCompacted> cacheRef = ignite.cache(BuildRefDao.TEAMCITY_BUILD_CACHE_NAME);
         IgniteCache<Long, FatBuildCompacted> cacheFat = ignite.cache(FatBuildDao.TEAMCITY_FAT_BUILD_CACHE_NAME);
 
-        String oldBuild = IgnitePersistentTeamcity.ignCacheNme(IgnitePersistentTeamcity.BUILDS,
apacheSrvName);
-        IgniteCache<String, Build> buildCache = ignite.cache(oldBuild);
+        String cacheOldBuild = IgnitePersistentTeamcity.ignCacheNme(IgnitePersistentTeamcity.BUILDS,
apacheSrvName);
+        IgniteCache<String, Build> buildCache = ignite.cache(cacheOldBuild);
         cacheRef.forEach(
             entry -> {
                 BuildRefCompacted buildRef = entry.getValue();
@@ -80,13 +82,20 @@ public class RemoteClientTmpHelper {
                         dumpBuildRef(buildId, buildRef);
                         dumpFatBuild(cacheFat, apache, buildId);
                         String href = ITeamcity.buildHref(buildId);
-                        Build fatBuild = buildCache.get(href);
-                        if (fatBuild != null)
-                            dumpOldBuild(buildId, href, fatBuild);
+                        Build oldBuild = buildCache.get(href);
+                        if (oldBuild != null)
+                            dumpOldBuild(buildId, href, oldBuild);
+
+                        inconsistent.incrementAndGet();
+
+                        if(!fat.isOutdatedEntityVersion())
+                            Preconditions.checkState(false, oldBuild);
                     }
                 }
             }
         );
+
+        return inconsistent.get();
     }
 
     public static void dumpOldBuild(int buildId, String href, Build fatBuild) {
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java
index 8ca0b24..f513809 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java
@@ -53,7 +53,7 @@ public class CompareBuildsService {
     public List<String> tests0(String srvId, Integer buildId, ICredentialsProv prov
) {
         IAnalyticsEnabledTeamcity teamcity = helper.server(srvId, prov);
 
-        String hrefById = teamcity.getBuildHrefById(buildId);
+        String hrefById = ITeamcity.buildHref(buildId);
         BuildRef buildRef = new BuildRef();
 
         buildRef.setId(buildId);
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
index 1d0b4bc..0a2ed37 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
@@ -441,10 +441,8 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
         FatBuildCompacted existingBuild = getFatBuildFromIgnite(buildId);
 
         if (mode == SyncMode.NONE) {
-            if (existingBuild != null)
-                return existingBuild;
-            else
-                return new FatBuildCompacted().setFakeStub(true); // providing fake builds
+            // providing fake builds
+            return existingBuild != null ? existingBuild : new FatBuildCompacted().setFakeStub(true);
         }
 
         FatBuildCompacted savedVer = buildSync.loadBuild(conn, buildId, existingBuild, mode);
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
index 1a50a5d..3f4ff2b 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java
@@ -50,7 +50,10 @@ import java.util.stream.Stream;
 @Persisted
 public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEntity {
     /** Latest version. */
-    private static final int LATEST_VERSION = 5;
+    public static final short LATEST_VERSION = 6;
+
+    /** Latest version. */
+    public static final short VER_FULL_DATA_BUT_ID_CONFLICTS_POSSIBLE = 5;
 
     /** Default branch flag offset. */
     public static final int DEF_BR_F = 0;
@@ -66,7 +69,14 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
 
     public static final int[] EMPTY = new int[0];
 
-    /** Entity fields version. */
+    /**
+     * Entity fields version.
+     * <ul>
+     * <li>{@link #VER_FULL_DATA_BUT_ID_CONFLICTS_POSSIBLE} - fully supported field
set, tests, problems. </li>
+     * <li>6 - done double check if build ID is consistent with a key. If this check
passes, version is set to 6, if
+     * not-build is deleted.</li>
+     * </ul>
+     */
     private short _ver = LATEST_VERSION;
 
     /** Start date. The number of milliseconds since January 1, 1970, 00:00:00 GMT */
@@ -495,4 +505,8 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn
         invocation.changesPresent(changes().length > 0 ? 1 : 0);
         return invocation;
     }
+
+    public void setVersion(short version) {
+        this._ver = version;
+    }
 }
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
index 9979bed..ccafc68 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
@@ -119,6 +119,12 @@ public class FatBuildDao {
         buildsCache.put(buildIdToCacheKey(srvIdMaskHigh, buildId), newBuild);
     }
 
+
+    @AutoProfiling
+    public boolean removeFatBuild(int srvIdMaskHigh, int buildId) {
+       return  buildsCache.remove(buildIdToCacheKey(srvIdMaskHigh, buildId));
+    }
+
     public static int[] extractChangeIds(@NotNull ChangesList changesList) {
         return changesList.changes().stream().mapToInt(
                         ch -> {
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
index 753a12f..e4a4439 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
@@ -16,6 +16,7 @@
  */
 package org.apache.ignite.ci.teamcity.ignited.fatbuild;
 
+import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
 import com.google.common.base.Throwables;
 import java.util.stream.Stream;
@@ -300,6 +301,21 @@ public class ProactiveFatBuildSync {
         final String srvName = conn.serverId();
         final int srvIdMask = ITeamcityIgnited.serverIdToInt(srvName);
 
+        if (existingBuild != null && existingBuild.isOutdatedEntityVersion()) {
+            int ver = existingBuild.version();
+            if (ver == FatBuildCompacted.VER_FULL_DATA_BUT_ID_CONFLICTS_POSSIBLE) {
+                if (Objects.equals(buildId, existingBuild.id()))
+                    existingBuild.setVersion(FatBuildCompacted.LATEST_VERSION);
+                else {
+                    logger.warn("Build inconsistency found in the DB, removing build " +
existingBuild.getId());
+
+                    existingBuild = null;
+
+                    fatBuildDao.removeFatBuild(srvIdMask, buildId);
+                }
+            }
+        }
+
         Build build;
         List<TestOccurrencesFull> tests = new ArrayList<>();
         List<ProblemOccurrence> problems = null;
@@ -309,9 +325,12 @@ public class ProactiveFatBuildSync {
             build = conn.getBuild(buildId);
 
             if (build.isFakeStub())
-                build.setCancelled();
+                build.setCancelled(); // probably now it will not happen because of direct
connection to TC.
+            else
+                Preconditions.checkState(Objects.equals(build.getId(), buildId),
+                    "Build IDs are not consistent: returned " + build.getId() + " queued
is " + buildId);
 
-            if(build.testOccurrences != null && !build.isComposite()) { // don't
query tests for compoite
+            if (build.testOccurrences != null && !build.isComposite()) { // don't
query tests for compoite
                 String nextHref = null;
                 do {
                     TestOccurrencesFull page = conn.getTestsPage(buildId, nextHref, true);


Mime
View raw message