ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dpav...@apache.org
Subject [ignite-teamcity-bot] branch ignite-9542-new-run-stripe updated: IGNITE-9542 Full missed run history search implemented
Date Tue, 27 Nov 2018 15:48:59 GMT
This is an automated email from the ASF dual-hosted git repository.

dpavlov pushed a commit to branch ignite-9542-new-run-stripe
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git


The following commit(s) were added to refs/heads/ignite-9542-new-run-stripe by this push:
     new da0ca86  IGNITE-9542 Full missed run history search implemented
da0ca86 is described below

commit da0ca866b3ff32dc791d290040f82faed936c449
Author: Dmitriy Pavlov <dpavlov@apache.org>
AuthorDate: Tue Nov 27 18:48:58 2018 +0300

    IGNITE-9542 Full missed run history search implemented
---
 .../ci/teamcity/ignited/TeamcityIgnitedImpl.java   | 19 ++---
 .../ci/teamcity/ignited/buildref/BuildRefSync.java | 23 ++++++
 .../ci/teamcity/ignited/fatbuild/FatBuildDao.java  |  2 +-
 .../ignited/fatbuild/ProactiveFatBuildSync.java    |  1 -
 .../teamcity/ignited/runhist/RunHistCompacted.java |  5 ++
 .../ignited/runhist/RunHistCompactedDao.java       |  4 --
 .../ci/teamcity/ignited/runhist/RunHistSync.java   | 84 ++++++++++++++++++++--
 .../ignited/IgnitedTcInMemoryIntegrationTest.java  | 63 ++++++++++++----
 8 files changed, 168 insertions(+), 33 deletions(-)

diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
index 77a0893..e1022cd 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java
@@ -95,6 +95,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
     /** Run history DAO. */
     @Inject private RunHistCompactedDao runHistCompactedDao;
 
+    /** Run history sync. */
     @Inject private RunHistSync runHistSync;
 
     /** Strings compactor. */
@@ -331,6 +332,11 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
 
     public void ensureActualizeRequested() {
         scheduler.sheduleNamed(taskName("actualizeRecentBuildRefs"), this::actualizeRecentBuildRefs,
2, TimeUnit.MINUTES);
+
+        // schedule find missing later
+        buildSync.invokeLaterFindMissingByBuildRef(srvNme, conn);
+
+        runHistSync.invokeLaterFindMissingHistory(srvNme);
     }
 
     /** {@inheritDoc} */
@@ -397,13 +403,13 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
         FatBuildCompacted savedVer = buildSync.loadBuild(conn, buildId, existingBuild, mode);
 
         //build was modified, probably we need also to update reference accordingly
-        if (savedVer != null)
-            buildRefDao.save(srvIdMaskHigh, new BuildRefCompacted(savedVer));
+        if (savedVer == null)
+            return existingBuild;
 
-        if (savedVer != null)
-            runHistSync.saveToHistoryLater(srvIdMaskHigh, buildId, savedVer);
+        buildRefDao.save(srvIdMaskHigh, new BuildRefCompacted(savedVer));
+        runHistSync.saveToHistoryLater(srvIdMaskHigh, buildId, savedVer);
 
-        return savedVer == null ? existingBuild : savedVer;
+        return savedVer;
     }
 
     protected FatBuildCompacted getFatBuildFromIgnite(int buildId) {
@@ -442,9 +448,6 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited {
      *
      */
     void actualizeRecentBuildRefs() {
-        // schedule find missing later
-        buildSync.invokeLaterFindMissingByBuildRef(srvNme, conn);
-
         List<BuildRefCompacted> running = buildRefDao.getQueuedAndRunning(srvIdMaskHigh);
 
         Set<Integer> paginateUntil = new HashSet<>();
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/buildref/BuildRefSync.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/buildref/BuildRefSync.java
new file mode 100644
index 0000000..fce67c7
--- /dev/null
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/buildref/BuildRefSync.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.ci.teamcity.ignited.buildref;
+
+/**
+ *
+ */
+public class BuildRefSync {
+}
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
index aa34f3f..2c4df80 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildDao.java
@@ -113,7 +113,7 @@ public class FatBuildDao {
     }
 
     @AutoProfiling
-    protected void putFatBuild(long srvIdMaskHigh, int buildId, FatBuildCompacted newBuild)
{
+    public void putFatBuild(long srvIdMaskHigh, int buildId, FatBuildCompacted newBuild)
{
         buildsCache.put(buildIdToCacheKey(srvIdMaskHigh, buildId), newBuild);
     }
 
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
index 8c95421..6a63289 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/ProactiveFatBuildSync.java
@@ -229,7 +229,6 @@ public class ProactiveFatBuildSync {
                 () -> findMissingBuildsFromBuildRef(srvName, conn), 360, TimeUnit.MINUTES);
     }
 
-
     /**
      *
      * @param conn TC connection to load data
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompacted.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompacted.java
index 96adae4..f93d618 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompacted.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompacted.java
@@ -20,11 +20,16 @@ package org.apache.ignite.ci.teamcity.ignited.runhist;
 import com.google.common.base.MoreObjects;
 import org.apache.ignite.cache.query.annotations.QuerySqlField;
 import org.apache.ignite.ci.analysis.IVersionedEntity;
+import org.apache.ignite.ci.db.Persisted;
 import org.apache.ignite.ci.teamcity.ignited.IRunHistory;
 
 import javax.annotation.Nullable;
 import java.util.List;
 
+/**
+ *
+ */
+@Persisted
 public class RunHistCompacted implements IVersionedEntity, IRunHistory {
     /** Latest version. */
     private static final int LATEST_VERSION = 1;
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompactedDao.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompactedDao.java
index c2d1e5d..46768db 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompactedDao.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompactedDao.java
@@ -60,10 +60,6 @@ public class RunHistCompactedDao {
         testHistCache = ignite.getOrCreateCache(cfg);
     }
 
-    public void save(RunHistKey k, RunHistCompacted v) {
-        testHistCache.put(k, v);
-    }
-
     public IRunHistory getTestRunHist(int srvIdMaskHigh, String name, String branch) {
         final Integer testName = compactor.getStringIdIfPresent(name);
         if (testName == null)
diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistSync.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistSync.java
index aefaf61..1408f80 100644
--- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistSync.java
+++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistSync.java
@@ -17,20 +17,26 @@
 
 package org.apache.ignite.ci.teamcity.ignited.runhist;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.ignite.ci.di.AutoProfiling;
 import org.apache.ignite.ci.di.MonitoredTask;
 import org.apache.ignite.ci.di.scheduler.IScheduler;
+import org.apache.ignite.ci.teamcity.ignited.BuildRefDao;
 import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
+import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited;
 import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
 import org.apache.ignite.internal.util.GridConcurrentHashSet;
 
 import javax.annotation.concurrent.GuardedBy;
 import javax.inject.Inject;
 import java.util.HashMap;
 import java.util.Map;
+import org.jetbrains.annotations.NotNull;
 
 /**
  *
@@ -41,7 +47,14 @@ public class RunHistSync {
 
     @Inject private IScheduler scheduler;
 
-    @Inject private RunHistCompactedDao dao;
+    @Inject private RunHistCompactedDao histDao;
+
+    /** Build reference DAO. */
+    @Inject private BuildRefDao buildRefDao;
+
+    /** Build DAO. */
+    @Inject private FatBuildDao fatBuildDao;
+
 
     /** Build to save to history. */
     @GuardedBy("this")
@@ -53,10 +66,7 @@ public class RunHistSync {
      * @param build Build.
      */
     public void saveToHistoryLater(int srvId, int buildId, FatBuildCompacted build) {
-        if (!build.isFinished(compactor))
-            return;
-
-        if (build.isCancelled(compactor))
+        if (!validForStatistics(build))
             return;
 
         synchronized (this) {
@@ -95,7 +105,7 @@ public class RunHistSync {
                 build.getAllTests().forEach(t -> {
                     Invocation inv = t.toInvocation(compactor, build);
 
-                    final Boolean res = dao.addInvocation(srvId, t, build.id(), build.branchName(),
inv);
+                    final Boolean res = histDao.addInvocation(srvId, t, build.id(), build.branchName(),
inv);
 
                     if(Boolean.FALSE.equals(res))
                         duplicates.incrementAndGet();
@@ -109,6 +119,68 @@ public class RunHistSync {
                 + " invocations saved to DB " + duplicates.get() + " duplicates";
     }
 
+    public void invokeLaterFindMissingHistory(String srvName) {
+            scheduler.sheduleNamed(taskName("findMissingHistFromBuildRef", srvName),
+                () -> findMissingHistFromBuildRef(srvName), 360, TimeUnit.MINUTES);
+    }
+
+    @NotNull
+    private String taskName(String taskName, String srvName) {
+        return RunHistSync.class.getSimpleName() +"." + taskName + "." + srvName;
+    }
+
+    @SuppressWarnings({"WeakerAccess", "UnusedReturnValue"})
+    @MonitoredTask(name = "Find missing Build History", nameExtArgsIndexes = {0})
+    @AutoProfiling
+    protected String findMissingHistFromBuildRef(String srvId ) {
+        int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvId);
+
+        final int[] buildRefKeys = buildRefDao.getAllIds(srvIdMaskHigh);
+
+        List<Integer> buildsIdsToLoad = new ArrayList<>();
+        int totalAskedToLoad = 0;
+
+        for (int buildRefKey : buildRefKeys) {
+            if (!fatBuildDao.containsKey(srvIdMaskHigh, buildRefKey))
+                continue; // FAT Build not yet avaiable, skipping hist loading
+
+            if (buildsIdsToLoad.size() >= 100) {
+                totalAskedToLoad += buildsIdsToLoad.size();
+                scheduleHistLoad(srvIdMaskHigh,  buildsIdsToLoad);
+                buildsIdsToLoad.clear();
+            }
+            buildsIdsToLoad.add(buildRefKey);
+        }
+
+        if (!buildsIdsToLoad.isEmpty()) {
+            totalAskedToLoad += buildsIdsToLoad.size();
+            scheduleHistLoad(srvIdMaskHigh, buildsIdsToLoad);
+        }
+
+        return "Invoked later load for " + totalAskedToLoad + " builds from " + srvId;
+    }
+
+    private void scheduleHistLoad(int srvIdMaskHigh, List<Integer> load) {
+        //todo implement
+        System.err.println("scheduleHistLoad: " + load.toString());
+
+        load.forEach(id -> {
+            FatBuildCompacted fatBuild = fatBuildDao.getFatBuild(srvIdMaskHigh, id);
+
+            if (validForStatistics(fatBuild))
+                saveToHistoryLater(srvIdMaskHigh, fatBuild.id(), fatBuild);
+            else
+                System.err.println("Build is not valid for stat: " + fatBuild.toString());
+        });
+    }
+
+    private boolean validForStatistics(FatBuildCompacted fatBuild) {
+        return fatBuild != null
+            && !fatBuild.isFakeStub()
+            && !fatBuild.isCancelled(compactor)
+            && fatBuild.isFinished(compactor);
+    }
+
     /**
      * Scope of work: builds to be loaded from a connection.
      */
diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
index dbbc09c..0a5e013 100644
--- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
+++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/teamcity/ignited/IgnitedTcInMemoryIntegrationTest.java
@@ -348,10 +348,8 @@ public class IgnitedTcInMemoryIntegrationTest {
 
     @Test
     public void testRunHistSaveLoad() {
-        final String srvId = "apache";
 
-        TeamcityIgnitedModule module = new TeamcityIgnitedModule();
-        Injector injector = Guice.createInjector(module, new AbstractModule() {
+        Injector injector = Guice.createInjector(new TeamcityIgnitedModule(), new AbstractModule()
{
             @Override
             protected void configure() {
                 bind(Ignite.class).toInstance(ignite);
@@ -359,11 +357,10 @@ public class IgnitedTcInMemoryIntegrationTest {
             }
         });
 
+        injector.getInstance(RunHistCompactedDao.class).init();
         final IStringCompactor c = injector.getInstance(IStringCompactor.class);
 
-        final RunHistCompactedDao dao = injector.getInstance(RunHistCompactedDao.class);
-        dao.init();
-
+        final String srvId = "apache";
         final String btId = "RunAll";
         final String branch = ITeamcity.DEFAULT;
 
@@ -372,21 +369,61 @@ public class IgnitedTcInMemoryIntegrationTest {
 
         final Map<Integer, FatBuildCompacted> buildsMap = tst.apacheBuilds();
 
-        Map<RunHistKey, RunHistCompacted> resMap = new HashMap<>();
+        final RunHistSync histSync = injector.getInstance(RunHistSync.class);
+        buildsMap.forEach((id, build) -> histSync.saveToHistoryLater(ITeamcityIgnited.serverIdToInt(srvId),
id, build));
 
-        TeamcityIgnitedMock.initHistory(c, resMap, buildsMap, U.safeAbs(srvId.hashCode()));
+        final ITeamcityIgnitedProvider inst = injector.getInstance(ITeamcityIgnitedProvider.class);
+        final ITeamcityIgnited srv = inst.server(srvId, Mockito.mock(ICredentialsProv.class));
+        final IRunHistory testRunHist = srv.getTestRunHist(new TestInBranch(PrChainsProcessorTest.TEST_FLAKY_IN_MASTER,
branch));
 
-        System.out.println(resMap);
+        assertNotNull(testRunHist);
+        assertEquals(0.5, testRunHist.getFailRate(), 0.1);
+    }
 
-       // resMap.forEach(dao::save);
 
-        final RunHistSync histSync = injector.getInstance(RunHistSync.class);
-        buildsMap.forEach((id, build) -> {
-            histSync.saveToHistoryLater(U.safeAbs(srvId.hashCode()), id, build);
+
+    @Test
+    public void testHistoryBackgroundUpdateWorks() {
+        Injector injector = Guice.createInjector(new TeamcityIgnitedModule(), new AbstractModule()
{
+            @Override
+            protected void configure() {
+                bind(Ignite.class).toInstance(ignite);
+                bind(IScheduler.class).to(DirectExecNoWaitSheduler.class).in(new SingletonScope());
+            }
         });
 
+        injector.getInstance(RunHistCompactedDao.class).init();
+
+        final String srvId = "apache";
+        final String btId = "RunAll";
+        final String branch = ITeamcity.DEFAULT;
+
         final ITeamcityIgnitedProvider inst = injector.getInstance(ITeamcityIgnitedProvider.class);
         final ITeamcityIgnited srv = inst.server(srvId, Mockito.mock(ICredentialsProv.class));
+
+        FatBuildDao fatBuildDao = injector.getInstance(FatBuildDao.class);
+        fatBuildDao.init();
+
+        BuildRefDao buildRefDao = injector.getInstance(BuildRefDao.class);
+        buildRefDao.init();
+
+        final IStringCompactor c = injector.getInstance(IStringCompactor.class);
+
+
+        final PrChainsProcessorTest tst = new PrChainsProcessorTest();
+        tst.initBuildChain(c, btId, branch);
+
+        final Map<Integer, FatBuildCompacted> buildsMap = tst.apacheBuilds();
+
+        buildsMap.forEach((id, build) -> {
+            int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvId);
+            fatBuildDao.putFatBuild(srvIdMaskHigh, id, build);
+            buildRefDao.save(srvIdMaskHigh, new BuildRefCompacted(build));
+        });
+
+        final RunHistSync histSync = injector.getInstance(RunHistSync.class);
+        histSync.invokeLaterFindMissingHistory(srvId);
+
         final IRunHistory testRunHist = srv.getTestRunHist(new TestInBranch(PrChainsProcessorTest.TEST_FLAKY_IN_MASTER,
branch));
 
         assertNotNull(testRunHist);


Mime
View raw message