aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jco...@apache.org
Subject aurora git commit: Revert "Improving job update query performance."
Date Thu, 28 Jan 2016 22:19:40 GMT
Repository: aurora
Updated Branches:
  refs/heads/master 165a617f8 -> fbd29a176


Revert "Improving job update query performance."

This reverts commit fee5943a95c4f08e148dc5f1366486a8c23d5773.

We discovered a bug when deploying this commit that caused corruption of the update store.

Reviewed at https://reviews.apache.org/r/42922/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/fbd29a17
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/fbd29a17
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/fbd29a17

Branch: refs/heads/master
Commit: fbd29a176c38ce9bac643794daa217056138c448
Parents: 165a617
Author: Joshua Cohen <jcohen@apache.org>
Authored: Thu Jan 28 16:19:14 2016 -0600
Committer: Joshua Cohen <jcohen@apache.org>
Committed: Thu Jan 28 16:19:14 2016 -0600

----------------------------------------------------------------------
 .../org/apache/aurora/benchmark/JobUpdates.java | 31 +------
 .../aurora/benchmark/UpdateStoreBenchmarks.java | 92 +++++---------------
 .../storage/db/JobUpdateDetailsMapper.xml       | 57 ++++++------
 3 files changed, 52 insertions(+), 128 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/aurora/blob/fbd29a17/src/jmh/java/org/apache/aurora/benchmark/JobUpdates.java
----------------------------------------------------------------------
diff --git a/src/jmh/java/org/apache/aurora/benchmark/JobUpdates.java b/src/jmh/java/org/apache/aurora/benchmark/JobUpdates.java
index 50044e1..0c9dbae 100644
--- a/src/jmh/java/org/apache/aurora/benchmark/JobUpdates.java
+++ b/src/jmh/java/org/apache/aurora/benchmark/JobUpdates.java
@@ -47,7 +47,6 @@ final class JobUpdates {
     private static final String USER = "user";
     private int numEvents = 1;
     private int numInstanceEvents = 5000;
-    private int numInstanceOverrides = 1;
 
     Builder setNumEvents(int newCount) {
       numEvents = newCount;
@@ -59,11 +58,6 @@ final class JobUpdates {
       return this;
     }
 
-    Builder setNumInstanceOverrides(int newCount) {
-      numInstanceOverrides = newCount;
-      return this;
-    }
-
     Set<IJobUpdateDetails> build(int count) {
       ImmutableSet.Builder<IJobUpdateDetails> result = ImmutableSet.builder();
       for (int i = 0; i < count; i++) {
@@ -84,9 +78,10 @@ final class JobUpdates {
                     .setMaxPerInstanceFailures(1)
                     .setMinWaitInInstanceRunningMs(1)
                     .setRollbackOnFailure(true)
-                    .setWaitForBatchCompletion(false)
-                    .setUpdateOnlyTheseInstances(ranges(numInstanceOverrides)))
-                .setInitialState(instanceConfigs(task, numInstanceOverrides))
+                    .setWaitForBatchCompletion(false))
+                .setInitialState(ImmutableSet.of(new InstanceTaskConfig()
+                    .setTask(task)
+                    .setInstances(ImmutableSet.of(new Range(0, 10)))))
                 .setDesiredState(new InstanceTaskConfig()
                     .setTask(task)
                     .setInstances(ImmutableSet.of(new Range(0, numInstanceEvents)))));
@@ -112,24 +107,6 @@ final class JobUpdates {
       return result.build();
     }
 
-    private static Set<InstanceTaskConfig> instanceConfigs(TaskConfig task, int count)
{
-      ImmutableSet.Builder<InstanceTaskConfig> result = ImmutableSet.builder();
-      for (int i = 0; i < count; i++) {
-        result.add(new InstanceTaskConfig(task, ImmutableSet.of(new Range(i, i))));
-      }
-
-      return result.build();
-    }
-
-    private static Set<Range> ranges(int count) {
-      ImmutableSet.Builder<Range> result = ImmutableSet.builder();
-      for (int i = 0; i < count; i++) {
-        result.add(new Range(i, i));
-      }
-
-      return result.build();
-    }
-
     private static String string(int numChars) {
       char[] chars = new char[numChars];
       Arrays.fill(chars, 'a');

http://git-wip-us.apache.org/repos/asf/aurora/blob/fbd29a17/src/jmh/java/org/apache/aurora/benchmark/UpdateStoreBenchmarks.java
----------------------------------------------------------------------
diff --git a/src/jmh/java/org/apache/aurora/benchmark/UpdateStoreBenchmarks.java b/src/jmh/java/org/apache/aurora/benchmark/UpdateStoreBenchmarks.java
index 92849d9..1d8986b 100644
--- a/src/jmh/java/org/apache/aurora/benchmark/UpdateStoreBenchmarks.java
+++ b/src/jmh/java/org/apache/aurora/benchmark/UpdateStoreBenchmarks.java
@@ -48,7 +48,6 @@ import org.openjdk.jmh.annotations.TearDown;
 import org.openjdk.jmh.annotations.Warmup;
 
 public class UpdateStoreBenchmarks {
-  private static final String USER = "user";
 
   @BenchmarkMode(Mode.Throughput)
   @OutputTimeUnit(TimeUnit.SECONDS)
@@ -57,6 +56,7 @@ public class UpdateStoreBenchmarks {
   @Fork(1)
   @State(Scope.Thread)
   public static class JobDetailsBenchmark {
+    private static final String USER = "user";
     private Storage storage;
     private Set<IJobUpdateKey> keys;
 
@@ -71,10 +71,29 @@ public class UpdateStoreBenchmarks {
     @Setup(Level.Iteration)
     public void setUpIteration() {
       storage.write((NoResult.Quiet) storeProvider -> {
+        JobUpdateStore.Mutable updateStore = storeProvider.getJobUpdateStore();
         Set<IJobUpdateDetails> updates =
             new JobUpdates.Builder().setNumInstanceEvents(instances).build(1);
 
-        keys = saveToStore(updates, storeProvider);
+        ImmutableSet.Builder<IJobUpdateKey> keyBuilder = ImmutableSet.builder();
+        for (IJobUpdateDetails details : updates) {
+          IJobUpdateKey key = details.getUpdate().getSummary().getKey();
+          keyBuilder.add(key);
+          String lockToken = UUID.randomUUID().toString();
+          storeProvider.getLockStore().saveLock(
+              ILock.build(new Lock(LockKey.job(key.getJob().newBuilder()), lockToken, USER,
0L)));
+
+          updateStore.saveJobUpdate(details.getUpdate(), Optional.of(lockToken));
+
+          for (IJobUpdateEvent updateEvent : details.getUpdateEvents()) {
+            updateStore.saveJobUpdateEvent(key, updateEvent);
+          }
+
+          for (IJobInstanceUpdateEvent instanceEvent : details.getInstanceEvents()) {
+            updateStore.saveJobInstanceUpdateEvent(key, instanceEvent);
+          }
+        }
+        keys = keyBuilder.build();
       });
     }
 
@@ -92,73 +111,4 @@ public class UpdateStoreBenchmarks {
           Iterables.getOnlyElement(keys)).get());
     }
   }
-
-  @BenchmarkMode(Mode.Throughput)
-  @OutputTimeUnit(TimeUnit.SECONDS)
-  @Warmup(iterations = 1, time = 10, timeUnit = TimeUnit.SECONDS)
-  @Measurement(iterations = 5, time = 5, timeUnit = TimeUnit.SECONDS)
-  @Fork(1)
-  @State(Scope.Thread)
-  public static class JobInstructionsBenchmark {
-    private Storage storage;
-    private Set<IJobUpdateKey> keys;
-
-    @Param({"1", "10", "100", "1000"})
-    private int instanceOverrides;
-
-    @Setup(Level.Trial)
-    public void setUp() {
-      storage = DbUtil.createStorage();
-    }
-
-    @Setup(Level.Iteration)
-    public void setUpIteration() {
-      storage.write((NoResult.Quiet) storeProvider -> {
-        Set<IJobUpdateDetails> updates =
-            new JobUpdates.Builder().setNumInstanceOverrides(instanceOverrides).build(1);
-
-        keys = saveToStore(updates, storeProvider);
-      });
-    }
-
-    @TearDown(Level.Iteration)
-    public void tearDownIteration() {
-      storage.write((NoResult.Quiet) storeProvider -> {
-        storeProvider.getJobUpdateStore().deleteAllUpdatesAndEvents();
-        storeProvider.getLockStore().deleteLocks();
-      });
-    }
-
-    @Benchmark
-    public IJobUpdateDetails run() throws TException {
-      return storage.read(store -> store.getJobUpdateStore().fetchJobUpdateDetails(
-          Iterables.getOnlyElement(keys)).get());
-    }
-  }
-
-  private static Set<IJobUpdateKey> saveToStore(
-      Set<IJobUpdateDetails> updates,
-      Storage.MutableStoreProvider storeProvider) {
-
-    JobUpdateStore.Mutable updateStore = storeProvider.getJobUpdateStore();
-    ImmutableSet.Builder<IJobUpdateKey> keyBuilder = ImmutableSet.builder();
-    for (IJobUpdateDetails details : updates) {
-      IJobUpdateKey key = details.getUpdate().getSummary().getKey();
-      keyBuilder.add(key);
-      String lockToken = UUID.randomUUID().toString();
-      storeProvider.getLockStore().saveLock(
-          ILock.build(new Lock(LockKey.job(key.getJob().newBuilder()), lockToken, USER, 0L)));
-
-      updateStore.saveJobUpdate(details.getUpdate(), Optional.of(lockToken));
-
-      for (IJobUpdateEvent updateEvent : details.getUpdateEvents()) {
-        updateStore.saveJobUpdateEvent(key, updateEvent);
-      }
-
-      for (IJobInstanceUpdateEvent instanceEvent : details.getInstanceEvents()) {
-        updateStore.saveJobInstanceUpdateEvent(key, instanceEvent);
-      }
-    }
-    return keyBuilder.build();
-  }
 }

http://git-wip-us.apache.org/repos/asf/aurora/blob/fbd29a17/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml
----------------------------------------------------------------------
diff --git a/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml
b/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml
index 8438f72..fba7d4f 100644
--- a/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml
+++ b/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml
@@ -170,9 +170,9 @@
   <resultMap id="jobUpdateSettingsMap" type="org.apache.aurora.gen.JobUpdateSettings">
     <id column="id" />
     <collection property="updateOnlyTheseInstances"
-                select="selectInstanceOverrides"
-                column="id"
-                foreignColumn="update_row_id" />
+                resultMap="rangeMap"
+                columnPrefix="r_"
+                notNullColumn="id" />
   </resultMap>
 
   <resultMap
@@ -182,9 +182,9 @@
     <association property="desiredState" resultMap="instanceConfigMap" columnPrefix="ditc_"
/>
     <association property="settings" resultMap="jobUpdateSettingsMap" columnPrefix="juse_"/>
     <collection property="initialState"
-                select="selectInstanceConfigs"
-                column="id"
-                foreignColumn="update_row_id" />
+                resultMap="instanceConfigMap"
+                columnPrefix="iitc_"
+                notNullColumn="id" />
   </resultMap>
 
   <resultMap id="jobUpdateMap" type="org.apache.aurora.scheduler.storage.db.views.DbJobUpdate">
@@ -350,12 +350,23 @@
       cn.task_config_row_id AS jui_ditc_task_config_row_id,
       di.id AS jui_ditc_r_id,
       di.first AS jui_ditc_r_first,
-      di.last AS jui_ditc_r_last
+      di.last AS jui_ditc_r_last,
+      co.id AS jui_iitc_id,
+      co.task_config_row_id AS jui_iitc_task_config_row_id,
+      ci.id AS jui_iitc_r_id,
+      ci.first AS jui_iitc_r_first,
+      ci.last AS jui_iitc_r_last,
+      io.id AS jui_juse_r_id,
+      io.first AS jui_juse_r_first,
+      io.last AS jui_juse_r_last
   </sql>
 
   <sql id="job_update_outer_joins">
     LEFT OUTER JOIN job_update_configs AS cn ON cn.update_row_id = u.id AND cn.is_new = TRUE
+    LEFT OUTER JOIN job_update_configs AS co ON co.update_row_id = u.id AND co.is_new = FALSE
+    LEFT OUTER JOIN job_update_configs_to_instances AS ci ON ci.config_id = co.id
     LEFT OUTER JOIN job_updates_to_desired_instances AS di ON di.update_row_id = u.id
+    LEFT OUTER JOIN job_updates_to_instance_overrides AS io ON io.update_row_id = u.id
   </sql>
 
   <sql id="lock_outer_join">
@@ -391,7 +402,15 @@
       cn.task_config_row_id AS ditc_task_config_row_id,
       di.id AS ditc_r_id,
       di.first AS ditc_r_first,
-      di.last AS ditc_r_last
+      di.last AS ditc_r_last,
+      co.id AS iitc_id,
+      co.task_config_row_id AS iitc_task_config_row_id,
+      ci.id AS iitc_r_id,
+      ci.first AS iitc_r_first,
+      ci.last AS iitc_r_last,
+      io.id AS juse_r_id,
+      io.first AS juse_r_first,
+      io.last AS juse_r_last
     FROM job_updates AS u
     <include refid="job_key_inner_join" />
     <include refid="job_update_outer_joins" />
@@ -431,28 +450,6 @@
     WHERE <include refid="filter_by_update_key"/>
   </select>
 
-  <select id="selectInstanceConfigs" resultMap="instanceConfigMap">
-    SELECT
-      co.id AS id,
-      co.task_config_row_id AS task_config_row_id,
-      ci.id AS r_id,
-      ci.first AS r_first,
-      ci.last AS r_last
-    FROM job_update_configs as co
-    INNER JOIN job_update_configs_to_instances AS ci ON ci.config_id = co.id
-    WHERE co.update_row_id = #{id}
-      AND co.is_new = FALSE
-  </select>
-
-  <select id="selectInstanceOverrides" resultMap="rangeMap">
-    SELECT
-      o.id,
-      o.first,
-      o.last
-    FROM job_updates_to_instance_overrides as o
-    WHERE update_row_id = #{id}
-  </select>
-
   <select id="selectInstanceUpdateEvents" resultMap="jobInstanceUpdateMap">
     SELECT
       e.id,


Mime
View raw message