aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ma...@apache.org
Subject aurora git commit: Improving JobUpdateDetails fetch performance.
Date Thu, 09 Apr 2015 23:23:40 GMT
Repository: aurora
Updated Branches:
  refs/heads/master 02717ec6a -> 8b9a3649a


Improving JobUpdateDetails fetch performance.

Bugs closed: AURORA-1263

Reviewed at https://reviews.apache.org/r/32990/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/8b9a3649
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/8b9a3649
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/8b9a3649

Branch: refs/heads/master
Commit: 8b9a3649abe5e6cd57d5cd01313932ff7c6cbcef
Parents: 02717ec
Author: Maxim Khutornenko <maxim@apache.org>
Authored: Thu Apr 9 16:20:17 2015 -0700
Committer: Maxim Khutornenko <maxim@apache.org>
Committed: Thu Apr 9 16:20:17 2015 -0700

----------------------------------------------------------------------
 .../aurora/benchmark/UpdateStoreBenchmarks.java | 174 +++++++++++++++++++
 .../storage/db/JobUpdateDetailsMapper.xml       |  61 ++++---
 2 files changed, 211 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/aurora/blob/8b9a3649/src/jmh/java/org/apache/aurora/benchmark/UpdateStoreBenchmarks.java
----------------------------------------------------------------------
diff --git a/src/jmh/java/org/apache/aurora/benchmark/UpdateStoreBenchmarks.java b/src/jmh/java/org/apache/aurora/benchmark/UpdateStoreBenchmarks.java
new file mode 100644
index 0000000..c7456aa
--- /dev/null
+++ b/src/jmh/java/org/apache/aurora/benchmark/UpdateStoreBenchmarks.java
@@ -0,0 +1,174 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.benchmark;
+
+import java.util.Arrays;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableSet;
+import com.google.inject.AbstractModule;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.twitter.common.inject.Bindings;
+import com.twitter.common.util.Clock;
+
+import org.apache.aurora.gen.ExecutorConfig;
+import org.apache.aurora.gen.InstanceTaskConfig;
+import org.apache.aurora.gen.JobInstanceUpdateEvent;
+import org.apache.aurora.gen.JobKey;
+import org.apache.aurora.gen.JobUpdate;
+import org.apache.aurora.gen.JobUpdateAction;
+import org.apache.aurora.gen.JobUpdateEvent;
+import org.apache.aurora.gen.JobUpdateInstructions;
+import org.apache.aurora.gen.JobUpdateKey;
+import org.apache.aurora.gen.JobUpdateSettings;
+import org.apache.aurora.gen.JobUpdateStatus;
+import org.apache.aurora.gen.JobUpdateSummary;
+import org.apache.aurora.gen.Lock;
+import org.apache.aurora.gen.LockKey;
+import org.apache.aurora.gen.Range;
+import org.apache.aurora.gen.TaskConfig;
+import org.apache.aurora.scheduler.storage.JobUpdateStore;
+import org.apache.aurora.scheduler.storage.Storage;
+import org.apache.aurora.scheduler.storage.db.DbModule;
+import org.apache.aurora.scheduler.storage.entities.IJobInstanceUpdateEvent;
+import org.apache.aurora.scheduler.storage.entities.IJobUpdate;
+import org.apache.aurora.scheduler.storage.entities.IJobUpdateDetails;
+import org.apache.aurora.scheduler.storage.entities.IJobUpdateEvent;
+import org.apache.aurora.scheduler.storage.entities.IJobUpdateKey;
+import org.apache.aurora.scheduler.storage.entities.ILock;
+import org.apache.aurora.scheduler.storage.mem.MemStorage;
+import org.apache.aurora.scheduler.storage.mem.MemStorageModule;
+import org.apache.thrift.TException;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+public class UpdateStoreBenchmarks {
+
+  @BenchmarkMode(Mode.Throughput)
+  @OutputTimeUnit(TimeUnit.SECONDS)
+  @Warmup(iterations = 1, time = 10, timeUnit = TimeUnit.SECONDS)
+  @Measurement(iterations = 5, time = 5, timeUnit = TimeUnit.SECONDS)
+  @Fork(1)
+  @State(Scope.Thread)
+  public static class JobDetailsBenchmark {
+    private static final String USER = "user";
+    private Storage storage;
+    private IJobUpdateKey key;
+
+    @Param({"1000", "5000", "10000"})
+    private int instances;
+
+    @Setup(Level.Trial)
+    public void setUp() {
+      Injector injector = Guice.createInjector(
+          new AbstractModule() {
+            @Override
+            protected void configure() {
+              bind(Clock.class).toInstance(Clock.SYSTEM_CLOCK);
+            }
+          },
+          new MemStorageModule(Bindings.KeyFactory.PLAIN),
+          new DbModule(Bindings.annotatedKeyFactory(MemStorage.Delegated.class)));
+      storage = injector.getInstance(Storage.class);
+      storage.prepare();
+    }
+
+    @Setup(Level.Iteration)
+    public void setUpIteration() {
+      final JobKey job = new JobKey("role", "env", UUID.randomUUID().toString());
+      key = IJobUpdateKey.build(new JobUpdateKey()
+          .setJob(job)
+          .setId(UUID.randomUUID().toString()));
+
+      TaskConfig task = new TaskConfig()
+          .setJob(job)
+          .setExecutorConfig(new ExecutorConfig("cfg", string(10000)));
+
+      final JobUpdate update = new JobUpdate()
+          .setSummary(new JobUpdateSummary()
+              .setKey(key.newBuilder())
+              .setUpdateId(key.getId())
+              .setJobKey(job)
+              .setUser(USER))
+          .setInstructions(new JobUpdateInstructions()
+              .setSettings(new JobUpdateSettings()
+                  .setUpdateGroupSize(100)
+                  .setMaxFailedInstances(1)
+                  .setMaxPerInstanceFailures(1)
+                  .setMaxWaitToInstanceRunningMs(1)
+                  .setMinWaitInInstanceRunningMs(1)
+                  .setRollbackOnFailure(true)
+                  .setWaitForBatchCompletion(false))
+              .setInitialState(ImmutableSet.of(new InstanceTaskConfig()
+                  .setTask(task)
+                  .setInstances(ImmutableSet.of(new Range(0, 10)))))
+              .setDesiredState(new InstanceTaskConfig()
+                  .setTask(task)
+                  .setInstances(ImmutableSet.of(new Range(0, instances)))));
+
+      storage.write(new Storage.MutateWork.NoResult.Quiet() {
+        @Override
+        protected void execute(Storage.MutableStoreProvider storeProvider) {
+          String lockToken = UUID.randomUUID().toString();
+          storeProvider.getLockStore().saveLock(
+              ILock.build(new Lock(LockKey.job(job), lockToken, USER, 0L)));
+
+          JobUpdateStore.Mutable updateStore = storeProvider.getJobUpdateStore();
+          updateStore.saveJobUpdate(IJobUpdate.build(update), Optional.of(lockToken));
+          updateStore.saveJobUpdateEvent(
+              key,
+              IJobUpdateEvent.build(new JobUpdateEvent(JobUpdateStatus.ROLLING_FORWARD, 0L)
+                  .setUser(USER)
+                  .setMessage("message")));
+
+          for (int i = 0; i < instances; i++) {
+            updateStore.saveJobInstanceUpdateEvent(
+                key,
+                IJobInstanceUpdateEvent.build(
+                    new JobInstanceUpdateEvent(i, 0L, JobUpdateAction.INSTANCE_UPDATING)));
+          }
+        }
+      });
+    }
+
+    private static String string(int numChars) {
+      char[] chars = new char[numChars];
+      Arrays.fill(chars, 'a');
+      return new String(chars);
+    }
+
+    @Benchmark
+    public IJobUpdateDetails run() throws TException {
+      return storage.read(new Storage.Work.Quiet<IJobUpdateDetails>() {
+        @Override
+        public IJobUpdateDetails apply(Storage.StoreProvider store) {
+          return store.getJobUpdateStore().fetchJobUpdateDetails(key).get();
+        }
+      });
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/aurora/blob/8b9a3649/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml
----------------------------------------------------------------------
diff --git a/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml
b/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml
index 4536cd3..f76f9a9 100644
--- a/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml
+++ b/src/main/resources/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.xml
@@ -206,24 +206,26 @@
             typeHandler="org.apache.aurora.scheduler.storage.db.typehandlers.JobUpdateActionTypeHandler"/>
   </resultMap>
 
+  <resultMap id="jobUpdateEventMap" type="org.apache.aurora.gen.JobUpdateEvent">
+    <id column="id" />
+    <result property="status"
+            column="status"
+            typeHandler="org.apache.aurora.scheduler.storage.db.typehandlers.JobUpdateStatusTypeHandler"/>
+  </resultMap>
+
+
   <resultMap id="jobUpdateDetailsMap" type="org.apache.aurora.gen.storage.StoredJobUpdateDetails">
     <id column="u_id" />
     <association property="details.update" resultMap="jobUpdateMap" />
-    <!--Using notNullColumn attribute is required below as LEFT JOIN with empty right
side
-    will produce an empty row.-->
     <collection property="details.updateEvents"
-                ofType="org.apache.aurora.gen.JobUpdateEvent"
-                columnPrefix="e_"
-                notNullColumn="id">
-      <id column="id" />
-      <result property="status"
-              column="status"
-              typeHandler="org.apache.aurora.scheduler.storage.db.typehandlers.JobUpdateStatusTypeHandler"
/>
+                select="selectUpdateEvents"
+                column="u_id"
+                foreignColumn="update_row_id">
     </collection>
     <collection property="details.instanceEvents"
-                resultMap="jobInstanceUpdateMap"
-                columnPrefix="i_"
-                notNullColumn="id">
+                select="selectInstanceEvents"
+                column="u_id"
+                foreignColumn="update_row_id">
     </collection>
   </resultMap>
 
@@ -393,22 +395,11 @@
   <sql id="unscoped_details_select">
     SELECT
       <include refid="job_update_columns" />,
-      e.id AS e_id,
-      e.status AS e_status,
-      e.user AS e_user,
-      e.timestamp_ms AS e_timestamp_ms,
-      e.message AS e_message,
-      i.id AS i_id,
-      i.action AS i_action,
-      i.instance_id AS i_instance_id,
-      i.timestamp_ms AS i_timestamp_ms,
       l.lock_token AS lock_token
     FROM job_updates AS u
     <include refid="job_key_inner_join" />
     <include refid="timestamps_inner_joins" />
     <include refid="job_update_outer_joins" />
-    LEFT OUTER JOIN job_update_events AS e ON e.update_row_id = u.id
-    LEFT OUTER JOIN job_instance_update_events AS i ON i.update_row_id = u.id
     <include refid="lock_outer_join" />
   </sql>
 
@@ -459,7 +450,6 @@
   <select id="selectDetails" resultMap="jobUpdateDetailsMap">
     <include refid="unscoped_details_select"/>
     WHERE <include refid="filter_by_update_key"/>
-    ORDER BY e_timestamp_ms, i_timestamp_ms
   </select>
 
   <select id="selectDetailsList" resultMap="jobUpdateDetailsMap">
@@ -494,6 +484,29 @@
     ORDER BY e.timestamp_ms
   </select>
 
+  <select id="selectUpdateEvents" resultMap="jobUpdateEventMap">
+    SELECT
+      e.id,
+      status,
+      timestamp_ms,
+      e.user,
+      message
+    FROM job_update_events as e
+    WHERE update_row_id = #{id}
+    ORDER BY e.timestamp_ms
+  </select>
+
+  <select id="selectInstanceEvents" resultMap="jobInstanceUpdateMap">
+    SELECT
+      e.id,
+      instance_id,
+      timestamp_ms,
+      action
+    FROM job_instance_update_events as e
+    WHERE update_row_id = #{id}
+    ORDER BY e.timestamp_ms
+  </select>
+
   <delete id="truncate">
     DELETE FROM job_updates;
   </delete>


Mime
View raw message