aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zma...@apache.org
Subject aurora git commit: Add benchmarks for `StateManagerImpl`.
Date Wed, 23 Nov 2016 22:14:17 GMT
Repository: aurora
Updated Branches:
  refs/heads/master 4797dfe33 -> b099e2f0f


Add benchmarks for `StateManagerImpl`.

`StateManagerImpl` is in the middle of every task state transition in the
scheduler. Performance improvements here could yield scheduling throughput
improvements across the board. This adds benchmarks for the two bulk APIs,
inserting pending tasks and deleting tasks. Sample output:

````
Benchmark                                               (numPendingTasks)  (numTasksToDelete)
  Mode  Cnt  Score   Error  Units
StateManagerBenchmarks.DeleteTasksBenchmark.run                       N/A                1000
 thrpt   10  2.510 ± 0.557  ops/s
StateManagerBenchmarks.DeleteTasksBenchmark.run                       N/A               10000
 thrpt   10  0.272 ± 0.030  ops/s
StateManagerBenchmarks.DeleteTasksBenchmark.run                       N/A               50000
 thrpt   10  0.053 ± 0.011  ops/s
StateManagerBenchmarks.InsertPendingTasksBenchmark.run               1000                
N/A  thrpt   10  2.446 ± 0.698  ops/s
StateManagerBenchmarks.InsertPendingTasksBenchmark.run              10000                
N/A  thrpt   10  0.246 ± 0.018  ops/s
StateManagerBenchmarks.InsertPendingTasksBenchmark.run              50000                
N/A  thrpt   10  0.041 ± 0.006  ops/s
````

Reviewed at https://reviews.apache.org/r/54011/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/b099e2f0
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/b099e2f0
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/b099e2f0

Branch: refs/heads/master
Commit: b099e2f0f50607a762af6e48f9986adc05b073b7
Parents: 4797dfe
Author: Zameer Manji <zmanji@apache.org>
Authored: Wed Nov 23 14:14:03 2016 -0800
Committer: Zameer Manji <zmanji@apache.org>
Committed: Wed Nov 23 14:14:03 2016 -0800

----------------------------------------------------------------------
 .../benchmark/StateManagerBenchmarks.java       | 193 +++++++++++++++++++
 1 file changed, 193 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/aurora/blob/b099e2f0/src/jmh/java/org/apache/aurora/benchmark/StateManagerBenchmarks.java
----------------------------------------------------------------------
diff --git a/src/jmh/java/org/apache/aurora/benchmark/StateManagerBenchmarks.java b/src/jmh/java/org/apache/aurora/benchmark/StateManagerBenchmarks.java
new file mode 100644
index 0000000..b4f14f1
--- /dev/null
+++ b/src/jmh/java/org/apache/aurora/benchmark/StateManagerBenchmarks.java
@@ -0,0 +1,193 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.benchmark;
+
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import com.google.common.collect.Iterables;
+import com.google.inject.AbstractModule;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+
+import org.apache.aurora.benchmark.fakes.FakeDriver;
+import org.apache.aurora.benchmark.fakes.FakeEventSink;
+import org.apache.aurora.benchmark.fakes.FakeRescheduleCalculator;
+import org.apache.aurora.common.inject.Bindings;
+import org.apache.aurora.common.stats.StatsProvider;
+import org.apache.aurora.common.util.Clock;
+import org.apache.aurora.gen.ScheduleStatus;
+import org.apache.aurora.scheduler.TaskIdGenerator;
+import org.apache.aurora.scheduler.async.AsyncModule;
+import org.apache.aurora.scheduler.events.EventSink;
+import org.apache.aurora.scheduler.mesos.Driver;
+import org.apache.aurora.scheduler.scheduling.RescheduleCalculator;
+import org.apache.aurora.scheduler.state.StateManager;
+import org.apache.aurora.scheduler.state.StateManagerImpl;
+import org.apache.aurora.scheduler.storage.Storage;
+import org.apache.aurora.scheduler.storage.TaskStore;
+import org.apache.aurora.scheduler.storage.db.DbModule;
+import org.apache.aurora.scheduler.storage.entities.IScheduledTask;
+import org.apache.aurora.scheduler.storage.entities.ITaskConfig;
+import org.apache.aurora.scheduler.testing.FakeStatsProvider;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
+public class StateManagerBenchmarks {
+
+  @BenchmarkMode(Mode.Throughput)
+  @OutputTimeUnit(TimeUnit.SECONDS)
+  @Warmup(iterations = 5, time = 10, timeUnit = TimeUnit.SECONDS)
+  @Measurement(iterations = 10, time = 30, timeUnit = TimeUnit.SECONDS)
+  @Fork(1)
+  @State(Scope.Thread)
+  public static class InsertPendingTasksBenchmark {
+    private StateManager manager;
+    private Storage storage;
+
+    @Param({"1000", "10000", "50000"})
+    private int numPendingTasks;
+
+    // Used to prevent job key collisions
+    private int numIterations = 0;
+
+    @Setup
+    public void setUp() {
+      Injector injector = getInjector();
+      manager = injector.getInstance(StateManager.class);
+      storage = injector.getInstance(Storage.class);
+      storage.prepare();
+    }
+
+    @TearDown
+    public void tearDown() {
+      storage.write(new Storage.MutateWork.NoResult.Quiet() {
+        @Override
+        public void execute(Storage.MutableStoreProvider storeProvider) throws RuntimeException
{
+          storeProvider.getUnsafeTaskStore().deleteAllTasks();
+        }
+      });
+    }
+
+    @Benchmark
+    public Set<Integer> run() {
+      IScheduledTask task =
+          Iterables.getOnlyElement(new Tasks.Builder().setJob("iter_" + numIterations).build(1));
+      ITaskConfig config = task.getAssignedTask().getTask();
+      Set<Integer> taskIds =
+          IntStream.range(0, numPendingTasks).boxed().collect(Collectors.toSet());
+
+      numIterations++;
+
+      return storage.write((Storage.MutateWork.Quiet<Set<Integer>>) storeProvider
-> {
+        manager.insertPendingTasks(
+            storeProvider,
+            config,
+            taskIds
+        );
+
+        return taskIds;
+      });
+    }
+  }
+
+  @BenchmarkMode(Mode.Throughput)
+  @OutputTimeUnit(TimeUnit.SECONDS)
+  @Warmup(iterations = 5, time = 10, timeUnit = TimeUnit.SECONDS)
+  @Measurement(iterations = 10, time = 30, timeUnit = TimeUnit.SECONDS)
+  @Fork(1)
+  @State(Scope.Thread)
+  public static class DeleteTasksBenchmark {
+    private StateManager manager;
+    private Storage storage;
+    private Set<String> taskIds;
+
+    @Param({"1000", "10000", "50000"})
+    private int numTasksToDelete;
+
+    @Setup(Level.Trial)
+    public void setUpStorage() {
+      Injector injector = getInjector();
+      manager = injector.getInstance(StateManager.class);
+      storage = injector.getInstance(Storage.class);
+      storage.prepare();
+    }
+
+    // JMH warns heavily against using `Invocation` but this test seems to meet the requirements
+    // of using it. Each benchmark will take more than one ms and it avoids awkward logic
to
+    // setup storage before the benchmark.
+    @Setup(Level.Invocation)
+    public void setUp() {
+      storage.write(new Storage.MutateWork.NoResult.Quiet() {
+        @Override
+        public void execute(Storage.MutableStoreProvider storeProvider) throws RuntimeException
{
+          taskIds = bulkInsertTasks(numTasksToDelete, storeProvider.getUnsafeTaskStore());
+        }
+      });
+    }
+
+    @Benchmark
+    public Set<String> run() {
+      return storage.write((Storage.MutateWork.Quiet<Set<String>>) storeProvider
-> {
+        manager.deleteTasks(storeProvider, taskIds);
+        return taskIds;
+      });
+    }
+  }
+
+  private static Set<String> bulkInsertTasks(int num, TaskStore.Mutable store) {
+    Set<IScheduledTask> tasks =
+        new Tasks.Builder().setScheduleStatus(ScheduleStatus.FINISHED).build(num);
+    store.saveTasks(tasks);
+
+    return tasks.stream().map(t -> t.getAssignedTask().getTaskId()).collect(Collectors.toSet());
+  }
+
+  private static Injector getInjector() {
+    return Guice.createInjector(
+        new AbstractModule() {
+          @Override
+          protected void configure() {
+            bind(Clock.class).toInstance(Clock.SYSTEM_CLOCK);
+            bind(Driver.class).toInstance(new FakeDriver());
+            bind(EventSink.class).toInstance(new FakeEventSink());
+            // We want to measure the throughput of the state manager so we fake out the
+            // rescheduling calculator.
+            bind(RescheduleCalculator.class).toInstance(new FakeRescheduleCalculator());
+            bind(TaskIdGenerator.class).to(TaskIdGenerator.TaskIdGeneratorImpl.class);
+            // This is what we want to benchmark
+            bind(StateManager.class).to(StateManagerImpl.class);
+            // This is needed for storage
+            bind(StatsProvider.class).toInstance(new FakeStatsProvider());
+          }
+        },
+        DbModule.productionModule(Bindings.KeyFactory.PLAIN),
+        // This is needed for storage
+        new AsyncModule()
+    );
+  }
+}


Mime
View raw message