aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s...@apache.org
Subject aurora git commit: Expose Thrift server request workload stats
Date Mon, 30 Jan 2017 13:00:50 GMT
Repository: aurora
Updated Branches:
  refs/heads/master 1217fc87c -> 7be7ad6f1


Expose Thrift server request workload stats

This patch introduces a number of stats that measure the workload generated by
Thrift server requests.

Current Thrift server stats expose the number and timing of requests received
by the server. However, they fail to reflect the size of the requests. This is
limiting us in having an accurate view of the workload handled by the scheduler.
For example, every call to `restartShards()` is recorded as one event despite
the fact that a request might only restart one shard while another request might
seek to restart 1K shards.

Bugs closed: AURORA-1826

Reviewed at https://reviews.apache.org/r/55089/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/7be7ad6f
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/7be7ad6f
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/7be7ad6f

Branch: refs/heads/master
Commit: 7be7ad6f1b0b09c042efc551b0e3e4f2e5d1f530
Parents: 1217fc8
Author: Mehrdad Nurolahzade <mehrdad@nurolahzade.com>
Authored: Mon Jan 30 14:00:15 2017 +0100
Committer: Stephan Erb <serb@apache.org>
Committed: Mon Jan 30 14:00:15 2017 +0100

----------------------------------------------------------------------
 .../thrift/SchedulerThriftInterface.java        |  88 ++++++++++++++--
 .../aop/ThriftStatsExporterInterceptor.java     |  39 ++++++-
 .../scheduler/thrift/aop/ThriftWorkload.java    |  67 ++++++++++++
 .../thrift/SchedulerThriftInterfaceTest.java    |  91 +++++++++++++++-
 .../aop/ThriftStatsExporterInterceptorTest.java | 105 ++++++++++++++++++-
 5 files changed, 374 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/aurora/blob/7be7ad6f/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java b/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java
index 16b1b52..a40114d 100644
--- a/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java
+++ b/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java
@@ -17,6 +17,7 @@ import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
 
 import javax.annotation.Nullable;
 import javax.inject.Inject;
@@ -36,6 +37,7 @@ import com.google.common.collect.Multimap;
 import com.google.common.collect.Multimaps;
 import com.google.common.collect.Range;
 
+import org.apache.aurora.common.stats.StatsProvider;
 import org.apache.aurora.gen.ConfigRewrite;
 import org.apache.aurora.gen.DrainHostsResult;
 import org.apache.aurora.gen.EndMaintenanceResult;
@@ -114,6 +116,7 @@ import org.apache.aurora.scheduler.storage.entities.IScheduledTask;
 import org.apache.aurora.scheduler.storage.entities.ITaskConfig;
 import org.apache.aurora.scheduler.storage.log.ThriftBackfill;
 import org.apache.aurora.scheduler.thrift.aop.AnnotatedAuroraAdmin;
+import org.apache.aurora.scheduler.thrift.aop.ThriftWorkload;
 import org.apache.aurora.scheduler.thrift.auth.DecoratedThrift;
 import org.apache.aurora.scheduler.updater.JobDiff;
 import org.apache.aurora.scheduler.updater.JobUpdateController;
@@ -155,6 +158,30 @@ class SchedulerThriftInterface implements AnnotatedAuroraAdmin {
   // delimiters.
   @VisibleForTesting
   static final int MAX_TASK_ID_LENGTH = 255 - 90;
+  @VisibleForTesting
+  static final String STAT_PREFIX = "thrift_workload_";
+  @VisibleForTesting
+  static final String CREATE_JOB = STAT_PREFIX + "createJob";
+  @VisibleForTesting
+  static final String CREATE_OR_UPDATE_CRON = STAT_PREFIX + "createOrUpdateCronTemplate";
+  @VisibleForTesting
+  static final String KILL_TASKS = STAT_PREFIX + "killTasks";
+  @VisibleForTesting
+  static final String RESTART_SHARDS = STAT_PREFIX + "restartShards";
+  @VisibleForTesting
+  static final String START_MAINTENANCE = STAT_PREFIX + "startMaintenance";
+  @VisibleForTesting
+  static final String DRAIN_HOSTS = STAT_PREFIX + "drainHosts";
+  @VisibleForTesting
+  static final String MAINTENANCE_STATUS = STAT_PREFIX + "maintenanceStatus";
+  @VisibleForTesting
+  static final String END_MAINTENANCE = STAT_PREFIX + "endMaintenance";
+  @VisibleForTesting
+  static final String REWRITE_CONFIGS = STAT_PREFIX + "rewriteConfigs";
+  @VisibleForTesting
+  static final String ADD_INSTANCES = STAT_PREFIX + "addInstances";
+  @VisibleForTesting
+  static final String START_JOB_UPDATE = STAT_PREFIX + "startJobUpdate";
 
   private static final Logger LOG = LoggerFactory.getLogger(SchedulerThriftInterface.class);
 
@@ -175,6 +202,18 @@ class SchedulerThriftInterface implements AnnotatedAuroraAdmin {
   private final AuditMessages auditMessages;
   private final TaskReconciler taskReconciler;
 
+  private final AtomicLong createJobCounter;
+  private final AtomicLong createOrUpdateCronCounter;
+  private final AtomicLong killTasksCounter;
+  private final AtomicLong restartShardsCounter;
+  private final AtomicLong startMaintenanceCounter;
+  private final AtomicLong drainHostsCounter;
+  private final AtomicLong maintenanceStatusCounter;
+  private final AtomicLong endMaintenanceCounter;
+  private final AtomicLong rewriteConfigsCounter;
+  private final AtomicLong addInstancesCounter;
+  private final AtomicLong startJobUpdateCounter;
+
   @Inject
   SchedulerThriftInterface(
       ConfigurationManager configurationManager,
@@ -192,7 +231,8 @@ class SchedulerThriftInterface implements AnnotatedAuroraAdmin {
       JobUpdateController jobUpdateController,
       ReadOnlyScheduler.Iface readOnlyScheduler,
       AuditMessages auditMessages,
-      TaskReconciler taskReconciler) {
+      TaskReconciler taskReconciler,
+      StatsProvider statsProvider) {
 
     this.configurationManager = requireNonNull(configurationManager);
     this.thresholds = requireNonNull(thresholds);
@@ -210,6 +250,18 @@ class SchedulerThriftInterface implements AnnotatedAuroraAdmin {
     this.readOnlyScheduler = requireNonNull(readOnlyScheduler);
     this.auditMessages = requireNonNull(auditMessages);
     this.taskReconciler = requireNonNull(taskReconciler);
+
+    this.createJobCounter = statsProvider.makeCounter(CREATE_JOB);
+    this.createOrUpdateCronCounter = statsProvider.makeCounter(CREATE_OR_UPDATE_CRON);
+    this.killTasksCounter = statsProvider.makeCounter(KILL_TASKS);
+    this.restartShardsCounter = statsProvider.makeCounter(RESTART_SHARDS);
+    this.startMaintenanceCounter = statsProvider.makeCounter(START_MAINTENANCE);
+    this.drainHostsCounter = statsProvider.makeCounter(DRAIN_HOSTS);
+    this.maintenanceStatusCounter = statsProvider.makeCounter(MAINTENANCE_STATUS);
+    this.endMaintenanceCounter = statsProvider.makeCounter(END_MAINTENANCE);
+    this.rewriteConfigsCounter = statsProvider.makeCounter(REWRITE_CONFIGS);
+    this.addInstancesCounter = statsProvider.makeCounter(ADD_INSTANCES);
+    this.startJobUpdateCounter = statsProvider.makeCounter(START_JOB_UPDATE);
   }
 
   @Override
@@ -248,6 +300,7 @@ class SchedulerThriftInterface implements AnnotatedAuroraAdmin {
             storeProvider,
             template,
             sanitized.getInstanceIds());
+        createJobCounter.addAndGet(sanitized.getInstanceIds().size());
 
         return ok();
       } catch (LockException e) {
@@ -310,6 +363,7 @@ class SchedulerThriftInterface implements AnnotatedAuroraAdmin {
           checkJobExists(storeProvider, jobKey);
           cronJobManager.createJob(SanitizedCronJob.from(sanitized));
         }
+        createOrUpdateCronCounter.addAndGet(count);
 
         return ok();
       } catch (LockException e) {
@@ -364,36 +418,43 @@ class SchedulerThriftInterface implements AnnotatedAuroraAdmin {
   }
 
   // TODO(William Farner): Provide status information about cron jobs here.
+  @ThriftWorkload
   @Override
   public Response getTasksStatus(TaskQuery query) throws TException {
     return readOnlyScheduler.getTasksStatus(query);
   }
 
+  @ThriftWorkload
   @Override
   public Response getTasksWithoutConfigs(TaskQuery query) throws TException {
     return readOnlyScheduler.getTasksWithoutConfigs(query);
   }
 
+  @ThriftWorkload
   @Override
   public Response getPendingReason(TaskQuery query) throws TException {
     return readOnlyScheduler.getPendingReason(query);
   }
 
+  @ThriftWorkload
   @Override
   public Response getConfigSummary(JobKey job) throws TException {
     return readOnlyScheduler.getConfigSummary(job);
   }
 
+  @ThriftWorkload
   @Override
   public Response getRoleSummary() throws TException {
     return readOnlyScheduler.getRoleSummary();
   }
 
+  @ThriftWorkload
   @Override
   public Response getJobSummary(@Nullable String maybeNullRole) throws TException {
     return readOnlyScheduler.getJobSummary(maybeNullRole);
   }
 
+  @ThriftWorkload
   @Override
   public Response getJobs(@Nullable String maybeNullRole) throws TException {
     return readOnlyScheduler.getJobs(maybeNullRole);
@@ -446,17 +507,20 @@ class SchedulerThriftInterface implements AnnotatedAuroraAdmin {
 
       LOG.info("Killing tasks matching " + query);
 
-      boolean tasksKilled = false;
+      int tasksKilled = 0;
       for (String taskId : Tasks.ids(tasks)) {
-        tasksKilled |= StateChangeResult.SUCCESS == stateManager.changeState(
+        if (StateChangeResult.SUCCESS == stateManager.changeState(
             storeProvider,
             taskId,
             Optional.absent(),
             ScheduleStatus.KILLING,
-            auditMessages.killedByRemoteUser());
+            auditMessages.killedByRemoteUser())) {
+          ++tasksKilled;
+        }
       }
+      killTasksCounter.addAndGet(tasksKilled);
 
-      return tasksKilled
+      return tasksKilled > 0
           ? response.setResponseCode(OK)
           : addMessage(response, OK, NO_TASKS_TO_KILL_MESSAGE);
     });
@@ -489,6 +553,8 @@ class SchedulerThriftInterface implements AnnotatedAuroraAdmin {
             ScheduleStatus.RESTARTING,
             auditMessages.restartedByRemoteUser());
       }
+      restartShardsCounter.addAndGet(shardIds.size());
+
       return ok();
     });
   }
@@ -516,6 +582,7 @@ class SchedulerThriftInterface implements AnnotatedAuroraAdmin {
 
   @Override
   public Response startMaintenance(Hosts hosts) {
+    startMaintenanceCounter.addAndGet(hosts.getHostNamesSize());
     return ok(Result.startMaintenanceResult(
         new StartMaintenanceResult()
             .setStatuses(IHostStatus.toBuildersSet(
@@ -524,6 +591,7 @@ class SchedulerThriftInterface implements AnnotatedAuroraAdmin {
 
   @Override
   public Response drainHosts(Hosts hosts) {
+    drainHostsCounter.addAndGet(hosts.getHostNamesSize());
     return ok(Result.drainHostsResult(
         new DrainHostsResult().setStatuses(IHostStatus.toBuildersSet(
             maintenance.drain(hosts.getHostNames())))));
@@ -531,6 +599,7 @@ class SchedulerThriftInterface implements AnnotatedAuroraAdmin {
 
   @Override
   public Response maintenanceStatus(Hosts hosts) {
+    maintenanceStatusCounter.addAndGet(hosts.getHostNamesSize());
     return ok(Result.maintenanceStatusResult(
         new MaintenanceStatusResult().setStatuses(IHostStatus.toBuildersSet(
             maintenance.getStatus(hosts.getHostNames())))));
@@ -538,6 +607,7 @@ class SchedulerThriftInterface implements AnnotatedAuroraAdmin {
 
   @Override
   public Response endMaintenance(Hosts hosts) {
+    endMaintenanceCounter.addAndGet(hosts.getHostNamesSize());
     return ok(Result.endMaintenanceResult(
         new EndMaintenanceResult()
             .setStatuses(IHostStatus.toBuildersSet(
@@ -620,6 +690,8 @@ class SchedulerThriftInterface implements AnnotatedAuroraAdmin {
         Optional<String> error = rewriteConfig(IConfigRewrite.build(command), storeProvider);
         if (error.isPresent()) {
           errors.add(error.get());
+        } else {
+          rewriteConfigsCounter.incrementAndGet();
         }
       }
 
@@ -790,6 +862,7 @@ class SchedulerThriftInterface implements AnnotatedAuroraAdmin {
             quotaManager.checkInstanceAddition(task, instanceIds.size(), storeProvider));
 
         stateManager.insertPendingTasks(storeProvider, task, instanceIds);
+        addInstancesCounter.addAndGet(instanceIds.size());
 
         return response.setResponseCode(OK);
       } catch (LockException e) {
@@ -800,7 +873,7 @@ class SchedulerThriftInterface implements AnnotatedAuroraAdmin {
     });
   }
 
-  public Optional<IJobConfiguration> getCronJob(StoreProvider storeProvider, IJobKey jobKey) {
+  private Optional<IJobConfiguration> getCronJob(StoreProvider storeProvider, IJobKey jobKey) {
     requireNonNull(jobKey);
     return storeProvider.getCronJobStore().fetchJob(jobKey);
   }
@@ -954,6 +1027,7 @@ class SchedulerThriftInterface implements AnnotatedAuroraAdmin {
         jobUpdateController.start(
             update,
             new AuditData(remoteUserName, Optional.fromNullable(message)));
+        startJobUpdateCounter.addAndGet(request.getInstanceCount());
         return response.setResponseCode(OK)
             .setResult(Result.startJobUpdateResult(
                 new StartJobUpdateResult(update.getSummary().getKey().newBuilder())
@@ -1037,11 +1111,13 @@ class SchedulerThriftInterface implements AnnotatedAuroraAdmin {
     }
   }
 
+  @ThriftWorkload
   @Override
   public Response getJobUpdateSummaries(JobUpdateQuery mutableQuery) throws TException {
     return readOnlyScheduler.getJobUpdateSummaries(mutableQuery);
   }
 
+  @ThriftWorkload
   @Override
   public Response getJobUpdateDetails(JobUpdateKey key, JobUpdateQuery query) throws TException {
     return readOnlyScheduler.getJobUpdateDetails(key, query);

http://git-wip-us.apache.org/repos/asf/aurora/blob/7be7ad6f/src/main/java/org/apache/aurora/scheduler/thrift/aop/ThriftStatsExporterInterceptor.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/aop/ThriftStatsExporterInterceptor.java b/src/main/java/org/apache/aurora/scheduler/thrift/aop/ThriftStatsExporterInterceptor.java
index d57f910..aad4190 100644
--- a/src/main/java/org/apache/aurora/scheduler/thrift/aop/ThriftStatsExporterInterceptor.java
+++ b/src/main/java/org/apache/aurora/scheduler/thrift/aop/ThriftStatsExporterInterceptor.java
@@ -14,7 +14,9 @@
 package org.apache.aurora.scheduler.thrift.aop;
 
 import java.lang.reflect.Method;
+import java.util.concurrent.atomic.AtomicLong;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
@@ -23,30 +25,59 @@ import org.aopalliance.intercept.MethodInterceptor;
 import org.aopalliance.intercept.MethodInvocation;
 import org.apache.aurora.common.stats.SlidingStats;
 import org.apache.aurora.common.stats.Stats;
+import org.apache.aurora.gen.Response;
+import org.apache.aurora.gen.ResponseCode;
+import org.apache.aurora.scheduler.thrift.aop.ThriftWorkload.ThriftWorkloadCounter;
 
 /**
  * A method interceptor that exports counterStats about thrift calls.
  */
 class ThriftStatsExporterInterceptor implements MethodInterceptor {
 
-  private final LoadingCache<Method, SlidingStats> stats =
+  @VisibleForTesting
+  static final String TIMING_STATS_NAME_TEMPLATE = "scheduler_thrift_%s";
+  @VisibleForTesting
+  static final String WORKLOAD_STATS_NAME_TEMPLATE = "scheduler_thrift_workload_%s";
+
+  private final LoadingCache<Method, SlidingStats> timingStats =
       CacheBuilder.newBuilder().build(new CacheLoader<Method, SlidingStats>() {
         @Override
         public SlidingStats load(Method method) {
           return new SlidingStats(
-              Stats.normalizeName(String.format("scheduler_thrift_%s", method.getName())),
+              Stats.normalizeName(String.format(TIMING_STATS_NAME_TEMPLATE, method.getName())),
               "nanos");
         }
       });
 
+  private final LoadingCache<Method, AtomicLong> workloadStats =
+      CacheBuilder.newBuilder().build(new CacheLoader<Method, AtomicLong>() {
+        @Override
+        public AtomicLong load(Method method) {
+          return Stats.exportLong(
+              Stats.normalizeName(String.format(WORKLOAD_STATS_NAME_TEMPLATE, method.getName())));
+        }
+      });
+
   @Override
   public Object invoke(MethodInvocation invocation) throws Throwable {
-    SlidingStats stat = stats.get(invocation.getMethod());
+    Method method = invocation.getMethod();
+    SlidingStats stat = timingStats.getUnchecked(method);
     long start = System.nanoTime();
+    Response response = null;
     try {
-      return invocation.proceed();
+      response = (Response) invocation.proceed();
     } finally {
       stat.accumulate(System.nanoTime() - start);
+      if (response != null
+          && response.getResponseCode() == ResponseCode.OK
+          && method.isAnnotationPresent(ThriftWorkload.class)) {
+
+        ThriftWorkloadCounter counter = method.getAnnotation(ThriftWorkload.class)
+            .value()
+            .newInstance();
+        workloadStats.getUnchecked(method).addAndGet(counter.apply(response.getResult()));
+      }
     }
+    return response;
   }
 }

http://git-wip-us.apache.org/repos/asf/aurora/blob/7be7ad6f/src/main/java/org/apache/aurora/scheduler/thrift/aop/ThriftWorkload.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/aop/ThriftWorkload.java b/src/main/java/org/apache/aurora/scheduler/thrift/aop/ThriftWorkload.java
new file mode 100644
index 0000000..bb927e8
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/thrift/aop/ThriftWorkload.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.scheduler.thrift.aop;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import java.util.function.Function;
+
+import org.apache.aurora.gen.Result;
+
+/**
+ * Marks a scheduler thrift server method as a target for workload measuring.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.METHOD)
+public @interface ThriftWorkload {
+
+  /**
+   * Translates Thrift server call {@link Result} to an {@literal Integer} that represents the
+   * amount of workload generated by the call.
+   */
+  interface ThriftWorkloadCounter extends Function<Result, Integer> { }
+
+  /**
+   * Specify the class that will do the thrift load mapping.
+   */
+  Class<? extends ThriftWorkloadCounter> value() default ThriftWorkloadCounterImpl.class;
+
+  class ThriftWorkloadCounterImpl implements ThriftWorkloadCounter {
+    @Override
+    public Integer apply(Result result) {
+      int count = 0;
+      if (result.isSetScheduleStatusResult()) {
+        count = result.getScheduleStatusResult().getTasksSize();
+      } else if (result.isSetGetPendingReasonResult()) {
+        count = result.getGetPendingReasonResult().getReasonsSize();
+      } else if (result.isSetConfigSummaryResult()) {
+        count = result.getConfigSummaryResult().getSummary().getGroupsSize();
+      } else if (result.isSetRoleSummaryResult()) {
+        count = result.getRoleSummaryResult().getSummariesSize();
+      } else if (result.isSetJobSummaryResult()) {
+        count = result.getJobSummaryResult().getSummariesSize();
+      } else if (result.isSetGetJobsResult()) {
+        count = result.getGetJobsResult().getConfigsSize();
+      } else if (result.isSetGetJobUpdateSummariesResult()) {
+        count = result.getGetJobUpdateSummariesResult().getUpdateSummariesSize();
+      } else if (result.isSetGetJobUpdateDetailsResult()) {
+        count = result.getGetJobUpdateDetailsResult().getDetailsListSize();
+      }
+      return count;
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/aurora/blob/7be7ad6f/src/test/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java b/src/test/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java
index b28cd24..5262e57 100644
--- a/src/test/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java
@@ -107,6 +107,7 @@ import org.apache.aurora.scheduler.storage.entities.IResourceAggregate;
 import org.apache.aurora.scheduler.storage.entities.IScheduledTask;
 import org.apache.aurora.scheduler.storage.entities.ITaskConfig;
 import org.apache.aurora.scheduler.storage.testing.StorageTestUtil;
+import org.apache.aurora.scheduler.testing.FakeStatsProvider;
 import org.apache.aurora.scheduler.updater.JobUpdateController;
 import org.apache.aurora.scheduler.updater.JobUpdateController.AuditData;
 import org.apache.aurora.scheduler.updater.UpdateInProgressException;
@@ -152,9 +153,20 @@ import static org.apache.aurora.scheduler.thrift.Fixtures.nonProductionTask;
 import static org.apache.aurora.scheduler.thrift.Fixtures.okResponse;
 import static org.apache.aurora.scheduler.thrift.Fixtures.productionTask;
 import static org.apache.aurora.scheduler.thrift.Fixtures.response;
+import static org.apache.aurora.scheduler.thrift.SchedulerThriftInterface.ADD_INSTANCES;
+import static org.apache.aurora.scheduler.thrift.SchedulerThriftInterface.CREATE_JOB;
+import static org.apache.aurora.scheduler.thrift.SchedulerThriftInterface.CREATE_OR_UPDATE_CRON;
+import static org.apache.aurora.scheduler.thrift.SchedulerThriftInterface.DRAIN_HOSTS;
+import static org.apache.aurora.scheduler.thrift.SchedulerThriftInterface.END_MAINTENANCE;
+import static org.apache.aurora.scheduler.thrift.SchedulerThriftInterface.KILL_TASKS;
+import static org.apache.aurora.scheduler.thrift.SchedulerThriftInterface.MAINTENANCE_STATUS;
 import static org.apache.aurora.scheduler.thrift.SchedulerThriftInterface.MAX_TASK_ID_LENGTH;
 import static org.apache.aurora.scheduler.thrift.SchedulerThriftInterface.NOOP_JOB_UPDATE_MESSAGE;
 import static org.apache.aurora.scheduler.thrift.SchedulerThriftInterface.NO_CRON;
+import static org.apache.aurora.scheduler.thrift.SchedulerThriftInterface.RESTART_SHARDS;
+import static org.apache.aurora.scheduler.thrift.SchedulerThriftInterface.REWRITE_CONFIGS;
+import static org.apache.aurora.scheduler.thrift.SchedulerThriftInterface.START_JOB_UPDATE;
+import static org.apache.aurora.scheduler.thrift.SchedulerThriftInterface.START_MAINTENANCE;
 import static org.apache.aurora.scheduler.thrift.SchedulerThriftInterface.jobAlreadyExistsMessage;
 import static org.apache.aurora.scheduler.thrift.SchedulerThriftInterface.noCronScheduleMessage;
 import static org.apache.aurora.scheduler.thrift.SchedulerThriftInterface.notScheduledCronMessage;
@@ -192,6 +204,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
   private ReadOnlyScheduler.Iface readOnlyScheduler;
   private AuditMessages auditMessages;
   private TaskReconciler taskReconciler;
+  private FakeStatsProvider statsProvider;
 
   @Before
   public void setUp() throws Exception {
@@ -210,6 +223,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     readOnlyScheduler = createMock(ReadOnlyScheduler.Iface.class);
     auditMessages = createMock(AuditMessages.class);
     taskReconciler = createMock(TaskReconciler.class);
+    statsProvider = new FakeStatsProvider();
 
     thrift = getResponseProxy(
         new SchedulerThriftInterface(
@@ -228,7 +242,8 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
             jobUpdateController,
             readOnlyScheduler,
             auditMessages,
-            taskReconciler));
+            taskReconciler,
+            statsProvider));
   }
 
   private static AuroraAdmin.Iface getResponseProxy(AuroraAdmin.Iface realThrift) {
@@ -282,6 +297,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
 
     assertOkResponse(thrift.createJob(makeProdJob()));
+    assertEquals(1L, statsProvider.getLongValue(CREATE_JOB));
   }
 
   @Test
@@ -303,6 +319,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
 
     assertOkResponse(thrift.createJob(job.newBuilder()));
+    assertEquals(1L, statsProvider.getLongValue(CREATE_JOB));
   }
 
   @Test
@@ -312,6 +329,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
 
     assertEquals(invalidResponse(NO_CRON), thrift.createJob(job.newBuilder()));
+    assertEquals(0L, statsProvider.getLongValue(CREATE_JOB));
   }
 
   @Test
@@ -320,6 +338,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
 
     assertResponse(INVALID_REQUEST, thrift.createJob(job.newBuilder()));
+    assertEquals(0L, statsProvider.getLongValue(CREATE_JOB));
   }
 
   @Test
@@ -331,6 +350,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
 
     assertResponse(LOCK_ERROR, thrift.createJob(job.newBuilder()));
+    assertEquals(0L, statsProvider.getLongValue(CREATE_JOB));
   }
 
   @Test
@@ -342,6 +362,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
 
     assertResponse(INVALID_REQUEST, thrift.createJob(job.newBuilder()));
+    assertEquals(0L, statsProvider.getLongValue(CREATE_JOB));
   }
 
   @Test
@@ -354,6 +375,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
 
     assertResponse(INVALID_REQUEST, thrift.createJob(job.newBuilder()));
+    assertEquals(0L, statsProvider.getLongValue(CREATE_JOB));
   }
 
   @Test
@@ -372,6 +394,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
 
     assertResponse(INVALID_REQUEST, thrift.createJob(job.newBuilder()));
+    assertEquals(0L, statsProvider.getLongValue(CREATE_JOB));
   }
 
   @Test
@@ -392,6 +415,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
 
     assertResponse(INVALID_REQUEST, thrift.createJob(job.newBuilder()));
+    assertEquals(0L, statsProvider.getLongValue(CREATE_JOB));
   }
 
   @Test
@@ -408,6 +432,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
 
     assertResponse(INVALID_REQUEST, thrift.createJob(job.newBuilder()));
+    assertEquals(0L, statsProvider.getLongValue(CREATE_JOB));
   }
 
   private void assertMessageMatches(Response response, String string) {
@@ -423,6 +448,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     JobConfiguration job =
         new JobConfiguration().setKey(JOB_KEY.newBuilder()).setOwner(IDENTITY);
     assertResponse(INVALID_REQUEST, thrift.createJob(job));
+    assertEquals(0L, statsProvider.getLongValue(CREATE_JOB));
   }
 
   @Test
@@ -435,6 +461,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     Response response = thrift.createJob(job);
     assertResponse(INVALID_REQUEST, response);
     assertMessageMatches(response, ConfigurationManager.NO_EXECUTOR_OR_CONTAINER);
+    assertEquals(0L, statsProvider.getLongValue(CREATE_JOB));
   }
 
   @Test
@@ -445,6 +472,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
 
     assertResponse(INVALID_REQUEST, thrift.createJob(job));
+    assertEquals(0L, statsProvider.getLongValue(CREATE_JOB));
   }
 
   @Test
@@ -455,6 +483,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
 
     assertResponse(INVALID_REQUEST, thrift.createJob(job));
+    assertEquals(0L, statsProvider.getLongValue(CREATE_JOB));
   }
 
   @Test
@@ -466,6 +495,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     task.setRamMb(0);
     task.setDiskMb(0);
     assertResponse(INVALID_REQUEST, thrift.createJob(makeJob(task)));
+    assertEquals(0L, statsProvider.getLongValue(CREATE_JOB));
   }
 
   @Test
@@ -474,6 +504,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
 
     TaskConfig task = productionTask().setNumCpus(0.0);
     assertResponse(INVALID_REQUEST, thrift.createJob(makeJob(task)));
+    assertEquals(0L, statsProvider.getLongValue(CREATE_JOB));
   }
 
   @Test
@@ -482,6 +513,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
 
     TaskConfig task = productionTask().setRamMb(-123);
     assertResponse(INVALID_REQUEST, thrift.createJob(makeJob(task)));
+    assertEquals(0L, statsProvider.getLongValue(CREATE_JOB));
   }
 
   @Test
@@ -490,6 +522,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
 
     TaskConfig task = productionTask().setDiskMb(0);
     assertResponse(INVALID_REQUEST, thrift.createJob(makeJob(task)));
+    assertEquals(0L, statsProvider.getLongValue(CREATE_JOB));
   }
 
   @Test
@@ -541,6 +574,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
 
     assertOkResponse(thrift.createJob(job));
+    assertEquals(1L, statsProvider.getLongValue(CREATE_JOB));
   }
 
   @Test
@@ -550,6 +584,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     TaskConfig task = nonProductionTask();
     task.setConstraints(ImmutableSet.of(dedicatedConstraint(ImmutableSet.of("mesos"))));
     assertResponse(INVALID_REQUEST, thrift.createJob(makeJob(task)));
+    assertEquals(0L, statsProvider.getLongValue(CREATE_JOB));
   }
 
   @Test
@@ -559,6 +594,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     TaskConfig task = nonProductionTask();
     task.setConstraints(ImmutableSet.of(dedicatedConstraint(1)));
     assertResponse(INVALID_REQUEST, thrift.createJob(makeJob(task)));
+    assertEquals(0L, statsProvider.getLongValue(CREATE_JOB));
   }
 
   @Test
@@ -568,6 +604,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     TaskConfig task = nonProductionTask();
     task.setConstraints(ImmutableSet.of(dedicatedConstraint(ImmutableSet.of("mesos", "test"))));
     assertResponse(INVALID_REQUEST, thrift.createJob(makeJob(task)));
+    assertEquals(0L, statsProvider.getLongValue(CREATE_JOB));
   }
 
   private IScheduledTask buildTaskForJobUpdate(int instanceId) {
@@ -618,6 +655,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
 
     assertOkResponse(thrift.killTasks(JOB_KEY.newBuilder(), null));
+    assertEquals(1L, statsProvider.getLongValue(KILL_TASKS));
   }
 
   @Test
@@ -630,6 +668,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
 
     assertOkResponse(thrift.killTasks(JOB_KEY.newBuilder(), ImmutableSet.of(1)));
+    assertEquals(1L, statsProvider.getLongValue(KILL_TASKS));
   }
 
   @Test
@@ -648,6 +687,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     assertResponse(
         LOCK_ERROR,
         thrift.killTasks(JOB_KEY.newBuilder(), null));
+    assertEquals(0L, statsProvider.getLongValue(KILL_TASKS));
   }
 
   @Test
@@ -660,6 +700,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     Response response = thrift.killTasks(JOB_KEY.newBuilder(), null);
     assertOkResponse(response);
     assertMessageMatches(response, SchedulerThriftInterface.NO_TASKS_TO_KILL_MESSAGE);
+    assertEquals(0L, statsProvider.getLongValue(KILL_TASKS));
   }
 
   @Test
@@ -798,6 +839,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
 
     assertOkResponse(
         thrift.restartShards(JOB_KEY.newBuilder(), shards));
+    assertEquals(1L, statsProvider.getLongValue(RESTART_SHARDS));
   }
 
   @Test
@@ -812,6 +854,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     assertResponse(
         LOCK_ERROR,
         thrift.restartShards(JOB_KEY.newBuilder(), shards));
+    assertEquals(0L, statsProvider.getLongValue(KILL_TASKS));
   }
 
   @Test
@@ -826,6 +869,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     assertResponse(
         INVALID_REQUEST,
         thrift.restartShards(JOB_KEY.newBuilder(), shards));
+    assertEquals(0L, statsProvider.getLongValue(KILL_TASKS));
   }
 
   @Test
@@ -840,6 +884,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
 
     assertOkResponse(thrift.replaceCronTemplate(CRON_JOB));
+    assertEquals(1L, statsProvider.getLongValue(CREATE_OR_UPDATE_CRON));
   }
 
   @Test
@@ -849,6 +894,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
 
     assertResponse(LOCK_ERROR, thrift.replaceCronTemplate(CRON_JOB));
+    assertEquals(0L, statsProvider.getLongValue(CREATE_OR_UPDATE_CRON));
   }
 
   @Test
@@ -865,6 +911,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
 
     assertResponse(INVALID_REQUEST, thrift.replaceCronTemplate(CRON_JOB));
+    assertEquals(0L, statsProvider.getLongValue(CREATE_OR_UPDATE_CRON));
   }
 
   @Test
@@ -1011,6 +1058,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
         ImmutableList.of(ConfigRewrite.instanceRewrite(
             new InstanceConfigRewrite(instance, productionTask(), productionTask()))));
     assertResponse(WARNING, thrift.rewriteConfigs(request));
+    assertEquals(0L, statsProvider.getLongValue(REWRITE_CONFIGS));
   }
 
   @Test
@@ -1019,6 +1067,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
 
     RewriteConfigsRequest request = new RewriteConfigsRequest(ImmutableList.of());
     assertResponse(INVALID_REQUEST, thrift.rewriteConfigs(request));
+    assertEquals(0L, statsProvider.getLongValue(REWRITE_CONFIGS));
   }
 
   @Test(expected = RuntimeException.class)
@@ -1048,6 +1097,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
                 .setOwner(rewrittenIdentity)
                 .setKey(rewrittenJobKey)))));
     assertResponse(WARNING, thrift.rewriteConfigs(request));
+    assertEquals(0L, statsProvider.getLongValue(REWRITE_CONFIGS));
   }
 
   @Test
@@ -1068,6 +1118,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
         ImmutableList.of(ConfigRewrite.instanceRewrite(
             new InstanceConfigRewrite(instance, modifiedConfig, modifiedConfig))));
     assertResponse(WARNING, thrift.rewriteConfigs(request));
+    assertEquals(0L, statsProvider.getLongValue(REWRITE_CONFIGS));
   }
 
   @Test
@@ -1092,6 +1143,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
         ImmutableList.of(ConfigRewrite.instanceRewrite(
             new InstanceConfigRewrite(instanceKey, storedConfig, modifiedConfig.newBuilder()))));
     assertOkResponse(thrift.rewriteConfigs(request));
+    assertEquals(1L, statsProvider.getLongValue(REWRITE_CONFIGS));
   }
 
   @Test
@@ -1117,6 +1169,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
         ImmutableList.of(ConfigRewrite.instanceRewrite(
             new InstanceConfigRewrite(instanceKey, config, config))));
     assertResponse(WARNING, thrift.rewriteConfigs(request));
+    assertEquals(0L, statsProvider.getLongValue(REWRITE_CONFIGS));
   }
 
   @Test
@@ -1133,6 +1186,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
         ImmutableList.of(ConfigRewrite.jobRewrite(
             new JobConfigRewrite(newJob, newJob))));
     assertResponse(WARNING, thrift.rewriteConfigs(request));
+    assertEquals(0L, statsProvider.getLongValue(REWRITE_CONFIGS));
   }
 
   @Test
@@ -1149,6 +1203,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
         ImmutableList.of(ConfigRewrite.jobRewrite(
             new JobConfigRewrite(oldJob, newJob))));
     assertResponse(WARNING, thrift.rewriteConfigs(request));
+    assertEquals(0L, statsProvider.getLongValue(REWRITE_CONFIGS));
   }
 
   @Test
@@ -1166,6 +1221,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
         ImmutableList.of(ConfigRewrite.jobRewrite(
             new JobConfigRewrite(oldJob, newJob))));
     assertOkResponse(thrift.rewriteConfigs(request));
+    assertEquals(1L, statsProvider.getLongValue(REWRITE_CONFIGS));
   }
 
   @Test
@@ -1175,6 +1231,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     TaskConfig task = nonProductionTask();
     task.setConstraints(ImmutableSet.of(dedicatedConstraint(ImmutableSet.of("mesos"))));
     assertResponse(INVALID_REQUEST, thrift.createJob(makeJob(task)));
+    assertEquals(0L, statsProvider.getLongValue(CREATE_JOB));
   }
 
   private static Set<IHostStatus> status(String host, MaintenanceMode mode) {
@@ -1203,24 +1260,30 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
         IHostStatus.toBuildersSet(none),
         thrift.maintenanceStatus(hosts).getResult().getMaintenanceStatusResult()
             .getStatuses());
+    assertEquals(1L, statsProvider.getLongValue(MAINTENANCE_STATUS));
     assertEquals(
         IHostStatus.toBuildersSet(scheduled),
         thrift.startMaintenance(hosts).getResult().getStartMaintenanceResult()
             .getStatuses());
+    assertEquals(1L, statsProvider.getLongValue(START_MAINTENANCE));
     assertEquals(
         IHostStatus.toBuildersSet(draining),
         thrift.drainHosts(hosts).getResult().getDrainHostsResult().getStatuses());
+    assertEquals(1L, statsProvider.getLongValue(DRAIN_HOSTS));
     assertEquals(
         IHostStatus.toBuildersSet(draining),
         thrift.maintenanceStatus(hosts).getResult().getMaintenanceStatusResult()
             .getStatuses());
+    assertEquals(2L, statsProvider.getLongValue(MAINTENANCE_STATUS));
     assertEquals(
         IHostStatus.toBuildersSet(drained),
         thrift.maintenanceStatus(hosts).getResult().getMaintenanceStatusResult()
             .getStatuses());
+    assertEquals(3L, statsProvider.getLongValue(MAINTENANCE_STATUS));
     assertEquals(
         IHostStatus.toBuildersSet(none),
         thrift.endMaintenance(hosts).getResult().getEndMaintenanceResult().getStatuses());
+    assertEquals(1L, statsProvider.getLongValue(END_MAINTENANCE));
   }
 
   private static Response okEmptyResponse() {
@@ -1331,6 +1394,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
 
     assertOkResponse(thrift.addInstances(INSTANCE_KEY, 2));
+    assertEquals(2L, statsProvider.getLongValue(ADD_INSTANCES));
   }
 
   @Test
@@ -1344,6 +1408,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     assertEquals(
         invalidResponse(SchedulerThriftInterface.INVALID_INSTANCE_ID),
         thrift.addInstances(INSTANCE_KEY, 2));
+    assertEquals(0L, statsProvider.getLongValue(ADD_INSTANCES));
   }
 
   @Test
@@ -1357,6 +1422,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     assertEquals(
         invalidResponse(SchedulerThriftInterface.INVALID_INSTANCE_COUNT),
         thrift.addInstances(INSTANCE_KEY, 0));
+    assertEquals(0L, statsProvider.getLongValue(ADD_INSTANCES));
   }
 
   @Test
@@ -1386,6 +1452,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
 
     assertResponse(LOCK_ERROR, thrift.addInstances(INSTANCE_KEY, 1));
+    assertEquals(0L, statsProvider.getLongValue(ADD_INSTANCES));
   }
 
   @Test
@@ -1405,6 +1472,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
 
     assertResponse(INVALID_REQUEST, thrift.addInstances(INSTANCE_KEY, 1));
+    assertEquals(0L, statsProvider.getLongValue(ADD_INSTANCES));
   }
 
   @Test
@@ -1421,6 +1489,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
 
     assertResponse(INVALID_REQUEST, thrift.addInstances(INSTANCE_KEY, 1));
+    assertEquals(0L, statsProvider.getLongValue(ADD_INSTANCES));
   }
 
   @Test
@@ -1442,6 +1511,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
 
     assertResponse(INVALID_REQUEST, thrift.addInstances(INSTANCE_KEY, 1));
+    assertEquals(0L, statsProvider.getLongValue(ADD_INSTANCES));
   }
 
   @Test
@@ -1489,6 +1559,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     assertEquals(
         new StartJobUpdateResult(UPDATE_KEY.newBuilder()).setUpdateSummary(
         update.getSummary().newBuilder()), response.getResult().getStartJobUpdateResult());
+    assertEquals(6L, statsProvider.getLongValue(START_JOB_UPDATE));
   }
 
   private void expectJobUpdateQuotaCheck(QuotaCheckResult result) {
@@ -1537,6 +1608,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     assertEquals(
         new StartJobUpdateResult(UPDATE_KEY.newBuilder()).setUpdateSummary(expected.getSummary()),
         response.getResult().getStartJobUpdateResult());
+    assertEquals(1L, statsProvider.getLongValue(START_JOB_UPDATE));
   }
 
   @Test(expected = NullPointerException.class)
@@ -1563,6 +1635,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     assertEquals(
         invalidResponse(SchedulerThriftInterface.INVALID_GROUP_SIZE),
         thrift.startJobUpdate(updateRequest, AUDIT_MESSAGE));
+    assertEquals(0L, statsProvider.getLongValue(START_JOB_UPDATE));
   }
 
   @Test
@@ -1575,6 +1648,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     assertEquals(
         invalidResponse(SchedulerThriftInterface.INVALID_MAX_INSTANCE_FAILURES),
         thrift.startJobUpdate(updateRequest, AUDIT_MESSAGE));
+    assertEquals(0L, statsProvider.getLongValue(START_JOB_UPDATE));
   }
 
   @Test
@@ -1588,6 +1662,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     assertEquals(
         invalidResponse(SchedulerThriftInterface.TOO_MANY_POTENTIAL_FAILED_INSTANCES),
         thrift.startJobUpdate(updateRequest, AUDIT_MESSAGE));
+    assertEquals(0L, statsProvider.getLongValue(START_JOB_UPDATE));
   }
 
   @Test
@@ -1600,6 +1675,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     assertEquals(
         invalidResponse(SchedulerThriftInterface.INVALID_MAX_FAILED_INSTANCES),
         thrift.startJobUpdate(updateRequest, AUDIT_MESSAGE));
+    assertEquals(0L, statsProvider.getLongValue(START_JOB_UPDATE));
   }
 
   @Test
@@ -1612,6 +1688,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     assertEquals(
         invalidResponse(SchedulerThriftInterface.INVALID_MIN_WAIT_TO_RUNNING),
         thrift.startJobUpdate(updateRequest, AUDIT_MESSAGE));
+    assertEquals(0L, statsProvider.getLongValue(START_JOB_UPDATE));
   }
 
   @Test
@@ -1619,6 +1696,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
     JobUpdateRequest request = buildJobUpdateRequest(populatedTask().setIsService(false));
     assertResponse(INVALID_REQUEST, thrift.startJobUpdate(request, AUDIT_MESSAGE));
+    assertEquals(0L, statsProvider.getLongValue(START_JOB_UPDATE));
   }
 
   @Test
@@ -1631,6 +1709,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     assertEquals(
         invalidResponse(SchedulerThriftInterface.INVALID_PULSE_TIMEOUT),
         thrift.startJobUpdate(updateRequest, AUDIT_MESSAGE));
+    assertEquals(0L, statsProvider.getLongValue(START_JOB_UPDATE));
   }
 
   @Test
@@ -1640,6 +1719,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
 
     control.replay();
     assertEquals(invalidResponse(NO_CRON), thrift.startJobUpdate(request, AUDIT_MESSAGE));
+    assertEquals(0L, statsProvider.getLongValue(START_JOB_UPDATE));
   }
 
   @Test
@@ -1648,6 +1728,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
 
     control.replay();
     assertResponse(INVALID_REQUEST, thrift.startJobUpdate(request, AUDIT_MESSAGE));
+    assertEquals(0L, statsProvider.getLongValue(START_JOB_UPDATE));
   }
 
   @Test
@@ -1671,6 +1752,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     assertEquals(
         NOOP_JOB_UPDATE_MESSAGE,
         Iterables.getOnlyElement(response.getDetails()).getMessage());
+    assertEquals(0L, statsProvider.getLongValue(START_JOB_UPDATE));
   }
 
   @Test
@@ -1693,6 +1775,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
     JobUpdateRequest request = buildJobUpdateRequest(IJobUpdate.build(builder));
     assertResponse(INVALID_REQUEST, thrift.startJobUpdate(request, AUDIT_MESSAGE));
+    assertEquals(0L, statsProvider.getLongValue(START_JOB_UPDATE));
   }
 
   @Test
@@ -1732,6 +1815,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
     JobUpdateRequest request = buildJobUpdateRequest(IJobUpdate.build(builder));
     assertResponse(OK, thrift.startJobUpdate(request, AUDIT_MESSAGE));
+    assertEquals(3L, statsProvider.getLongValue(START_JOB_UPDATE));
   }
 
   @Test
@@ -1747,6 +1831,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
 
     assertResponse(INVALID_REQUEST, thrift.startJobUpdate(request, AUDIT_MESSAGE));
+    assertEquals(0L, statsProvider.getLongValue(START_JOB_UPDATE));
   }
 
   @Test
@@ -1764,6 +1849,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
 
     assertResponse(INVALID_REQUEST, thrift.startJobUpdate(request, AUDIT_MESSAGE));
+    assertEquals(0L, statsProvider.getLongValue(START_JOB_UPDATE));
   }
 
   @Test
@@ -1783,6 +1869,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     control.replay();
 
     assertResponse(INVALID_REQUEST, thrift.startJobUpdate(request, AUDIT_MESSAGE));
+    assertEquals(0L, statsProvider.getLongValue(START_JOB_UPDATE));
   }
 
   @Test
@@ -1811,6 +1898,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     assertResponse(
         INVALID_REQUEST,
         thrift.startJobUpdate(buildJobUpdateRequest(update), AUDIT_MESSAGE));
+    assertEquals(0L, statsProvider.getLongValue(START_JOB_UPDATE));
   }
 
   @Test
@@ -1842,6 +1930,7 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
     assertEquals(
         new StartJobUpdateResult(UPDATE_KEY.newBuilder()).setUpdateSummary(
             update.getSummary().newBuilder()), response.getResult().getStartJobUpdateResult());
+    assertEquals(0L, statsProvider.getLongValue(START_JOB_UPDATE));
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/aurora/blob/7be7ad6f/src/test/java/org/apache/aurora/scheduler/thrift/aop/ThriftStatsExporterInterceptorTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/thrift/aop/ThriftStatsExporterInterceptorTest.java b/src/test/java/org/apache/aurora/scheduler/thrift/aop/ThriftStatsExporterInterceptorTest.java
index 9c40ec5..a62ac93 100644
--- a/src/test/java/org/apache/aurora/scheduler/thrift/aop/ThriftStatsExporterInterceptorTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/thrift/aop/ThriftStatsExporterInterceptorTest.java
@@ -19,9 +19,11 @@ import com.google.inject.Guice;
 import com.google.inject.Injector;
 import com.google.inject.matcher.Matchers;
 
+import org.aopalliance.intercept.MethodInvocation;
 import org.apache.aurora.common.stats.Stats;
 import org.apache.aurora.common.testing.easymock.EasyMockTest;
 import org.apache.aurora.gen.GetJobsResult;
+import org.apache.aurora.gen.JobConfiguration;
 import org.apache.aurora.gen.Response;
 import org.apache.aurora.gen.Result;
 import org.apache.aurora.scheduler.thrift.auth.DecoratedThrift;
@@ -29,9 +31,14 @@ import org.junit.Before;
 import org.junit.Test;
 
 import static org.apache.aurora.gen.ResponseCode.OK;
+import static org.apache.aurora.scheduler.thrift.Responses.error;
+import static org.apache.aurora.scheduler.thrift.Responses.ok;
 import static org.easymock.EasyMock.expect;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertSame;
+import static org.junit.Assert.fail;
 
 public class ThriftStatsExporterInterceptorTest extends EasyMockTest {
 
@@ -56,6 +63,7 @@ public class ThriftStatsExporterInterceptorTest extends EasyMockTest {
       }
     });
     decoratedThrift = injector.getInstance(AnnotatedAuroraAdmin.class);
+    addTearDown(Stats::flush);
   }
 
   @Test
@@ -68,10 +76,97 @@ public class ThriftStatsExporterInterceptorTest extends EasyMockTest {
     control.replay();
 
     assertSame(response, decoratedThrift.getJobs(ROLE));
-    assertNotNull(Stats.getVariable("scheduler_thrift_getJobs_events"));
-    assertNotNull(Stats.getVariable("scheduler_thrift_getJobs_events_per_sec"));
-    assertNotNull(Stats.getVariable("scheduler_thrift_getJobs_nanos_per_event"));
-    assertNotNull(Stats.getVariable("scheduler_thrift_getJobs_nanos_total"));
-    assertNotNull(Stats.getVariable("scheduler_thrift_getJobs_nanos_total_per_sec"));
+    String statName = timingStatName("getJobs");
+    assertEquals(1L, Stats.getVariable(statName + "_events").read());
+    assertNotNull(Stats.getVariable(statName + "_events_per_sec"));
+    assertNotNull(Stats.getVariable(statName + "_nanos_per_event"));
+    assertNotNull(Stats.getVariable(statName + "_nanos_total"));
+    assertNotNull(Stats.getVariable(statName + "_nanos_total_per_sec"));
   }
+
+  @Test
+  public void testMeasuredMethod() throws Throwable {
+    MethodInvocation invocation = createMock(MethodInvocation.class);
+
+    expect(invocation.getMethod())
+        .andReturn(InterceptedClass.class.getDeclaredMethod("measuredMethod"));
+    expect(invocation.proceed()).andReturn(ok().setResult(Result.getJobsResult(
+        new GetJobsResult(ImmutableSet.of(new JobConfiguration())))));
+
+    control.replay();
+
+    new ThriftStatsExporterInterceptor().invoke(invocation);
+    assertEquals(1L, Stats.getVariable(workloadStatName("measuredMethod")).read());
+  }
+
+  @Test
+  public void testUnmeasuredMethod() throws Throwable {
+    MethodInvocation invocation = createMock(MethodInvocation.class);
+
+    expect(invocation.getMethod())
+        .andReturn(InterceptedClass.class.getDeclaredMethod("unmeasuredMethod"));
+    expect(invocation.proceed()).andReturn(ok().setResult(Result.getJobsResult(
+        new GetJobsResult(ImmutableSet.of(new JobConfiguration())))));
+
+    control.replay();
+
+    new ThriftStatsExporterInterceptor().invoke(invocation);
+    assertNull(Stats.getVariable(workloadStatName("unmeasuredMethod")));
+  }
+
+  @Test
+  public void testExceptionalMeasuredMethod() throws Throwable {
+    MethodInvocation invocation = createMock(MethodInvocation.class);
+
+    expect(invocation.getMethod())
+        .andReturn(InterceptedClass.class.getDeclaredMethod("measuredMethod"));
+    expect(invocation.proceed()).andThrow(new Exception());
+
+    control.replay();
+
+    try {
+      new ThriftStatsExporterInterceptor().invoke(invocation);
+      fail("Should not be reached");
+    } catch (Exception e) {
+      assertNull(Stats.getVariable(workloadStatName("measuredMethod")));
+    }
+  }
+
+  @Test
+  public void testMeasuredMethodWithErrorResponse() throws Throwable {
+    MethodInvocation invocation = createMock(MethodInvocation.class);
+
+    expect(invocation.getMethod())
+        .andReturn(InterceptedClass.class.getDeclaredMethod("measuredMethod"));
+    expect(invocation.proceed()).andReturn(error("ERROR"));
+
+    control.replay();
+
+    new ThriftStatsExporterInterceptor().invoke(invocation);
+    assertNull(Stats.getVariable(workloadStatName("measuredMethod")));
+  }
+
+  private static class InterceptedClass {
+    @ThriftWorkload
+    public Response measuredMethod() {
+      throw new UnsupportedOperationException("Should not be called.");
+    }
+
+    public Response unmeasuredMethod() {
+      throw new UnsupportedOperationException("Should not be called.");
+    }
+  }
+
+  private static String timingStatName(String methodName) {
+    return statName(ThriftStatsExporterInterceptor.TIMING_STATS_NAME_TEMPLATE, methodName);
+  }
+
+  private static String workloadStatName(String methodName) {
+    return statName(ThriftStatsExporterInterceptor.WORKLOAD_STATS_NAME_TEMPLATE, methodName);
+  }
+
+  private static String statName(String template, String methodName) {
+    return String.format(template, methodName);
+  }
+
 }


Mime
View raw message