aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kevi...@apache.org
Subject git commit: Add getConfigGroupSummary API endpoint
Date Mon, 23 Jun 2014 23:22:36 GMT
Repository: incubator-aurora
Updated Branches:
  refs/heads/master 69110a7b4 -> 1dd3691d7


Add getConfigGroupSummary API endpoint

Add getConfigGroupSummary API endpoint, which returns a TaskConfig ->
Set<InstanceId> struct for active tasks. This will let us paginate
active tasks, or drop TaskConfig from the getTasksStatus response in the
future.

Testing Done:
./gradlew build
./gradlew FlagSchemaChanges

Bugs closed: AURORA-519

Reviewed at https://reviews.apache.org/r/22429/


Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/1dd3691d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/1dd3691d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/1dd3691d

Branch: refs/heads/master
Commit: 1dd3691d77b740629f3e69f63ae40b8775b3fd79
Parents: 69110a7
Author: David McLaughlin <david@dmclaughlin.com>
Authored: Mon Jun 23 15:44:52 2014 -0700
Committer: Kevin Sweeney <kevints@apache.org>
Committed: Mon Jun 23 16:22:20 2014 -0700

----------------------------------------------------------------------
 .../thrift/SchedulerThriftInterface.java        | 36 ++++++++++++
 .../thrift/org/apache/aurora/gen/api.thrift     | 20 +++++++
 .../thrift/SchedulerThriftInterfaceTest.java    | 59 ++++++++++++++++++++
 .../scheduler/thrift/aop/ForwardingThrift.java  |  5 ++
 .../org/apache/aurora/gen/api.thrift.md5        |  2 +-
 5 files changed, 121 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/1dd3691d/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java
b/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java
index 7cc435f..5d21e5e 100644
--- a/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java
+++ b/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java
@@ -16,6 +16,7 @@ package org.apache.aurora.scheduler.thrift;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Set;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -49,7 +50,10 @@ import org.apache.aurora.auth.SessionValidator.AuthFailedException;
 import org.apache.aurora.gen.AcquireLockResult;
 import org.apache.aurora.gen.AddInstancesConfig;
 import org.apache.aurora.gen.AuroraAdmin;
+import org.apache.aurora.gen.ConfigGroup;
 import org.apache.aurora.gen.ConfigRewrite;
+import org.apache.aurora.gen.ConfigSummary;
+import org.apache.aurora.gen.ConfigSummaryResult;
 import org.apache.aurora.gen.DrainHostsResult;
 import org.apache.aurora.gen.EndMaintenanceResult;
 import org.apache.aurora.gen.GetJobsResult;
@@ -123,6 +127,7 @@ import org.apache.aurora.scheduler.storage.entities.ITaskConfig;
 import org.apache.aurora.scheduler.thrift.auth.DecoratedThrift;
 import org.apache.aurora.scheduler.thrift.auth.Requires;
 import org.apache.commons.lang.StringUtils;
+import org.apache.thrift.TException;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.twitter.common.base.MorePreconditions.checkNotBlank;
@@ -445,6 +450,37 @@ class SchedulerThriftInterface implements AuroraAdmin.Iface {
     return IScheduledTask.toBuildersList(tasks);
   }
 
+  private static final Function<Entry<ITaskConfig, Collection<Integer>>, ConfigGroup>
+      CONFIG_TO_GROUP = new Function<Entry<ITaskConfig, Collection<Integer>>,
ConfigGroup>() {
+
+    @Override
+    public ConfigGroup apply(Entry<ITaskConfig, Collection<Integer>> input) {
+      return new ConfigGroup(input.getKey().newBuilder(), ImmutableSet.copyOf(input.getValue()));
+    }
+  };
+
+  @Override
+  public Response getConfigSummary(JobKey job) throws TException {
+    IJobKey jobKey = JobKeys.assertValid(IJobKey.build(job));
+
+    Set<IScheduledTask> activeTasks =
+        Storage.Util.weaklyConsistentFetchTasks(storage, Query.jobScoped(jobKey).active());
+
+    Iterable<IAssignedTask> assignedTasks =
+        Iterables.transform(activeTasks, Tasks.SCHEDULED_TO_ASSIGNED);
+    Map<Integer, ITaskConfig> tasksByInstance = Maps.transformValues(
+        Maps.uniqueIndex(assignedTasks, Tasks.ASSIGNED_TO_INSTANCE_ID),
+        Tasks.ASSIGNED_TO_INFO);
+    Multimap<ITaskConfig, Integer> instancesByDetails = Multimaps.invertFrom(
+        Multimaps.forMap(tasksByInstance),
+        HashMultimap.<ITaskConfig, Integer>create());
+    Iterable<ConfigGroup> groups = Iterables.transform(
+        instancesByDetails.asMap().entrySet(), CONFIG_TO_GROUP);
+
+    ConfigSummary summary = new ConfigSummary(job, ImmutableSet.copyOf(groups));
+    return okResponse(Result.configSummaryResult(new ConfigSummaryResult().setSummary(summary)));
+  }
+
   @Override
   public Response getRoleSummary() {
     Multimap<String, IJobKey> jobsByRole = mapByRole(

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/1dd3691d/src/main/thrift/org/apache/aurora/gen/api.thrift
----------------------------------------------------------------------
diff --git a/src/main/thrift/org/apache/aurora/gen/api.thrift b/src/main/thrift/org/apache/aurora/gen/api.thrift
index 4a468d7..8ee43fa 100644
--- a/src/main/thrift/org/apache/aurora/gen/api.thrift
+++ b/src/main/thrift/org/apache/aurora/gen/api.thrift
@@ -226,6 +226,16 @@ struct AddInstancesConfig {
   3: set<i32> instanceIds
 }
 
+struct ConfigGroup {
+  1: TaskConfig config
+  2: set<i32> instanceIds
+}
+
+struct ConfigSummary {
+  1: JobKey key
+  2: set<ConfigGroup> groups
+}
+
 struct PopulateJobResult {
   1: set<TaskConfig> populated
 }
@@ -432,10 +442,15 @@ struct JobSummaryResult {
   1: set<JobSummary> summaries
 }
 
+
 struct GetLocksResult {
   1: set<Lock> locks
 }
 
+struct ConfigSummaryResult {
+  1: ConfigSummary summary
+}
+
 // meta-data about the thrift server that is wrapped around every thrift response
 struct ServerInfo {
   1: string clusterName
@@ -459,6 +474,8 @@ union Result {
   17: RoleSummaryResult roleSummaryResult
   18: JobSummaryResult jobSummaryResult
   19: GetLocksResult getLocksResult
+  20: ConfigSummaryResult configSummaryResult
+
 }
 
 struct ResponseDetail {
@@ -491,6 +508,9 @@ service ReadOnlyScheduler {
   // This is an interim solution until we have a better way to query TaskConfigs (AURORA-541).
   Response getTasksWithoutConfigs(1: TaskQuery query)
 
+  // Fetches the configuration summary of active tasks for the specified job.
+  Response getConfigSummary(1: JobKey job)
+
   // Fetches the status of jobs.
   // ownerRole is optional, in which case all jobs are returned.
   Response getJobs(1: string ownerRole)

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/1dd3691d/src/test/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java
b/src/test/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java
index 3807c12..2b9edd4 100644
--- a/src/test/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java
@@ -30,6 +30,7 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
 import com.google.inject.AbstractModule;
 import com.google.inject.Guice;
 import com.google.inject.Injector;
@@ -46,7 +47,10 @@ import org.apache.aurora.gen.APIVersion;
 import org.apache.aurora.gen.AddInstancesConfig;
 import org.apache.aurora.gen.AssignedTask;
 import org.apache.aurora.gen.AuroraAdmin;
+import org.apache.aurora.gen.ConfigGroup;
 import org.apache.aurora.gen.ConfigRewrite;
+import org.apache.aurora.gen.ConfigSummary;
+import org.apache.aurora.gen.ConfigSummaryResult;
 import org.apache.aurora.gen.Constraint;
 import org.apache.aurora.gen.ExecutorConfig;
 import org.apache.aurora.gen.HostStatus;
@@ -1314,6 +1318,61 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
   }
 
   @Test
+  public void testGetConfigSummary() throws Exception {
+    IJobKey key = JobKeys.from("test", "test", "test");
+
+    TaskConfig firstGroupTask = defaultTask(true);
+    TaskConfig secondGroupTask = defaultTask(true).setNumCpus(2);
+
+    IScheduledTask first1 = IScheduledTask.build(new ScheduledTask()
+        .setAssignedTask(new AssignedTask().setTask(firstGroupTask).setInstanceId(0)));
+
+    IScheduledTask first2 = IScheduledTask.build(new ScheduledTask()
+        .setAssignedTask(new AssignedTask().setTask(firstGroupTask).setInstanceId(1)));
+
+    IScheduledTask second = IScheduledTask.build(new ScheduledTask()
+        .setAssignedTask(new AssignedTask().setTask(secondGroupTask).setInstanceId(2)));
+
+    storageUtil.expectTaskFetch(Query.jobScoped(key).active(), first1, first2, second);
+
+    ConfigGroup group1 = new ConfigGroup()
+        .setConfig(firstGroupTask)
+        .setInstanceIds(Sets.newHashSet(0, 1));
+    ConfigGroup group2 = new ConfigGroup()
+        .setConfig(secondGroupTask)
+        .setInstanceIds(Sets.newHashSet(2));
+
+    ConfigSummary summary = new ConfigSummary()
+        .setKey(key.newBuilder())
+        .setGroups(Sets.newHashSet(group1, group2));
+
+    ConfigSummaryResult expected = new ConfigSummaryResult().setSummary(summary);
+
+    control.replay();
+
+    Response response = assertOkResponse(thrift.getConfigSummary(key.newBuilder()));
+    assertEquals(expected, response.getResult().getConfigSummaryResult());
+  }
+
+  @Test
+  public void testEmptyConfigSummary() throws Exception {
+    IJobKey key = JobKeys.from("test", "test", "test");
+
+    storageUtil.expectTaskFetch(Query.jobScoped(key).active(), ImmutableSet.<IScheduledTask>of());
+
+    ConfigSummary summary = new ConfigSummary()
+        .setKey(key.newBuilder())
+        .setGroups(Sets.<ConfigGroup>newHashSet());
+
+    ConfigSummaryResult expected = new ConfigSummaryResult().setSummary(summary);
+
+    control.replay();
+
+    Response response = assertOkResponse(thrift.getConfigSummary(key.newBuilder()));
+    assertEquals(expected, response.getResult().getConfigSummaryResult());
+  }
+
+  @Test
   public void testGetRoleSummary() throws Exception {
     final String BAZ_ROLE = "baz_role";
     final Identity BAZ_ROLE_IDENTITY = new Identity(BAZ_ROLE, USER);

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/1dd3691d/src/test/java/org/apache/aurora/scheduler/thrift/aop/ForwardingThrift.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/thrift/aop/ForwardingThrift.java b/src/test/java/org/apache/aurora/scheduler/thrift/aop/ForwardingThrift.java
index 9991e3c..a746c48 100644
--- a/src/test/java/org/apache/aurora/scheduler/thrift/aop/ForwardingThrift.java
+++ b/src/test/java/org/apache/aurora/scheduler/thrift/aop/ForwardingThrift.java
@@ -115,6 +115,11 @@ abstract class ForwardingThrift implements AuroraAdmin.Iface {
   }
 
   @Override
+  public Response getConfigSummary(JobKey key) throws TException {
+    return delegate.getConfigSummary(key);
+  }
+
+  @Override
   public Response createJob(JobConfiguration description, Lock lock, SessionKey session)
       throws TException {
 

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/1dd3691d/src/test/resources/org/apache/aurora/gen/api.thrift.md5
----------------------------------------------------------------------
diff --git a/src/test/resources/org/apache/aurora/gen/api.thrift.md5 b/src/test/resources/org/apache/aurora/gen/api.thrift.md5
index a201d3c..3245817 100644
--- a/src/test/resources/org/apache/aurora/gen/api.thrift.md5
+++ b/src/test/resources/org/apache/aurora/gen/api.thrift.md5
@@ -1 +1 @@
-8f819cbaa88a131143aa3884d449ed7a
+121eacd83eb70fe4053abc37a7143e95


Mime
View raw message