aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s...@apache.org
Subject aurora git commit: Display reservations and persistent volumes in /offers debug http endpoint
Date Sat, 23 Jul 2016 16:54:56 GMT
Repository: aurora
Updated Branches:
  refs/heads/master e67c6a732 -> 9c316d324


Display reservations and persistent volumes in /offers debug http endpoint

Bugs closed: AURORA-1736

Reviewed at https://reviews.apache.org/r/50052/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/9c316d32
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/9c316d32
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/9c316d32

Branch: refs/heads/master
Commit: 9c316d324c88281ba97b332a427901f69bf705df
Parents: e67c6a7
Author: Mehrdad Nurolahzade <mehrdad@nurolahzade.com>
Authored: Sat Jul 23 18:48:48 2016 +0200
Committer: Stephan Erb <serb@apache.org>
Committed: Sat Jul 23 18:48:48 2016 +0200

----------------------------------------------------------------------
 RELEASE-NOTES.md                                |  12 ++
 config/legacy_untested_classes.txt              |   1 -
 .../apache/aurora/scheduler/http/Offers.java    |  93 +++--------
 .../aurora/scheduler/http/OffersTest.java       | 156 +++++++++++++++++++
 4 files changed, 186 insertions(+), 76 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/aurora/blob/9c316d32/RELEASE-NOTES.md
----------------------------------------------------------------------
diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md
index 29d224d..ca98d7a 100644
--- a/RELEASE-NOTES.md
+++ b/RELEASE-NOTES.md
@@ -1,3 +1,15 @@
+0.16.0 (Not yet released)
+======
+
+### New/updated:
+
+- The `/offers` endpoint has been modified to display attributes of resource offers as received
+  from Mesos. This has affected rendering of some of the existing attributes. Furthermore,
it now
+  dumps additional offer attributes including [reservations](http://mesos.apache.org/documentation/latest/reservation/)
+  and [persistent volumes](http://mesos.apache.org/documentation/latest/persistent-volume/).
+
+### Deprecations and removals:
+
 0.15.0
 ======
 

http://git-wip-us.apache.org/repos/asf/aurora/blob/9c316d32/config/legacy_untested_classes.txt
----------------------------------------------------------------------
diff --git a/config/legacy_untested_classes.txt b/config/legacy_untested_classes.txt
index 1ea2183..ee4d3d7 100644
--- a/config/legacy_untested_classes.txt
+++ b/config/legacy_untested_classes.txt
@@ -26,7 +26,6 @@ org/apache/aurora/scheduler/http/api/security/Kerberos5Realm
 org/apache/aurora/scheduler/http/JerseyTemplateServlet
 org/apache/aurora/scheduler/http/LogConfig
 org/apache/aurora/scheduler/http/LogConfig$LoggerConfig
-org/apache/aurora/scheduler/http/Offers
 org/apache/aurora/scheduler/http/Offers$1
 org/apache/aurora/scheduler/http/Offers$2
 org/apache/aurora/scheduler/http/Offers$3

http://git-wip-us.apache.org/repos/asf/aurora/blob/9c316d32/src/main/java/org/apache/aurora/scheduler/http/Offers.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/Offers.java b/src/main/java/org/apache/aurora/scheduler/http/Offers.java
index 80f0824..f22ca6e 100644
--- a/src/main/java/org/apache/aurora/scheduler/http/Offers.java
+++ b/src/main/java/org/apache/aurora/scheduler/http/Offers.java
@@ -13,9 +13,9 @@
  */
 package org.apache.aurora.scheduler.http;
 
-import java.util.Map;
 import java.util.Objects;
-
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
 import javax.inject.Inject;
 import javax.ws.rs.GET;
 import javax.ws.rs.Path;
@@ -23,18 +23,13 @@ import javax.ws.rs.Produces;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 
-import com.google.common.base.Function;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.ImmutableMap;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.PropertyNamingStrategy;
+import com.hubspot.jackson.datatype.protobuf.ProtobufModule;
 
-import org.apache.aurora.scheduler.HostOffer;
 import org.apache.aurora.scheduler.offers.OfferManager;
-import org.apache.mesos.Protos.Attribute;
-import org.apache.mesos.Protos.ExecutorID;
-import org.apache.mesos.Protos.Resource;
-import org.apache.mesos.Protos.Value.Range;
-
-import static org.apache.mesos.Protos.Offer;
 
 /**
  * Servlet that exposes resource offers that the scheduler is currently retaining.
@@ -43,10 +38,15 @@ import static org.apache.mesos.Protos.Offer;
 public class Offers {
 
   private final OfferManager offerManager;
+  private final ObjectMapper mapper;
 
   @Inject
   Offers(OfferManager offerManager) {
     this.offerManager = Objects.requireNonNull(offerManager);
+    mapper = new ObjectMapper()
+        .registerModule(new ProtobufModule())
+        .setPropertyNamingStrategy(PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES)
+        .setSerializationInclusion(JsonInclude.Include.NON_NULL);
   }
 
   /**
@@ -56,69 +56,12 @@ public class Offers {
    */
   @GET
   @Produces(MediaType.APPLICATION_JSON)
-  public Response getOffers() {
+  public Response getOffers() throws JsonProcessingException {
     return Response.ok(
-        FluentIterable.from(offerManager.getOffers()).transform(TO_BEAN).toList()).build();
+        mapper.writeValueAsString(
+            StreamSupport.stream(offerManager.getOffers().spliterator(), false)
+                .map(o -> o.getOffer())
+                .collect(Collectors.toList())))
+        .build();
   }
-
-  private static final Function<ExecutorID, String> EXECUTOR_ID_TOSTRING = ExecutorID::getValue;
-
-  private static final Function<Range, Object> RANGE_TO_BEAN =
-      range -> range.getBegin() + "-" + range.getEnd();
-
-  private static final Function<Attribute, Object> ATTRIBUTE_TO_BEAN =
-      attr -> {
-        ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder();
-        builder.put("name", attr.getName());
-        if (attr.hasScalar()) {
-          builder.put("scalar", attr.getScalar().getValue());
-        }
-        if (attr.hasRanges()) {
-          builder.put("ranges", immutable(attr.getRanges().getRangeList(), RANGE_TO_BEAN));
-        }
-        if (attr.hasSet()) {
-          builder.put("set", attr.getSet().getItemList());
-        }
-        if (attr.hasText()) {
-          builder.put("text", attr.getText().getValue());
-        }
-        return builder.build();
-      };
-
-  private static final Function<Resource, Object> RESOURCE_TO_BEAN =
-      resource -> {
-        ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder();
-        builder.put("name", resource.getName());
-        if (resource.hasScalar()) {
-          builder.put("scalar", resource.getScalar().getValue());
-        }
-        if (resource.hasRanges()) {
-          builder.put("ranges", immutable(resource.getRanges().getRangeList(), RANGE_TO_BEAN));
-        }
-        if (resource.hasSet()) {
-          builder.put("set", resource.getSet().getItemList());
-        }
-        if (resource.hasRevocable()) {
-          builder.put("revocable", "true");
-        }
-        return builder.build();
-      };
-
-  private static <A, B> Iterable<B> immutable(Iterable<A> iterable, Function<A,
B> transform) {
-    return FluentIterable.from(iterable).transform(transform).toList();
-  }
-
-  private static final Function<HostOffer, Map<String, ?>> TO_BEAN =
-      hostOffer -> {
-        Offer offer = hostOffer.getOffer();
-        return ImmutableMap.<String, Object>builder()
-            .put("id", offer.getId().getValue())
-            .put("framework_id", offer.getFrameworkId().getValue())
-            .put("slave_id", offer.getSlaveId().getValue())
-            .put("hostname", offer.getHostname())
-            .put("resources", immutable(offer.getResourcesList(), RESOURCE_TO_BEAN))
-            .put("attributes", immutable(offer.getAttributesList(), ATTRIBUTE_TO_BEAN))
-            .put("executor_ids", immutable(offer.getExecutorIdsList(), EXECUTOR_ID_TOSTRING))
-            .build();
-      };
 }

http://git-wip-us.apache.org/repos/asf/aurora/blob/9c316d32/src/test/java/org/apache/aurora/scheduler/http/OffersTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/http/OffersTest.java b/src/test/java/org/apache/aurora/scheduler/http/OffersTest.java
new file mode 100644
index 0000000..9e35732
--- /dev/null
+++ b/src/test/java/org/apache/aurora/scheduler/http/OffersTest.java
@@ -0,0 +1,156 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.scheduler.http;
+
+import java.util.List;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.core.Response;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.PropertyNamingStrategy;
+import com.google.common.collect.ImmutableSet;
+import com.google.gson.Gson;
+import com.hubspot.jackson.datatype.protobuf.ProtobufModule;
+
+import org.apache.aurora.common.testing.easymock.EasyMockTest;
+import org.apache.aurora.gen.HostAttributes;
+import org.apache.aurora.scheduler.HostOffer;
+import org.apache.aurora.scheduler.offers.OfferManager;
+import org.apache.aurora.scheduler.storage.entities.IHostAttributes;
+import org.apache.mesos.Protos;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.apache.aurora.gen.MaintenanceMode.NONE;
+import static org.easymock.EasyMock.expect;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class OffersTest extends EasyMockTest {
+  private Offers offers;
+  private OfferManager offerManager;
+
+  @Before
+  public void setUp() {
+    offerManager = createMock(OfferManager.class);
+    offers = new Offers(offerManager);
+  }
+
+  @Test
+  public void testNoOffers() throws Exception {
+    expect(offerManager.getOffers()).andReturn(ImmutableSet.of());
+
+    control.replay();
+
+    Response response = offers.getOffers();
+    assertEquals(HttpServletResponse.SC_OK, response.getStatus());
+    assertTrue(toList(response.getEntity().toString()).isEmpty());
+  }
+
+  @Test
+  public void testOneOffer() throws Exception {
+    HostOffer offer = new HostOffer(
+        Protos.Offer.newBuilder()
+            .setId(Protos.OfferID.newBuilder().setValue("offer_id"))
+            .setFrameworkId(Protos.FrameworkID.newBuilder().setValue("framework_id"))
+            .setSlaveId(Protos.SlaveID.newBuilder().setValue("slave_id"))
+            .setHostname("host_name")
+            .addResources(Protos.Resource.newBuilder()
+                .setName("cpus")
+                .setType(Protos.Value.Type.SCALAR)
+                .setScalar(Protos.Value.Scalar.newBuilder().setValue(1.0).build())
+                .setReservation(Protos.Resource.ReservationInfo.newBuilder()
+                    .setLabels(Protos.Labels.newBuilder()
+                        .addLabels(Protos.Label.newBuilder()
+                            .setKey("key")
+                            .setValue("value"))
+                        .build())
+                    .build())
+                .build())
+            .addResources(Protos.Resource.newBuilder()
+                .setName("mem")
+                .setType(Protos.Value.Type.SCALAR)
+                .setScalar(Protos.Value.Scalar.newBuilder().setValue(128.0).build())
+                .setReservation(Protos.Resource.ReservationInfo.newBuilder().build())
+                .build())
+            .addResources(Protos.Resource.newBuilder()
+                .setName("disk")
+                .setType(Protos.Value.Type.SCALAR)
+                .setScalar(Protos.Value.Scalar.newBuilder().setValue(128.0).build())
+                .setReservation(Protos.Resource.ReservationInfo.newBuilder()
+                    .setLabels(Protos.Labels.newBuilder()
+                        .addLabels(Protos.Label.newBuilder()
+                            .setKey("key"))
+                        .build())
+                    .build())
+                .setDisk(Protos.Resource.DiskInfo.newBuilder()
+                    .setPersistence(Protos.Resource.DiskInfo.Persistence.newBuilder()
+                        .setId("volume")
+                        .build())
+                    .setVolume(Protos.Volume.newBuilder()
+                        .setContainerPath("path")
+                        .setMode(Protos.Volume.Mode.RW)
+                        .setImage(Protos.Image.newBuilder()
+                            .setType(Protos.Image.Type.DOCKER)
+                            .setDocker(Protos.Image.Docker.newBuilder()
+                                .setName("image")
+                                .build())
+                            .build())
+                        .build())
+                    .setSource(Protos.Resource.DiskInfo.Source.newBuilder()
+                        .setType(Protos.Resource.DiskInfo.Source.Type.PATH)
+                        .setPath(Protos.Resource.DiskInfo.Source.Path.newBuilder()
+                            .setRoot("root")
+                            .build())
+                        .build())
+                    .build())
+                .build())
+            .addResources(Protos.Resource.newBuilder()
+                .setName("gpus")
+                .setType(Protos.Value.Type.SCALAR)
+                .setScalar(Protos.Value.Scalar.newBuilder().setValue(4.0).build())
+                .build())
+            .addResources(Protos.Resource.newBuilder()
+                .setName("ports")
+                .setType(Protos.Value.Type.RANGES)
+                .setRanges(Protos.Value.Ranges.newBuilder()
+                    .addRange(Protos.Value.Range.newBuilder()
+                        .setBegin(31000)
+                        .setEnd(32000)
+                        .build())
+                    .build())
+                .build())
+            .build(),
+        IHostAttributes.build(new HostAttributes().setMode(NONE)));
+
+    expect(offerManager.getOffers()).andReturn(ImmutableSet.of(offer));
+
+    control.replay();
+
+    Response response = offers.getOffers();
+    assertEquals(HttpServletResponse.SC_OK, response.getStatus());
+    ObjectMapper mapper = new ObjectMapper()
+        .registerModule(new ProtobufModule())
+        .setPropertyNamingStrategy(
+            PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
+    assertEquals(
+        offer.getOffer(),
+        mapper.readValue(response.getEntity().toString(), Protos.Offer.class));
+  }
+
+  private static List toList(String json) {
+    return new Gson().fromJson(json, List.class);
+  }
+
+}


Mime
View raw message