aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ma...@apache.org
Subject aurora git commit: Fixing resource slot counting.
Date Thu, 19 May 2016 15:27:17 GMT
Repository: aurora
Updated Branches:
  refs/heads/master 3cbff4117 -> 9ad153f0f


Fixing resource slot counting.

Reviewed at https://reviews.apache.org/r/47561/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/9ad153f0
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/9ad153f0
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/9ad153f0

Branch: refs/heads/master
Commit: 9ad153f0f474649669b0c9fa2debd516a9a4db5a
Parents: 3cbff41
Author: Maxim Khutornenko <maxim@apache.org>
Authored: Thu May 19 08:27:00 2016 -0700
Committer: Maxim Khutornenko <maxim@apache.org>
Committed: Thu May 19 08:27:00 2016 -0700

----------------------------------------------------------------------
 .../org/apache/aurora/benchmark/Offers.java     |  4 +--
 .../aurora/benchmark/SchedulingBenchmarks.java  |  3 +++
 .../java/org/apache/aurora/benchmark/Tasks.java | 10 +++++++
 .../aurora/scheduler/resources/ResourceBag.java | 28 +++++++++++++-------
 .../scheduler/resources/ResourceBagTest.java    | 17 +++++++++---
 .../scheduler/stats/SlotSizeCounterTest.java    | 22 +++++++++++++++
 6 files changed, 70 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/aurora/blob/9ad153f0/src/jmh/java/org/apache/aurora/benchmark/Offers.java
----------------------------------------------------------------------
diff --git a/src/jmh/java/org/apache/aurora/benchmark/Offers.java b/src/jmh/java/org/apache/aurora/benchmark/Offers.java
index 1dabe15..144f47e 100644
--- a/src/jmh/java/org/apache/aurora/benchmark/Offers.java
+++ b/src/jmh/java/org/apache/aurora/benchmark/Offers.java
@@ -99,8 +99,8 @@ final class Offers {
         Protos.Offer offer = Protos.Offer.newBuilder()
             .addAllResources(ImmutableSet.of(
                 makeScalar(CPUS, cpu),
-                makeScalar(RAM_MB, ram.getValue()),
-                makeScalar(DISK_MB, disk.getValue()),
+                makeScalar(RAM_MB, ram.as(Data.MB)),
+                makeScalar(DISK_MB, disk.as(Data.MB)),
                 makeRange(
                     PORTS,
                     IntStream.range(1, ports).boxed().collect(Collectors.toSet()))))

http://git-wip-us.apache.org/repos/asf/aurora/blob/9ad153f0/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java
----------------------------------------------------------------------
diff --git a/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java b/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java
index 1aa484c..65f5edc 100644
--- a/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java
+++ b/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java
@@ -33,6 +33,7 @@ import org.apache.aurora.common.quantity.Time;
 import org.apache.aurora.common.stats.StatsProvider;
 import org.apache.aurora.common.util.Clock;
 import org.apache.aurora.common.util.testing.FakeClock;
+import org.apache.aurora.gen.ServerInfo;
 import org.apache.aurora.scheduler.HostOffer;
 import org.apache.aurora.scheduler.TaskIdGenerator;
 import org.apache.aurora.scheduler.TierModule;
@@ -60,6 +61,7 @@ import org.apache.aurora.scheduler.storage.Storage.MutateWork.NoResult;
 import org.apache.aurora.scheduler.storage.db.DbUtil;
 import org.apache.aurora.scheduler.storage.entities.IHostAttributes;
 import org.apache.aurora.scheduler.storage.entities.IScheduledTask;
+import org.apache.aurora.scheduler.storage.entities.IServerInfo;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -157,6 +159,7 @@ public class SchedulingBenchmarks {
               bind(Clock.class).toInstance(clock);
               bind(StatsProvider.class).toInstance(new FakeStatsProvider());
               bind(EventSink.class).toInstance(eventBus::post);
+              bind(IServerInfo.class).toInstance(IServerInfo.build(new ServerInfo("jmh",
"")));
             }
           }
       );

http://git-wip-us.apache.org/repos/asf/aurora/blob/9ad153f0/src/jmh/java/org/apache/aurora/benchmark/Tasks.java
----------------------------------------------------------------------
diff --git a/src/jmh/java/org/apache/aurora/benchmark/Tasks.java b/src/jmh/java/org/apache/aurora/benchmark/Tasks.java
index 4c4b03d..e548a09 100644
--- a/src/jmh/java/org/apache/aurora/benchmark/Tasks.java
+++ b/src/jmh/java/org/apache/aurora/benchmark/Tasks.java
@@ -15,6 +15,7 @@ package org.apache.aurora.benchmark;
 
 import java.util.Set;
 
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
 
@@ -32,6 +33,10 @@ import org.apache.aurora.scheduler.base.TaskTestUtil;
 import org.apache.aurora.scheduler.storage.entities.IJobKey;
 import org.apache.aurora.scheduler.storage.entities.IScheduledTask;
 
+import static org.apache.aurora.gen.Resource.diskMb;
+import static org.apache.aurora.gen.Resource.numCpus;
+import static org.apache.aurora.gen.Resource.ramMb;
+
 /**
  * Task factory.
  */
@@ -140,12 +145,17 @@ final class Tasks {
         builder.getAssignedTask()
             .setInstanceId(i)
             .setTaskId(taskId);
+        builder.getAssignedTask().setAssignedPorts(ImmutableMap.of());
         builder.getAssignedTask().getTask()
             .setConstraints(constraints.build())
             .setNumCpus(cpu)
             .setRamMb(ram.as(Data.MB))
             .setDiskMb(disk.as(Data.MB))
             .setProduction(isProduction)
+            .setResources(ImmutableSet.of(
+                numCpus(cpu),
+                ramMb(ram.as(Data.MB)),
+                diskMb(disk.as(Data.MB))))
             .setRequestedPorts(ImmutableSet.of());
         tasks.add(IScheduledTask.build(builder));
       }

http://git-wip-us.apache.org/repos/asf/aurora/blob/9ad153f0/src/main/java/org/apache/aurora/scheduler/resources/ResourceBag.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/resources/ResourceBag.java b/src/main/java/org/apache/aurora/scheduler/resources/ResourceBag.java
index 3cff946..211f1fc 100644
--- a/src/main/java/org/apache/aurora/scheduler/resources/ResourceBag.java
+++ b/src/main/java/org/apache/aurora/scheduler/resources/ResourceBag.java
@@ -15,12 +15,14 @@ package org.apache.aurora.scheduler.resources;
 
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.function.BinaryOperator;
 import java.util.function.Predicate;
 import java.util.stream.Stream;
 
 import com.google.common.base.MoreObjects;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Sets;
 
 import static java.util.stream.Collectors.toMap;
 
@@ -132,7 +134,9 @@ public class ResourceBag {
 
   /**
    * Divides this by other bag contents.
-   *
+   * <p>
+   * Note: any missing or zero resource values in {@code other} will result in
+   * {@code java.lang.Double.POSITIVE_INFINITY}.
    * @param other Other bag to divide by.
    * @return Result of division.
    */
@@ -173,16 +177,22 @@ public class ResourceBag {
         .collect(toMap(Map.Entry::getKey, Map.Entry::getValue)));
   }
 
+  /**
+   * Applies {@code operator} to this and {@code other} resource values. Any missing resource
type
+   * values on either side will get substituted with 0.0 before applying the operator.
+   *
+   * @param other Other ResourceBag.
+   * @param operator Operator to apply.
+   * @return Operation result.
+   */
   private ResourceBag binaryOp(ResourceBag other, BinaryOperator<Double> operator)
{
     ImmutableMap.Builder<ResourceType, Double> builder = ImmutableMap.builder();
-    for (Map.Entry<ResourceType, Double> entry : resourceVectors.entrySet()) {
-      // Apply binary operator only on matching keys from the other. If there is no match,
keep the
-      // current value unchanged.
-      builder.put(
-          entry.getKey(),
-          other.getResourceVectors().containsKey(entry.getKey())
-              ? operator.apply(entry.getValue(), other.getResourceVectors().get(entry.getKey()))
-              : entry.getValue());
+    Set<ResourceType> resourceTypes =
+        Sets.union(resourceVectors.keySet(), other.getResourceVectors().keySet());
+    for (ResourceType type : resourceTypes) {
+      Double left = resourceVectors.getOrDefault(type, 0.0);
+      Double right = other.getResourceVectors().getOrDefault(type, 0.0);
+      builder.put(type, operator.apply(left, right));
     }
 
     return new ResourceBag(builder.build());

http://git-wip-us.apache.org/repos/asf/aurora/blob/9ad153f0/src/test/java/org/apache/aurora/scheduler/resources/ResourceBagTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/resources/ResourceBagTest.java b/src/test/java/org/apache/aurora/scheduler/resources/ResourceBagTest.java
index b8e6474..c163826 100644
--- a/src/test/java/org/apache/aurora/scheduler/resources/ResourceBagTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/resources/ResourceBagTest.java
@@ -17,6 +17,8 @@ import com.google.common.collect.ImmutableMap;
 
 import org.junit.Test;
 
+import static java.lang.Double.POSITIVE_INFINITY;
+
 import static org.apache.aurora.scheduler.resources.ResourceBag.IS_NEGATIVE;
 import static org.apache.aurora.scheduler.resources.ResourceBag.LARGE;
 import static org.apache.aurora.scheduler.resources.ResourceBag.MEDIUM;
@@ -58,12 +60,21 @@ public class ResourceBagTest {
   @Test
   public void testKeyMismatch() {
     assertEquals(
-        new ResourceBag(ImmutableMap.of(CPUS, 9.0)),
+        new ResourceBag(ImmutableMap.of(CPUS, 9.0, RAM_MB, 16384.0, DISK_MB, 32768.0)),
         new ResourceBag(ImmutableMap.of(CPUS, 1.0)).add(LARGE));
 
+    // Check add with mismatched keys is still commutative.
+    assertEquals(
+        new ResourceBag(ImmutableMap.of(CPUS, 9.0, RAM_MB, 16384.0, DISK_MB, 32768.0)),
+        LARGE.add(new ResourceBag(ImmutableMap.of(CPUS, 1.0))));
+
+    // Check divide with missing values on the right.
     assertEquals(
-        LARGE.add(new ResourceBag(ImmutableMap.of(CPUS, 1.0))),
-        new ResourceBag(ImmutableMap.of(CPUS, 9.0, RAM_MB, 16384.0, DISK_MB, 32768.0)));
+        new ResourceBag(ImmutableMap.of(
+            CPUS, 1.0,
+            RAM_MB, POSITIVE_INFINITY,
+            DISK_MB, POSITIVE_INFINITY)),
+        LARGE.divide(new ResourceBag(ImmutableMap.of(CPUS, 8.0))));
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/aurora/blob/9ad153f0/src/test/java/org/apache/aurora/scheduler/stats/SlotSizeCounterTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/stats/SlotSizeCounterTest.java b/src/test/java/org/apache/aurora/scheduler/stats/SlotSizeCounterTest.java
index e529e67..6c532e6 100644
--- a/src/test/java/org/apache/aurora/scheduler/stats/SlotSizeCounterTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/stats/SlotSizeCounterTest.java
@@ -142,6 +142,28 @@ public class SlotSizeCounterTest extends EasyMockTest {
   }
 
   @Test
+  public void testMissingResourceVector() {
+    expectStatExport();
+    expectGetSlots(
+        new MachineResource(
+            ResourceTestUtil.bag(ImmutableMap.of(RAM_MB, 65536.0, DISK_MB, 65536.0)),
+            false,
+            false));
+
+    control.replay();
+
+    slotCounter.run();
+    assertEquals(0, smallCounter.get());
+    assertEquals(0, smallDedicatedCounter.get());
+    assertEquals(0, smallRevocableCounter.get());
+    assertEquals(0, smallDedicatedRevocableCounter.get());
+    assertEquals(0, largeCounter.get());
+    assertEquals(0, largeDedicatedCounter.get());
+    assertEquals(0, largeRevocableCounter.get());
+    assertEquals(0, largeDedicatedRevocableCounter.get());
+  }
+
+  @Test
   public void testCountSlots() {
     expectStatExport();
     expectGetSlots(


Mime
View raw message