Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: subru@apache.org
To: common-commits@hadoop.apache.org
Date: Tue, 30 Aug 2016 22:05:34 -0000
Message-Id: <e6c8afc79be8464b9a3dfa4f7fabc0bc@git.apache.org>
In-Reply-To: <3126957f2246440baf5fc57b79e5c80e@git.apache.org>
References: <3126957f2246440baf5fc57b79e5c80e@git.apache.org>
Subject: [15/50] [abbrv] hadoop git commit: YARN-5327. API changes required to
 support recurring reservations in the YARN ReservationSystem. (Sangeetha Abdu
 Jyothi via Subru).
archived-at: Tue, 30 Aug 2016 22:05:27 -0000

YARN-5327. API changes required to support recurring reservations in the YARN ReservationSystem. (Sangeetha Abdu Jyothi via Subru).


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b930dc3e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b930dc3e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b930dc3e

Branch: refs/heads/YARN-2915
Commit: b930dc3ec06afa479a249490976e3e127d201706
Parents: e806db7
Author: Subru Krishnan <subu@apache.org>
Authored: Fri Aug 26 16:58:47 2016 -0700
Committer: Subru Krishnan <subu@apache.org>
Committed: Fri Aug 26 16:58:47 2016 -0700

----------------------------------------------------------------------
 .../yarn/api/records/ReservationDefinition.java | 49 +++++++++++++++-
 .../src/main/proto/yarn_protos.proto            |  1 +
 .../impl/pb/ReservationDefinitionPBImpl.java    | 21 ++++++-
 .../reservation/ReservationInputValidator.java  | 14 +++++
 .../TestReservationInputValidator.java          | 61 ++++++++++++++++++++
 5 files changed, 143 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b930dc3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationDefinition.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationDefinition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationDefinition.java
index 10f592a..8ef881b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationDefinition.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationDefinition.java
@@ -37,13 +37,24 @@ public abstract class ReservationDefinition {
   @Public
   @Unstable
   public static ReservationDefinition newInstance(long arrival, long deadline,
-      ReservationRequests reservationRequests, String name) {
+      ReservationRequests reservationRequests, String name,
+      String recurrenceExpression) {
     ReservationDefinition rDefinition =
         Records.newRecord(ReservationDefinition.class);
     rDefinition.setArrival(arrival);
     rDefinition.setDeadline(deadline);
     rDefinition.setReservationRequests(reservationRequests);
     rDefinition.setReservationName(name);
+    rDefinition.setRecurrenceExpression(recurrenceExpression);
+    return rDefinition;
+  }
+
+  @Public
+  @Unstable
+  public static ReservationDefinition newInstance(long arrival, long deadline,
+      ReservationRequests reservationRequests, String name) {
+    ReservationDefinition rDefinition =
+        newInstance(arrival, deadline, reservationRequests, name, "0");
     return rDefinition;
   }
 
@@ -134,4 +145,40 @@ public abstract class ReservationDefinition {
   @Evolving
   public abstract void setReservationName(String name);
 
+  /**
+   * Get the recurrence of this reservation representing the time period of
+   * the periodic job. Currently, only long values are supported. Later,
+   * support for regular expressions denoting arbitrary recurrence patterns
+   * (e.g., every Tuesday and Thursday) will be added.
+   * Recurrence is represented in milliseconds for periodic jobs.
+   * Recurrence is 0 for non-periodic jobs. Periodic jobs are valid until they
+   * are explicitly cancelled and have higher priority than non-periodic jobs
+   * (during initial placement and replanning). Periodic job allocations are
+   * consistent across runs (flexibility in allocation is leveraged only during
+   * initial placement, allocations remain consistent thereafter).
+   *
+   * @return recurrence of this reservation
+   */
+  @Public
+  @Evolving
+  public abstract String getRecurrenceExpression();
+
+  /**
+   * Set the recurrence of this reservation representing the time period of
+   * the periodic job. Currently, only long values are supported. Later,
+   * support for regular expressions denoting arbitrary recurrence patterns
+   * (e.g., every Tuesday and Thursday) will be added.
+   * Recurrence is represented in milliseconds for periodic jobs.
+   * Recurrence is 0 for non-periodic jobs. Periodic jobs are valid until they
+   * are explicitly cancelled and have higher priority than non-periodic jobs
+   * (during initial placement and replanning). Periodic job allocations are
+   * consistent across runs (flexibility in allocation is leveraged only during
+   * initial placement, allocations remain consistent thereafter).
+   *
+   * @param recurrenceExpression recurrence interval of this reservation
+   */
+  @Public
+  @Evolving
+  public abstract void setRecurrenceExpression(String recurrenceExpression);
+
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b930dc3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 8a6ae3f..2cc1784 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -477,6 +477,7 @@ message ReservationDefinitionProto {
   optional int64 arrival = 2;
   optional int64 deadline = 3;
   optional string reservation_name = 4;
+  optional string recurrence_expression = 5 [default = "0"];
 }
 
 message ResourceAllocationRequestProto {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b930dc3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationDefinitionPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationDefinitionPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationDefinitionPBImpl.java
index cbe7ccd..b30cd2a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationDefinitionPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationDefinitionPBImpl.java
@@ -162,8 +162,9 @@ public class ReservationDefinitionPBImpl extends ReservationDefinition {
   @Override
   public String toString() {
     return "{Arrival: " + getArrival() + ", Deadline: " + getDeadline()
-        + ", Reservation Name: " + getReservationName() + ", Resources: "
-        + getReservationRequests() + "}";
+        + ", Reservation Name: " + getReservationName()
+        + ", Recurrence expression: " + getRecurrenceExpression()
+        + ", Resources: " + getReservationRequests() + "}";
   }
 
   @Override
@@ -181,4 +182,20 @@ public class ReservationDefinitionPBImpl extends ReservationDefinition {
     return false;
   }
 
+  @Override
+  public String getRecurrenceExpression() {
+    ReservationDefinitionProtoOrBuilder p = viaProto ? proto : builder;
+    if (p.hasRecurrenceExpression()) {
+      String recurrenceExpression = p.getRecurrenceExpression();
+      return recurrenceExpression;
+    } else {
+      return "0";
+    }
+  }
+
+  @Override
+  public void setRecurrenceExpression(String recurrenceExpression) {
+    builder.setRecurrenceExpression(recurrenceExpression);
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b930dc3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java
index 8ebde0a..068894f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java
@@ -155,6 +155,20 @@ public class ReservationInputValidator {
           "validate reservation input definition", "ClientRMService", message);
       throw RPCUtil.getRemoteException(message);
     }
+    // check that the recurrence is a positive long value.
+    String recurrenceExpression = contract.getRecurrenceExpression();
+    try {
+      Long recurrence = Long.parseLong(recurrenceExpression);
+      if (recurrence < 0) {
+        message = "Negative Period : " + recurrenceExpression + ". Please try"
+            + " again with a non-negative long value as period";
+        throw RPCUtil.getRemoteException(message);
+      }
+    } catch (NumberFormatException e) {
+      message = "Invalid period " + recurrenceExpression + ". Please try"
+          + " again with a non-negative long value as period";
+      throw RPCUtil.getRemoteException(message);
+    }
   }
 
   private Plan getPlanFromQueue(ReservationSystem reservationSystem, String

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b930dc3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationInputValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationInputValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationInputValidator.java
index bd40ccd..031a767 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationInputValidator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationInputValidator.java
@@ -270,6 +270,59 @@ public class TestReservationInputValidator {
   }
 
   @Test
+  public void testSubmitReservationValidRecurrenceExpression() {
+    ReservationSubmissionRequest request =
+        createSimpleReservationSubmissionRequest(1, 1, 1, 5, 3, "600000");
+    plan = null;
+    try {
+      plan =
+          rrValidator.validateReservationSubmissionRequest(rSystem, request,
+              ReservationSystemTestUtil.getNewReservationId());
+    } catch (YarnException e) {
+      Assert.fail(e.getMessage());
+    }
+    Assert.assertNotNull(plan);
+  }
+
+  @Test
+  public void testSubmitReservationNegativeRecurrenceExpression() {
+    ReservationSubmissionRequest request =
+        createSimpleReservationSubmissionRequest(1, 1, 1, 5, 3, "-1234");
+    plan = null;
+    try {
+      plan =
+          rrValidator.validateReservationSubmissionRequest(rSystem, request,
+              ReservationSystemTestUtil.getNewReservationId());
+      Assert.fail();
+    } catch (YarnException e) {
+      Assert.assertNull(plan);
+      String message = e.getMessage();
+      Assert.assertTrue(message
+          .startsWith("Negative Period : "));
+      LOG.info(message);
+    }
+  }
+
+  @Test
+  public void testSubmitReservationInvalidRecurrenceExpression() {
+    ReservationSubmissionRequest request =
+        createSimpleReservationSubmissionRequest(1, 1, 1, 5, 3, "123abc");
+    plan = null;
+    try {
+      plan =
+          rrValidator.validateReservationSubmissionRequest(rSystem, request,
+              ReservationSystemTestUtil.getNewReservationId());
+      Assert.fail();
+    } catch (YarnException e) {
+      Assert.assertNull(plan);
+      String message = e.getMessage();
+      Assert.assertTrue(message
+          .startsWith("Invalid period "));
+      LOG.info(message);
+    }
+  }
+
+  @Test
   public void testUpdateReservationNormal() {
     ReservationUpdateRequest request =
         createSimpleReservationUpdateRequest(1, 1, 1, 5, 3);
@@ -625,12 +678,20 @@ public class TestReservationInputValidator {
   private ReservationSubmissionRequest createSimpleReservationSubmissionRequest(
       int numRequests, int numContainers, long arrival, long deadline,
       long duration) {
+    return createSimpleReservationSubmissionRequest(numRequests, numContainers,
+        arrival, deadline, duration, "0");
+  }
+
+  private ReservationSubmissionRequest createSimpleReservationSubmissionRequest(
+      int numRequests, int numContainers, long arrival, long deadline,
+      long duration, String recurrence) {
     // create a request with a single atomic ask
     ReservationSubmissionRequest request =
         new ReservationSubmissionRequestPBImpl();
     ReservationDefinition rDef = new ReservationDefinitionPBImpl();
     rDef.setArrival(arrival);
     rDef.setDeadline(deadline);
+    rDef.setRecurrenceExpression(recurrence);
     if (numRequests > 0) {
       ReservationRequests reqs = new ReservationRequestsPBImpl();
       rDef.setReservationRequests(reqs);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org