Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 2BF3618667 for ; Mon, 7 Dec 2015 21:39:23 +0000 (UTC) Received: (qmail 40476 invoked by uid 500); 7 Dec 2015 21:39:16 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 40411 invoked by uid 500); 7 Dec 2015 21:39:16 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 40402 invoked by uid 99); 7 Dec 2015 21:39:16 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 07 Dec 2015 21:39:16 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 6FEBEE0939; Mon, 7 Dec 2015 21:39:16 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: curino@apache.org To: common-commits@hadoop.apache.org Message-Id: <0405780219594120882773a4337a99b8@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-4248. REST API for submit/update/delete Reservations. (curino) Date: Mon, 7 Dec 2015 21:39:16 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-2 7c604c3d0 -> 4ac156441 YARN-4248. REST API for submit/update/delete Reservations. (curino) (cherry picked from commit c25a6354598ec855bec7f695a7c3eed8794cd381) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4ac15644 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4ac15644 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4ac15644 Branch: refs/heads/branch-2 Commit: 4ac1564418597e2ebbaa80600eb5f002b1431281 Parents: 7c604c3 Author: = Authored: Mon Dec 7 13:33:28 2015 -0800 Committer: = Committed: Mon Dec 7 13:38:38 2015 -0800 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 2 + .../resourcemanager/webapp/RMWebServices.java | 304 +++++++++++ .../webapp/dao/ReservationDefinitionInfo.java | 82 +++ .../dao/ReservationDeleteRequestInfo.java | 49 ++ .../dao/ReservationDeleteResponseInfo.java | 36 ++ .../webapp/dao/ReservationRequestInfo.java | 78 +++ .../webapp/dao/ReservationRequestsInfo.java | 63 +++ .../dao/ReservationSubmissionRequestInfo.java | 60 +++ .../dao/ReservationSubmissionResponseInfo.java | 54 ++ .../dao/ReservationUpdateRequestInfo.java | 60 +++ .../dao/ReservationUpdateResponseInfo.java | 37 ++ .../webapp/TestRMWebServicesReservation.java | 517 +++++++++++++++++++ .../src/test/resources/delete-reservation.json | 3 + .../src/test/resources/submit-reservation.json | 31 ++ .../src/test/resources/update-reservation.json | 31 ++ 15 files changed, 1407 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac15644/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index f956cbd..45e9934 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -550,6 +550,8 @@ Release 2.8.0 - UNRELEASED YARN-3456. Improve handling of incomplete TimelineEntities. (Varun Saxena via rohithsharmaks) + YARN-4248. REST API for submit/update/delete Reservations. (curino) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac15644/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 2da477d..b744765 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -98,6 +98,11 @@ import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.api.records.ReservationDefinition; +import org.apache.hadoop.yarn.api.records.ReservationId; +import org.apache.hadoop.yarn.api.records.ReservationRequest; +import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter; +import org.apache.hadoop.yarn.api.records.ReservationRequests; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -145,6 +150,19 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsEntr import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsEntryList; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDefinitionInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDeleteRequestInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDeleteResponseInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationRequestInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationRequestsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationSubmissionResponseInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationSubmissionRequestInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationUpdateRequestInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationUpdateResponseInfo; +import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse; +import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; @@ -1822,4 +1840,290 @@ public class RMWebServices { } return token; } + + /** + * Function to submit a Reservation to the RM. + * + * @param resContext provides information to construct the + * ReservationSubmissionRequest + * @param hsr the servlet request + * @return Response containing the status code + * @throws AuthorizationException + * @throws IOException + * @throws InterruptedException + */ + @POST + @Path("/reservation/submit") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response submitReservation( + ReservationSubmissionRequestInfo resContext, + @Context HttpServletRequest hsr) throws AuthorizationException, + IOException, InterruptedException { + + init(); + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); + if (callerUGI == null) { + throw new AuthorizationException("Unable to obtain user name, " + + "user not authenticated"); + } + if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) { + String msg = "The default static user cannot carry out this operation."; + return Response.status(Status.FORBIDDEN).entity(msg).build(); + } + + final ReservationSubmissionRequest reservation = + createReservationSubmissionRequest(resContext); + + ReservationSubmissionResponseInfo resRespInfo; + try { + resRespInfo = + callerUGI.doAs( + new PrivilegedExceptionAction() { + @Override + public ReservationSubmissionResponseInfo run() + throws IOException, YarnException { + ReservationSubmissionResponse tempRes = + rm.getClientRMService().submitReservation(reservation); + return new ReservationSubmissionResponseInfo(tempRes); + } + }); + } catch (UndeclaredThrowableException ue) { + if (ue.getCause() instanceof YarnException) { + throw new BadRequestException(ue.getCause().getMessage()); + } + LOG.info("Submit reservation request failed", ue); + throw ue; + } + + return Response.status(Status.OK).entity(resRespInfo).build(); + } + + private ReservationSubmissionRequest createReservationSubmissionRequest( + ReservationSubmissionRequestInfo resContext) { + + // defending against a couple of common submission format problems + if (resContext == null) { + throw new BadRequestException( + "Input ReservationSubmissionContext should not be null"); + } + ReservationDefinitionInfo resInfo = resContext.getReservationDefinition(); + if (resInfo == null) { + throw new BadRequestException( + "Input ReservationDefinition should not be null"); + } + + ReservationRequestsInfo resReqsInfo = resInfo.getReservationRequests(); + + if (resReqsInfo == null || resReqsInfo.getReservationRequest() == null + || resReqsInfo.getReservationRequest().size() == 0) { + throw new BadRequestException("The ReservationDefinition should" + + " contain at least one ReservationRequest"); + } + + ReservationRequestInterpreter[] values = + ReservationRequestInterpreter.values(); + ReservationRequestInterpreter resInt = + values[resReqsInfo.getReservationRequestsInterpreter()]; + List list = new ArrayList(); + + for (ReservationRequestInfo resReqInfo : resReqsInfo + .getReservationRequest()) { + ResourceInfo rInfo = resReqInfo.getCapability(); + Resource capability = + Resource.newInstance(rInfo.getMemory(), rInfo.getvCores()); + int numContainers = resReqInfo.getNumContainers(); + int minConcurrency = resReqInfo.getMinConcurrency(); + long duration = resReqInfo.getDuration(); + ReservationRequest rr = + ReservationRequest.newInstance(capability, numContainers, + minConcurrency, duration); + list.add(rr); + } + ReservationRequests reqs = ReservationRequests.newInstance(list, resInt); + ReservationDefinition rDef = + ReservationDefinition.newInstance(resInfo.getArrival(), + resInfo.getDeadline(), reqs, resInfo.getReservationName()); + ReservationSubmissionRequest request = + ReservationSubmissionRequest.newInstance(rDef, resContext.getQueue()); + + return request; + } + + /** + * Function to update a Reservation to the RM. + * + * @param resContext provides information to construct the + * ReservationUpdateRequest + * @param hsr the servlet request + * @return Response containing the status code + * @throws AuthorizationException + * @throws IOException + * @throws InterruptedException + */ + @POST + @Path("/reservation/update") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response updateReservation(ReservationUpdateRequestInfo resContext, + @Context HttpServletRequest hsr) throws AuthorizationException, + IOException, InterruptedException { + + init(); + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); + if (callerUGI == null) { + throw new AuthorizationException("Unable to obtain user name, " + + "user not authenticated"); + } + if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) { + String msg = "The default static user cannot carry out this operation."; + return Response.status(Status.FORBIDDEN).entity(msg).build(); + } + + final ReservationUpdateRequest reservation = + createReservationUpdateRequest(resContext); + + ReservationUpdateResponseInfo resRespInfo; + try { + resRespInfo = + callerUGI.doAs( + new PrivilegedExceptionAction() { + @Override + public ReservationUpdateResponseInfo run() throws IOException, + YarnException { + rm.getClientRMService().updateReservation(reservation); + return new ReservationUpdateResponseInfo(); + } + }); + } catch (UndeclaredThrowableException ue) { + if (ue.getCause() instanceof YarnException) { + throw new BadRequestException(ue.getCause().getMessage()); + } + LOG.info("Update reservation request failed", ue); + throw ue; + } + + return Response.status(Status.OK).entity(resRespInfo).build(); + } + + private ReservationUpdateRequest createReservationUpdateRequest( + ReservationUpdateRequestInfo resContext) throws IOException { + + // defending against a couple of common submission format problems + if (resContext == null) { + throw new BadRequestException( + "Input ReservationSubmissionContext should not be null"); + } + ReservationDefinitionInfo resInfo = resContext.getReservationDefinition(); + if (resInfo == null) { + throw new BadRequestException( + "Input ReservationDefinition should not be null"); + } + ReservationRequestsInfo resReqsInfo = resInfo.getReservationRequests(); + if (resReqsInfo == null || resReqsInfo.getReservationRequest() == null + || resReqsInfo.getReservationRequest().size() == 0) { + throw new BadRequestException("The ReservationDefinition should" + + " contain at least one ReservationRequest"); + } + if (resContext.getReservationId() == null) { + throw new BadRequestException( + "Update operations must specify an existing ReservaitonId"); + } + + ReservationRequestInterpreter[] values = + ReservationRequestInterpreter.values(); + ReservationRequestInterpreter resInt = + values[resReqsInfo.getReservationRequestsInterpreter()]; + List list = new ArrayList(); + + for (ReservationRequestInfo resReqInfo : resReqsInfo + .getReservationRequest()) { + ResourceInfo rInfo = resReqInfo.getCapability(); + Resource capability = + Resource.newInstance(rInfo.getMemory(), rInfo.getvCores()); + int numContainers = resReqInfo.getNumContainers(); + int minConcurrency = resReqInfo.getMinConcurrency(); + long duration = resReqInfo.getDuration(); + ReservationRequest rr = + ReservationRequest.newInstance(capability, numContainers, + minConcurrency, duration); + list.add(rr); + } + ReservationRequests reqs = ReservationRequests.newInstance(list, resInt); + ReservationDefinition rDef = + ReservationDefinition.newInstance(resInfo.getArrival(), + resInfo.getDeadline(), reqs, resInfo.getReservationName()); + ReservationUpdateRequest request = + ReservationUpdateRequest.newInstance(rDef, ReservationId + .parseReservationId(resContext.getReservationId())); + + return request; + } + + /** + * Function to delete a Reservation to the RM. + * + * @param resContext provides information to construct + * the ReservationDeleteRequest + * @param hsr the servlet request + * @return Response containing the status code + * @throws AuthorizationException + * @throws IOException + * @throws InterruptedException + */ + @POST + @Path("/reservation/delete") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response deleteReservation(ReservationDeleteRequestInfo resContext, + @Context HttpServletRequest hsr) throws AuthorizationException, + IOException, InterruptedException { + + init(); + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); + if (callerUGI == null) { + throw new AuthorizationException("Unable to obtain user name, " + + "user not authenticated"); + } + if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) { + String msg = "The default static user cannot carry out this operation."; + return Response.status(Status.FORBIDDEN).entity(msg).build(); + } + + final ReservationDeleteRequest reservation = + createReservationDeleteRequest(resContext); + + ReservationDeleteResponseInfo resRespInfo; + try { + resRespInfo = + callerUGI.doAs( + new PrivilegedExceptionAction() { + @Override + public ReservationDeleteResponseInfo run() throws IOException, + YarnException { + rm.getClientRMService().deleteReservation(reservation); + return new ReservationDeleteResponseInfo(); + } + }); + } catch (UndeclaredThrowableException ue) { + if (ue.getCause() instanceof YarnException) { + throw new BadRequestException(ue.getCause().getMessage()); + } + LOG.info("Update reservation request failed", ue); + throw ue; + } + + return Response.status(Status.OK).entity(resRespInfo).build(); + } + + private ReservationDeleteRequest createReservationDeleteRequest( + ReservationDeleteRequestInfo resContext) throws IOException { + + ReservationDeleteRequest request = + ReservationDeleteRequest.newInstance(ReservationId + .parseReservationId(resContext.getReservationId())); + + return request; + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac15644/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationDefinitionInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationDefinitionInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationDefinitionInfo.java new file mode 100644 index 0000000..ff82e75 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationDefinitionInfo.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * Simple class that represent a reservation definition. + */ +@XmlRootElement(name = "reservation-definition") +@XmlAccessorType(XmlAccessType.FIELD) +public class ReservationDefinitionInfo { + + @XmlElement(name = "arrival") + private long arrival; + + @XmlElement(name = "deadline") + private long deadline; + + @XmlElement(name = "reservation-requests") + private ReservationRequestsInfo reservationRequests; + + @XmlElement(name = "reservation-name") + private String reservationName; + + public ReservationDefinitionInfo() { + + } + + public long getArrival() { + return arrival; + } + + public void setArrival(long arrival) { + this.arrival = arrival; + } + + public long getDeadline() { + return deadline; + } + + public void setDeadline(long deadline) { + this.deadline = deadline; + } + + public ReservationRequestsInfo getReservationRequests() { + return reservationRequests; + } + + public void setReservationRequests( + ReservationRequestsInfo reservationRequests) { + this.reservationRequests = reservationRequests; + } + + public String getReservationName() { + return reservationName; + } + + public void setReservationName(String reservationName) { + this.reservationName = reservationName; + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac15644/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationDeleteRequestInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationDeleteRequestInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationDeleteRequestInfo.java new file mode 100644 index 0000000..2a6bde3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationDeleteRequestInfo.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * Simple class represent the request of deleting a given reservation, + * selected by its id. + */ +@XmlRootElement(name = "reservation-delete-context") +@XmlAccessorType(XmlAccessType.FIELD) +public class ReservationDeleteRequestInfo { + + @XmlElement(name = "reservation-id") + private String reservationId; + + public ReservationDeleteRequestInfo() { + reservationId = null; + } + + public String getReservationId() { + return reservationId; + } + + public void setReservationId(String reservationId) { + this.reservationId = reservationId; + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac15644/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationDeleteResponseInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationDeleteResponseInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationDeleteResponseInfo.java new file mode 100644 index 0000000..51377a2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationDeleteResponseInfo.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * Simple class that represent a reponse to a delete operation. + */ +@XmlRootElement(name = "reservation-delete-response") +@XmlAccessorType(XmlAccessType.FIELD) +public class ReservationDeleteResponseInfo { + + public ReservationDeleteResponseInfo() { + + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac15644/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationRequestInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationRequestInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationRequestInfo.java new file mode 100644 index 0000000..ab82f3f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationRequestInfo.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * Simple class representing a reservation request. + */ +@XmlRootElement(name = "reservation-definition") +@XmlAccessorType(XmlAccessType.FIELD) +public class ReservationRequestInfo { + + @XmlElement(name = "capability") + private ResourceInfo capability; + @XmlElement(name = "min-concurrency") + private int minConcurrency; + @XmlElement(name = "num-containers") + private int numContainers; + @XmlElement(name = "duration") + private long duration; + + public ReservationRequestInfo() { + + } + + public ResourceInfo getCapability() { + return capability; + } + + public void setCapability(ResourceInfo capability) { + this.capability = capability; + } + + public int getMinConcurrency() { + return minConcurrency; + } + + public void setMinConcurrency(int minConcurrency) { + this.minConcurrency = minConcurrency; + } + + public int getNumContainers() { + return numContainers; + } + + public void setNumContainers(int numContainers) { + this.numContainers = numContainers; + } + + public long getDuration() { + return duration; + } + + public void setDuration(long duration) { + this.duration = duration; + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac15644/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationRequestsInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationRequestsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationRequestsInfo.java new file mode 100644 index 0000000..42cdb0e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationRequestsInfo.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.ArrayList; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * Simple class representing a list of ReservationRequest and the + * interpreter which capture the semantic of this list (all/any/order). + */ +@XmlRootElement(name = "reservation-definition") +@XmlAccessorType(XmlAccessType.FIELD) +public class ReservationRequestsInfo { + + @XmlElement(name = "reservation-request-interpreter") + private int reservationRequestsInterpreter; + @XmlElement(name = "reservation-request") + private ArrayList reservationRequest; + + public ReservationRequestsInfo() { + + } + + public int getReservationRequestsInterpreter() { + return reservationRequestsInterpreter; + } + + public void setReservationRequestsInterpreter( + int reservationRequestsInterpreter) { + this.reservationRequestsInterpreter = reservationRequestsInterpreter; + } + + public ArrayList getReservationRequest() { + return reservationRequest; + } + + public void setReservationRequest( + ArrayList reservationRequest) { + this.reservationRequest = reservationRequest; + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac15644/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationSubmissionRequestInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationSubmissionRequestInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationSubmissionRequestInfo.java new file mode 100644 index 0000000..701370d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationSubmissionRequestInfo.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * Simple class to allow users to send information required to create an + * ReservationSubmissionContext which can then be used to submit a reservation. + */ +@XmlRootElement(name = "reservation-submission-context") +@XmlAccessorType(XmlAccessType.FIELD) +public class ReservationSubmissionRequestInfo { + + @XmlElement(name = "queue") + private String queue; + + @XmlElement(name = "reservation-definition") + private ReservationDefinitionInfo reservationDefinition; + + public ReservationSubmissionRequestInfo() { + } + + public String getQueue() { + return queue; + } + + public void setQueue(String queue) { + this.queue = queue; + } + + public ReservationDefinitionInfo getReservationDefinition() { + return reservationDefinition; + } + + public void setReservationDefinition( + ReservationDefinitionInfo reservationDefinition) { + this.reservationDefinition = reservationDefinition; + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac15644/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationSubmissionResponseInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationSubmissionResponseInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationSubmissionResponseInfo.java new file mode 100644 index 0000000..943390b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationSubmissionResponseInfo.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse; + +/** + * Simple class that represent a response to a reservation submission. + */ +@XmlRootElement(name = "reservation-submission-response") +@XmlAccessorType(XmlAccessType.FIELD) +public class ReservationSubmissionResponseInfo { + + @XmlElement(name = "reservation-id") + private String reservationId; + + public ReservationSubmissionResponseInfo() { + + } + + public ReservationSubmissionResponseInfo( + ReservationSubmissionResponse response) { + this.reservationId = response.getReservationId().toString(); + } + + public String getReservationId() { + return reservationId; + } + + public void setReservationId(String reservationId) { + this.reservationId = reservationId; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac15644/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationUpdateRequestInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationUpdateRequestInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationUpdateRequestInfo.java new file mode 100644 index 0000000..cde24f4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationUpdateRequestInfo.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * Simple class to allow users to send information required to update an + * existing reservation. + */ +@XmlRootElement(name = "reservation-update-context") +@XmlAccessorType(XmlAccessType.FIELD) +public class ReservationUpdateRequestInfo { + + @XmlElement(name = "reservation-id") + private String reservationId; + + @XmlElement(name = "reservation-definition") + private ReservationDefinitionInfo reservationDefinition; + + public ReservationUpdateRequestInfo() { + } + + public String getReservationId() { + return reservationId; + } + + public void setReservationId(String reservationId) { + this.reservationId = reservationId; + } + + public ReservationDefinitionInfo getReservationDefinition() { + return reservationDefinition; + } + + public void setReservationDefinition( + ReservationDefinitionInfo reservationDefinition) { + this.reservationDefinition = reservationDefinition; + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac15644/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationUpdateResponseInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationUpdateResponseInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationUpdateResponseInfo.java new file mode 100644 index 0000000..0a5cf59 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationUpdateResponseInfo.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * Simple class that represent the response to a reservation update + * request. + */ +@XmlRootElement(name = "reservation-update-response") +@XmlAccessorType(XmlAccessType.FIELD) +public class ReservationUpdateResponseInfo { + + public ReservationUpdateResponseInfo() { + + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac15644/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java new file mode 100644 index 0000000..bb0db81 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java @@ -0,0 +1,517 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringReader; +import java.net.URL; +import java.util.Arrays; +import java.util.Collection; +import java.util.Enumeration; +import java.util.Properties; + +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.ws.rs.core.MediaType; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.authentication.server.AuthenticationFilter; +import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler; +import org.apache.hadoop.yarn.api.records.ReservationId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.MockNM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDeleteRequestInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationSubmissionRequestInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationUpdateRequestInfo; +import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; +import org.apache.hadoop.yarn.webapp.JerseyTestBase; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Singleton; +import com.google.inject.servlet.GuiceServletContextListener; +import com.google.inject.servlet.ServletModule; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.ClientResponse.Status; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.api.client.config.DefaultClientConfig; +import com.sun.jersey.api.json.JSONConfiguration; +import com.sun.jersey.api.json.JSONJAXBContext; +import com.sun.jersey.api.json.JSONUnmarshaller; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.sun.jersey.test.framework.WebAppDescriptor; + +@RunWith(Parameterized.class) +public class TestRMWebServicesReservation extends JerseyTestBase { + private static MockRM rm; + + private static Injector injector; + private String webserviceUserName = "testuser"; + + private boolean setAuthFilter = false; + + private static final String TEST_DIR = new File(System.getProperty( + "test.build.data", "/tmp")).getAbsolutePath(); + private static final String FS_ALLOC_FILE = new File(TEST_DIR, + "test-fs-queues.xml").getAbsolutePath(); + + public static class GuiceServletConfig extends GuiceServletContextListener { + + @Override + protected Injector getInjector() { + return injector; + } + } + + /* + * Helper class to allow testing of RM web services which require + * authorization Add this class as a filter in the Guice injector for the + * MockRM + */ + + @Singleton + public static class TestRMCustomAuthFilter extends AuthenticationFilter { + + @Override + protected Properties getConfiguration(String configPrefix, + FilterConfig filterConfig) throws ServletException { + Properties props = new Properties(); + Enumeration names = filterConfig.getInitParameterNames(); + while (names.hasMoreElements()) { + String name = (String) names.nextElement(); + if (name.startsWith(configPrefix)) { + String value = filterConfig.getInitParameter(name); + props.put(name.substring(configPrefix.length()), value); + } + } + props.put(AuthenticationFilter.AUTH_TYPE, "simple"); + props.put(PseudoAuthenticationHandler.ANONYMOUS_ALLOWED, "false"); + return props; + } + + } + + private abstract class TestServletModule extends ServletModule { + public Configuration conf = new Configuration(); + + public abstract void configureScheduler(); + + @Override + protected void configureServlets() { + configureScheduler(); + bind(JAXBContextResolver.class); + bind(RMWebServices.class); + bind(GenericExceptionHandler.class); + conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); + Configuration conf = new Configuration(); + conf.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true); + conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + CapacitySchedulerConfiguration csconf = + new CapacitySchedulerConfiguration(conf); + String[] queues = { "default", "dedicated" }; + csconf.setQueues("root", queues); + csconf.setCapacity("root.default", 50.0f); + csconf.setCapacity("root.dedicated", 50.0f); + csconf.setReservable("root.dedicated", true); + + rm = new MockRM(csconf); + bind(ResourceManager.class).toInstance(rm); + if (setAuthFilter) { + filter("/*").through(TestRMCustomAuthFilter.class); + } + serve("/*").with(GuiceContainer.class); + } + } + + private class CapTestServletModule extends TestServletModule { + @Override + public void configureScheduler() { + conf.set("yarn.resourcemanager.scheduler.class", + CapacityScheduler.class.getName()); + } + } + + private class FairTestServletModule extends TestServletModule { + @Override + public void configureScheduler() { + try { + PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(" someuser "); + out.println(" "); + out.println(" someuser "); + out.println(" "); + out.println(" "); + out.println(" someuser "); + out.println(" "); + out.println(""); + out.println(""); + out.close(); + } catch (IOException e) { + } + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE); + conf.set("yarn.resourcemanager.scheduler.class", + FairScheduler.class.getName()); + } + } + + private Injector getNoAuthInjectorCap() { + return Guice.createInjector(new CapTestServletModule() { + @Override + protected void configureServlets() { + setAuthFilter = false; + super.configureServlets(); + } + }); + } + + private Injector getSimpleAuthInjectorCap() { + return Guice.createInjector(new CapTestServletModule() { + @Override + protected void configureServlets() { + setAuthFilter = true; + conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); + // set the admin acls otherwise all users are considered admins + // and we can't test authorization + conf.setStrings(YarnConfiguration.YARN_ADMIN_ACL, "testuser1"); + super.configureServlets(); + } + }); + } + + private Injector getNoAuthInjectorFair() { + return Guice.createInjector(new FairTestServletModule() { + @Override + protected void configureServlets() { + setAuthFilter = false; + super.configureServlets(); + } + }); + } + + private Injector getSimpleAuthInjectorFair() { + return Guice.createInjector(new FairTestServletModule() { + @Override + protected void configureServlets() { + setAuthFilter = true; + conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); + // set the admin acls otherwise all users are considered admins + // and we can't test authorization + conf.setStrings(YarnConfiguration.YARN_ADMIN_ACL, "testuser1"); + super.configureServlets(); + } + }); + } + + @Parameters + public static Collection guiceConfigs() { + return Arrays.asList(new Object[][] { { 0 }, { 1 }, { 2 }, { 3 } }); + } + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + } + + public TestRMWebServicesReservation(int run) { + super(new WebAppDescriptor.Builder( + "org.apache.hadoop.yarn.server.resourcemanager.webapp") + .contextListenerClass(GuiceServletConfig.class) + .filterClass(com.google.inject.servlet.GuiceFilter.class) + .clientConfig(new DefaultClientConfig(JAXBContextResolver.class)) + .contextPath("jersey-guice-filter").servletPath("/").build()); + switch (run) { + case 0: + default: + // No Auth Capacity Scheduler + injector = getNoAuthInjectorCap(); + break; + case 1: + // Simple Auth Capacity Scheduler + injector = getSimpleAuthInjectorCap(); + break; + case 2: + // No Auth Fair Scheduler + injector = getNoAuthInjectorFair(); + break; + case 3: + // Simple Auth Fair Scheduler + injector = getSimpleAuthInjectorFair(); + break; + } + } + + private boolean isAuthenticationEnabled() { + return setAuthFilter; + } + + private WebResource constructWebResource(WebResource r, String... paths) { + WebResource rt = r; + for (String path : paths) { + rt = rt.path(path); + } + if (isAuthenticationEnabled()) { + rt = rt.queryParam("user.name", webserviceUserName); + } + return rt; + } + + private WebResource constructWebResource(String... paths) { + WebResource r = resource(); + WebResource ws = r.path("ws").path("v1").path("cluster"); + return this.constructWebResource(ws, paths); + } + + @After + @Override + public void tearDown() throws Exception { + if (rm != null) { + rm.stop(); + } + super.tearDown(); + } + + @Test + public void testSubmitReservation() throws JSONException, Exception { + rm.start(); + for (int i = 0; i < 100; i++) { + MockNM amNodeManager = + rm.registerNode("127.0.0." + i + ":1234", 100 * 1024); + amNodeManager.nodeHeartbeat(true); + } + ReservationId rid = + testSubmissionReservationHelper("reservation/submit", + MediaType.APPLICATION_JSON); + if (this.isAuthenticationEnabled()) { + assertNotNull(rid); + } + rm.stop(); + } + + @Test + public void testFailedSubmitReservation() throws JSONException, Exception { + rm.start(); + // setup a cluster too small to accept the reservation + for (int i = 0; i < 1; i++) { + MockNM amNodeManager = + rm.registerNode("127.0.0." + i + ":1234", 100 * 1024); + amNodeManager.nodeHeartbeat(true); + } + ReservationId rid = + testSubmissionReservationHelper("reservation/submit", + MediaType.APPLICATION_JSON); + assertNull(rid); + rm.stop(); + } + + @Test + public void testUpdateReservation() throws JSONException, Exception { + rm.start(); + for (int i = 0; i < 100; i++) { + MockNM amNodeManager = + rm.registerNode("127.0.0." + i + ":1234", 100 * 1024); + amNodeManager.nodeHeartbeat(true); + } + ReservationId rid = + testSubmissionReservationHelper("reservation/submit", + MediaType.APPLICATION_JSON); + if (this.isAuthenticationEnabled()) { + assertNotNull(rid); + } + testUpdateReservationHelper("reservation/update", rid, + MediaType.APPLICATION_JSON); + + rm.stop(); + } + + @Test + public void testDeleteReservation() throws JSONException, Exception { + rm.start(); + for (int i = 0; i < 100; i++) { + MockNM amNodeManager = + rm.registerNode("127.0.0." + i + ":1234", 100 * 1024); + amNodeManager.nodeHeartbeat(true); + } + ReservationId rid = + testSubmissionReservationHelper("reservation/submit", + MediaType.APPLICATION_JSON); + if (this.isAuthenticationEnabled()) { + assertNotNull(rid); + } + testDeleteReservationHelper("reservation/delete", rid, + MediaType.APPLICATION_JSON); + + rm.stop(); + } + + private ReservationId testSubmissionReservationHelper(String path, + String media) throws JSONException, Exception { + + String reservationJson = loadJsonFile("submit-reservation.json"); + + JSONJAXBContext jc = + new JSONJAXBContext(JSONConfiguration.mapped() + .build(), ReservationSubmissionRequestInfo.class); + JSONUnmarshaller unmarshaller = jc.createJSONUnmarshaller(); + ReservationSubmissionRequestInfo rsci = + unmarshaller.unmarshalFromJSON(new StringReader(reservationJson), + ReservationSubmissionRequestInfo.class); + + Thread.sleep(1000); + ClientResponse response = + constructWebResource(path).entity(rsci, MediaType.APPLICATION_JSON) + .accept(media).post(ClientResponse.class); + + if (!this.isAuthenticationEnabled()) { + assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus()); + return null; + } + + System.out.println("RESPONSE:" + response); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + + assertEquals("incorrect number of elements", 1, json.length()); + ReservationId rid = null; + try { + rid = ReservationId.parseReservationId(json.getString("reservation-id")); + assertEquals("incorrect return value", rid.getId(), 1); + } catch (JSONException j) { + // failure is possible and is checked outside + } + return rid; + } + + private void testUpdateReservationHelper(String path, + ReservationId reservationId, String media) throws JSONException, + Exception { + + String reservationJson = loadJsonFile("update-reservation.json"); + + JSONJAXBContext jc = + new JSONJAXBContext(JSONConfiguration.mapped() + .build(), ReservationUpdateRequestInfo.class); + JSONUnmarshaller unmarshaller = jc.createJSONUnmarshaller(); + ReservationUpdateRequestInfo rsci = + unmarshaller.unmarshalFromJSON(new StringReader(reservationJson), + ReservationUpdateRequestInfo.class); + if (this.isAuthenticationEnabled()) { + // only works when previous submit worked + if(rsci.getReservationId() == null) { + throw new IOException("Incorrectly parsed the reservatinId"); + } + rsci.setReservationId(reservationId.toString()); + } + + Thread.sleep(1000); + ClientResponse response = + constructWebResource(path).entity(rsci, MediaType.APPLICATION_JSON) + .accept(media).post(ClientResponse.class); + + if (!this.isAuthenticationEnabled()) { + assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus()); + return; + } + + System.out.println("RESPONSE:" + response); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + assertEquals(Status.OK, response.getClientResponseStatus()); + + } + + private String loadJsonFile(String filename) throws IOException { + ClassLoader cL = Thread.currentThread().getContextClassLoader(); + if (cL == null) { + cL = Configuration.class.getClassLoader(); + } + URL submitURI = cL.getResource(filename); + + String reservationJson = + FileUtils.readFileToString(new File(submitURI.getFile())); + return reservationJson; + } + + private void testDeleteReservationHelper(String path, + ReservationId reservationId, String media) throws JSONException, + Exception { + + String reservationJson = loadJsonFile("delete-reservation.json"); + + JSONJAXBContext jc = + new JSONJAXBContext(JSONConfiguration.mapped() + .build(), ReservationDeleteRequestInfo.class); + JSONUnmarshaller unmarshaller = jc.createJSONUnmarshaller(); + ReservationDeleteRequestInfo rsci = + unmarshaller.unmarshalFromJSON(new StringReader(reservationJson), + ReservationDeleteRequestInfo.class); + if (this.isAuthenticationEnabled()) { + // only works when previous submit worked + if(rsci.getReservationId() == null) { + throw new IOException("Incorrectly parsed the reservatinId"); + } + rsci.setReservationId(reservationId.toString()); + } + + Thread.sleep(1000); + ClientResponse response = + constructWebResource(path).entity(rsci, MediaType.APPLICATION_JSON) + .accept(media).post(ClientResponse.class); + + if (!this.isAuthenticationEnabled()) { + assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus()); + return; + } + + System.out.println("RESPONSE:" + response); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + assertEquals(Status.OK, response.getClientResponseStatus()); + + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac15644/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/delete-reservation.json ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/delete-reservation.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/delete-reservation.json new file mode 100644 index 0000000..205d3d4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/delete-reservation.json @@ -0,0 +1,3 @@ +{ + "reservation-id" : "reservation_12341234_1" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac15644/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/submit-reservation.json ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/submit-reservation.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/submit-reservation.json new file mode 100644 index 0000000..573d317 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/submit-reservation.json @@ -0,0 +1,31 @@ +{ + "queue" : "dedicated", + "reservation-definition" : { + "arrival" : 1765541532000, + "deadline" : 1765542252000, + "reservation-name" : "res_1", + "reservation-requests" : { + "reservation-request-interpreter" : 0, + "reservation-request" : [ + { + "duration" : 60, + "num-containers" : 220, + "min-concurrency" : 220, + "capability" : { + "memory" : 1024, + "vCores" : 1 + } + }, + { + "duration" : 120, + "num-containers" : 110, + "min-concurrency" : 110, + "capability" : { + "memory" : 1024, + "vCores" : 1 + } + } + ] + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac15644/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/update-reservation.json ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/update-reservation.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/update-reservation.json new file mode 100644 index 0000000..df0df21 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/update-reservation.json @@ -0,0 +1,31 @@ +{ + "reservation-id" : "reservation_12341234_1", + "reservation-definition" : { + "arrival" : 1765541532000, + "deadline" : 1765542252000, + "reservation-name" : "res_1", + "reservation-requests" : { + "reservation-request-interpreter" : 0, + "reservation-request" : [ + { + "duration" : 60, + "num-containers" : 100, + "min-concurrency" : 1, + "capability" : { + "memory" : 1024, + "vCores" : 1 + } + }, + { + "duration" : 120, + "num-containers" : 40, + "min-concurrency" : 1, + "capability" : { + "memory" : 1024, + "vCores" : 1 + } + } + ] + } + } +} \ No newline at end of file