Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 448C218980 for ; Mon, 7 Dec 2015 09:51:20 +0000 (UTC) Received: (qmail 94318 invoked by uid 500); 7 Dec 2015 09:51:20 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 94252 invoked by uid 500); 7 Dec 2015 09:51:20 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 94237 invoked by uid 99); 7 Dec 2015 09:51:20 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 07 Dec 2015 09:51:20 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id C7381DFDC5; Mon, 7 Dec 2015 09:51:19 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: rohithsharmaks@apache.org To: common-commits@hadoop.apache.org Message-Id: <682eca41d6e04ca9aa9cd05bb4f1178c@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-3456. Improve handling of incomplete TimelineEntities. (Varun Saxena via rohithsharmaks) Date: Mon, 7 Dec 2015 09:51:19 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-2.8 db69c5f74 -> 03e615ba5 YARN-3456. Improve handling of incomplete TimelineEntities. (Varun Saxena via rohithsharmaks) (cherry picked from commit 01a641bc447c464b2830d58addd482f47dbd92ae) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/03e615ba Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/03e615ba Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/03e615ba Branch: refs/heads/branch-2.8 Commit: 03e615ba55938c26f04715a735760db92ef4e645 Parents: db69c5f Author: rohithsharmaks Authored: Mon Dec 7 15:16:56 2015 +0530 Committer: rohithsharmaks Committed: Mon Dec 7 15:20:50 2015 +0530 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 +++ .../client/api/impl/TimelineClientImpl.java | 8 ++++++-- .../client/api/impl/TestTimelineClient.java | 9 +++++++++ .../server/timeline/TimelineDataManager.java | 6 +++++- .../timeline/webapp/TimelineWebServices.java | 2 ++ .../webapp/TestTimelineWebServices.java | 21 ++++++++++++++++++++ 6 files changed, 46 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/03e615ba/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 9bab6c48..c91b3f2 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -525,6 +525,9 @@ Release 2.8.0 - UNRELEASED YARN-4358. Reservation System: Improve relationship between SharingPolicy and ReservationAgent. (Carlo Curino via asuresh) + YARN-3456. Improve handling of incomplete TimelineEntities. (Varun Saxena + via rohithsharmaks) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not http://git-wip-us.apache.org/repos/asf/hadoop/blob/03e615ba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index 04c84ca..019c7a5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -29,7 +29,6 @@ import java.net.URL; import java.net.URLConnection; import java.security.GeneralSecurityException; import java.security.PrivilegedExceptionAction; -import java.util.Arrays; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.HttpsURLConnection; @@ -301,7 +300,12 @@ public class TimelineClientImpl extends TimelineClient { public TimelinePutResponse putEntities( TimelineEntity... entities) throws IOException, YarnException { TimelineEntities entitiesContainer = new TimelineEntities(); - entitiesContainer.addEntities(Arrays.asList(entities)); + for (TimelineEntity entity : entities) { + if (entity.getEntityId() == null || entity.getEntityType() == null) { + throw new YarnException("Incomplete entity without entity id/type"); + } + entitiesContainer.addEntity(entity); + } ClientResponse resp = doPosting(entitiesContainer, null); return resp.getEntity(TimelinePutResponse.class); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/03e615ba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java index 859a6c9..4c74c61 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java @@ -96,6 +96,15 @@ public class TestTimelineClient { } @Test + public void testPostIncompleteEntities() throws Exception { + try { + client.putEntities(new TimelineEntity()); + Assert.fail("Exception should have been thrown"); + } catch (YarnException e) { + } + } + + @Test public void testPostEntitiesNoResponse() throws Exception { mockEntityClientResponse( client, ClientResponse.Status.INTERNAL_SERVER_ERROR, false, false); http://git-wip-us.apache.org/repos/asf/hadoop/blob/03e615ba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineDataManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineDataManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineDataManager.java index 459fd4e..23ff8e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineDataManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineDataManager.java @@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field; import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager; +import org.apache.hadoop.yarn.webapp.BadRequestException; import com.google.common.annotations.VisibleForTesting; @@ -338,7 +339,10 @@ public class TimelineDataManager extends AbstractService { entity.getDomainId().length() == 0) { entity.setDomainId(DEFAULT_DOMAIN_ID); } - + if (entity.getEntityId() == null || entity.getEntityType() == null) { + throw new BadRequestException("Incomplete entity without entity" + + " id/type"); + } // check if there is existing entity TimelineEntity existingEntity = null; try { http://git-wip-us.apache.org/repos/asf/hadoop/blob/03e615ba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java index 90f4d39..eb47ef2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java @@ -229,6 +229,8 @@ public class TimelineWebServices { } try { return timelineDataManager.postEntities(entities, callerUGI); + } catch (BadRequestException bre) { + throw bre; } catch (Exception e) { LOG.error("Error putting entities", e); throw new WebApplicationException(e, http://git-wip-us.apache.org/repos/asf/hadoop/blob/03e615ba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java index ab7cffd..74466a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java @@ -503,6 +503,27 @@ public class TestTimelineWebServices extends JerseyTestBase { } @Test + public void testPostIncompleteEntities() throws Exception { + TimelineEntities entities = new TimelineEntities(); + TimelineEntity entity1 = new TimelineEntity(); + entity1.setEntityId("test id 1"); + entity1.setEntityType("test type 1"); + entity1.setStartTime(System.currentTimeMillis()); + entity1.setDomainId("domain_id_1"); + entities.addEntity(entity1); + // Add an entity with no id or type. + entities.addEntity(new TimelineEntity()); + WebResource r = resource(); + // One of the entities has no id or type. HTTP 400 will be returned + ClientResponse response = r.path("ws").path("v1").path("timeline") + .queryParam("user.name", "tester").accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON).post(ClientResponse.class, entities); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + assertEquals(ClientResponse.Status.BAD_REQUEST, + response.getClientResponseStatus()); + } + + @Test public void testPostEntitiesWithYarnACLsEnabled() throws Exception { AdminACLsManager oldAdminACLsManager = timelineACLsManager.setAdminACLsManager(adminACLsManager);