Return-Path:
X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io
Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io
Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183])
by cust-asf2.ponee.io (Postfix) with ESMTP id 5EEE9200AE3
for ; Thu, 5 May 2016 01:45:24 +0200 (CEST)
Received: by cust-asf.ponee.io (Postfix)
id 5D8FB160A02; Wed, 4 May 2016 23:45:24 +0000 (UTC)
Delivered-To: archive-asf-public@cust-asf.ponee.io
Received: from mail.apache.org (hermes.apache.org [140.211.11.3])
by cust-asf.ponee.io (Postfix) with SMTP id 397171609FF
for ; Thu, 5 May 2016 01:45:22 +0200 (CEST)
Received: (qmail 47982 invoked by uid 500); 4 May 2016 23:45:15 -0000
Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
List-Help:
List-Unsubscribe:
List-Post:
List-Id:
Delivered-To: mailing list common-commits@hadoop.apache.org
Received: (qmail 47090 invoked by uid 99); 4 May 2016 23:45:14 -0000
Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23)
by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 04 May 2016 23:45:14 +0000
Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33)
id B4763DFF75; Wed, 4 May 2016 23:45:14 +0000 (UTC)
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: gtcarrera9@apache.org
To: common-commits@hadoop.apache.org
Date: Wed, 04 May 2016 23:45:29 -0000
Message-Id:
In-Reply-To: <02efc7cfd02b4c32a5bd3100cc827b43@git.apache.org>
References: <02efc7cfd02b4c32a5bd3100cc827b43@git.apache.org>
X-Mailer: ASF-Git Admin Mailer
Subject: [16/50] [abbrv] hadoop git commit: YARN-4238. createdTime and
modifiedTime is not reported while publishing entities to ATSv2. (Varun
Saxena via Naganarasimha G R)
archived-at: Wed, 04 May 2016 23:45:24 -0000
YARN-4238. createdTime and modifiedTime is not reported while publishing entities to ATSv2. (Varun Saxena via Naganarasimha G R)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fd369a54
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fd369a54
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fd369a54
Branch: refs/heads/YARN-2928
Commit: fd369a54e3cc08b812746f7c339826b0c4c7977f
Parents: 2269e78
Author: Naganarasimha
Authored: Wed Jan 27 11:59:40 2016 +0530
Committer: Li Lu
Committed: Wed May 4 16:24:12 2016 -0700
----------------------------------------------------------------------
.../jobhistory/JobHistoryEventHandler.java | 47 ++++++----
.../records/timelineservice/TimelineEntity.java | 19 ----
.../TestTimelineServiceRecords.java | 1 -
.../monitor/ContainersMonitorImpl.java | 6 +-
.../timelineservice/NMTimelinePublisher.java | 6 +-
.../TestSystemMetricsPublisherForV2.java | 34 +++++--
.../reader/TimelineReaderManager.java | 7 +-
.../reader/TimelineReaderWebServices.java | 51 ++++-------
.../storage/FileSystemTimelineReaderImpl.java | 21 +----
.../storage/HBaseTimelineReaderImpl.java | 8 +-
.../storage/HBaseTimelineWriterImpl.java | 4 -
.../PhoenixOfflineAggregationWriterImpl.java | 9 +-
.../timelineservice/storage/TimelineReader.java | 32 +++----
.../storage/application/ApplicationColumn.java | 5 --
.../storage/application/ApplicationTable.java | 11 +--
.../storage/entity/EntityColumn.java | 5 --
.../storage/entity/EntityTable.java | 11 +--
.../storage/reader/ApplicationEntityReader.java | 23 +----
.../reader/FlowActivityEntityReader.java | 7 +-
.../storage/reader/FlowRunEntityReader.java | 7 +-
.../storage/reader/GenericEntityReader.java | 22 +----
.../storage/reader/TimelineEntityReader.java | 5 --
.../reader/TimelineEntityReaderFactory.java | 27 +++---
.../reader/TestTimelineReaderWebServices.java | 48 +---------
.../TestFileSystemTimelineReaderImpl.java | 92 +++++--------------
.../TestFileSystemTimelineWriterImpl.java | 3 +-
.../storage/TestHBaseTimelineStorage.java | 95 +++++++-------------
...TestPhoenixOfflineAggregationWriterImpl.java | 1 -
.../storage/flow/TestFlowDataGenerator.java | 1 -
.../flow/TestHBaseStorageFlowActivity.java | 6 +-
.../storage/flow/TestHBaseStorageFlowRun.java | 9 +-
31 files changed, 197 insertions(+), 426 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index d88588c..a10872a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -1135,10 +1135,10 @@ public class JobHistoryEventHandler extends AbstractService
// jobId, timestamp and entityType.
private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
createJobEntity(HistoryEvent event, long timestamp, JobId jobId,
- String entityType) {
-
- org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
- createBaseEntity(event, timestamp, entityType);
+ String entityType, boolean setCreatedTime) {
+
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
+ createBaseEntity(event, timestamp, entityType, setCreatedTime);
entity.setId(jobId.toString());
return entity;
}
@@ -1146,8 +1146,9 @@ public class JobHistoryEventHandler extends AbstractService
// create BaseEntity from HistoryEvent with adding other info, like:
// timestamp and entityType.
private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
- createBaseEntity(HistoryEvent event, long timestamp, String entityType) {
- org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent tEvent =
+ createBaseEntity(HistoryEvent event, long timestamp, String entityType,
+ boolean setCreatedTime) {
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent tEvent =
event.toTimelineEvent();
tEvent.setTimestamp(timestamp);
@@ -1155,6 +1156,9 @@ public class JobHistoryEventHandler extends AbstractService
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity();
entity.addEvent(tEvent);
entity.setType(entityType);
+ if (setCreatedTime) {
+ entity.setCreatedTime(timestamp);
+ }
return entity;
}
@@ -1162,9 +1166,10 @@ public class JobHistoryEventHandler extends AbstractService
// taskId, jobId, timestamp, entityType and relatedJobEntity.
private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
createTaskEntity(HistoryEvent event, long timestamp, String taskId,
- String entityType, String relatedJobEntity, JobId jobId) {
- org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
- createBaseEntity(event, timestamp, entityType);
+ String entityType, String relatedJobEntity, JobId jobId,
+ boolean setCreatedTime) {
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
+ createBaseEntity(event, timestamp, entityType, setCreatedTime);
entity.setId(taskId);
entity.addIsRelatedToEntity(relatedJobEntity, jobId.toString());
return entity;
@@ -1175,9 +1180,9 @@ public class JobHistoryEventHandler extends AbstractService
private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
createTaskAttemptEntity(HistoryEvent event, long timestamp,
String taskAttemptId, String entityType, String relatedTaskEntity,
- String taskId) {
- org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
- createBaseEntity(event, timestamp, entityType);
+ String taskId, boolean setCreatedTime) {
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
+ createBaseEntity(event, timestamp, entityType, setCreatedTime);
entity.setId(taskAttemptId);
entity.addIsRelatedToEntity(relatedTaskEntity, taskId);
return entity;
@@ -1188,10 +1193,13 @@ public class JobHistoryEventHandler extends AbstractService
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity tEntity = null;
String taskId = null;
String taskAttemptId = null;
+ boolean setCreatedTime = false;
switch (event.getEventType()) {
// Handle job events
case JOB_SUBMITTED:
+ setCreatedTime = true;
+ break;
case JOB_STATUS_CHANGED:
case JOB_INFO_CHANGED:
case JOB_INITED:
@@ -1206,6 +1214,7 @@ public class JobHistoryEventHandler extends AbstractService
break;
// Handle task events
case TASK_STARTED:
+ setCreatedTime = true;
taskId = ((TaskStartedEvent)event).getTaskId().toString();
break;
case TASK_FAILED:
@@ -1218,8 +1227,13 @@ public class JobHistoryEventHandler extends AbstractService
taskId = ((TaskFinishedEvent)event).getTaskId().toString();
break;
case MAP_ATTEMPT_STARTED:
- case CLEANUP_ATTEMPT_STARTED:
case REDUCE_ATTEMPT_STARTED:
+ setCreatedTime = true;
+ taskId = ((TaskAttemptStartedEvent)event).getTaskId().toString();
+ taskAttemptId = ((TaskAttemptStartedEvent)event).
+ getTaskAttemptId().toString();
+ break;
+ case CLEANUP_ATTEMPT_STARTED:
case SETUP_ATTEMPT_STARTED:
taskId = ((TaskAttemptStartedEvent)event).getTaskId().toString();
taskAttemptId = ((TaskAttemptStartedEvent)event).
@@ -1258,17 +1272,18 @@ public class JobHistoryEventHandler extends AbstractService
if (taskId == null) {
// JobEntity
tEntity = createJobEntity(event, timestamp, jobId,
- MAPREDUCE_JOB_ENTITY_TYPE);
+ MAPREDUCE_JOB_ENTITY_TYPE, setCreatedTime);
} else {
if (taskAttemptId == null) {
// TaskEntity
tEntity = createTaskEntity(event, timestamp, taskId,
- MAPREDUCE_TASK_ENTITY_TYPE, MAPREDUCE_JOB_ENTITY_TYPE, jobId);
+ MAPREDUCE_TASK_ENTITY_TYPE, MAPREDUCE_JOB_ENTITY_TYPE,
+ jobId, setCreatedTime);
} else {
// TaskAttemptEntity
tEntity = createTaskAttemptEntity(event, timestamp, taskAttemptId,
MAPREDUCE_TASK_ATTEMPT_ENTITY_TYPE, MAPREDUCE_TASK_ENTITY_TYPE,
- taskId);
+ taskId, setCreatedTime);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
index 0701001..dcf2473 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
@@ -140,7 +140,6 @@ public class TimelineEntity implements Comparable {
private HashMap> isRelatedToEntities = new HashMap<>();
private HashMap> relatesToEntities = new HashMap<>();
private long createdTime;
- private long modifiedTime;
public TimelineEntity() {
identifier = new Identifier();
@@ -505,24 +504,6 @@ public class TimelineEntity implements Comparable {
}
}
- @XmlElement(name = "modifiedtime")
- public long getModifiedTime() {
- if (real == null) {
- return modifiedTime;
- } else {
- return real.getModifiedTime();
- }
- }
-
- @JsonSetter("modifiedtime")
- public void setModifiedTime(long modifiedTime) {
- if (real == null) {
- this.modifiedTime = modifiedTime;
- } else {
- real.setModifiedTime(modifiedTime);
- }
- }
-
public boolean isValid() {
return (getId() != null && getType() != null);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
index 7c9acf2..51ec762 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
@@ -139,7 +139,6 @@ public class TestTimelineServiceRecords {
event1, event2);
entity.setCreatedTime(0L);
- entity.setModifiedTime(1L);
entity.addRelatesToEntity("test type 2", "test id 2");
entity.addRelatesToEntity("test type 3", "test id 3");
entity.addIsRelatedToEntity("test type 4", "test id 4");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 739d363..da4aac2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -463,7 +463,6 @@ public class ContainersMonitorImpl extends AbstractService implements
pTree.updateProcessTree(); // update process-tree
long currentVmemUsage = pTree.getVirtualMemorySize();
long currentPmemUsage = pTree.getRssMemorySize();
- long currentTime = System.currentTimeMillis();
// if machine has 6 cores and 3 are used,
// cpuUsagePercentPerCore should be 300% and
@@ -568,9 +567,8 @@ public class ContainersMonitorImpl extends AbstractService implements
NMTimelinePublisher nmMetricsPublisher =
container.getNMTimelinePublisher();
if (nmMetricsPublisher != null) {
- nmMetricsPublisher.reportContainerResourceUsage(
- container, currentTime, pId, currentPmemUsage,
- cpuUsageTotalCoresPercentage);
+ nmMetricsPublisher.reportContainerResourceUsage(container, pId,
+ currentPmemUsage, cpuUsageTotalCoresPercentage);
}
} catch (Exception e) {
// Log the exception and proceed to the next container.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
index 69de433..affaaae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
@@ -113,9 +113,8 @@ public class NMTimelinePublisher extends CompositeService {
}
@SuppressWarnings("unchecked")
- public void reportContainerResourceUsage(Container container,
- long createdTime, String pId, Long pmemUsage,
- Float cpuUsageTotalCoresPercentage) {
+ public void reportContainerResourceUsage(Container container, String pId,
+ Long pmemUsage, Float cpuUsageTotalCoresPercentage) {
if (pmemUsage != ResourceCalculatorProcessTree.UNAVAILABLE ||
cpuUsageTotalCoresPercentage !=
ResourceCalculatorProcessTree.UNAVAILABLE) {
@@ -164,6 +163,7 @@ public class NMTimelinePublisher extends CompositeService {
tEvent.setTimestamp(timestamp);
entity.addEvent(tEvent);
+ entity.setCreatedTime(timestamp);
putEntity(entity, containerId.getApplicationAttemptId().getApplicationId());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
index 9100366..1a58cb2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.metrics;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
@@ -44,10 +45,15 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
+import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
@@ -58,6 +64,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptS
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector;
+import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineReaderImpl;
import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.junit.AfterClass;
@@ -201,8 +208,7 @@ public class TestSystemMetricsPublisherForV2 {
+ FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
File appFile = new File(outputDirApp, timelineServiceFileName);
Assert.assertTrue(appFile.exists());
- Assert.assertEquals("Expected 3 events to be published", 3,
- getNumOfNonEmptyLines(appFile));
+ verifyEntity(appFile, 3, ApplicationMetricsConstants.CREATED_EVENT_TYPE);
}
@Test(timeout = 10000)
@@ -236,8 +242,7 @@ public class TestSystemMetricsPublisherForV2 {
+ FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
File appFile = new File(outputDirApp, timelineServiceFileName);
Assert.assertTrue(appFile.exists());
- Assert.assertEquals("Expected 2 events to be published", 2,
- getNumOfNonEmptyLines(appFile));
+ verifyEntity(appFile,2, AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE);
}
@Test(timeout = 10000)
@@ -268,8 +273,8 @@ public class TestSystemMetricsPublisherForV2 {
+ FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
File appFile = new File(outputDirApp, timelineServiceFileName);
Assert.assertTrue(appFile.exists());
- Assert.assertEquals("Expected 2 events to be published", 2,
- getNumOfNonEmptyLines(appFile));
+ verifyEntity(appFile, 2,
+ ContainerMetricsConstants.CREATED_IN_RM_EVENT_TYPE);
}
private RMApp createAppAndRegister(ApplicationId appId) {
@@ -282,20 +287,31 @@ public class TestSystemMetricsPublisherForV2 {
return app;
}
- private long getNumOfNonEmptyLines(File entityFile) throws IOException {
+ private static void verifyEntity(File entityFile, long expectedEvents,
+ String eventForCreatedTime) throws IOException {
BufferedReader reader = null;
String strLine;
long count = 0;
try {
reader = new BufferedReader(new FileReader(entityFile));
while ((strLine = reader.readLine()) != null) {
- if (strLine.trim().length() > 0)
+ if (strLine.trim().length() > 0) {
+ TimelineEntity entity = FileSystemTimelineReaderImpl.
+ getTimelineRecordFromJSON(strLine.trim(), TimelineEntity.class);
+ for (TimelineEvent event : entity.getEvents()) {
+ if (event.getId().equals(eventForCreatedTime)) {
+ assertTrue(entity.getCreatedTime() > 0);
+ break;
+ }
+ }
count++;
+ }
}
} finally {
reader.close();
}
- return count;
+ assertEquals("Expected " + expectedEvents + " events to be published",
+ count, expectedEvents);
}
private String getTimelineEntityDir(RMApp app) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
index 37f1671..65d251d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
@@ -68,16 +68,15 @@ public class TimelineReaderManager extends AbstractService {
Set getEntities(String userId, String clusterId,
String flowName, Long flowRunId, String appId, String entityType,
Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Long modifiedTimeBegin, Long modifiedTimeEnd,
Map> relatesTo, Map> isRelatedTo,
Map infoFilters, Map configFilters,
Set metricFilters, Set eventFilters,
EnumSet fieldsToRetrieve) throws IOException {
String cluster = getClusterID(clusterId, getConfig());
return reader.getEntities(userId, cluster, flowName, flowRunId, appId,
- entityType, limit, createdTimeBegin, createdTimeEnd, modifiedTimeBegin,
- modifiedTimeEnd, relatesTo, isRelatedTo, infoFilters, configFilters,
- metricFilters, eventFilters, null, null, fieldsToRetrieve);
+ entityType, limit, createdTimeBegin, createdTimeEnd, relatesTo,
+ isRelatedTo, infoFilters, configFilters, metricFilters, eventFilters,
+ null, null, fieldsToRetrieve);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
index f0283a5..a054ee5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
@@ -331,8 +331,6 @@ public class TimelineReaderWebServices {
@QueryParam("limit") String limit,
@QueryParam("createdtimestart") String createdTimeStart,
@QueryParam("createdtimeend") String createdTimeEnd,
- @QueryParam("modifiedtimestart") String modifiedTimeStart,
- @QueryParam("modifiedtimeend") String modifiedTimeEnd,
@QueryParam("relatesto") String relatesTo,
@QueryParam("isrelatedto") String isRelatedTo,
@QueryParam("infofilters") String infofilters,
@@ -341,9 +339,9 @@ public class TimelineReaderWebServices {
@QueryParam("eventfilters") String eventfilters,
@QueryParam("fields") String fields) {
return getEntities(req, res, null, appId, entityType, userId, flowName,
- flowRunId, limit, createdTimeStart, createdTimeEnd, modifiedTimeStart,
- modifiedTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters,
- metricfilters, eventfilters, fields);
+ flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo,
+ isRelatedTo, infofilters, conffilters, metricfilters, eventfilters,
+ fields);
}
/**
@@ -364,8 +362,6 @@ public class TimelineReaderWebServices {
@QueryParam("limit") String limit,
@QueryParam("createdtimestart") String createdTimeStart,
@QueryParam("createdtimeend") String createdTimeEnd,
- @QueryParam("modifiedtimestart") String modifiedTimeStart,
- @QueryParam("modifiedtimeend") String modifiedTimeEnd,
@QueryParam("relatesto") String relatesTo,
@QueryParam("isrelatedto") String isRelatedTo,
@QueryParam("infofilters") String infofilters,
@@ -387,8 +383,7 @@ public class TimelineReaderWebServices {
parseStr(userId), parseStr(clusterId), parseStr(flowName),
parseLongStr(flowRunId), parseStr(appId), parseStr(entityType),
parseLongStr(limit), parseLongStr(createdTimeStart),
- parseLongStr(createdTimeEnd), parseLongStr(modifiedTimeStart),
- parseLongStr(modifiedTimeEnd),
+ parseLongStr(createdTimeEnd),
parseKeyStrValuesStr(relatesTo, COMMA_DELIMITER, COLON_DELIMITER),
parseKeyStrValuesStr(isRelatedTo, COMMA_DELIMITER, COLON_DELIMITER),
parseKeyStrValueObj(infofilters, COMMA_DELIMITER, COLON_DELIMITER),
@@ -398,7 +393,7 @@ public class TimelineReaderWebServices {
parseFieldsStr(fields, COMMA_DELIMITER));
} catch (Exception e) {
handleException(e, url, startTime,
- "createdTime or modifiedTime start/end or limit or flowrunid");
+ "createdTime start/end or limit or flowrunid");
}
long endTime = Time.monotonicNow();
if (entities == null) {
@@ -585,8 +580,8 @@ public class TimelineReaderWebServices {
entities = timelineReaderManager.getEntities(
parseStr(userId), parseStr(clusterId), parseStr(flowName), null, null,
TimelineEntityType.YARN_FLOW_RUN.toString(), parseLongStr(limit),
- parseLongStr(createdTimeStart), parseLongStr(createdTimeEnd), null,
- null, null, null, null, null, null, null,
+ parseLongStr(createdTimeStart), parseLongStr(createdTimeEnd),
+ null, null, null, null, null, null,
parseFieldsStr(fields, COMMA_DELIMITER));
} catch (Exception e) {
handleException(e, url, startTime, "createdTime start/end or limit");
@@ -668,7 +663,7 @@ public class TimelineReaderWebServices {
null, parseStr(clusterId), null, null, null,
TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), parseLongStr(limit),
range.dateStart, range.dateEnd, null, null, null, null, null, null,
- null, null, parseFieldsStr(fields, COMMA_DELIMITER));
+ parseFieldsStr(fields, COMMA_DELIMITER));
} catch (Exception e) {
handleException(e, url, startTime, "limit");
}
@@ -760,8 +755,6 @@ public class TimelineReaderWebServices {
@QueryParam("limit") String limit,
@QueryParam("createdtimestart") String createdTimeStart,
@QueryParam("createdtimeend") String createdTimeEnd,
- @QueryParam("modifiedtimestart") String modifiedTimeStart,
- @QueryParam("modifiedtimeend") String modifiedTimeEnd,
@QueryParam("relatesto") String relatesTo,
@QueryParam("isrelatedto") String isRelatedTo,
@QueryParam("infofilters") String infofilters,
@@ -771,9 +764,9 @@ public class TimelineReaderWebServices {
@QueryParam("fields") String fields) {
return getEntities(req, res, null, null,
TimelineEntityType.YARN_APPLICATION.toString(), userId, flowName,
- flowRunId, limit, createdTimeStart, createdTimeEnd, modifiedTimeStart,
- modifiedTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters,
- metricfilters, eventfilters, fields);
+ flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo,
+ isRelatedTo, infofilters, conffilters, metricfilters, eventfilters,
+ fields);
}
/**
@@ -794,8 +787,6 @@ public class TimelineReaderWebServices {
@QueryParam("limit") String limit,
@QueryParam("createdtimestart") String createdTimeStart,
@QueryParam("createdtimeend") String createdTimeEnd,
- @QueryParam("modifiedtimestart") String modifiedTimeStart,
- @QueryParam("modifiedtimeend") String modifiedTimeEnd,
@QueryParam("relatesto") String relatesTo,
@QueryParam("isrelatedto") String isRelatedTo,
@QueryParam("infofilters") String infofilters,
@@ -805,9 +796,9 @@ public class TimelineReaderWebServices {
@QueryParam("fields") String fields) {
return getEntities(req, res, clusterId, null,
TimelineEntityType.YARN_APPLICATION.toString(), userId, flowName,
- flowRunId, limit, createdTimeStart, createdTimeEnd, modifiedTimeStart,
- modifiedTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters,
- metricfilters, eventfilters, fields);
+ flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo,
+ isRelatedTo, infofilters, conffilters, metricfilters, eventfilters,
+ fields);
}
/**
@@ -827,8 +818,6 @@ public class TimelineReaderWebServices {
@QueryParam("limit") String limit,
@QueryParam("createdtimestart") String createdTimeStart,
@QueryParam("createdtimeend") String createdTimeEnd,
- @QueryParam("modifiedtimestart") String modifiedTimeStart,
- @QueryParam("modifiedtimeend") String modifiedTimeEnd,
@QueryParam("relatesto") String relatesTo,
@QueryParam("isrelatedto") String isRelatedTo,
@QueryParam("infofilters") String infofilters,
@@ -838,9 +827,8 @@ public class TimelineReaderWebServices {
@QueryParam("fields") String fields) {
return getEntities(req, res, null, null,
TimelineEntityType.YARN_APPLICATION.toString(), userId, flowName,
- null, limit, createdTimeStart, createdTimeEnd, modifiedTimeStart,
- modifiedTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters,
- metricfilters, eventfilters, fields);
+ null, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo,
+ infofilters, conffilters, metricfilters, eventfilters, fields);
}
/**
@@ -860,8 +848,6 @@ public class TimelineReaderWebServices {
@QueryParam("limit") String limit,
@QueryParam("createdtimestart") String createdTimeStart,
@QueryParam("createdtimeend") String createdTimeEnd,
- @QueryParam("modifiedtimestart") String modifiedTimeStart,
- @QueryParam("modifiedtimeend") String modifiedTimeEnd,
@QueryParam("relatesto") String relatesTo,
@QueryParam("isrelatedto") String isRelatedTo,
@QueryParam("infofilters") String infofilters,
@@ -871,8 +857,7 @@ public class TimelineReaderWebServices {
@QueryParam("fields") String fields) {
return getEntities(req, res, clusterId, null,
TimelineEntityType.YARN_APPLICATION.toString(), userId, flowName,
- null, limit, createdTimeStart, createdTimeEnd, modifiedTimeStart,
- modifiedTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters,
- metricfilters, eventfilters, fields);
+ null, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo,
+ infofilters, conffilters, metricfilters, eventfilters, fields);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
index 36b3f0d..72c9249 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
@@ -188,7 +188,6 @@ public class FileSystemTimelineReaderImpl extends AbstractService
TimelineEntity entityToBeReturned = new TimelineEntity();
entityToBeReturned.setIdentifier(entity.getIdentifier());
entityToBeReturned.setCreatedTime(entity.getCreatedTime());
- entityToBeReturned.setModifiedTime(entity.getModifiedTime());
if (fieldsToRetrieve != null) {
fillFields(entityToBeReturned, entity, fieldsToRetrieve);
}
@@ -206,9 +205,6 @@ public class FileSystemTimelineReaderImpl extends AbstractService
if (entity2.getCreatedTime() > 0) {
entity1.setCreatedTime(entity2.getCreatedTime());
}
- if (entity2.getModifiedTime() > 0) {
- entity1.setModifiedTime(entity2.getModifiedTime());
- }
for (Entry configEntry : entity2.getConfigs().entrySet()) {
entity1.addConfig(configEntry.getKey(), configEntry.getValue());
}
@@ -268,8 +264,7 @@ public class FileSystemTimelineReaderImpl extends AbstractService
}
private Set getEntities(File dir, String entityType,
- Long limit, Long createdTimeBegin,
- Long createdTimeEnd, Long modifiedTimeBegin, Long modifiedTimeEnd,
+ Long limit, Long createdTimeBegin, Long createdTimeEnd,
Map> relatesTo, Map> isRelatedTo,
Map infoFilters, Map configFilters,
Set metricFilters, Set eventFilters,
@@ -284,12 +279,6 @@ public class FileSystemTimelineReaderImpl extends AbstractService
if (createdTimeEnd == null || createdTimeEnd <= 0) {
createdTimeEnd = Long.MAX_VALUE;
}
- if (modifiedTimeBegin == null || modifiedTimeBegin <= 0) {
- modifiedTimeBegin = 0L;
- }
- if (modifiedTimeEnd == null || modifiedTimeEnd <= 0) {
- modifiedTimeEnd = Long.MAX_VALUE;
- }
// First sort the selected entities based on created/start time.
Map> sortedEntities =
@@ -318,10 +307,6 @@ public class FileSystemTimelineReaderImpl extends AbstractService
createdTimeEnd)) {
continue;
}
- if (!isTimeInRange(entity.getModifiedTime(), modifiedTimeBegin,
- modifiedTimeEnd)) {
- continue;
- }
if (relatesTo != null && !relatesTo.isEmpty() &&
!TimelineStorageUtils
.matchRelations(entity.getRelatesToEntities(), relatesTo)) {
@@ -413,7 +398,6 @@ public class FileSystemTimelineReaderImpl extends AbstractService
public Set getEntities(String userId, String clusterId,
String flowName, Long flowRunId, String appId, String entityType,
Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Long modifiedTimeBegin, Long modifiedTimeEnd,
Map> relatesTo, Map> isRelatedTo,
Map infoFilters, Map configFilters,
Set metricFilters, Set eventFilters,
@@ -424,8 +408,7 @@ public class FileSystemTimelineReaderImpl extends AbstractService
File dir =
new File(new File(rootPath, ENTITIES_DIR),
clusterId + "/" + flowRunPath + "/" + appId + "/" + entityType);
- return getEntities(dir, entityType, limit,
- createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
+ return getEntities(dir, entityType, limit, createdTimeBegin, createdTimeEnd,
relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
eventFilters, confsToRetrieve, metricsToRetrieve, fieldsToRetrieve);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
index bc48cbe..0ce9a94 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
@@ -81,7 +81,6 @@ public class HBaseTimelineReaderImpl
public Set getEntities(String userId, String clusterId,
String flowName, Long flowRunId, String appId, String entityType,
Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Long modifiedTimeBegin, Long modifiedTimeEnd,
Map> relatesTo, Map> isRelatedTo,
Map infoFilters, Map configFilters,
Set metricFilters, Set eventFilters,
@@ -90,10 +89,9 @@ public class HBaseTimelineReaderImpl
TimelineEntityReader reader =
TimelineEntityReaderFactory.createMultipleEntitiesReader(userId,
clusterId, flowName, flowRunId, appId, entityType, limit,
- createdTimeBegin, createdTimeEnd, modifiedTimeBegin,
- modifiedTimeEnd, relatesTo, isRelatedTo, infoFilters, configFilters,
- metricFilters, eventFilters, confsToRetrieve, metricsToRetrieve,
- fieldsToRetrieve);
+ createdTimeBegin, createdTimeEnd, relatesTo, isRelatedTo,
+ infoFilters, configFilters, metricFilters, eventFilters,
+ confsToRetrieve, metricsToRetrieve, fieldsToRetrieve);
return reader.readEntities(hbaseConf, conn);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index a57be55..2a82ccb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -315,8 +315,6 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
ApplicationColumn.ID.store(rowKey, applicationTable, null, te.getId());
ApplicationColumn.CREATED_TIME.store(rowKey, applicationTable, null,
te.getCreatedTime());
- ApplicationColumn.MODIFIED_TIME.store(rowKey, applicationTable, null,
- te.getModifiedTime());
ApplicationColumn.FLOW_VERSION.store(rowKey, applicationTable, null,
flowVersion);
Map info = te.getInfo();
@@ -331,8 +329,6 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
EntityColumn.TYPE.store(rowKey, entityTable, null, te.getType());
EntityColumn.CREATED_TIME.store(rowKey, entityTable, null,
te.getCreatedTime());
- EntityColumn.MODIFIED_TIME.store(rowKey, entityTable, null,
- te.getModifiedTime());
EntityColumn.FLOW_VERSION.store(rowKey, entityTable, null, flowVersion);
Map info = te.getInfo();
if (info != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java
index 4c1352c..b5834c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java
@@ -143,10 +143,10 @@ public class PhoenixOfflineAggregationWriterImpl
TimelineWriteResponse response = new TimelineWriteResponse();
String sql = "UPSERT INTO " + info.getTableName()
+ " (" + StringUtils.join(info.getPrimaryKeyList(), ",")
- + ", created_time, modified_time, metric_names) "
+ + ", created_time, metric_names) "
+ "VALUES ("
+ StringUtils.repeat("?,", info.getPrimaryKeyList().length)
- + "?, ?, ?)";
+ + "?, ?)";
if (LOG.isDebugEnabled()) {
LOG.debug("TimelineEntity write SQL: " + sql);
}
@@ -162,7 +162,6 @@ public class PhoenixOfflineAggregationWriterImpl
}
int idx = info.setStringsForPrimaryKey(ps, context, null, 1);
ps.setLong(idx++, entity.getCreatedTime());
- ps.setLong(idx++, entity.getModifiedTime());
ps.setString(idx++, StringUtils.join(formattedMetrics.keySet().toArray(),
AGGREGATION_STORAGE_SEPARATOR));
ps.execute();
@@ -197,7 +196,7 @@ public class PhoenixOfflineAggregationWriterImpl
+ OfflineAggregationInfo.FLOW_AGGREGATION_TABLE_NAME
+ "(user VARCHAR NOT NULL, cluster VARCHAR NOT NULL, "
+ "flow_name VARCHAR NOT NULL, "
- + "created_time UNSIGNED_LONG, modified_time UNSIGNED_LONG, "
+ + "created_time UNSIGNED_LONG, "
+ METRIC_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARBINARY, "
+ "metric_names VARCHAR, info_keys VARCHAR "
+ "CONSTRAINT pk PRIMARY KEY("
@@ -206,7 +205,7 @@ public class PhoenixOfflineAggregationWriterImpl
sql = "CREATE TABLE IF NOT EXISTS "
+ OfflineAggregationInfo.USER_AGGREGATION_TABLE_NAME
+ "(user VARCHAR NOT NULL, cluster VARCHAR NOT NULL, "
- + "created_time UNSIGNED_LONG, modified_time UNSIGNED_LONG, "
+ + "created_time UNSIGNED_LONG, "
+ METRIC_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARBINARY, "
+ "metric_names VARCHAR, info_keys VARCHAR "
+ "CONSTRAINT pk PRIMARY KEY(user, cluster))";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
index 56bd3a0..a5a271e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
@@ -87,8 +87,8 @@ public interface TimelineReader extends Service {
* @param fieldsToRetrieve
* Specifies which fields of the entity object to retrieve(optional), see
* {@link Field}. If null, retrieves 4 fields namely entity id,
- * entity type, entity created time and entity modified time. All
- * entities will be returned if {@link Field#ALL} is specified.
+ * entity type and entity created time. All entities will be returned if
+ * {@link Field#ALL} is specified.
* @return a {@link TimelineEntity} instance or null. The entity will
* contain the metadata plus the given fields to retrieve.
* @throws IOException
@@ -101,13 +101,13 @@ public interface TimelineReader extends Service {
/**
* The API to search for a set of entities of the given the entity type in
* the scope of the given context which matches the given predicates. The
- * predicates include the created/modified time window, limit to number of
- * entities to be returned, and the entities can be filtered by checking
- * whether they contain the given info/configs entries in the form of
- * key/value pairs, given metrics in the form of metricsIds and its relation
- * with metric values given events in the form of the Ids, and whether they
- * relate to/are related to other entities. For those parameters which have
- * multiple entries, the qualified entity needs to meet all or them.
+ * predicates include the created time window, limit to number of entities to
+ * be returned, and the entities can be filtered by checking whether they
+ * contain the given info/configs entries in the form of key/value pairs,
+ * given metrics in the form of metricsIds and its relation with metric
+ * values, given events in the form of the Ids, and whether they relate to/are
+ * related to other entities. For those parameters which have multiple
+ * entries, the qualified entity needs to meet all or them.
*
* @param userId
* Context user Id(optional).
@@ -130,12 +130,6 @@ public interface TimelineReader extends Service {
* @param createdTimeEnd
* Matched entities should not be created after this timestamp (optional).
* If null or <=0, defaults to {@link Long#MAX_VALUE}.
- * @param modifiedTimeBegin
- * Matched entities should not be modified before this timestamp
- * (optional). If null or <=0, defaults to 0.
- * @param modifiedTimeEnd
- * Matched entities should not be modified after this timestamp (optional).
- * If null or <=0, defaults to {@link Long#MAX_VALUE}.
* @param relatesTo
* Matched entities should relate to given entities (optional).
* @param isRelatedTo
@@ -173,19 +167,17 @@ public interface TimelineReader extends Service {
* @param fieldsToRetrieve
* Specifies which fields of the entity object to retrieve(optional), see
* {@link Field}. If null, retrieves 4 fields namely entity id,
- * entity type, entity created time and entity modified time. All
- * entities will be returned if {@link Field#ALL} is specified.
+ * entity type and entity created time. All entities will be returned if
+ * {@link Field#ALL} is specified.
* @return A set of {@link TimelineEntity} instances of the given entity type
* in the given context scope which matches the given predicates
* ordered by created time, descending. Each entity will only contain the
- * metadata(id, type, created and modified times) plus the given fields to
- * retrieve.
+ * metadata(id, type and created time) plus the given fields to retrieve.
* @throws IOException
*/
Set getEntities(String userId, String clusterId,
String flowName, Long flowRunId, String appId, String entityType,
Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Long modifiedTimeBegin, Long modifiedTimeEnd,
Map> relatesTo, Map> isRelatedTo,
Map infoFilters, Map configFilters,
Set metricFilters, Set eventFilters,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
index 802626d..c03c9b6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
@@ -44,11 +44,6 @@ public enum ApplicationColumn implements Column {
CREATED_TIME(ApplicationColumnFamily.INFO, "created_time"),
/**
- * When it was modified.
- */
- MODIFIED_TIME(ApplicationColumnFamily.INFO, "modified_time"),
-
- /**
* The version of the flow that this app belongs to.
*/
FLOW_VERSION(ApplicationColumnFamily.INFO, "flow_version");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
index ee5ff83..681c200 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
@@ -50,15 +50,12 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBas
* | flowName! | created_time: | @timestamp1 | |
* | flowRunId! | 1392993084018 | | configKey2: |
* | AppId | | metriciD1: | configValue2 |
- * | | modified_time: | metricValue2 | |
- * | | 1392995081012 | @timestamp2 | |
+ * | | i!infoKey: | metricValue2 | |
+ * | | infoValue | @timestamp2 | |
* | | | | |
- * | | i!infoKey: | metricId2: | |
- * | | infoValue | metricValue1 | |
+ * | | r!relatesToKey: | metricId2: | |
+ * | | id3=id4=id5 | metricValue1 | |
* | | | @timestamp2 | |
- * | | r!relatesToKey: | | |
- * | | id3=id4=id5 | | |
- * | | | | |
* | | s!isRelatedToKey: | | |
* | | id7=id9=id6 | | |
* | | | | |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
index e12b6e0..deb8bd5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
@@ -49,11 +49,6 @@ public enum EntityColumn implements Column {
CREATED_TIME(EntityColumnFamily.INFO, "created_time"),
/**
- * When it was modified.
- */
- MODIFIED_TIME(EntityColumnFamily.INFO, "modified_time"),
-
- /**
* The version of the flow that this entity belongs to.
*/
FLOW_VERSION(EntityColumnFamily.INFO, "flow_version");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
index 15d4465..d7d770b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
@@ -48,16 +48,13 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBas
* | clusterId! | | metricValue1 | configValue1 |
* | flowName! | type:entityType | @timestamp1 | |
* | flowRunId! | | | configKey2: |
- * | AppId! | created_time: | metriciD1: | configValue2 |
+ * | AppId! | created_time: | metricId1: | configValue2 |
* | entityType!| 1392993084018 | metricValue2 | |
* | entityId | | @timestamp2 | |
- * | | modified_time: | | |
- * | | 1392995081012 | metricId2: | |
+ * | | i!infoKey: | | |
+ * | | infoValue | metricId1: | |
* | | | metricValue1 | |
- * | | i!infoKey: | @timestamp2 | |
- * | | infoValue | | |
- * | | | | |
- * | | r!relatesToKey: | | |
+ * | | r!relatesToKey: | @timestamp2 | |
* | | id3=id4=id5 | | |
* | | | | |
* | | s!isRelatedToKey | | |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
index 181ec81..4d61076 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
@@ -63,17 +63,15 @@ class ApplicationEntityReader extends GenericEntityReader {
public ApplicationEntityReader(String userId, String clusterId,
String flowName, Long flowRunId, String appId, String entityType,
Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Long modifiedTimeBegin, Long modifiedTimeEnd,
Map> relatesTo, Map> isRelatedTo,
Map infoFilters, Map configFilters,
Set metricFilters, Set eventFilters,
TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
EnumSet fieldsToRetrieve) {
super(userId, clusterId, flowName, flowRunId, appId, entityType, limit,
- createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
- relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
- eventFilters, confsToRetrieve, metricsToRetrieve, fieldsToRetrieve,
- true);
+ createdTimeBegin, createdTimeEnd, relatesTo, isRelatedTo, infoFilters,
+ configFilters, metricFilters, eventFilters, confsToRetrieve,
+ metricsToRetrieve, fieldsToRetrieve, true);
}
public ApplicationEntityReader(String userId, String clusterId,
@@ -230,12 +228,6 @@ class ApplicationEntityReader extends GenericEntityReader {
if (createdTimeEnd == null) {
createdTimeEnd = DEFAULT_END_TIME;
}
- if (modifiedTimeBegin == null) {
- modifiedTimeBegin = DEFAULT_BEGIN_TIME;
- }
- if (modifiedTimeEnd == null) {
- modifiedTimeEnd = DEFAULT_END_TIME;
- }
}
}
@@ -278,15 +270,6 @@ class ApplicationEntityReader extends GenericEntityReader {
return null;
}
- // fetch modified time
- Number modifiedTime =
- (Number)ApplicationColumn.MODIFIED_TIME.readResult(result);
- entity.setModifiedTime(modifiedTime.longValue());
- if (!singleEntityRead && (entity.getModifiedTime() < modifiedTimeBegin ||
- entity.getModifiedTime() > modifiedTimeEnd)) {
- return null;
- }
-
// fetch is related to entities
boolean checkIsRelatedTo = isRelatedTo != null && isRelatedTo.size() > 0;
if (fieldsToRetrieve.contains(Field.ALL) ||
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
index 52ceef8..048f608 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
@@ -52,15 +52,14 @@ class FlowActivityEntityReader extends TimelineEntityReader {
public FlowActivityEntityReader(String userId, String clusterId,
String flowName, Long flowRunId, String appId, String entityType,
Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Long modifiedTimeBegin, Long modifiedTimeEnd,
Map> relatesTo, Map> isRelatedTo,
Map infoFilters, Map configFilters,
Set metricFilters, Set eventFilters,
EnumSet fieldsToRetrieve) {
super(userId, clusterId, flowName, flowRunId, appId, entityType, limit,
- createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
- relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
- eventFilters, null, null, fieldsToRetrieve, true);
+ createdTimeBegin, createdTimeEnd, relatesTo, isRelatedTo, infoFilters,
+ configFilters, metricFilters, eventFilters, null, null,
+ fieldsToRetrieve, true);
}
public FlowActivityEntityReader(String userId, String clusterId,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
index 6286ee1..4f50b02 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
@@ -61,16 +61,15 @@ class FlowRunEntityReader extends TimelineEntityReader {
public FlowRunEntityReader(String userId, String clusterId,
String flowName, Long flowRunId, String appId, String entityType,
Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Long modifiedTimeBegin, Long modifiedTimeEnd,
Map> relatesTo, Map> isRelatedTo,
Map infoFilters, Map configFilters,
Set metricFilters, Set eventFilters,
TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
EnumSet fieldsToRetrieve) {
super(userId, clusterId, flowName, flowRunId, appId, entityType, limit,
- createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
- relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
- eventFilters, null, metricsToRetrieve, fieldsToRetrieve, true);
+ createdTimeBegin, createdTimeEnd, relatesTo, isRelatedTo, infoFilters,
+ configFilters, metricFilters, eventFilters, null, metricsToRetrieve,
+ fieldsToRetrieve, true);
}
public FlowRunEntityReader(String userId, String clusterId,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
index f3f380c..990856f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
@@ -79,17 +79,15 @@ class GenericEntityReader extends TimelineEntityReader {
public GenericEntityReader(String userId, String clusterId,
String flowName, Long flowRunId, String appId, String entityType,
Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Long modifiedTimeBegin, Long modifiedTimeEnd,
Map> relatesTo, Map> isRelatedTo,
Map infoFilters, Map configFilters,
Set metricFilters, Set eventFilters,
TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
EnumSet fieldsToRetrieve, boolean sortedKeys) {
super(userId, clusterId, flowName, flowRunId, appId, entityType, limit,
- createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
- relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
- eventFilters, confsToRetrieve, metricsToRetrieve, fieldsToRetrieve,
- sortedKeys);
+ createdTimeBegin, createdTimeEnd, relatesTo, isRelatedTo, infoFilters,
+ configFilters, metricFilters, eventFilters, confsToRetrieve,
+ metricsToRetrieve, fieldsToRetrieve, sortedKeys);
}
public GenericEntityReader(String userId, String clusterId,
@@ -257,12 +255,6 @@ class GenericEntityReader extends TimelineEntityReader {
if (createdTimeEnd == null) {
createdTimeEnd = DEFAULT_END_TIME;
}
- if (modifiedTimeBegin == null) {
- modifiedTimeBegin = DEFAULT_BEGIN_TIME;
- }
- if (modifiedTimeEnd == null) {
- modifiedTimeEnd = DEFAULT_END_TIME;
- }
}
}
@@ -314,14 +306,6 @@ class GenericEntityReader extends TimelineEntityReader {
return null;
}
- // fetch modified time
- Number modifiedTime = (Number)EntityColumn.MODIFIED_TIME.readResult(result);
- entity.setModifiedTime(modifiedTime.longValue());
- if (!singleEntityRead && (entity.getModifiedTime() < modifiedTimeBegin ||
- entity.getModifiedTime() > modifiedTimeEnd)) {
- return null;
- }
-
// fetch is related to entities
boolean checkIsRelatedTo = isRelatedTo != null && isRelatedTo.size() > 0;
if (fieldsToRetrieve.contains(Field.ALL) ||
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
index e801466..bc86b6d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
@@ -64,8 +64,6 @@ public abstract class TimelineEntityReader {
protected Long limit;
protected Long createdTimeBegin;
protected Long createdTimeEnd;
- protected Long modifiedTimeBegin;
- protected Long modifiedTimeEnd;
protected Map> relatesTo;
protected Map> isRelatedTo;
protected Map infoFilters;
@@ -94,7 +92,6 @@ public abstract class TimelineEntityReader {
protected TimelineEntityReader(String userId, String clusterId,
String flowName, Long flowRunId, String appId, String entityType,
Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Long modifiedTimeBegin, Long modifiedTimeEnd,
Map> relatesTo, Map> isRelatedTo,
Map infoFilters, Map configFilters,
Set metricFilters, Set eventFilters,
@@ -112,8 +109,6 @@ public abstract class TimelineEntityReader {
this.limit = limit;
this.createdTimeBegin = createdTimeBegin;
this.createdTimeEnd = createdTimeEnd;
- this.modifiedTimeBegin = modifiedTimeBegin;
- this.modifiedTimeEnd = modifiedTimeEnd;
this.relatesTo = relatesTo;
this.isRelatedTo = isRelatedTo;
this.infoFilters = infoFilters;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
index c77897a..2e2c652 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
@@ -62,7 +62,6 @@ public class TimelineEntityReaderFactory {
public static TimelineEntityReader createMultipleEntitiesReader(String userId,
String clusterId, String flowName, Long flowRunId, String appId,
String entityType, Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Long modifiedTimeBegin, Long modifiedTimeEnd,
Map> relatesTo, Map> isRelatedTo,
Map infoFilters, Map configFilters,
Set metricFilters, Set eventFilters,
@@ -72,29 +71,25 @@ public class TimelineEntityReaderFactory {
// table are application, flow run, and flow activity entities
if (TimelineEntityType.YARN_APPLICATION.matches(entityType)) {
return new ApplicationEntityReader(userId, clusterId, flowName, flowRunId,
- appId, entityType, limit, createdTimeBegin, createdTimeEnd,
- modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo,
- infoFilters, configFilters, metricFilters, eventFilters, confs,
- metrics, fieldsToRetrieve);
+ appId, entityType, limit, createdTimeBegin, createdTimeEnd, relatesTo,
+ isRelatedTo, infoFilters, configFilters, metricFilters, eventFilters,
+ confs, metrics, fieldsToRetrieve);
} else if (TimelineEntityType.YARN_FLOW_ACTIVITY.matches(entityType)) {
return new FlowActivityEntityReader(userId, clusterId, flowName, flowRunId,
- appId, entityType, limit, createdTimeBegin, createdTimeEnd,
- modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo,
- infoFilters, configFilters, metricFilters, eventFilters,
+ appId, entityType, limit, createdTimeBegin, createdTimeEnd, relatesTo,
+ isRelatedTo, infoFilters, configFilters, metricFilters, eventFilters,
fieldsToRetrieve);
} else if (TimelineEntityType.YARN_FLOW_RUN.matches(entityType)) {
return new FlowRunEntityReader(userId, clusterId, flowName, flowRunId,
- appId, entityType, limit, createdTimeBegin, createdTimeEnd,
- modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo,
- infoFilters, configFilters, metricFilters, eventFilters, confs,
- metrics, fieldsToRetrieve);
+ appId, entityType, limit, createdTimeBegin, createdTimeEnd, relatesTo,
+ isRelatedTo, infoFilters, configFilters, metricFilters, eventFilters,
+ confs, metrics, fieldsToRetrieve);
} else {
// assume we're dealing with a generic entity read
return new GenericEntityReader(userId, clusterId, flowName, flowRunId,
- appId, entityType, limit, createdTimeBegin, createdTimeEnd,
- modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo,
- infoFilters, configFilters, metricFilters, eventFilters, confs,
- metrics, fieldsToRetrieve, false);
+ appId, entityType, limit, createdTimeBegin, createdTimeEnd, relatesTo,
+ isRelatedTo, infoFilters, configFilters, metricFilters, eventFilters,
+ confs, metrics, fieldsToRetrieve, false);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
index 9c74e2d..1aefdca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
@@ -174,9 +174,8 @@ public class TestTimelineReaderWebServices {
assertEquals("id_1", entity.getId());
assertEquals("app", entity.getType());
assertEquals(1425016502000L, entity.getCreatedTime());
- assertEquals(1425016503000L, entity.getModifiedTime());
// Default view i.e. when no fields are specified, entity contains only
- // entity id, entity type, created and modified time.
+ // entity id, entity type and created time.
assertEquals(0, entity.getConfigs().size());
assertEquals(0, entity.getMetrics().size());
} finally {
@@ -198,7 +197,6 @@ public class TestTimelineReaderWebServices {
assertEquals("id_1", entity.getId());
assertEquals("app", entity.getType());
assertEquals(1425016502000L, entity.getCreatedTime());
- assertEquals(1425016503000L, entity.getModifiedTime());
} finally {
client.destroy();
}
@@ -382,50 +380,6 @@ public class TestTimelineReaderWebServices {
}
@Test
- public void testGetEntitiesBasedOnModifiedTime() throws Exception {
- Client client = createClient();
- try {
- URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/entities/cluster1/app1/app?modifiedtimestart=1425016502090"
- + "&modifiedtimeend=1425016503020");
- ClientResponse resp = getResponse(client, uri);
- Set entities =
- resp.getEntity(new GenericType>(){});
- assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
- assertNotNull(entities);
- assertEquals(2, entities.size());
- assertTrue("Entities with id_1 and id_4 should have been" +
- " present in response.",
- entities.contains(newEntity("app", "id_1")) &&
- entities.contains(newEntity("app", "id_4")));
-
- uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
- "entities/cluster1/app1/app?modifiedtimeend=1425016502090");
- resp = getResponse(client, uri);
- entities = resp.getEntity(new GenericType>(){});
- assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
- assertNotNull(entities);
- assertEquals(2, entities.size());
- assertTrue("Entities with id_2 and id_3 should have been " +
- "present in response.",
- entities.contains(newEntity("app", "id_2")) &&
- entities.contains(newEntity("app", "id_3")));
-
- uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
- "entities/cluster1/app1/app?modifiedtimestart=1425016503005");
- resp = getResponse(client, uri);
- entities = resp.getEntity(new GenericType>(){});
- assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
- assertNotNull(entities);
- assertEquals(1, entities.size());
- assertTrue("Entity with id_4 should have been present in response.",
- entities.contains(newEntity("app", "id_4")));
- } finally {
- client.destroy();
- }
- }
-
- @Test
public void testGetEntitiesByRelations() throws Exception {
Client client = createClient();
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org