tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jeag...@apache.org
Subject [07/53] tez git commit: Upmerging to master
Date Wed, 10 Dec 2014 03:33:30 GMT
Upmerging to master


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/fb365f41
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/fb365f41
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/fb365f41

Branch: refs/heads/master
Commit: fb365f41e3ac2f8dad5065d93644a326206bcda2
Parents: ca3015d 2103129
Author: Jonathan Eagles <jeagles@gmail.com>
Authored: Thu Oct 2 15:45:29 2014 -0500
Committer: Jonathan Eagles <jeagles@gmail.com>
Committed: Thu Oct 2 15:45:29 2014 -0500

----------------------------------------------------------------------
 CHANGES.txt                                     |  55 +-
 .../java/org/apache/tez/client/TezClient.java   |   6 +-
 .../org/apache/tez/client/TezClientUtils.java   |  59 +--
 .../org/apache/tez/common/ATSConstants.java     |  94 ++++
 .../main/java/org/apache/tez/dag/api/DAG.java   | 136 ++---
 .../tez/dag/api/DAGNotRunningException.java     |  34 ++
 .../apache/tez/dag/api/DagTypeConverters.java   |  29 ++
 .../apache/tez/dag/api/DataSinkDescriptor.java  |  48 ++
 .../tez/dag/api/DataSourceDescriptor.java       |  44 ++
 .../java/org/apache/tez/dag/api/Vertex.java     |   7 +-
 .../tez/dag/api/client/DAGClientImpl.java       | 477 +++++++++++++++++
 .../dag/api/client/DAGClientTimelineImpl.java   | 508 ++++++++++++++++++
 .../apache/tez/dag/api/client/VertexStatus.java |   2 +-
 .../dag/api/client/rpc/DAGClientRPCImpl.java    | 228 +-------
 tez-api/src/main/proto/DAGApiRecords.proto      |   1 +
 .../org/apache/tez/client/TestTezClient.java    |  37 +-
 .../apache/tez/client/TestTezClientUtils.java   |   4 +-
 .../apache/tez/common/TestTezCommonUtils.java   |  54 +-
 .../org/apache/tez/dag/api/TestDAGPlan.java     |   8 +-
 .../org/apache/tez/dag/api/TestDAGVerify.java   |  81 +--
 .../tez/dag/api/client/TestATSHttpClient.java   | 167 ++++++
 .../tez/dag/api/client/rpc/TestDAGClient.java   |  24 +-
 .../tez/common/EnvironmentUpdateUtils.java      | 127 +++++
 .../tez/dag/utils/EnvironmentUpdateUtils.java   |  92 ----
 .../tez/common/TestEnvironmentUpdateUtils.java  |  34 ++
 .../dag/utils/TestEnvironmentUpdateUtils.java   |  34 --
 .../java/org/apache/tez/client/LocalClient.java |   2 +-
 .../tez/dag/api/client/DAGClientHandler.java    |  25 +-
 .../java/org/apache/tez/dag/app/AppContext.java |   3 +
 .../org/apache/tez/dag/app/DAGAppMaster.java    |   7 +
 .../org/apache/tez/dag/app/RecoveryParser.java  |   1 +
 .../java/org/apache/tez/dag/app/dag/DAG.java    |   2 +
 .../apache/tez/dag/app/dag/impl/DAGImpl.java    |  11 +
 .../dag/impl/ImmediateStartVertexManager.java   |   9 +-
 .../apache/tez/dag/app/dag/impl/VertexImpl.java |  97 ++--
 .../app/launcher/LocalContainerLauncher.java    |   6 +-
 .../dag/app/rm/LocalTaskSchedulerService.java   |  23 +-
 .../dag/app/rm/YarnTaskSchedulerService.java    | 122 ++---
 .../app/rm/container/AMContainerHelpers.java    |   9 +-
 .../dag/app/rm/container/AMContainerImpl.java   |   8 +-
 .../tez/dag/history/events/DAGStartedEvent.java |   4 +
 .../dag/history/events/VertexFinishedEvent.java |  24 +-
 .../dag/history/events/VertexStartedEvent.java  |   4 +
 .../impl/HistoryEventJsonConversion.java        |   2 +-
 .../tez/dag/history/utils/ATSConstants.java     |  76 ---
 .../apache/tez/dag/history/utils/DAGUtils.java  |   1 +
 .../apache/tez/runtime/task/TezTaskRunner.java  |  29 +-
 .../apache/tez/dag/app/MockDAGAppMaster.java    |  18 +-
 .../tez/dag/app/TestMockDAGAppMaster.java       | 138 +++++
 .../tez/dag/app/dag/impl/TestDAGImpl.java       |   2 +-
 .../tez/dag/app/dag/impl/TestVertexImpl.java    | 153 +++++-
 .../dag/app/dag/impl/TestVertexRecovery.java    |   4 +-
 .../app/rm/TestLocalTaskSchedulerService.java   |  68 +++
 .../TestHistoryEventsProtoConversion.java       |   4 +-
 .../impl/TestHistoryEventJsonConversion.java    |   2 +-
 .../tez/dag/history/utils/TestDAGUtils.java     |   2 +-
 .../org/apache/tez/examples/ExampleDriver.java  |   6 +-
 .../apache/tez/examples/HashJoinExample.java    | 403 +++++++++++++++
 .../org/apache/tez/examples/JoinDataGen.java    |   1 -
 .../org/apache/tez/examples/JoinExample.java    | 364 -------------
 .../org/apache/tez/examples/JoinValidate.java   |   4 +-
 .../tez/examples/SortMergeJoinExample.java      | 374 ++++++++++++++
 .../org/apache/tez/mapreduce/input/MRInput.java |  30 +-
 .../apache/tez/mapreduce/output/MROutput.java   |  16 +-
 .../tez/mapreduce/hadoop/TestMRHelpers.java     |  20 +-
 .../ats/HistoryEventTimelineConversion.java     |  12 +-
 .../ats/TestHistoryEventTimelineConversion.java |   2 +-
 .../vertexmanager/ShuffleVertexManager.java     |  73 ++-
 .../library/common/shuffle/impl/Fetcher.java    |   5 +-
 .../runtime/library/common/sort/impl/IFile.java |  14 +-
 .../library/shuffle/common/HttpConnection.java  |   8 +-
 .../shuffle/common/impl/ShuffleManager.java     |   4 +-
 .../src/main/proto/ShufflePayloads.proto        |   5 +-
 .../vertexmanager/TestShuffleVertexManager.java | 109 +++-
 .../library/output/TestOnFileSortedOutput.java  |   2 +-
 .../library/shuffle/common/TestFetcher.java     |   2 +-
 .../tez/mapreduce/examples/MRRSleepJob.java     |  14 +-
 .../examples/TestOrderedWordCount.java          |   9 +
 .../org/apache/tez/test/MiniTezCluster.java     |  43 ++
 .../org/apache/tez/test/TestAMRecovery.java     |   4 +-
 .../org/apache/tez/test/TestDAGRecovery.java    |   2 -
 .../org/apache/tez/test/TestDAGRecovery2.java   |   2 -
 .../tez/test/TestExceptionPropagation.java      | 516 +++++++++++++++++++
 .../java/org/apache/tez/test/TestTezJobs.java   | 149 +++++-
 84 files changed, 4181 insertions(+), 1258 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/fb365f41/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 8dabca0,0be2a65..ced524a
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -19,17 -38,19 +38,21 @@@ ALL CHANGE
    TEZ-1580. Change TestOrderedWordCount to optionally use MR configs.
    TEZ-1524. Resolve user group information only if ACLs are enabled.
    TEZ-1581. GroupByOrderByMRRTest no longer functional.
-   TEZ-1563. TezClient.submitDAGSession alters DAG local resources regardless of DAG submission.
    TEZ-1157. Optimize broadcast shuffle to download data only once per host. 
 +  TEZ-1594. Initial TezUI into TEZ-8 branch
 +  TEZ-1595. Document timeline server setup for use with the Tez UI
- 
- Release 0.5.1: Unreleased
- 
- INCOMPATIBLE CHANGES
-   TEZ-1539. Change InputInitializerEvent semantics to SEND_ONCE_ON_TASK_SUCCESS
- 
- ALL CHANGES
+   TEZ-1607. support mr envs in mrrsleep and testorderedwordcount
+   TEZ-1499. Add SortMergeJoinExample to tez-examples
+   TEZ-1613. Decrease running time for TestAMRecovery
+   TEZ-1240. Add system test for propagation of diagnostics for errors
+   TEZ-1618. LocalTaskSchedulerService.getTotalResources() and getAvailableResources() can
get 
+   negative if JVM memory is larger than 2GB
+   TEZ-1611. Change DataSource/Sink to be able to supply URIs for credentials
+   TEZ-1592. Vertex should wait for all initializers to finish before moving to INITED state
+   TEZ-1612. ShuffleVertexManager's EdgeManager should not hard code source num tasks
+   TEZ-1555. TestTezClientUtils.validateSetTezJarLocalResourcesDefinedButEmpty
+   failing on Windows
+   TEZ-1609. Add hostname to logIdentifiers of fetchers for easy debugging
    TEZ-1494. DAG hangs waiting for ShuffleManager.getNextInput()
    TEZ-1515. Remove usage of ResourceBundles in Counters.
    TEZ-1527. Fix indentation of Vertex status in DAGClient output.


Mime
View raw message