tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jeag...@apache.org
Subject tez git commit: TEZ-3378. Move Shuffle Handler configuration into the Tez namespace (Eric Badger via jeagles)
Date Wed, 27 Jul 2016 19:50:29 GMT
Repository: tez
Updated Branches:
  refs/heads/TEZ-3334 333d19322 -> 50541ffff


TEZ-3378. Move Shuffle Handler configuration into the Tez namespace (Eric Badger via jeagles)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/50541fff
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/50541fff
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/50541fff

Branch: refs/heads/TEZ-3334
Commit: 50541ffffdd84d599571791b758ca8184e558632
Parents: 333d193
Author: Jonathan Eagles <jeagles@yahoo-inc.com>
Authored: Wed Jul 27 14:50:16 2016 -0500
Committer: Jonathan Eagles <jeagles@yahoo-inc.com>
Committed: Wed Jul 27 14:50:16 2016 -0500

----------------------------------------------------------------------
 TEZ-3334-CHANGES.txt                            |  1 +
 .../org/apache/tez/dag/api/TezConstants.java    |  2 +-
 .../apache/tez/auxservices/ShuffleHandler.java  | 36 ++++++++++----------
 .../library/common/shuffle/ShuffleUtils.java    |  2 +-
 4 files changed, 21 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/50541fff/TEZ-3334-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-3334-CHANGES.txt b/TEZ-3334-CHANGES.txt
index c5a16b3..7a41a1f 100644
--- a/TEZ-3334-CHANGES.txt
+++ b/TEZ-3334-CHANGES.txt
@@ -4,6 +4,7 @@ Apache Tez Change Log
 INCOMPATIBLE CHANGES:
 
 ALL CHANGES:
+  TEZ-3378. Move Shuffle Handler configuration into the Tez namespace
   TEZ-3377. Remove ShuffleHandler dependency on mapred.FadvisedChunkedFile and mapred.FadvisedFileRegion
   TEZ-3380. Shuffle Handler: Replace primitive wrapper's valueOf method with parse* method
to avoid unnecessary boxing/unboxing
   TEZ-3355. Tez Custom Shuffle Handler POC

http://git-wip-us.apache.org/repos/asf/tez/blob/50541fff/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
index 6e1cb2d..43f09a2 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
@@ -74,7 +74,7 @@ public class TezConstants {
    * The service id for the NodeManager plugin used to share intermediate data
    * between vertices.
    */
-  public static final String TEZ_SHUFFLE_HANDLER_SERVICE_ID = "mapreduce_shuffle";
+  public static final String TEZ_SHUFFLE_HANDLER_SERVICE_ID = "tez_shuffle";
   
   public static final String TEZ_PREWARM_DAG_NAME_PREFIX = "TezPreWarmDAG";
 

http://git-wip-us.apache.org/repos/asf/tez/blob/50541fff/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java
b/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java
index cccd1a0..ca49ecc 100644
--- a/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java
+++ b/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java
@@ -146,10 +146,10 @@ public class ShuffleHandler extends AuxiliaryService {
   private static final Log LOG = LogFactory.getLog(ShuffleHandler.class);
   private static final Log AUDITLOG =
       LogFactory.getLog(ShuffleHandler.class.getName()+".audit");
-  public static final String SHUFFLE_MANAGE_OS_CACHE = "mapreduce.shuffle.manage.os.cache";
+  public static final String SHUFFLE_MANAGE_OS_CACHE = "tez.shuffle.manage.os.cache";
   public static final boolean DEFAULT_SHUFFLE_MANAGE_OS_CACHE = true;
 
-  public static final String SHUFFLE_READAHEAD_BYTES = "mapreduce.shuffle.readahead.bytes";
+  public static final String SHUFFLE_READAHEAD_BYTES = "tez.shuffle.readahead.bytes";
   public static final int DEFAULT_SHUFFLE_READAHEAD_BYTES = 4 * 1024 * 1024;
   public static final String USERCACHE = "usercache";
   public static final String APPCACHE = "appcache";
@@ -160,7 +160,7 @@ public class ShuffleHandler extends AuxiliaryService {
       "^.*(?:connection.*reset|connection.*closed|broken.*pipe).*$",
       Pattern.CASE_INSENSITIVE);
 
-  private static final String STATE_DB_NAME = "mapreduce_shuffle_state";
+  private static final String STATE_DB_NAME = "tez_shuffle_state";
   private static final String STATE_DB_SCHEMA_VERSION_KEY = "shuffle-schema-version";
   protected static final Version CURRENT_VERSION_INFO =
       Version.newInstance(1, 0);
@@ -191,45 +191,45 @@ public class ShuffleHandler extends AuxiliaryService {
 
   private DB stateDb = null;
 
-  public static final String MAPREDUCE_SHUFFLE_SERVICEID =
-      "mapreduce_shuffle";
+  public static final String TEZ_SHUFFLE_SERVICEID =
+      "tez_shuffle";
 
-  public static final String SHUFFLE_PORT_CONFIG_KEY = "mapreduce.shuffle.port";
-  public static final int DEFAULT_SHUFFLE_PORT = 13562;
+  public static final String SHUFFLE_PORT_CONFIG_KEY = "tez.shuffle.port";
+  public static final int DEFAULT_SHUFFLE_PORT = 13563;
 
   public static final String SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED =
-      "mapreduce.shuffle.connection-keep-alive.enable";
+      "tez.shuffle.connection-keep-alive.enable";
   public static final boolean DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED = false;
 
   public static final String SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT =
-      "mapreduce.shuffle.connection-keep-alive.timeout";
+      "tez.shuffle.connection-keep-alive.timeout";
   public static final int DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT = 5; //seconds
 
   public static final String SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE =
-      "mapreduce.shuffle.mapoutput-info.meta.cache.size";
+      "tez.shuffle.mapoutput-info.meta.cache.size";
   public static final int DEFAULT_SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE =
       1000;
 
   public static final String CONNECTION_CLOSE = "close";
 
   public static final String SUFFLE_SSL_FILE_BUFFER_SIZE_KEY =
-    "mapreduce.shuffle.ssl.file.buffer.size";
+    "tez.shuffle.ssl.file.buffer.size";
 
   public static final int DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE = 60 * 1024;
 
-  public static final String MAX_SHUFFLE_CONNECTIONS = "mapreduce.shuffle.max.connections";
+  public static final String MAX_SHUFFLE_CONNECTIONS = "tez.shuffle.max.connections";
   public static final int DEFAULT_MAX_SHUFFLE_CONNECTIONS = 0; // 0 implies no limit
 
-  public static final String MAX_SHUFFLE_THREADS = "mapreduce.shuffle.max.threads";
+  public static final String MAX_SHUFFLE_THREADS = "tez.shuffle.max.threads";
   // 0 implies Netty default of 2 * number of available processors
   public static final int DEFAULT_MAX_SHUFFLE_THREADS = 0;
 
   public static final String SHUFFLE_BUFFER_SIZE =
-      "mapreduce.shuffle.transfer.buffer.size";
+      "tez.shuffle.transfer.buffer.size";
   public static final int DEFAULT_SHUFFLE_BUFFER_SIZE = 128 * 1024;
 
   public static final String  SHUFFLE_TRANSFERTO_ALLOWED =
-      "mapreduce.shuffle.transferTo.allowed";
+      "tez.shuffle.transferTo.allowed";
   public static final boolean DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED = true;
   public static final boolean WINDOWS_DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED =
       false;
@@ -238,14 +238,14 @@ public class ShuffleHandler extends AuxiliaryService {
    open simultaneously during shuffle
    */
   public static final String SHUFFLE_MAX_SESSION_OPEN_FILES =
-      "mapreduce.shuffle.max.session-open-files";
+      "tez.shuffle.max.session-open-files";
   public static final int DEFAULT_SHUFFLE_MAX_SESSION_OPEN_FILES = 3;
 
   boolean connectionKeepAliveEnabled = false;
   int connectionKeepAliveTimeOut;
   int mapOutputMetaInfoCacheSize;
 
-  @Metrics(about="Shuffle output metrics", context="mapred")
+  @Metrics(about="Shuffle output metrics", context="mapred", name="tez")
   static class ShuffleMetrics implements ChannelFutureListener {
     @Metric("Shuffle output in bytes")
         MutableCounterLong shuffleOutputBytes;
@@ -368,7 +368,7 @@ public class ShuffleHandler extends AuxiliaryService {
   }
 
   ShuffleHandler(MetricsSystem ms) {
-    super(MAPREDUCE_SHUFFLE_SERVICEID);
+    super(TEZ_SHUFFLE_SERVICEID);
     metrics = ms.register(new ShuffleMetrics());
   }
 

http://git-wip-us.apache.org/repos/asf/tez/blob/50541fff/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java
index d74e447..e194298 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java
@@ -73,7 +73,7 @@ import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.DetailedP
 public class ShuffleUtils {
 
   private static final Logger LOG = LoggerFactory.getLogger(ShuffleUtils.class);
-  public static final String SHUFFLE_HANDLER_SERVICE_ID = "mapreduce_shuffle";
+  public static final String SHUFFLE_HANDLER_SERVICE_ID = "tez_shuffle";
   private static final long MB = 1024l * 1024l;
 
   //Shared by multiple threads


Mime
View raw message