phoenix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jamestay...@apache.org
Subject phoenix git commit: PHOENIX-4732 Improve config defaults for async secondary index MR jobs
Date Tue, 08 May 2018 22:57:02 GMT
Repository: phoenix
Updated Branches:
  refs/heads/4.x-HBase-0.98 b09f26b73 -> 0482c539a


PHOENIX-4732 Improve config defaults for async secondary index MR jobs


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/0482c539
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/0482c539
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/0482c539

Branch: refs/heads/4.x-HBase-0.98
Commit: 0482c539a16d3b404f65ad9ccd9664d0d479e3cc
Parents: b09f26b
Author: James Taylor <jtaylor@salesforce.com>
Authored: Tue May 8 15:26:44 2018 -0700
Committer: James Taylor <jtaylor@salesforce.com>
Committed: Tue May 8 15:26:44 2018 -0700

----------------------------------------------------------------------
 .../index/automation/PhoenixMRJobSubmitter.java | 48 ++++++++++++++++++--
 .../apache/phoenix/util/PhoenixMRJobUtil.java   | 18 +++-----
 2 files changed, 52 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/0482c539/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/automation/PhoenixMRJobSubmitter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/automation/PhoenixMRJobSubmitter.java
b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/automation/PhoenixMRJobSubmitter.java
index 9c447e8..d86802a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/automation/PhoenixMRJobSubmitter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/automation/PhoenixMRJobSubmitter.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
 import org.apache.phoenix.mapreduce.index.IndexTool;
@@ -53,8 +54,8 @@ import org.apache.phoenix.schema.PTable.IndexType;
 import org.apache.phoenix.schema.PTableType;
 import org.apache.phoenix.schema.types.PDate;
 import org.apache.phoenix.util.PhoenixMRJobUtil;
-import org.apache.phoenix.util.UpgradeUtil;
 import org.apache.phoenix.util.PhoenixMRJobUtil.MR_SCHEDULER_TYPE;
+import org.apache.phoenix.util.UpgradeUtil;
 import org.apache.phoenix.util.ZKBasedMasterElectionUtil;
 import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONObject;
@@ -69,8 +70,28 @@ public class PhoenixMRJobSubmitter {
     // Lock to elect a master node that submits the Phoenix Secondary Index MR Jobs
     private static final String PHOENIX_LOCKS_PARENT =
             "/phoenix/automated-mr-index-build-leader-election";
+    public static final String PHOENIX_MR_CLIENT_SCANNER_TIMEOUT_PERIOD =
+            "phoenix.mr.client.scanner.timeout.period";
+    public static final String PHOENIX_MR_RPC_TIMEOUT =
+            "phoenix.mr.rpc.timeout";
+    public static final String PHOENIX_MR_TASK_TIMEOUT =
+            "phoenix.mr.task.timeout";
+    public static final String PHOENIX_MR_CLIENT_RETRIES_NUMBER =
+            "phoenix.mr.client.retries.number";
+    public static final String PHOENIX_MR_CLIENT_PAUSE =
+            "phoenix.mr.client.retries.number";
+    public static final String PHOENIX_MR_ZK_RECOVERY_RETRY =
+            "phoenix.mr.zk.recovery.retry";
     private static final String AUTO_INDEX_BUILD_LOCK_NAME = "ActiveStandbyElectorLock";
-
+    private static final int DEFAULT_TIMEOUT_IN_MILLIS = 600000;
+    public static final int DEFAULT_MR_CLIENT_SCANNER_TIMEOUT_PERIOD = DEFAULT_TIMEOUT_IN_MILLIS;
+    public static final int DEFAULT_MR_RPC_TIMEOUT = DEFAULT_TIMEOUT_IN_MILLIS;
+    public static final int DEFAULT_MR_TASK_TIMEOUT = DEFAULT_TIMEOUT_IN_MILLIS;
+    // Reduced HBase/Zookeeper Client Retries
+    public static final int DEFAULT_MR_CLIENT_RETRIES_NUMBER = 10;
+    public static final int DEFAULT_MR_CLIENT_PAUSE = 1000;
+    public static final int DEFAULT_MR_ZK_RECOVERY_RETRY = 1;
+    
     public static final String CANDIDATE_INDEX_INFO_QUERY = "SELECT "
             + PhoenixDatabaseMetaData.INDEX_TYPE + ","
             + PhoenixDatabaseMetaData.DATA_TABLE_NAME + ", "
@@ -103,7 +124,28 @@ public class PhoenixMRJobSubmitter {
         }
         this.conf = conf;
 
-        PhoenixMRJobUtil.updateTimeoutsToFailFast(conf);
+        // Have Phoenix specific properties for defaults to enable potential override
+        conf.setLong(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 
+                conf.getLong(PHOENIX_MR_CLIENT_SCANNER_TIMEOUT_PERIOD,
+                        DEFAULT_MR_CLIENT_SCANNER_TIMEOUT_PERIOD));
+        conf.setLong(HConstants.HBASE_RPC_TIMEOUT_KEY, 
+                conf.getLong(PHOENIX_MR_RPC_TIMEOUT,
+                        DEFAULT_MR_RPC_TIMEOUT));
+        conf.setLong(MRJobConfig.TASK_TIMEOUT, 
+                conf.getLong(PHOENIX_MR_TASK_TIMEOUT,
+                        DEFAULT_MR_TASK_TIMEOUT));
+
+        // Reduced HBase Client Retries
+        conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 
+                conf.getInt(PHOENIX_MR_CLIENT_RETRIES_NUMBER,
+                        DEFAULT_MR_CLIENT_RETRIES_NUMBER));
+        conf.setInt(HConstants.HBASE_CLIENT_PAUSE, 
+                conf.getInt(PHOENIX_MR_CLIENT_PAUSE,
+                        DEFAULT_MR_CLIENT_PAUSE));
+        conf.setInt("zookeeper.recovery.retry", 
+                conf.getInt(PHOENIX_MR_ZK_RECOVERY_RETRY,
+                        DEFAULT_MR_ZK_RECOVERY_RETRY));
+        
         String schedulerType =
                 conf.get(PhoenixMRJobUtil.PHOENIX_MR_SCHEDULER_TYPE_NAME,
                     MR_SCHEDULER_TYPE.NONE.toString());

http://git-wip-us.apache.org/repos/asf/phoenix/blob/0482c539/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixMRJobUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixMRJobUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixMRJobUtil.java
index f12d49d..13eae98 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixMRJobUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixMRJobUtil.java
@@ -52,19 +52,17 @@ public class PhoenixMRJobUtil {
     private static final String ACTIVE_STANDBY_ELECTOR_LOCK = "ActiveStandbyElectorLock";
     private static final String RM_APPS_GET_ENDPOINT = "/ws/v1/cluster/apps";
 
-    // Reduced HBase Client Retries
-    private static final int CLIENT_RETRIES_NUMBER = 2;
-    private static final long CLIENT_PAUSE_TIME = 1000;
-    private static final int ZOOKEEPER_RECOVERY_RETRY_COUNT = 1;
-
     public static final String PHOENIX_INDEX_MR_QUEUE_NAME_PROPERTY =
             "phoenix.index.mr.scheduler.capacity.queuename";
     public static final String PHOENIX_INDEX_MR_MAP_MEMORY_PROPERTY =
             "phoenix.index.mr.scheduler.capacity.mapMemoryMB";
+    public static final String PHOENIX_MR_CONCURRENT_MAP_LIMIT_PROPERTY =
+    		"phoenix.mr.concurrent.map.limit";
 
     // Default MR Capacity Scheduler Configurations for Phoenix MR Index Build
     // Jobs
     public static final String DEFAULT_QUEUE_NAME = "default";
+    public static final int DEFAULT_MR_CONCURRENT_MAP_LIMIT = 20;
     public static final int DEFAULT_MAP_MEMROY_MB = 5120;
     public static final String XMX_OPT = "-Xmx";
 
@@ -208,12 +206,6 @@ public class PhoenixMRJobUtil {
         return rmPort;
     }
 
-    public static void updateTimeoutsToFailFast(Configuration conf) {
-        conf.set("hbase.client.retries.number", String.valueOf(CLIENT_RETRIES_NUMBER));
-        conf.set("zookeeper.recovery.retry", String.valueOf(ZOOKEEPER_RECOVERY_RETRY_COUNT));
-        conf.set("hbase.client.pause", String.valueOf(CLIENT_PAUSE_TIME));
-    }
-
     /**
      * This method set the configuration values for Capacity scheduler.
      * @param conf - Configuration to which Capacity Queue information to be added
@@ -221,6 +213,10 @@ public class PhoenixMRJobUtil {
     public static void updateCapacityQueueInfo(Configuration conf) {
         conf.set(MRJobConfig.QUEUE_NAME,
             conf.get(PHOENIX_INDEX_MR_QUEUE_NAME_PROPERTY, DEFAULT_QUEUE_NAME));
+
+        conf.setInt(MRJobConfig.JOB_RUNNING_MAP_LIMIT,
+                conf.getInt(PHOENIX_MR_CONCURRENT_MAP_LIMIT_PROPERTY, DEFAULT_MR_CONCURRENT_MAP_LIMIT));
+
         int mapMemoryMB = conf.getInt(PHOENIX_INDEX_MR_MAP_MEMORY_PROPERTY, DEFAULT_MAP_MEMROY_MB);
 
         conf.setInt(MRJobConfig.MAP_MEMORY_MB, mapMemoryMB);


Mime
View raw message