tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hyun...@apache.org
Subject git commit: TAJO-141: Set on demand as the default cluster mode. (hyunsik)
Date Fri, 30 Aug 2013 06:57:20 GMT
Updated Branches:
  refs/heads/master ce84eba43 -> 466adcc57


TAJO-141: Set on demand as the default cluster mode. (hyunsik)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/466adcc5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/466adcc5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/466adcc5

Branch: refs/heads/master
Commit: 466adcc57f19005c6f3a9bbbc5cbf09a3d77fdc4
Parents: ce84eba
Author: Hyunsik Choi <hyunsik@apache.org>
Authored: Fri Aug 30 15:55:12 2013 +0900
Committer: Hyunsik Choi <hyunsik@apache.org>
Committed: Fri Aug 30 15:55:12 2013 +0900

----------------------------------------------------------------------
 .../main/java/org/apache/tajo/conf/TajoConf.java   |  2 +-
 .../tajo/master/TajoMasterClientService.java       |  2 +-
 .../org/apache/tajo/master/TajoMasterService.java  |  3 +--
 .../apache/tajo/master/querymaster/SubQuery.java   |  2 +-
 .../tajo/master/rm/YarnTajoResourceManager.java    |  2 +-
 .../src/main/resources/tajo-default.xml            | 17 ++---------------
 .../java/org/apache/tajo/TajoTestingCluster.java   | 12 ++++++++++++
 .../src/test/resources/tajo-default.xml            |  7 ++++++-
 tajo-dist/src/main/conf/tajo-env.sh                |  3 ++-
 .../org/apache/tajo/rpc/TestProtoAsyncRpc.java     |  4 +++-
 10 files changed, 30 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/466adcc5/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
index 2fe64e6..9bdda7e 100644
--- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
+++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
@@ -66,7 +66,7 @@ public class TajoConf extends YarnConfiguration {
     // Service Addresses
     TASKRUNNER_LISTENER_ADDRESS("tajo.master.taskrunnerlistener.addr", "0.0.0.0:0"), // used
internally
     CLIENT_SERVICE_ADDRESS("tajo.master.clientservice.addr", "127.0.0.1:9004"),
-    TAJO_MASTER_SERVICE_ADDRESS("tajo.master.manager.addr", "127.0.0.1:9005"),
+    TAJO_MASTER_SERVICE_ADDRESS("tajo.master.manager.addr", "0.0.0.0:9005"),
 
     //////////////////////////////////
     // Catalog Configuration

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/466adcc5/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
index 03426bf..e65c1f3 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
@@ -90,7 +90,7 @@ public class TajoMasterClientService extends AbstractService {
       LOG.error(e);
     }
     server.start();
-    bindAddress = server.getListenAddress();
+    bindAddress = NetUtils.getConnectAddress(server.getListenAddress());
     this.conf.setVar(ConfVars.CLIENT_SERVICE_ADDRESS, NetUtils.normalizeInetSocketAddress(bindAddress));
     LOG.info("Instantiated TajoMasterClientService at " + this.bindAddress);
     super.start();

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/466adcc5/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterService.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterService.java
b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterService.java
index 8d1bbe0..d518dce 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterService.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterService.java
@@ -58,7 +58,6 @@ public class TajoMasterService extends AbstractService {
 
   @Override
   public void start() {
-    // TODO resolve hostname
     String confMasterServiceAddr = conf.getVar(TajoConf.ConfVars.TAJO_MASTER_SERVICE_ADDRESS);
     InetSocketAddress initIsa = NetUtils.createSocketAddr(confMasterServiceAddr);
     try {
@@ -67,7 +66,7 @@ public class TajoMasterService extends AbstractService {
       LOG.error(e);
     }
     server.start();
-    bindAddress = server.getListenAddress();
+    bindAddress = NetUtils.getConnectAddress(server.getListenAddress());
     this.conf.setVar(TajoConf.ConfVars.TAJO_MASTER_SERVICE_ADDRESS,
         NetUtils.normalizeInetSocketAddress(bindAddress));
     LOG.info("Instantiated TajoMasterService at " + this.bindAddress);

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/466adcc5/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
index ce4c846..89d3fed 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
@@ -626,7 +626,7 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
 
       final Resource resource = Records.newRecord(Resource.class);
 
-      resource.setMemory(512);
+      resource.setMemory(2048);
 
       LOG.info("Request Container for " + subQuery.getId() + " containers=" + numRequest);
 

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/466adcc5/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/rm/YarnTajoResourceManager.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/rm/YarnTajoResourceManager.java
b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/rm/YarnTajoResourceManager.java
index 9bb153c..fe31bc6 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/rm/YarnTajoResourceManager.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/rm/YarnTajoResourceManager.java
@@ -268,7 +268,7 @@ public class YarnTajoResourceManager implements WorkerResourceManager
{
 
     final Resource resource = Records.newRecord(Resource.class);
     // TODO - get default value from conf
-    resource.setMemory(256);
+    resource.setMemory(2048);
     resource.setVirtualCores(1);
 
     Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>();

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/466adcc5/tajo-core/tajo-core-backend/src/main/resources/tajo-default.xml
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/resources/tajo-default.xml b/tajo-core/tajo-core-backend/src/main/resources/tajo-default.xml
index e8ad503..2025110 100644
--- a/tajo-core/tajo-core-backend/src/main/resources/tajo-default.xml
+++ b/tajo-core/tajo-core-backend/src/main/resources/tajo-default.xml
@@ -42,19 +42,6 @@
     <description>ms</description>
   </property>
 
-
-
-  <property>
-    <name>tajo.master.clientservice.addr</name>
-    <value>127.0.0.1:9004</value>
-  </property>
-
-  <property>
-    <name>tajo.master.manager.addr</name>
-    <value>127.0.0.1:9005</value>
-    <description>rpc port for tajo worker</description>
-  </property>
-
   <property>
     <name>tajo.query.session.timeout</name>
     <value>60000</value>
@@ -63,7 +50,7 @@
 
   <property>
     <name>tajo.resource.manager</name>
-    <value>org.apache.tajo.master.rm.TajoWorkerResourceManager</value>
+    <value>org.apache.tajo.master.rm.YarnTajoResourceManager</value>
     <description>This can be org.apache.tajo.master.rm.TajoWorkerResourceManager or
org.apache.tajo.master.rm.YarnTajoResourceManager</description>
   </property>
 
@@ -111,4 +98,4 @@
     <value>4</value>
     <description>The number of CPU cores on a worker</description>
   </property>
-</configuration>
+</configuration>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/466adcc5/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TajoTestingCluster.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TajoTestingCluster.java
b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TajoTestingCluster.java
index 2588e46..f031938 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TajoTestingCluster.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TajoTestingCluster.java
@@ -19,6 +19,7 @@
 package org.apache.tajo;
 
 import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
 import com.google.common.io.Closeables;
 import com.google.common.io.Files;
 import org.apache.commons.logging.Log;
@@ -37,6 +38,7 @@ import org.apache.tajo.conf.TajoConf;
 import org.apache.tajo.conf.TajoConf.ConfVars;
 import org.apache.tajo.master.TajoMaster;
 import org.apache.tajo.master.rm.TajoWorkerResourceManager;
+import org.apache.tajo.master.rm.YarnTajoResourceManager;
 import org.apache.tajo.util.NetUtils;
 import org.apache.tajo.worker.TajoWorker;
 
@@ -79,6 +81,16 @@ public class TajoTestingCluster {
 
 	public TajoTestingCluster() {
     this.conf = new TajoConf();
+    if (System.getProperty("tajo.resource.manager") != null) {
+      String testResourceManager = System.getProperty("tajo.resource.manager");
+      Preconditions.checkState(
+          testResourceManager.equals(TajoWorkerResourceManager.class.getCanonicalName())
||
+          testResourceManager.equals(YarnTajoResourceManager.class.getCanonicalName()),
+          "tajo.resource.manager must be either " + TajoWorkerResourceManager.class.getCanonicalName()
+ " or " +
+              YarnTajoResourceManager.class.getCanonicalName() +"."
+      );
+      this.conf.set("tajo.resource.manager", System.getProperty("tajo.resource.manager"));
+    }
     this.standbyWorkerMode =
         this.conf.get("tajo.resource.manager", TajoWorkerResourceManager.class.getCanonicalName())
             .indexOf(TajoWorkerResourceManager.class.getName()) >= 0;

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/466adcc5/tajo-core/tajo-core-backend/src/test/resources/tajo-default.xml
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/tajo-default.xml b/tajo-core/tajo-core-backend/src/test/resources/tajo-default.xml
index 20b80aa..aedaac3 100644
--- a/tajo-core/tajo-core-backend/src/test/resources/tajo-default.xml
+++ b/tajo-core/tajo-core-backend/src/test/resources/tajo-default.xml
@@ -39,6 +39,11 @@
   <property>
     <name>tajo.resource.manager</name>
     <value>org.apache.tajo.master.rm.TajoWorkerResourceManager</value>
-    <description>org.apache.tajo.master.rm.TajoWorkerResourceManager or org.apache.tajo.master.rm.YarnTajoResourceManager</description>
   </property>
+
+  <property>
+    <name>tajo.master.manager.addr</name>
+    <value>127.0.0.1</value>
+  </property>
+
 </configuration>

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/466adcc5/tajo-dist/src/main/conf/tajo-env.sh
----------------------------------------------------------------------
diff --git a/tajo-dist/src/main/conf/tajo-env.sh b/tajo-dist/src/main/conf/tajo-env.sh
index 2abc448..b70fe01 100755
--- a/tajo-dist/src/main/conf/tajo-env.sh
+++ b/tajo-dist/src/main/conf/tajo-env.sh
@@ -53,5 +53,6 @@ export TAJO_OPTS=-XX:+PrintGCTimeStamps
 # export TAJO_NICENESS=10
 
 # Tajo Distributed Execution Mode
-export TAJO_WORKER_STANDBY_MODE=true
+# the default mode is on-demand mode using YarnTajoResourceManager.
+# export TAJO_WORKER_STANDBY_MODE=yes
 

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/466adcc5/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestProtoAsyncRpc.java
----------------------------------------------------------------------
diff --git a/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestProtoAsyncRpc.java b/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestProtoAsyncRpc.java
index 69d68b8..a7bc4d0 100644
--- a/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestProtoAsyncRpc.java
+++ b/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestProtoAsyncRpc.java
@@ -27,6 +27,7 @@ import org.apache.tajo.rpc.test.TestProtos.EchoMessage;
 import org.apache.tajo.rpc.test.TestProtos.SumRequest;
 import org.apache.tajo.rpc.test.TestProtos.SumResponse;
 import org.apache.tajo.rpc.test.impl.DummyProtocolAsyncImpl;
+import org.apache.tajo.util.NetUtils;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -56,7 +57,8 @@ public class TestProtoAsyncRpc {
     server = new ProtoAsyncRpcServer(DummyProtocol.class,
         service, new InetSocketAddress(0));
     server.start();
-    client = new ProtoAsyncRpcClient(DummyProtocol.class, server.getListenAddress());
+    client = new ProtoAsyncRpcClient(DummyProtocol.class,
+        NetUtils.getConnectAddress(server.getListenAddress()));
     stub = client.getStub();
   }
 


Mime
View raw message