tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hyun...@apache.org
Subject [1/3] TAJO-261: Rearrange default port numbers and config names. (hyunsik)
Date Mon, 21 Oct 2013 06:38:17 GMT
Updated Branches:
  refs/heads/master 93b435d28 -> d51283284


http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/d5128328/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java
b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java
index d48ddf4..d89a2fc 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java
@@ -19,7 +19,6 @@
 package org.apache.tajo.storage;
 
 import com.google.protobuf.CodedInputStream;
-import com.google.protobuf.CodedOutputStream;
 import com.google.protobuf.Message;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -28,7 +27,9 @@ import org.apache.hadoop.fs.Path;
 import org.apache.tajo.catalog.TableMeta;
 import org.apache.tajo.catalog.statistics.TableStat;
 import org.apache.tajo.common.TajoDataTypes.DataType;
-import org.apache.tajo.datum.*;
+import org.apache.tajo.datum.DatumFactory;
+import org.apache.tajo.datum.NullDatum;
+import org.apache.tajo.datum.ProtobufDatumFactory;
 import org.apache.tajo.util.BitArray;
 
 import java.io.File;
@@ -191,21 +192,30 @@ public class RawFile {
             tuple.put(i, DatumFactory.createFloat8(buffer.getDouble()));
             break;
 
-          case TEXT : {
+          case TEXT :
             // TODO - shoud use CharsetEncoder / CharsetDecoder
-            byte [] rawBytes = getColumnBytes();
-            tuple.put(i, DatumFactory.createText(new String(rawBytes)));
+            //byte [] rawBytes = getColumnBytes();
+            int strSize2 = buffer.getInt();
+            byte [] strBytes2 = new byte[strSize2];
+            buffer.get(strBytes2);
+            tuple.put(i, DatumFactory.createText(new String(strBytes2)));
             break;
-          }
 
           case BLOB : {
-            byte [] rawBytes = getColumnBytes();
+            //byte [] rawBytes = getColumnBytes();
+            int byteSize = buffer.getInt();
+            byte [] rawBytes = new byte[byteSize];
+            buffer.get(rawBytes);
             tuple.put(i, DatumFactory.createBlob(rawBytes));
             break;
           }
 
           case PROTOBUF: {
-            byte [] rawBytes = getColumnBytes();
+            //byte [] rawBytes = getColumnBytes();
+            int byteSize = buffer.getInt();
+            byte [] rawBytes = new byte[byteSize];
+            buffer.get(rawBytes);
+
             ProtobufDatumFactory factory = ProtobufDatumFactory.get(columnTypes[i]);
             Message.Builder builder = factory.newBuilder();
             builder.mergeFrom(rawBytes);
@@ -427,19 +437,42 @@ public class RawFile {
             break;
 
           case TEXT:
-          case BLOB:
+            byte [] strBytes2 = t.get(i).asByteArray();
+            if (flushBufferAndReplace(recordOffset, strBytes2.length + 4)) {
+              recordOffset = 0;
+            }
+            buffer.putInt(strBytes2.length);
+            buffer.put(strBytes2);
+            break;
+
+          case BLOB : {
+            byte [] rawBytes = t.get(i).asByteArray();
+            if (flushBufferAndReplace(recordOffset, rawBytes.length + 4)) {
+              recordOffset = 0;
+            }
+            buffer.putInt(rawBytes.length);
+            buffer.put(rawBytes);
+            break;
+          }
+
           case PROTOBUF: {
-            byte [] lengthByte = new byte[4];
-            byte [] byteArray = t.get(i).asByteArray();
-            CodedOutputStream outputStream = CodedOutputStream.newInstance(lengthByte);
-            outputStream.writeUInt32NoTag(byteArray.length);
-            outputStream.flush();
-            int legnthByteLength = CodedOutputStream.computeInt32SizeNoTag(byteArray.length);
-            if (flushBufferAndReplace(recordOffset, byteArray.length + legnthByteLength))
{
+            // TODO - to be fixed
+//            byte [] lengthByte = new byte[4];
+//            byte [] byteArray = t.get(i).asByteArray();
+//            CodedOutputStream outputStream = CodedOutputStream.newInstance(lengthByte);
+//            outputStream.writeUInt32NoTag(byteArray.length);
+//            outputStream.flush();
+//            int legnthByteLength = CodedOutputStream.computeInt32SizeNoTag(byteArray.length);
+//            if (flushBufferAndReplace(recordOffset, byteArray.length + legnthByteLength))
{
+//              recordOffset = 0;
+//            }
+//            buffer.put(lengthByte, 0, legnthByteLength);
+            byte [] rawBytes = t.get(i).asByteArray();
+            if (flushBufferAndReplace(recordOffset, rawBytes.length + 4)) {
               recordOffset = 0;
             }
-            buffer.put(lengthByte, 0, legnthByteLength);
-            buffer.put(byteArray);
+            buffer.putInt(rawBytes.length);
+            buffer.put(rawBytes);
             break;
           }
 

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/d5128328/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/StorageManagerFactory.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/StorageManagerFactory.java
b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/StorageManagerFactory.java
index 8b7c2ca..5d6a298 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/StorageManagerFactory.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/StorageManagerFactory.java
@@ -18,6 +18,9 @@
 
 package org.apache.tajo.storage;
 
+import com.google.common.collect.Maps;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -28,49 +31,49 @@ import org.apache.tajo.storage.v2.StorageManagerV2;
 
 import java.io.IOException;
 import java.net.URI;
-import java.util.HashMap;
 import java.util.Map;
 
+import static org.apache.tajo.conf.TajoConf.ConfVars;
+
 public class StorageManagerFactory {
-  private static Map<String, AbstractStorageManager> storageManagers =
-      new HashMap<String, AbstractStorageManager>();
+  private static final Map<String, AbstractStorageManager> storageManagers = Maps.newConcurrentMap();
 
   public static AbstractStorageManager getStorageManager(TajoConf conf) throws IOException
{
     return getStorageManager(conf, null);
   }
 
   public static synchronized AbstractStorageManager getStorageManager (
-      TajoConf conf, Path dataRoot) throws IOException {
-    return getStorageManager(conf, dataRoot, conf.getBoolean("tajo.storage.manager.v2", false));
+      TajoConf conf, Path warehouseDir) throws IOException {
+    return getStorageManager(conf, warehouseDir, conf.getBoolVar(ConfVars.STORAGE_MANAGER_VERSION_2));
   }
 
   private static synchronized AbstractStorageManager getStorageManager (
-      TajoConf conf, Path dataRoot, boolean v2) throws IOException {
-    if(dataRoot != null) {
-      conf.setVar(TajoConf.ConfVars.ROOT_DIR, dataRoot.toString());
-    }
+      TajoConf conf, Path warehouseDir, boolean v2) throws IOException {
 
     URI uri;
-    if(dataRoot == null) {
-      uri = FileSystem.get(conf).getUri();
-    } else {
-      uri = dataRoot.toUri();
+    TajoConf localConf = new TajoConf(conf);
+    if (warehouseDir != null) {
+      localConf.setVar(ConfVars.WAREHOUSE_DIR, warehouseDir.toUri().toString());
     }
-    String key = "file".equals(uri.getScheme()) ? "file" : uri.getScheme() + uri.getHost()
+ uri.getPort();
+
+    uri = TajoConf.getWarehouseDir(localConf).toUri();
+
+    String key = "file".equals(uri.getScheme()) ? "file" : uri.toString();
 
     if(v2) {
       key += "_v2";
     }
 
     if(storageManagers.containsKey(key)) {
-      return storageManagers.get(key);
+      AbstractStorageManager sm = storageManagers.get(key);
+      return sm;
     } else {
-      AbstractStorageManager storageManager = null;
+      AbstractStorageManager storageManager;
 
       if(v2) {
-        storageManager = new StorageManagerV2(conf);
+        storageManager = new StorageManagerV2(localConf);
       } else {
-        storageManager = new StorageManager(conf);
+        storageManager = new StorageManager(localConf);
       }
 
       storageManagers.put(key, storageManager);

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/d5128328/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/DiskFileScanScheduler.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/DiskFileScanScheduler.java
b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/DiskFileScanScheduler.java
index 2f97d41..8ecf1de 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/DiskFileScanScheduler.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/DiskFileScanScheduler.java
@@ -29,6 +29,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
+import static org.apache.tajo.conf.TajoConf.ConfVars;
+
 public class DiskFileScanScheduler extends Thread {
   private static final Log LOG = LogFactory.getLog(DiskFileScanScheduler.class);
 
@@ -156,7 +158,7 @@ public class DiskFileScanScheduler extends Thread {
 
 	private void initScannerPool() {
 		// TODO finally implements heuristic, currently set with property
-		scanConcurrency = smContext.getConf().getInt("tajo.storage.manager.concurrency.perDisk",
1);
+		scanConcurrency = smContext.getConf().getIntVar(ConfVars.STORAGE_MANAGER_CONCURRENCY_PER_DISK);
 	}
 
   public int getTotalQueueSize() {

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/d5128328/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/RCFile.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/RCFile.java
b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/RCFile.java
index 7322c7a..acddf79 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/RCFile.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/RCFile.java
@@ -1438,7 +1438,6 @@ public class RCFile {
       seekToNextKeyBuffer();
       currentRecordLength = readRecordLength();
       if (currentRecordLength == -1) {
-    	  //System.out.println(">>>>>currentRecordLength is minus");
         keyInit = false;
         return -1;
       }

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/d5128328/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/RCFileScanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/RCFileScanner.java
b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/RCFileScanner.java
index a62ed4f..f2da623 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/RCFileScanner.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/RCFileScanner.java
@@ -275,7 +275,6 @@ public class RCFileScanner extends FileScannerV2 {
   public boolean isFetchProcessing() {
     //TODO row group size
     if(sin != null && sin.getAvaliableSize() > maxBytesPerSchedule * 3) {
-//			System.out.println(">>>>>sin.getAvaliableSize()>" + sin.getAvaliableSize());
       return true;
     } else {
       return false;

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/d5128328/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/ScanScheduler.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/ScanScheduler.java
b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/ScanScheduler.java
index 4cd04f3..8763cfa 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/ScanScheduler.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/ScanScheduler.java
@@ -25,6 +25,8 @@ import org.apache.tajo.storage.v2.StorageManagerV2.StorgaeManagerContext;
 import java.util.*;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import static org.apache.tajo.conf.TajoConf.ConfVars;
+
 public class ScanScheduler extends Thread {
   private static final Log LOG = LogFactory.getLog(ScanScheduler.class);
 
@@ -63,7 +65,7 @@ public class ScanScheduler extends Thread {
       LOG.error(e.getMessage(), e);
     }
 
-    final int reportInterval = context.getConf().getInt("tajo.disk.scheduler.report.interval",
60 * 1000);
+    final int reportInterval = context.getConf().getIntVar(ConfVars.STORAGE_MANAGER_DISK_SCHEDULER_REPORT_INTERVAL);
     if(reportInterval  > 0) {
       schedulerStatusReportThread = new Thread() {
         public void run() {

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/d5128328/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/StorageManagerV2.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/StorageManagerV2.java
b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/StorageManagerV2.java
index dcf523a..0625d28 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/StorageManagerV2.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/StorageManagerV2.java
@@ -112,7 +112,7 @@ public class StorageManagerV2 extends AbstractStorageManager {
 		}
 
 		public int getMaxReadBytesPerScheduleSlot() {
-			return conf.getInt("tajo.storage.manager.maxReadBytes", 8 * 1024 * 1024);		//8MB
+			return conf.getIntVar(TajoConf.ConfVars.STORAGE_MANAGER_DISK_SCHEDULER_MAX_READ_BYTES_PER_SLOT);
 		}
 
     public void requestFileScan(FileScannerV2 fileScanner) {

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/d5128328/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVCompression.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVCompression.java
b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVCompression.java
index 8827ded..e5dc00f 100644
--- a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVCompression.java
+++ b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVCompression.java
@@ -55,7 +55,7 @@ public class TestCSVCompression {
   public TestCSVCompression(CatalogProtos.StoreType type) throws IOException {
     this.storeType = type;
     conf = new TajoConf();
-    conf.set("tajo.storage.manager.v2", "true");
+    conf.setBoolVar(TajoConf.ConfVars.STORAGE_MANAGER_VERSION_2, true);
 
     testDir = CommonTestingUtil.getTestDir(TEST_PATH);
     fs = testDir.getFileSystem(conf);

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/d5128328/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVScanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVScanner.java
b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVScanner.java
index bf56943..f647a4c 100644
--- a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVScanner.java
+++ b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVScanner.java
@@ -36,6 +36,7 @@ import org.junit.Test;
 
 import java.io.IOException;
 
+import static org.apache.tajo.conf.TajoConf.ConfVars;
 import static org.junit.Assert.assertEquals;
 
 public class TestCSVScanner {
@@ -48,7 +49,7 @@ public class TestCSVScanner {
   @Before
   public void setUp() throws Exception {
     conf = new TajoConf();
-    conf.set("tajo.storage.manager.v2", "true");
+    conf.setBoolVar(ConfVars.STORAGE_MANAGER_VERSION_2, true);
     testDir = CommonTestingUtil.getTestDir(TEST_PATH);
     fs = testDir.getFileSystem(conf);
     sm = StorageManagerFactory.getStorageManager(conf, testDir);

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/d5128328/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestStorages.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestStorages.java
b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestStorages.java
index 3d1b6f7..91dd935 100644
--- a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestStorages.java
+++ b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestStorages.java
@@ -43,6 +43,7 @@ import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
 
+import static org.apache.tajo.conf.TajoConf.ConfVars;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -63,7 +64,7 @@ public class TestStorages {
     this.statsable = statsable;
 
     conf = new TajoConf();
-    conf.set("tajo.storage.manager.v2", "true");
+    conf.setBoolVar(ConfVars.STORAGE_MANAGER_VERSION_2, true);
 
     if (storeType == StoreType.RCFILE) {
       conf.setInt(RCFile.RECORD_INTERVAL_CONF_STR, 100);

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/d5128328/tajo-dist/src/main/bin/tajo
----------------------------------------------------------------------
diff --git a/tajo-dist/src/main/bin/tajo b/tajo-dist/src/main/bin/tajo
index fb62acf..d4ea664 100755
--- a/tajo-dist/src/main/bin/tajo
+++ b/tajo-dist/src/main/bin/tajo
@@ -31,8 +31,11 @@
 #                                    by doing 
 #                                    export TAJO_USER_CLASSPATH_FIRST=true
 #
-#   TAJO_HEAPSIZE  The maximum amount of heap to use, in MB. 
-#                    Default is 1000.
+#   TAJO_MASTER_HEAPSIZE  The maximum amount of heap to use, in MB. 
+#                         Default is 1000.
+#
+#   TAJO_WORKER_HEAPSIZE  The maximum amount of heap to use, in MB. 
+#                         Default is 1000.
 #
 #   TAJO_OPTS      Extra Java runtime options.
 
@@ -101,15 +104,22 @@ if [ "$JAVA_HOME" = "" ]; then
 fi
 
 JAVA=$JAVA_HOME/bin/java
-JAVA_HEAP_MAX=-Xmx1000m 
+JAVA_TAJO_MASTER_HEAP_MAX=-Xmx1000m 
+JAVA_WORKER_HEAP_MAX=-Xmx1000m 
 
 # check envvars which might override default args
-if [ "$TAJO_HEAPSIZE" != "" ]; then
-  #echo "run with heapsize $TAJO_HEAPSIZE"
-  JAVA_HEAP_MAX="-Xmx""$TAJO_HEAPSIZE""m"
-  #echo $JAVA_HEAP_MAX
+if [ "$TAJO_MASTER_HEAPSIZE" != "" ]; then
+  #echo "run with heapsize $TAJO_MASTER_HEAPSIZE"
+  JAVA_TAJO_MASTER_HEAP_MAX="-Xmx""$TAJO_MASTER_HEAPSIZE""m"
+  #echo $JAVA_MASTER_HEAP_MAX
+fi
+if [ "$TAJO_WORKER_HEAPSIZE" != "" ]; then
+  #echo "run with heapsize $TAJO_WORKER_HEAPSIZE"
+  JAVA_WORKER_HEAP_MAX="-Xmx""$TAJO_WORKER_HEAPSIZE""m"
+  #echo $JAVA_WORKER_HEAP_MAX
 fi
 
+
 ##############################################################################
 # Hadoop Version Checking Section Start
 ##############################################################################
@@ -273,10 +283,10 @@ if [ "$COMMAND" = "classpath" ] ; then
   exit
 elif [ "$COMMAND" = "master" ] ; then
   CLASS='org.apache.tajo.master.TajoMaster'
-  TAJO_OPTS="$TAJO_OPTS $TAJO_MASTER_OPTS"
+  TAJO_OPTS="$TAJO_OPTS $JAVA_TAJO_MASTER_HEAP_MAX $TAJO_MASTER_OPTS"
 elif [ "$COMMAND" = "worker" ] ; then
   CLASS='org.apache.tajo.worker.TajoWorker'
-  TAJO_OPTS="$TAJO_OPTS $TAJO_WORKER_OPTS"
+  TAJO_OPTS="$TAJO_OPTS $JAVA_WORKER_HEAP_MAX $TAJO_WORKER_OPTS"
 elif [ "$COMMAND" = "catalog" ] ; then
   CLASS='org.apache.tajo.catalog.CatalogServer'
   TAJO_OPTS="$TAJO_OPTS $TAJO_CATALOG_OPTS"
@@ -336,23 +346,5 @@ if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
 fi  
 TAJO_OPTS="$TAJO_OPTS -Dtajo.policy.file=$TAJO_POLICYFILE"
 
-# Check to see if we should start a secure datanode
-if [ "$starting_secure_dn" = "true" ]; then
-  if [ "$TAJO_PID_DIR" = "" ]; then
-    TAJO_SECURE_DN_PID="/tmp/tajo_secure_dn.pid"
-  else
-   TAJO_SECURE_DN_PID="$TAJO_PID_DIR/tajo_secure_dn.pid"
-  fi
-
-  exec "$TAJO_HOME/bin/jsvc" -Dproc_$COMMAND -outfile "$TAJO_LOG_DIR/jsvc.out" \
-                                               -errfile "$TAJO_LOG_DIR/jsvc.err" \
-                                               -pidfile "$TAJO_SECURE_DN_PID" \
-                                               -nodetach \
-                                               -user "$TAJO_SECURE_DN_USER" \
-                                               -cp "$CLASSPATH" \
-                                               $JAVA_HEAP_MAX $TAJO_OPTS \
-                                               org.apache.tajo.hdfs.server.datanode.SecureDataNodeStarter
"$@"
-else
-  # run it
-  exec "$JAVA" -Dproc_$COMMAND $JAVA_HEAP_MAX $TAJO_OPTS -classpath "$CLASSPATH" $CLASS "$@"
-fi
+# run it
+exec "$JAVA" -Dproc_$COMMAND $TAJO_OPTS -classpath "$CLASSPATH" $CLASS "$@"

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/d5128328/tajo-dist/src/main/conf/tajo-env.sh
----------------------------------------------------------------------
diff --git a/tajo-dist/src/main/conf/tajo-env.sh b/tajo-dist/src/main/conf/tajo-env.sh
index 3791904..3654fee 100755
--- a/tajo-dist/src/main/conf/tajo-env.sh
+++ b/tajo-dist/src/main/conf/tajo-env.sh
@@ -21,6 +21,9 @@
 # set JAVA_HOME in this file, so that it is correctly defined on
 # remote nodes.
 
+# Hadoop home. Required
+# export HADOOP_HOME=
+
 # The java implementation to use.  Required.
 # export JAVA_HOME=/usr
 
@@ -28,11 +31,13 @@
 # export TAJO_CLASSPATH=
 
 # The maximum amount of heap to use, in MB. Default is 1000.
-# export TAJO_HEAPSIZE=1000
+# export TAJO_MASTER_HEAPSIZE=1000
+
+# The maximum amount of heap to use, in MB. Default is 1000.
+# export TAJO_WORKER_HEAPSIZE=1000
 
 # Extra Java runtime options.  Empty by default.
 # export TAJO_OPTS=-server
-export TAJO_OPTS=-XX:+PrintGCTimeStamps
 
 # Extra TajoMaster's java runtime options for TajoMaster. Empty by default
 # export TAJO_MASTER_OPTS=
@@ -52,7 +57,5 @@ export TAJO_OPTS=-XX:+PrintGCTimeStamps
 # The scheduling priority for daemon processes.  See 'man nice'.
 # export TAJO_NICENESS=10
 
-# Tajo Distributed Execution Mode
-# the default mode is on-demand mode using YarnTajoResourceManager.
-# export TAJO_WORKER_STANDBY_MODE=true
-
+# Tajo cluster mode. the default mode is standby mode.
+export TAJO_WORKER_STANDBY_MODE=true

http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/d5128328/tajo-rpc/src/main/java/org/apache/tajo/util/NetUtils.java
----------------------------------------------------------------------
diff --git a/tajo-rpc/src/main/java/org/apache/tajo/util/NetUtils.java b/tajo-rpc/src/main/java/org/apache/tajo/util/NetUtils.java
deleted file mode 100644
index 6a43be3..0000000
--- a/tajo-rpc/src/main/java/org/apache/tajo/util/NetUtils.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.util;
-
-import java.net.*;
-
-public class NetUtils {
-  public static String normalizeInetSocketAddress(InetSocketAddress addr) {
-    return addr.getAddress().getHostAddress() + ":" + addr.getPort();
-  }
-
-  public static InetSocketAddress createSocketAddr(String addr) {
-    String [] splitted = addr.split(":");
-    return new InetSocketAddress(splitted[0], Integer.parseInt(splitted[1]));
-  }
-
-  /**
-   * Util method to build socket addr from either:
-   *   <host>
-   *   <host>:<port>
-   *   <fs>://<host>:<port>/<path>
-   */
-  public static InetSocketAddress createSocketAddr(String host, int port) {
-    return new InetSocketAddress(host, port);
-  }
-
-  public static InetSocketAddress createUnresolved(String addr) {
-    String [] splitted = addr.split(":");
-    return InetSocketAddress.createUnresolved(splitted[0], Integer.parseInt(splitted[1]));
-  }
-
-  /**
-   * Returns InetSocketAddress that a client can use to
-   * connect to the server. NettyServerBase.getListenerAddress() is not correct when
-   * the server binds to "0.0.0.0". This returns "hostname:port" of the server,
-   * or "127.0.0.1:port" when the getListenerAddress() returns "0.0.0.0:port".
-   *
-   * @param addr of a listener
-   * @return socket address that a client can use to connect to the server.
-   */
-  public static InetSocketAddress getConnectAddress(InetSocketAddress addr) {
-    if (!addr.isUnresolved() && addr.getAddress().isAnyLocalAddress()) {
-      try {
-        addr = new InetSocketAddress(InetAddress.getLocalHost(), addr.getPort());
-      } catch (UnknownHostException uhe) {
-        // shouldn't get here unless the host doesn't have a loopback iface
-        addr = new InetSocketAddress("127.0.0.1", addr.getPort());
-      }
-    }
-    return addr;
-  }
-
-  /**
-   * Given an InetAddress, checks to see if the address is a local address, by
-   * comparing the address with all the interfaces on the node.
-   * @param addr address to check if it is local node's address
-   * @return true if the address corresponds to the local node
-   */
-  public static boolean isLocalAddress(InetAddress addr) {
-    // Check if the address is any local or loop back
-    boolean local = addr.isAnyLocalAddress() || addr.isLoopbackAddress();
-
-    // Check if the address is defined on any interface
-    if (!local) {
-      try {
-        local = NetworkInterface.getByInetAddress(addr) != null;
-      } catch (SocketException e) {
-        local = false;
-      }
-    }
-    return local;
-  }
-
-  public static String normalizeHost(String host) {
-    try {
-      InetAddress address = InetAddress.getByName(host);
-      if (isLocalAddress(address)) {
-        return InetAddress.getLocalHost().getHostAddress();
-      } else {
-        return address.getHostAddress();
-      }
-    } catch (UnknownHostException e) {
-    }
-    return host;
-  }
-}
\ No newline at end of file


Mime
View raw message