tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jh...@apache.org
Subject git commit: TAJO-694: Bump up hadoop to 2.3.0. (jinho)
Date Fri, 28 Mar 2014 02:57:26 GMT
Repository: tajo
Updated Branches:
  refs/heads/branch-0.8.0 7c21fce88 -> bc98b27f5


TAJO-694: Bump up hadoop to 2.3.0. (jinho)


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/bc98b27f
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/bc98b27f
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/bc98b27f

Branch: refs/heads/branch-0.8.0
Commit: bc98b27f54653aaf50aeaded9951146b1c0e982d
Parents: 7c21fce
Author: jinossy <jinossy@gmail.com>
Authored: Fri Mar 28 11:56:48 2014 +0900
Committer: jinossy <jinossy@gmail.com>
Committed: Fri Mar 28 11:56:48 2014 +0900

----------------------------------------------------------------------
 BUILDING.txt                                    |   1 +
 CHANGES.txt                                     |   2 +
 tajo-core/tajo-core-pullserver/pom.xml          |  10 +-
 .../pullserver/listener/FileCloseListener.java  |  39 ++++++
 .../pullserver/listener/FileCloseListener.java  |  42 +++++++
 .../tajo/pullserver/FadvisedChunkedFile.java    |  80 -------------
 .../tajo/pullserver/FadvisedFileRegion.java     |  82 -------------
 .../tajo/pullserver/PullServerAuxService.java   |  12 +-
 .../tajo/pullserver/TajoPullServerService.java  |  12 +-
 tajo-dist/src/main/bin/tajo                     | 118 +++++++++----------
 .../main/sphinx/configuration/cluster_setup.rst |   2 +-
 .../sphinx/getting_started/prerequisites.rst    |   2 +-
 tajo-docs/src/main/sphinx/jdbc_driver.rst       |   2 +-
 tajo-project/pom.xml                            |  51 +++++++-
 14 files changed, 208 insertions(+), 247 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/bc98b27f/BUILDING.txt
----------------------------------------------------------------------
diff --git a/BUILDING.txt b/BUILDING.txt
index 8dcc179..0cf0a04 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -35,6 +35,7 @@ Maven build goals:
 
  Build options:
   * Use -Dtar to create a TAR with the distribution (using -Pdist)
+  * Use -Dhadoop.version to build with the specific hadoop version (-Dhadoop.version=2.3.0)
 
  Tests options:
   * Use -DskipTests to skip tests when running the following Maven goals:

http://git-wip-us.apache.org/repos/asf/tajo/blob/bc98b27f/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 5f4c656..7c3f379 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -574,6 +574,8 @@ Release 0.8.0 - unreleased
 
   TASKS
 
+    TAJO-694: Bump up hadoop to 2.3.0. (jinho)
+
     TAJO-684: Add functions about time. (Alvin Henrick via jihoon)
 
     TAJO-669: Add cluster setup documentation. (hyunsik)

http://git-wip-us.apache.org/repos/asf/tajo/blob/bc98b27f/tajo-core/tajo-core-pullserver/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-pullserver/pom.xml b/tajo-core/tajo-core-pullserver/pom.xml
index 0bdfed2..b5049be 100644
--- a/tajo-core/tajo-core-pullserver/pom.xml
+++ b/tajo-core/tajo-core-pullserver/pom.xml
@@ -34,11 +34,6 @@
   <dependencies>
     <dependency>
       <groupId>org.apache.tajo</groupId>
-      <artifactId>tajo-common</artifactId>
-      <scope>provided</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.tajo</groupId>
       <artifactId>tajo-catalog-common</artifactId>
       <scope>provided</scope>
     </dependency>
@@ -59,6 +54,11 @@
       <scope>provided</scope>
     </dependency>
     <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-shuffle</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
       <groupId>io.netty</groupId>
       <artifactId>netty</artifactId>
       <scope>compile</scope>

http://git-wip-us.apache.org/repos/asf/tajo/blob/bc98b27f/tajo-core/tajo-core-pullserver/src/main/hadoop-2.2.0/org/apache/tajo/pullserver/listener/FileCloseListener.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-pullserver/src/main/hadoop-2.2.0/org/apache/tajo/pullserver/listener/FileCloseListener.java
b/tajo-core/tajo-core-pullserver/src/main/hadoop-2.2.0/org/apache/tajo/pullserver/listener/FileCloseListener.java
new file mode 100644
index 0000000..b7f3e34
--- /dev/null
+++ b/tajo-core/tajo-core-pullserver/src/main/hadoop-2.2.0/org/apache/tajo/pullserver/listener/FileCloseListener.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.pullserver.listener;
+
+import org.apache.hadoop.mapred.FadvisedFileRegion;
+import org.jboss.netty.channel.ChannelFuture;
+import org.jboss.netty.channel.ChannelFutureListener;
+
+public class FileCloseListener implements ChannelFutureListener {
+
+  private FadvisedFileRegion filePart;
+
+  public FileCloseListener(FadvisedFileRegion filePart) {
+    this.filePart = filePart;
+  }
+
+  // TODO error handling; distinguish IO/connection failures,
+  //      attribute to appropriate spill output
+  @Override
+  public void operationComplete(ChannelFuture future) {
+    filePart.releaseExternalResources();
+  }
+}

http://git-wip-us.apache.org/repos/asf/tajo/blob/bc98b27f/tajo-core/tajo-core-pullserver/src/main/hadoop-2.3.0/org/apache/tajo/pullserver/listener/FileCloseListener.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-pullserver/src/main/hadoop-2.3.0/org/apache/tajo/pullserver/listener/FileCloseListener.java
b/tajo-core/tajo-core-pullserver/src/main/hadoop-2.3.0/org/apache/tajo/pullserver/listener/FileCloseListener.java
new file mode 100644
index 0000000..5b2d1b3
--- /dev/null
+++ b/tajo-core/tajo-core-pullserver/src/main/hadoop-2.3.0/org/apache/tajo/pullserver/listener/FileCloseListener.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.pullserver.listener;
+
+import org.apache.hadoop.mapred.FadvisedFileRegion;
+import org.jboss.netty.channel.ChannelFuture;
+import org.jboss.netty.channel.ChannelFutureListener;
+
+public class FileCloseListener implements ChannelFutureListener {
+
+  private FadvisedFileRegion filePart;
+
+  public FileCloseListener(FadvisedFileRegion filePart) {
+    this.filePart = filePart;
+  }
+
+  // TODO error handling; distinguish IO/connection failures,
+  //      attribute to appropriate spill output
+  @Override
+  public void operationComplete(ChannelFuture future) {
+    if(future.isSuccess()){
+      filePart.transferSuccessful();
+    }
+    filePart.releaseExternalResources();
+  }
+}

http://git-wip-us.apache.org/repos/asf/tajo/blob/bc98b27f/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/FadvisedChunkedFile.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/FadvisedChunkedFile.java
b/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/FadvisedChunkedFile.java
deleted file mode 100644
index ec34f00..0000000
--- a/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/FadvisedChunkedFile.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.pullserver;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.io.ReadaheadPool;
-import org.apache.hadoop.io.ReadaheadPool.ReadaheadRequest;
-import org.apache.hadoop.io.nativeio.NativeIO.POSIX;
-import org.jboss.netty.handler.stream.ChunkedFile;
-
-import java.io.FileDescriptor;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-
-public class FadvisedChunkedFile extends ChunkedFile {
-
-  private static final Log LOG = LogFactory.getLog(FadvisedChunkedFile.class);
-
-  private final boolean manageOsCache;
-  private final int readaheadLength;
-  private final ReadaheadPool readaheadPool;
-  private final FileDescriptor fd;
-  private final String identifier;
-
-  private ReadaheadRequest readaheadRequest;
-
-  public FadvisedChunkedFile(RandomAccessFile file, long position, long count,
-      int chunkSize, boolean manageOsCache, int readaheadLength,
-      ReadaheadPool readaheadPool, String identifier) throws IOException {
-    super(file, position, count, chunkSize);
-    this.manageOsCache = manageOsCache;
-    this.readaheadLength = readaheadLength;
-    this.readaheadPool = readaheadPool;
-    this.fd = file.getFD();
-    this.identifier = identifier;
-  }
-
-  @Override
-  public Object nextChunk() throws Exception {
-    if (manageOsCache && readaheadPool != null) {
-      readaheadRequest = readaheadPool
-          .readaheadStream(identifier, fd, getCurrentOffset(), readaheadLength,
-              getEndOffset(), readaheadRequest);
-    }
-    return super.nextChunk();
-  }
-
-  @Override
-  public void close() throws Exception {
-    if (readaheadRequest != null) {
-      readaheadRequest.cancel();
-    }
-    if (manageOsCache && getEndOffset() - getStartOffset() > 0) {
-      try {
-        POSIX.posixFadviseIfPossible(identifier, fd, getStartOffset(), getEndOffset()
-            - getStartOffset(), POSIX.POSIX_FADV_DONTNEED);
-      } catch (Throwable t) {
-        LOG.warn("Failed to manage OS cache for " + identifier, t);
-      }
-    }
-    super.close();
-  }
-}

http://git-wip-us.apache.org/repos/asf/tajo/blob/bc98b27f/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/FadvisedFileRegion.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/FadvisedFileRegion.java
b/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/FadvisedFileRegion.java
deleted file mode 100644
index 81e0bf1..0000000
--- a/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/FadvisedFileRegion.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.pullserver;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.io.ReadaheadPool;
-import org.apache.hadoop.io.ReadaheadPool.ReadaheadRequest;
-import org.apache.hadoop.io.nativeio.NativeIO.POSIX;
-import org.jboss.netty.channel.DefaultFileRegion;
-
-import java.io.FileDescriptor;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.nio.channels.WritableByteChannel;
-
-public class FadvisedFileRegion extends DefaultFileRegion {
-
-  private static final Log LOG = LogFactory.getLog(FadvisedFileRegion.class);
-
-  private final boolean manageOsCache;
-  private final int readaheadLength;
-  private final ReadaheadPool readaheadPool;
-  private final FileDescriptor fd;
-  private final String identifier;
-
-  private ReadaheadRequest readaheadRequest;
-
-  public FadvisedFileRegion(RandomAccessFile file, long position, long count,
-      boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool,
-      String identifier) throws IOException {
-    super(file.getChannel(), position, count);
-    this.manageOsCache = manageOsCache;
-    this.readaheadLength = readaheadLength;
-    this.readaheadPool = readaheadPool;
-    this.fd = file.getFD();
-    this.identifier = identifier;
-  }
-
-  @Override
-  public long transferTo(WritableByteChannel target, long position)
-      throws IOException {
-    if (manageOsCache && readaheadPool != null) {
-      readaheadRequest = readaheadPool.readaheadStream(identifier, fd,
-          getPosition() + position, readaheadLength,
-          getPosition() + getCount(), readaheadRequest);
-    }
-    return super.transferTo(target, position);
-  }
-
-  @Override
-  public void releaseExternalResources() {
-    if (readaheadRequest != null) {
-      readaheadRequest.cancel();
-    }
-    if (manageOsCache && getCount() > 0) {
-      try {
-        POSIX.posixFadviseIfPossible(identifier, fd, getPosition(), getCount(),
-            POSIX.POSIX_FADV_DONTNEED);
-      } catch (Throwable t) {
-        LOG.warn("Failed to manage OS cache for " + identifier, t);
-      }
-    }
-    super.releaseExternalResources();
-  }
-}

http://git-wip-us.apache.org/repos/asf/tajo/blob/bc98b27f/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/PullServerAuxService.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/PullServerAuxService.java
b/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/PullServerAuxService.java
index d098797..afdae23 100644
--- a/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/PullServerAuxService.java
+++ b/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/PullServerAuxService.java
@@ -31,6 +31,8 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.DataInputByteBuffer;
 import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.io.ReadaheadPool;
+import org.apache.hadoop.mapred.FadvisedChunkedFile;
+import org.apache.hadoop.mapred.FadvisedFileRegion;
 import org.apache.hadoop.metrics2.MetricsSystem;
 import org.apache.hadoop.metrics2.annotation.Metric;
 import org.apache.hadoop.metrics2.annotation.Metrics;
@@ -47,6 +49,7 @@ import org.apache.tajo.QueryId;
 import org.apache.tajo.catalog.Schema;
 import org.apache.tajo.conf.TajoConf;
 import org.apache.tajo.conf.TajoConf.ConfVars;
+import org.apache.tajo.pullserver.listener.FileCloseListener;
 import org.apache.tajo.pullserver.retriever.FileChunk;
 import org.apache.tajo.storage.RowStoreUtil.RowStoreDecoder;
 import org.apache.tajo.storage.Tuple;
@@ -490,14 +493,7 @@ public class PullServerAuxService extends AuxiliaryService {
             file.startOffset, file.length(), manageOsCache, readaheadLength,
             readaheadPool, file.getFile().getAbsolutePath());
         writeFuture = ch.write(partition);
-        writeFuture.addListener(new ChannelFutureListener() {
-          // TODO error handling; distinguish IO/connection failures,
-          //      attribute to appropriate spill output
-          @Override
-          public void operationComplete(ChannelFuture future) {
-            partition.releaseExternalResources();
-          }
-        });
+        writeFuture.addListener(new FileCloseListener(partition));
       } else {
         // HTTPS cannot be done with zero copy.
         final FadvisedChunkedFile chunk = new FadvisedChunkedFile(spill,

http://git-wip-us.apache.org/repos/asf/tajo/blob/bc98b27f/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java
b/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java
index c1fcef1..c416b66 100644
--- a/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java
+++ b/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java
@@ -30,6 +30,8 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.DataInputByteBuffer;
 import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.io.ReadaheadPool;
+import org.apache.hadoop.mapred.FadvisedChunkedFile;
+import org.apache.hadoop.mapred.FadvisedFileRegion;
 import org.apache.hadoop.metrics2.MetricsSystem;
 import org.apache.hadoop.metrics2.annotation.Metric;
 import org.apache.hadoop.metrics2.annotation.Metrics;
@@ -43,6 +45,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.tajo.catalog.Schema;
 import org.apache.tajo.conf.TajoConf;
 import org.apache.tajo.conf.TajoConf.ConfVars;
+import org.apache.tajo.pullserver.listener.FileCloseListener;
 import org.apache.tajo.pullserver.retriever.FileChunk;
 import org.apache.tajo.rpc.RpcChannelFactory;
 import org.apache.tajo.storage.RowStoreUtil.RowStoreDecoder;
@@ -487,14 +490,7 @@ public class TajoPullServerService extends AbstractService {
             file.startOffset, file.length(), manageOsCache, readaheadLength,
             readaheadPool, file.getFile().getAbsolutePath());
         writeFuture = ch.write(filePart);
-        writeFuture.addListener(new ChannelFutureListener() {
-          // TODO error handling; distinguish IO/connection failures,
-          //      attribute to appropriate spill output
-          @Override
-          public void operationComplete(ChannelFuture future) {
-            filePart.releaseExternalResources();
-          }
-        });
+        writeFuture.addListener(new FileCloseListener(filePart));
       } else {
         // HTTPS cannot be done with zero copy.
         final FadvisedChunkedFile chunk = new FadvisedChunkedFile(spill,

http://git-wip-us.apache.org/repos/asf/tajo/blob/bc98b27f/tajo-dist/src/main/bin/tajo
----------------------------------------------------------------------
diff --git a/tajo-dist/src/main/bin/tajo b/tajo-dist/src/main/bin/tajo
index 6b93203..77d4a02 100755
--- a/tajo-dist/src/main/bin/tajo
+++ b/tajo-dist/src/main/bin/tajo
@@ -183,51 +183,30 @@ HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_HOME/etc/hadoop}"
 # Hadoop Version Checking Section End
 ##############################################################################
 
-
 ##############################################################################
-# Find and Set Tajo CLASSPATH
+# Find and Set Hadoop CLASSPATH
 ##############################################################################
 
-# Add java common jars to TAJO_BASE_CLASSPATH
-TAJO_BASE_CLASSPATH="${JAVA_HOME}/lib/tools.jar"
-
-# add user-specified CLASSPATH firstly
-if [ "$TAJO_USER_CLASSPATH_FIRST" != "" ] && [ "$TAJO_CLASSPATH" != "" ] ; then
-  TAJO_BASE_CLASSPATH=${TAJO_BASE_CLASSPATH}:${TAJO_CLASSPATH}
-fi
-
-# for developers, add Tajo classes to TAJO_BASE_CLASSPATH
-if [ -d "$TAJO_HOME/target/classes" ]; then
-  TAJO_BASE_CLASSPATH=${TAJO_BASE_CLASSPATH}:$TAJO_HOME/target/classes
-fi
-if [ -d "$TAJO_HOME/target/test/classes" ]; then
-  TAJO_BASE_CLASSPATH=${TAJO_BASE_CLASSPATH}:$TAJO_HOME/target/test/classes
-fi
-
-# so that filenames w/ spaces are handled correctly in loops below
-IFS=$'\n'
-
-# TAJO_BASE_CLASSPATH contains $TAJO_CONF_DIR before containing jars.
-TAJO_BASE_CLASSPATH="${TAJO_CONF_DIR}"
+# HADOOP JAR DIRS
+HADOOP_MODULE_DIRS="$HADOOP_HOME/share/hadoop/common/lib
+$HADOOP_HOME/share/hadoop/common
+$HADOOP_HOME/share/hadoop/hdfs
+$HADOOP_HOME/share/hadoop/hdfs/lib
+$HADOOP_HOME/share/hadoop/yarn
+$HADOOP_HOME/share/hadoop/mapreduce"
 
-# Tajo Jar Directory
-TAJO_JAR_DIRS="$TAJO_HOME
-${TAJO_HOME}/lib"
+HADOOP_EXT_CLASSPATH="${HADOOP_CONF_DIR}"
 
-for d in $TAJO_JAR_DIRS; do
-  for j in `find $d/*.jar ! -name "*test*"`; do
-    TAJO_BASE_CLASSPATH=$TAJO_BASE_CLASSPATH:$j
-  done
-done
+for d in $HADOOP_MODULE_DIRS; do
+  CLASSPATH=${CLASSPATH}:$d/*
+done;
 
-# add user-specified CLASSPATH last
-if [ "$TAJO_USER_CLASSPATH_FIRST" = "" ] && [ "$TAJO_CLASSPATH" != "" ]; then
-  ${TAJO_BASE_CLASSPATH}=${CLASSPATH}:${TAJO_CLASSPATH}
-fi
+export HADOOP_EXT_CLASSPATH
 
-export TAJO_BASE_CLASSPATH
+# Append $HADOOP_JAR_CLASSPATH to $CLASSPATH
+CLASSPATH="${CLASSPATH}:${HADOOP_EXT_CLASSPATH}"
 
-CLASSPATH="${CLASSPATH}:${TAJO_BASE_CLASSPATH}"
+HDFS_LIBRARY_PATH="${HADOOP_HOME}/lib/native/"
 
 ##############################################################################
 # Find and Set Hive CLASSPATH
@@ -247,23 +226,23 @@ if [ -d ${HIVE_LIB} ]; then
     CLASSPATH=${CLASSPATH}:$f;
   done
 
-  for f in ${HIVE_LIB}/libfb303-*.jar; do
+  for f in ${HIVE_LIB}/libthrift-*.jar; do
     CLASSPATH=${CLASSPATH}:$f;
   done
 
-  for f in ${HIVE_LIB}/jdo-api-*.jar; do
-    CLASSPATH=${CLASSPATH}:$f;
+  for f in ${HIVE_LIB}/libfb*.jar; do
+      CLASSPATH=${CLASSPATH}:$f;
   done
 
-  for f in ${HIVE_LIB}/datanucleus-*.jar; do
+  for f in ${HIVE_LIB}/jdo*-api-*.jar; do
     CLASSPATH=${CLASSPATH}:$f;
   done
 
-  for f in $HIVE_HOME/hcatalog/share/hcatalog/hcatalog-core-*.jar; do
+  for f in ${HIVE_LIB}/datanucleus-*.jar; do
     CLASSPATH=${CLASSPATH}:$f;
   done
 
-  for f in $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-client-core-*.jar; do
+  for f in $HIVE_HOME/hcatalog/share/hcatalog/hcatalog-core-*.jar; do
     CLASSPATH=${CLASSPATH}:$f;
   done
 fi
@@ -273,29 +252,50 @@ if [ "${HIVE_JDBC_DRIVER_DIR}" != "" ]; then
 fi
 
 ##############################################################################
-# Find and Set Hadoop CLASSPATH
+# Find and Set Tajo CLASSPATH
 ##############################################################################
 
-# HADOOP JAR DIRS
-HADOOP_MODULE_DIRS="$HADOOP_HOME/share/hadoop/common/lib
-$HADOOP_HOME/share/hadoop/common
-$HADOOP_HOME/share/hadoop/hdfs
-$HADOOP_HOME/share/hadoop/hdfs/lib
-$HADOOP_HOME/share/hadoop/yarn/lib
-$HADOOP_HOME/share/hadoop/yarn"
+# Add java common jars to TAJO_BASE_CLASSPATH
+TAJO_BASE_CLASSPATH="${JAVA_HOME}/lib/tools.jar"
 
-HADOOP_EXT_CLASSPATH="${HADOOP_CONF_DIR}"
+# add user-specified CLASSPATH firstly
+if [ "$TAJO_USER_CLASSPATH_FIRST" != "" ] && [ "$TAJO_CLASSPATH" != "" ] ; then
+  TAJO_BASE_CLASSPATH=${TAJO_BASE_CLASSPATH}:${TAJO_CLASSPATH}
+fi
 
-for d in $HADOOP_MODULE_DIRS; do
-  CLASSPATH=${CLASSPATH}:$d/*
-done;
+# for developers, add Tajo classes to TAJO_BASE_CLASSPATH
+if [ -d "$TAJO_HOME/target/classes" ]; then
+  TAJO_BASE_CLASSPATH=${TAJO_BASE_CLASSPATH}:$TAJO_HOME/target/classes
+fi
+if [ -d "$TAJO_HOME/target/test/classes" ]; then
+  TAJO_BASE_CLASSPATH=${TAJO_BASE_CLASSPATH}:$TAJO_HOME/target/test/classes
+fi
 
-export HADOOP_EXT_CLASSPATH
+# so that filenames w/ spaces are handled correctly in loops below
+IFS=$'\n'
 
-# Append $HADOOP_JAR_CLASSPATH to $CLASSPATH
-CLASSPATH="${CLASSPATH}:${HADOOP_EXT_CLASSPATH}"
+# TAJO_BASE_CLASSPATH contains $TAJO_CONF_DIR before containing jars.
+TAJO_BASE_CLASSPATH="${TAJO_CONF_DIR}"
+
+# Tajo Jar Directory
+TAJO_JAR_DIRS="$TAJO_HOME
+${TAJO_HOME}/lib"
+
+for d in $TAJO_JAR_DIRS; do
+  for j in `find $d/*.jar ! -name "*test*"`; do
+    TAJO_BASE_CLASSPATH=$TAJO_BASE_CLASSPATH:$j
+  done
+done
+
+# add user-specified CLASSPATH last
+if [ "$TAJO_USER_CLASSPATH_FIRST" = "" ] && [ "$TAJO_CLASSPATH" != "" ]; then
+  ${TAJO_BASE_CLASSPATH}=${CLASSPATH}:${TAJO_CLASSPATH}
+fi
+
+export TAJO_BASE_CLASSPATH
+
+CLASSPATH="${CLASSPATH}:${TAJO_BASE_CLASSPATH}"
 
-HDFS_LIBRARY_PATH="${HADOOP_HOME}/lib/native/"
 ##############################################################################
 # Hadoop Home Configuration End
 ##############################################################################

http://git-wip-us.apache.org/repos/asf/tajo/blob/bc98b27f/tajo-docs/src/main/sphinx/configuration/cluster_setup.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/configuration/cluster_setup.rst b/tajo-docs/src/main/sphinx/configuration/cluster_setup.rst
index 89db12c..c95ee8a 100644
--- a/tajo-docs/src/main/sphinx/configuration/cluster_setup.rst
+++ b/tajo-docs/src/main/sphinx/configuration/cluster_setup.rst
@@ -28,7 +28,7 @@ Please add the following configs to tajo-site.xml file:
   </property>
 
   <property>
-    <name>tajo.master.client-rpc.addres</name>
+    <name>tajo.master.client-rpc.address</name>
     <value>hostname:26002</value>
   </property>
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/bc98b27f/tajo-docs/src/main/sphinx/getting_started/prerequisites.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/getting_started/prerequisites.rst b/tajo-docs/src/main/sphinx/getting_started/prerequisites.rst
index abfd04e..d4e20da 100644
--- a/tajo-docs/src/main/sphinx/getting_started/prerequisites.rst
+++ b/tajo-docs/src/main/sphinx/getting_started/prerequisites.rst
@@ -2,6 +2,6 @@
 Prerequisites
 **********************
 
- * Hadoop 2.2.0 and higher
+ * Hadoop 2.2.0 or higher
  * Java 1.6 or higher
  * Protocol buffer 2.5.0
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/bc98b27f/tajo-docs/src/main/sphinx/jdbc_driver.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/jdbc_driver.rst b/tajo-docs/src/main/sphinx/jdbc_driver.rst
index 306cdc1..515e509 100644
--- a/tajo-docs/src/main/sphinx/jdbc_driver.rst
+++ b/tajo-docs/src/main/sphinx/jdbc_driver.rst
@@ -117,7 +117,7 @@ In addition to the following JAR files, please don't forgot including
   * commons-logging-1.1.1.jar
   * guava-11.0.2.jar
   * protobuf-java-2.5.0.jar
-  * netty-3.6.2.Final.jar
+  * netty-3.6.6.Final.jar
   * commons-lang-2.5.jar
   * commons-configuration-1.6.jar
   * slf4j-api-1.7.5.jar

http://git-wip-us.apache.org/repos/asf/tajo/blob/bc98b27f/tajo-project/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-project/pom.xml b/tajo-project/pom.xml
index cae8602..aade02e 100644
--- a/tajo-project/pom.xml
+++ b/tajo-project/pom.xml
@@ -35,9 +35,9 @@
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-    <tajo.version>0.8.0-SNAPSHOT</tajo.version>
-    <hadoop.version>2.2.0</hadoop.version>
+    <hadoop.version>2.3.0</hadoop.version>
     <protobuf.version>2.5.0</protobuf.version>
+    <tajo.version>0.8.0-SNAPSHOT</tajo.version>
     <tajo.root>${project.parent.relativePath}/..</tajo.root>
   </properties>
 
@@ -575,6 +575,38 @@
         <artifactId>maven-surefire-report-plugin</artifactId>
         <version>2.15</version>
       </plugin>
+
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>build-helper-maven-plugin</artifactId>
+        <version>1.8</version>
+        <executions>
+          <execution>
+            <id>add-source</id>
+            <phase>generate-sources</phase>
+            <goals>
+              <goal>add-source</goal>
+            </goals>
+            <configuration>
+              <sources>
+                <source>src/main/hadoop-${hadoop.version}</source>
+              </sources>
+            </configuration>
+          </execution>
+          <execution>
+            <id>add-source-2.3</id>
+            <phase>prepare-package</phase>
+            <goals>
+              <goal>add-source</goal>
+            </goals>
+            <configuration>
+              <sources>
+                <source>src/main/hadoop-${hadoop.version}</source>
+              </sources>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 
@@ -747,6 +779,21 @@
         <type>test-jar</type>
         <scope>test</scope>
       </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-mapreduce-client-shuffle</artifactId>
+        <version>${hadoop.version}</version>
+        <exclusions>
+          <exclusion>
+            <artifactId>hadoop-mapreduce-client-core</artifactId>
+            <groupId>org.apache.hadoop</groupId>
+          </exclusion>
+          <exclusion>
+            <artifactId>hadoop-yarn-server-nodemanager</artifactId>
+            <groupId>org.apache.hadoop</groupId>
+          </exclusion>
+        </exclusions>
+      </dependency>
 
       <dependency>
         <groupId>com.google.protobuf</groupId>


Mime
View raw message