tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hyun...@apache.org
Subject [50/51] [partial] tajo git commit: TAJO-1761: Separate an integration unit test kit into an independent module.
Date Fri, 14 Aug 2015 14:30:29 GMT
http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 0b705e8..2cd18eb 100644
--- a/CHANGES
+++ b/CHANGES
@@ -32,6 +32,9 @@ Release 0.11.0 - unreleased
 
   IMPROVEMENT
 
+    TAJO-1761: Separate an integration unit test kit into an independent 
+    module. (hyunsik)
+
     TAJO-680: Improve the IN operator to support sub queries. (jihoon)
 
     TAJO-1751: Reduce the client connection timeout. (jinho)

http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 296830a..69eaebf 100644
--- a/pom.xml
+++ b/pom.xml
@@ -94,6 +94,8 @@
     <module>tajo-thirdparty/asm</module>
     <module>tajo-cli</module>
     <module>tajo-metrics</module>
+    <module>tajo-core-tests</module>
+    <module>tajo-cluster-tests</module>
   </modules>
 
   <build>

http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/tajo-cluster-tests/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-cluster-tests/pom.xml b/tajo-cluster-tests/pom.xml
new file mode 100644
index 0000000..20c43bd
--- /dev/null
+++ b/tajo-cluster-tests/pom.xml
@@ -0,0 +1,302 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <artifactId>tajo-project</artifactId>
+    <groupId>org.apache.tajo</groupId>
+    <version>0.11.0-SNAPSHOT</version>
+    <relativePath>../tajo-project</relativePath>
+  </parent>
+  <artifactId>tajo-cluster-tests</artifactId>
+  <packaging>jar</packaging>
+  <name>Tajo Cluster Tests</name>
+
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+  </properties>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.rat</groupId>
+        <artifactId>apache-rat-plugin</artifactId>
+        <executions>
+          <execution>
+            <phase>verify</phase>
+            <goals>
+              <goal>check</goal>
+            </goals>
+          </execution>
+        </executions>
+        <configuration>
+          <excludes>
+            <exclude>derby.log</exclude>
+            <exclude>benchmark/**</exclude>
+            <exclude>src/test/tpch/**</exclude>
+            <exclude>src/test/resources/dataset/**</exclude>
+            <exclude>src/test/resources/queries/**</exclude>
+            <exclude>src/test/resources/results/**</exclude>
+            <exclude>src/main/resources/META-INF/services/*</exclude>
+            <exclude>src/main/resources/webapps/static/js/*</exclude>
+          </excludes>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <configuration>
+          <systemProperties>
+            <tajo.test>TRUE</tajo.test>
+          </systemProperties>
+          <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8</argLine>
+        </configuration>
+      </plugin>
+      <plugin>
+        <artifactId>maven-deploy-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+        <version>2.3.1</version>
+        <configuration>
+          <excludes>
+            <exclude>LICENSE</exclude>
+          </excludes>
+          <archive>
+            <addMavenDescriptor>false</addMavenDescriptor>
+          </archive>
+        </configuration>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>test-jar</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>copy-dependencies</id>
+            <phase>package</phase>
+            <goals>
+              <goal>copy-dependencies</goal>
+            </goals>
+            <configuration>
+              <includeScope>runtime</includeScope>
+              <outputDirectory>${project.build.directory}/lib</outputDirectory>
+              <overWriteReleases>false</overWriteReleases>
+              <overWriteSnapshots>false</overWriteSnapshots>
+              <overWriteIfNewer>true</overWriteIfNewer>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.tajo</groupId>
+      <artifactId>tajo-common</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tajo</groupId>
+      <artifactId>tajo-core</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tajo</groupId>
+      <artifactId>tajo-algebra</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tajo</groupId>
+      <artifactId>tajo-catalog-common</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tajo</groupId>
+      <artifactId>tajo-plan</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tajo</groupId>
+      <artifactId>tajo-client</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tajo</groupId>
+      <artifactId>tajo-catalog-client</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tajo</groupId>
+      <artifactId>tajo-catalog-server</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tajo</groupId>
+      <artifactId>tajo-storage-common</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tajo</groupId>
+      <artifactId>tajo-storage-hdfs</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tajo</groupId>
+      <artifactId>tajo-storage-hbase</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tajo</groupId>
+      <artifactId>tajo-pullserver</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tajo</groupId>
+      <artifactId>tajo-rpc-protobuf</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tajo</groupId>
+      <artifactId>tajo-thirdparty-asm</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tajo</groupId>
+      <artifactId>tajo-ws-rs</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tajo</groupId>
+      <artifactId>tajo-metrics</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-minicluster</artifactId>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>commons-el</groupId>
+          <artifactId>commons-el</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>tomcat</groupId>
+          <artifactId>jasper-runtime</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>tomcat</groupId>
+          <artifactId>jasper-compiler</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.mortbay.jetty</groupId>
+          <artifactId>jsp-2.1-jetty</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.sun.jersey.jersey-test-framework</groupId>
+          <artifactId>jersey-test-framework-grizzly2</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs</artifactId>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>commons-el</groupId>
+          <artifactId>commons-el</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>tomcat</groupId>
+          <artifactId>jasper-runtime</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>tomcat</groupId>
+          <artifactId>jasper-compiler</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.mortbay.jetty</groupId>
+          <artifactId>jsp-2.1-jetty</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.sun.jersey.jersey-test-framework</groupId>
+          <artifactId>jersey-test-framework-grizzly2</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-api</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-common</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-server-common</artifactId>
+      <scope>provided</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.hbase</groupId>
+      <artifactId>hbase-server</artifactId>
+      <version>${hbase.version}</version>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hbase</groupId>
+      <artifactId>hbase-server</artifactId>
+      <version>${hbase.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hbase</groupId>
+      <artifactId>hbase-hadoop-compat</artifactId>
+      <version>${hbase.version}</version>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hbase</groupId>
+      <artifactId>hbase-hadoop2-compat</artifactId>
+      <version>${hbase.version}</version>
+      <type>test-jar</type>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <artifactId>jdk.tools</artifactId>
+          <groupId>jdk.tools</groupId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+  </dependencies>
+
+</project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/tajo-cluster-tests/src/test/java/org/apache/tajo/BackendTestingUtil.java
----------------------------------------------------------------------
diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/BackendTestingUtil.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/BackendTestingUtil.java
new file mode 100644
index 0000000..3bfd34d
--- /dev/null
+++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/BackendTestingUtil.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.tajo.catalog.CatalogUtil;
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.common.TajoDataTypes.Type;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.datum.DatumFactory;
+import org.apache.tajo.storage.*;
+
+import java.io.IOException;
+
+public class BackendTestingUtil {
+  public final static Schema mockupSchema;
+  public final static TableMeta mockupMeta;
+
+  static {
+    mockupSchema = new Schema();
+    mockupSchema.addColumn("deptname", Type.TEXT);
+    mockupSchema.addColumn("score", Type.INT4);
+    mockupMeta = CatalogUtil.newTableMeta("TEXT");
+  }
+
+  public static void writeTmpTable(TajoConf conf, Path tablePath)
+      throws IOException {
+    FileTablespace sm = TablespaceManager.getDefault();
+    Appender appender;
+
+    Path filePath = new Path(tablePath, "table.csv");
+    FileSystem fs = filePath.getFileSystem(conf);
+    if (fs.exists(tablePath)) {
+      fs.delete(tablePath, true);
+    }
+    fs.mkdirs(tablePath);
+
+    appender = sm.getAppender(mockupMeta, mockupSchema, filePath);
+    appender.init();
+
+    int deptSize = 10000;
+    int tupleNum = 100;
+    Tuple tuple;
+    for (int i = 0; i < tupleNum; i++) {
+      tuple = new VTuple(2);
+      String key = "test" + (i % deptSize);
+      tuple.put(0, DatumFactory.createText(key));
+      tuple.put(1, DatumFactory.createInt4(i + 1));
+      appender.addTuple(tuple);
+    }
+    appender.close();
+  }
+}

http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/tajo-cluster-tests/src/test/java/org/apache/tajo/HBaseTestClusterUtil.java
----------------------------------------------------------------------
diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/HBaseTestClusterUtil.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/HBaseTestClusterUtil.java
new file mode 100644
index 0000000..0072e97
--- /dev/null
+++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/HBaseTestClusterUtil.java
@@ -0,0 +1,184 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.master.ServerManager;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
+import org.apache.tajo.util.Bytes;
+
+import java.io.File;
+import java.io.IOException;
+
+import static org.apache.hadoop.hbase.HConstants.REPLICATION_ENABLE_KEY;
+
+public class HBaseTestClusterUtil {
+  private static final Log LOG = LogFactory.getLog(HBaseTestClusterUtil.class);
+  private Configuration conf;
+  private MiniHBaseCluster hbaseCluster;
+  private MiniZooKeeperCluster zkCluster;
+  private File testBaseDir;
+  public HBaseTestClusterUtil(Configuration conf, File testBaseDir) {
+    this.conf = conf;
+    this.testBaseDir = testBaseDir;
+  }
+  /**
+   * Returns the path to the default root dir the minicluster uses.
+   * Note: this does not cause the root dir to be created.
+   * @return Fully qualified path for the default hbase root dir
+   * @throws java.io.IOException
+   */
+  public Path getDefaultRootDirPath() throws IOException {
+    FileSystem fs = FileSystem.get(this.conf);
+    return new Path(fs.makeQualified(fs.getHomeDirectory()),"hbase");
+  }
+
+  /**
+   * Creates an hbase rootdir in user home directory.  Also creates hbase
+   * version file.  Normally you won't make use of this method.  Root hbasedir
+   * is created for you as part of mini cluster startup.  You'd only use this
+   * method if you were doing manual operation.
+   * @return Fully qualified path to hbase root dir
+   * @throws java.io.IOException
+   */
+  public Path createRootDir() throws IOException {
+    FileSystem fs = FileSystem.get(this.conf);
+    Path hbaseRootdir = getDefaultRootDirPath();
+    FSUtils.setRootDir(this.conf, hbaseRootdir);
+    fs.mkdirs(hbaseRootdir);
+    FSUtils.setVersion(fs, hbaseRootdir);
+    return hbaseRootdir;
+  }
+
+  public void stopHBaseCluster() throws IOException {
+    if (hbaseCluster != null) {
+      LOG.info("MiniHBaseCluster stopped");
+      hbaseCluster.shutdown();
+      hbaseCluster.waitUntilShutDown();
+      hbaseCluster = null;
+    }
+  }
+
+  public void startHBaseCluster() throws Exception {
+    if (zkCluster == null) {
+      startMiniZKCluster();
+    }
+    if (hbaseCluster != null) {
+      return;
+    }
+
+    System.setProperty("HBASE_ZNODE_FILE", testBaseDir + "/hbase_znode_file");
+    if (conf.getInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1) == -1) {
+      conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
+    }
+    if (conf.getInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, -1) == -1) {
+      conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 1);
+    }
+    conf.setBoolean(HConstants.REPLICATION_ENABLE_KEY, false);
+    createRootDir();
+
+    Configuration c = HBaseConfiguration.create(this.conf);
+    // randomize hbase info port
+    c.setInt(HConstants.MASTER_INFO_PORT, 0);
+
+    hbaseCluster = new MiniHBaseCluster(c, 1);
+
+    // Don't leave here till we've done a successful scan of the hbase:meta
+    HTable t = new HTable(c, TableName.META_TABLE_NAME);
+    ResultScanner s = t.getScanner(new Scan());
+    while (s.next() != null) {
+      continue;
+    }
+    s.close();
+    t.close();
+    LOG.info("MiniHBaseCluster started");
+
+  }
+
+  /**
+   * Start a mini ZK cluster. If the property "test.hbase.zookeeper.property.clientPort" is set
+   *  the port mentionned is used as the default port for ZooKeeper.
+   */
+  public MiniZooKeeperCluster startMiniZKCluster()
+      throws Exception {
+    File zkDataPath = new File(testBaseDir, "zk");
+    if (this.zkCluster != null) {
+      throw new IOException("Cluster already running at " + zkDataPath);
+    }
+    this.zkCluster = new MiniZooKeeperCluster(conf);
+    final int defPort = this.conf.getInt("test.hbase.zookeeper.property.clientPort", 0);
+    if (defPort > 0){
+      // If there is a port in the config file, we use it.
+      this.zkCluster.setDefaultClientPort(defPort);
+    }
+    int clientPort =  this.zkCluster.startup(zkDataPath, 1);
+    this.conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, Integer.toString(clientPort));
+    LOG.info("MiniZooKeeperCluster started");
+
+    return this.zkCluster;
+  }
+
+  public void stopZooKeeperCluster() throws IOException {
+    if (zkCluster != null) {
+      LOG.info("MiniZooKeeperCluster stopped");
+      zkCluster.shutdown();
+      zkCluster = null;
+    }
+  }
+
+  public Configuration getConf() {
+    return conf;
+  }
+
+  public MiniZooKeeperCluster getMiniZooKeeperCluster() {
+    return zkCluster;
+  }
+
+  public MiniHBaseCluster getMiniHBaseCluster() {
+    return hbaseCluster;
+  }
+
+  public HTableDescriptor getTableDescriptor(String tableName) throws IOException {
+    HBaseAdmin admin = new HBaseAdmin(conf);
+    try {
+      return admin.getTableDescriptor(Bytes.toBytes(tableName));
+    } finally {
+      admin.close();
+    }
+  }
+
+  public void createTable(HTableDescriptor hTableDesc) throws IOException {
+    HBaseAdmin admin = new HBaseAdmin(conf);
+    try {
+      admin.createTable(hTableDesc);
+    } finally {
+      admin.close();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/tajo-cluster-tests/src/test/java/org/apache/tajo/IntegrationTest.java
----------------------------------------------------------------------
diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/IntegrationTest.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/IntegrationTest.java
new file mode 100644
index 0000000..755b90a
--- /dev/null
+++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/IntegrationTest.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo;
+
+public interface IntegrationTest {
+}

http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/tajo-cluster-tests/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
----------------------------------------------------------------------
diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
new file mode 100644
index 0000000..f084138
--- /dev/null
+++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
@@ -0,0 +1,169 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.tajo.benchmark.TPCH;
+import org.apache.tajo.catalog.CatalogUtil;
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.TableDesc;
+import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.catalog.statistics.TableStats;
+import org.apache.tajo.client.TajoClient;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.engine.planner.global.MasterPlan;
+import org.apache.tajo.engine.query.QueryContext;
+import org.apache.tajo.exception.TajoException;
+import org.apache.tajo.session.Session;
+import org.apache.tajo.util.CommonTestingUtil;
+import org.apache.tajo.util.FileUtil;
+import org.apache.tajo.util.KeyValueSet;
+import org.apache.tajo.util.TajoIdUtils;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.sql.ResultSet;
+import java.util.UUID;
+
+public class LocalTajoTestingUtility {
+  private static final Log LOG = LogFactory.getLog(LocalTajoTestingUtility.class);
+
+  private TajoTestingCluster util;
+  private TajoConf conf;
+  private TajoClient client;
+
+  private static UserGroupInformation dummyUserInfo;
+
+  static {
+    try {
+      dummyUserInfo = UserGroupInformation.getCurrentUser();
+    } catch (IOException e) {
+      e.printStackTrace();
+    }
+  }
+
+  private static int taskAttemptId;
+
+  public static TaskAttemptId newTaskAttemptId() {
+    return QueryIdFactory.newTaskAttemptId(
+        QueryIdFactory.newTaskId(new MasterPlan(newQueryId(), null, null).newExecutionBlockId()), taskAttemptId++);
+  }
+  public static TaskAttemptId newTaskAttemptId(MasterPlan plan) {
+    return QueryIdFactory.newTaskAttemptId(QueryIdFactory.newTaskId(plan.newExecutionBlockId()), 0);
+  }
+
+  public static Session createDummySession() {
+    return new Session(UUID.randomUUID().toString(), dummyUserInfo.getUserName(), TajoConstants.DEFAULT_DATABASE_NAME);
+  }
+
+  public static QueryContext createDummyContext(TajoConf conf) {
+    QueryContext context = new QueryContext(conf, createDummySession());
+    context.putAll(CommonTestingUtil.getSessionVarsForTest().getAllKeyValus());
+    return context;
+  }
+
+  /**
+   * for test
+   * @return The generated QueryId
+   */
+  public synchronized static QueryId newQueryId() {
+    return QueryIdFactory.newQueryId(TajoIdUtils.MASTER_ID_FORMAT.format(0));
+  }
+
+  public void setup(String[] names,
+                    String[] tablepaths,
+                    Schema[] schemas,
+                    KeyValueSet option) throws Exception {
+    LOG.info("===================================================");
+    LOG.info("Starting Test Cluster.");
+    LOG.info("===================================================");
+
+    util = new TajoTestingCluster();
+    util.startMiniCluster(1);
+    conf = util.getConfiguration();
+    client = util.newTajoClient();
+
+    FileSystem fs = util.getDefaultFileSystem();
+    Path rootDir = TajoConf.getWarehouseDir(conf);
+    fs.mkdirs(rootDir);
+    for (int i = 0; i < tablepaths.length; i++) {
+      Path localPath = new Path(tablepaths[i]);
+      Path tablePath = new Path(rootDir, names[i]);
+      fs.mkdirs(tablePath);
+      Path dfsPath = new Path(tablePath, localPath.getName());
+      fs.copyFromLocalFile(localPath, dfsPath);
+      TableMeta meta = CatalogUtil.newTableMeta("TEXT", option);
+
+      // Add fake table statistic data to tables.
+      // It gives more various situations to unit tests.
+      TableStats stats = new TableStats();
+      stats.setNumBytes(TPCH.tableVolumes.get(names[i]));
+      TableDesc tableDesc = new TableDesc(
+          CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, names[i]), schemas[i], meta,
+          tablePath.toUri());
+      tableDesc.setStats(stats);
+      util.getMaster().getCatalog().createTable(tableDesc);
+    }
+
+    LOG.info("===================================================");
+    LOG.info("Test Cluster ready and test table created.");
+    LOG.info("===================================================");
+
+  }
+
+  public TajoTestingCluster getTestingCluster() {
+    return util;
+  }
+
+  public ResultSet execute(String query) throws TajoException {
+    return client.executeQueryAndGetResult(query);
+  }
+
+  public void shutdown() throws IOException {
+    if(client != null) {
+      client.close();
+    }
+    if(util != null) {
+      util.shutdownMiniCluster();
+    }
+  }
+
+  public static Path getResourcePath(String path, String suffix) {
+    URL resultBaseURL = ClassLoader.getSystemResource(path);
+    return new Path(resultBaseURL.toString(), suffix);
+  }
+
+  public static Path getResultPath(Class clazz, String fileName) {
+    return new Path (getResourcePath("results", clazz.getSimpleName()), fileName);
+  }
+
+  public static String getResultText(Class clazz, String fileName) throws IOException {
+    FileSystem localFS = FileSystem.getLocal(new Configuration());
+    Path path = getResultPath(clazz, fileName);
+    Preconditions.checkState(localFS.exists(path) && localFS.isFile(path));
+    return FileUtil.readTextFile(new File(path.toUri()));
+  }
+}

http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/tajo-cluster-tests/src/test/java/org/apache/tajo/NamedTest.java
----------------------------------------------------------------------
diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/NamedTest.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/NamedTest.java
new file mode 100644
index 0000000..f2c1bb7
--- /dev/null
+++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/NamedTest.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+@Retention(RetentionPolicy.RUNTIME)
+public @interface NamedTest {
+  String value();
+}

http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java
----------------------------------------------------------------------
diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java
new file mode 100644
index 0000000..c3c9b52
--- /dev/null
+++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java
@@ -0,0 +1,1030 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo;
+
+import com.google.protobuf.ServiceException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.tajo.algebra.*;
+import org.apache.tajo.annotation.Nullable;
+import org.apache.tajo.catalog.CatalogService;
+import org.apache.tajo.catalog.CatalogUtil;
+import org.apache.tajo.catalog.TableDesc;
+import org.apache.tajo.exception.UndefinedTableException;
+import org.apache.tajo.cli.tsql.ParsedResult;
+import org.apache.tajo.cli.tsql.SimpleParser;
+import org.apache.tajo.client.TajoClient;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.engine.parser.SQLAnalyzer;
+import org.apache.tajo.engine.query.QueryContext;
+import org.apache.tajo.exception.TajoException;
+import org.apache.tajo.master.GlobalEngine;
+import org.apache.tajo.plan.LogicalOptimizer;
+import org.apache.tajo.plan.LogicalPlan;
+import org.apache.tajo.plan.LogicalPlanner;
+import org.apache.tajo.plan.verifier.LogicalPlanVerifier;
+import org.apache.tajo.plan.verifier.PreLogicalPlanVerifier;
+import org.apache.tajo.plan.verifier.VerificationState;
+import org.apache.tajo.storage.StorageUtil;
+import org.apache.tajo.util.FileUtil;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.rules.TestName;
+import org.junit.rules.TestRule;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.annotation.*;
+import java.lang.reflect.Method;
+import java.net.URL;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.util.*;
+
+import static org.junit.Assert.*;
+
+/**
+ * (Note that this class is not thread safe. Do not execute maven test in any parallel mode.)
+ * <br />
+ * <code>QueryTestCaseBase</code> provides useful methods to easily execute queries and verify their results.
+ *
+ * This class basically uses four resource directories:
+ * <ul>
+ *   <li>src/test/resources/dataset - contains a set of data files. It contains sub directories, each of which
+ *   corresponds each test class. All data files in each sub directory can be used in the corresponding test class.</li>
+ *
+ *   <li>src/test/resources/queries - This is the query directory. It contains sub directories, each of which
+ *   corresponds each test class. All query files in each sub directory can be used in the corresponding test
+ *   class.</li>
+ *
+ *   <li>src/test/resources/results - This is the result directory. It contains sub directories, each of which
+ *   corresponds each test class. All result files in each sub directory can be used in the corresponding test class.
+ *   </li>
+ * </ul>
+ *
+ * For example, if you create a test class named <code>TestJoinQuery</code>, you should create a pair of query and
+ * result set directories as follows:
+ *
+ * <pre>
+ *   src-|
+ *       |- resources
+ *             |- dataset
+ *             |     |- TestJoinQuery
+ *             |              |- table1.tbl
+ *             |              |- table2.tbl
+ *             |
+ *             |- queries
+ *             |     |- TestJoinQuery
+ *             |              |- TestInnerJoin.sql
+ *             |              |- table1_ddl.sql
+ *             |              |- table2_ddl.sql
+ *             |
+ *             |- results
+ *                   |- TestJoinQuery
+ *                            |- TestInnerJoin.result
+ * </pre>
+ *
+ * <code>QueryTestCaseBase</code> basically provides the following methods:
+ * <ul>
+ *  <li><code>{@link #executeQuery()}</code> - executes a corresponding query and returns an ResultSet instance</li>
+ *  <li><code>{@link #executeFile(String)}</code> - executes a given query file included in the corresponding query
+ *  file in the current class's query directory</li>
+ *  <li><code>assertResultSet()</code> - check if the query result is equivalent to the expected result included
+ *  in the corresponding result file in the current class's result directory.</li>
+ *  <li><code>cleanQuery()</code> - clean up all resources</li>
+ *  <li><code>executeDDL()</code> - execute a DDL query like create or drop table.</li>
+ * </ul>
+ *
+ * In order to make use of the above methods, query files and results file must be as follows:
+ * <ul>
+ *  <li>Each query file must be located on the subdirectory whose structure must be src/resources/queries/${ClassName},
+ *  where ${ClassName} indicates an actual test class's simple name.</li>
+ *  <li>Each result file must be located on the subdirectory whose structure must be src/resources/results/${ClassName},
+ *  where ${ClassName} indicates an actual test class's simple name.</li>
+ * </ul>
+ *
+ * Especially, {@link #executeQuery() and {@link #assertResultSet(java.sql.ResultSet)} methods automatically finds
+ * a query file to be executed and a result to be compared, which are corresponding to the running class and method.
+ * For them, query and result files additionally must be follows as:
+ * <ul>
+ *  <li>Each result file must have the file extension '.result'</li>
+ *  <li>Each query file must have the file extension '.sql'.</li>
+ * </ul>
+ */
+public class QueryTestCaseBase {
+  private static final Log LOG = LogFactory.getLog(QueryTestCaseBase.class);
+  protected static final TpchTestBase testBase;
+  protected static final TajoTestingCluster testingCluster;
+  protected static TajoConf conf;
+  protected static TajoClient client;
+  protected static final CatalogService catalog;
+  protected static final SQLAnalyzer sqlParser;
+  protected static PreLogicalPlanVerifier verifier;
+  protected static LogicalPlanner planner;
+  protected static LogicalOptimizer optimizer;
+  protected static LogicalPlanVerifier postVerifier;
+
+  /** the base path of dataset directories */
+  protected static final Path datasetBasePath;
+  /** the base path of query directories */
+  protected static final Path queryBasePath;
+  /** the base path of result directories */
+  protected static final Path resultBasePath;
+
+  static {
+    testBase = TpchTestBase.getInstance();
+    testingCluster = testBase.getTestingCluster();
+    conf = testBase.getTestingCluster().getConfiguration();
+    catalog = testBase.getTestingCluster().getMaster().getCatalog();
+    URL datasetBaseURL = ClassLoader.getSystemResource("dataset");
+    datasetBasePath = new Path(datasetBaseURL.toString());
+    URL queryBaseURL = ClassLoader.getSystemResource("queries");
+    queryBasePath = new Path(queryBaseURL.toString());
+    URL resultBaseURL = ClassLoader.getSystemResource("results");
+    resultBasePath = new Path(resultBaseURL.toString());
+
+    GlobalEngine engine = testingCluster.getMaster().getContext().getGlobalEngine();
+    sqlParser = engine.getAnalyzer();
+    verifier = engine.getPreLogicalPlanVerifier();
+    planner = engine.getLogicalPlanner();
+    optimizer = engine.getLogicalOptimizer();
+    postVerifier = engine.getLogicalPlanVerifier();
+  }
+
+  /** It transiently contains created tables for the running test class. */
+  private static String currentDatabase;
+  private static Set<String> createdTableGlobalSet = new HashSet<String>();
+  // queries and results directory corresponding to subclass class.
+  protected Path currentQueryPath;
+  protected Path namedQueryPath;
+  protected Path currentResultPath;
+  protected Path currentDatasetPath;
+  protected Path namedDatasetPath;
+
+  protected FileSystem currentResultFS;
+
+  protected final String testParameter;
+
+  // for getting a method name
+  @Rule public TestName name = new TestName();
+
+  @BeforeClass
+  public static void setUpClass() throws Exception {
+    conf = testBase.getTestingCluster().getConfiguration();
+    client = testBase.getTestingCluster().newTajoClient();
+  }
+
+  @AfterClass
+  public static void tearDownClass() throws Exception {
+    for (String tableName : createdTableGlobalSet) {
+      client.updateQuery("DROP TABLE IF EXISTS " + CatalogUtil.denormalizeIdentifier(tableName));
+    }
+    createdTableGlobalSet.clear();
+
+    // if the current database is "default", shouldn't drop it.
+    if (!currentDatabase.equals(TajoConstants.DEFAULT_DATABASE_NAME)) {
+      for (String tableName : catalog.getAllTableNames(currentDatabase)) {
+        client.updateQuery("DROP TABLE IF EXISTS " + tableName);
+      }
+
+      client.selectDatabase(TajoConstants.DEFAULT_DATABASE_NAME);
+      client.dropDatabase(currentDatabase);
+    }
+    client.close();
+  }
+
+  @Before
+  public void printTestName() {
+    /* protect a travis stalled build */
+    System.out.println("Run: " + name.getMethodName() +
+        " Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())
+        / (1024 * 1024)) + " MBytes, Active Threads:" + Thread.activeCount());
+  }
+
+  public QueryTestCaseBase() {
+    // hive 0.12 does not support quoted identifier.
+    // So, we use lower case database names when Tajo uses HiveCatalogStore.
+    if (testingCluster.isHiveCatalogStoreRunning()) {
+      this.currentDatabase = getClass().getSimpleName().toLowerCase();
+    } else {
+      this.currentDatabase = getClass().getSimpleName();
+    }
+    testParameter = null;
+    init();
+  }
+
+  public QueryTestCaseBase(String currentDatabase) {
+    this(currentDatabase, null);
+  }
+
+  public QueryTestCaseBase(String currentDatabase, String testParameter) {
+    this.currentDatabase = currentDatabase;
+    this.testParameter = testParameter;
+    init();
+  }
+
+  private void init() {
+    String className = getClass().getSimpleName();
+    currentQueryPath = new Path(queryBasePath, className);
+    currentResultPath = new Path(resultBasePath, className);
+    currentDatasetPath = new Path(datasetBasePath, className);
+    NamedTest namedTest = getClass().getAnnotation(NamedTest.class);
+    if (namedTest != null) {
+      namedQueryPath = new Path(queryBasePath, namedTest.value());
+      namedDatasetPath = new Path(datasetBasePath, namedTest.value());
+    }
+
+    try {
+      // if the current database is "default", we don't need create it because it is already prepated at startup time.
+      if (!currentDatabase.equals(TajoConstants.DEFAULT_DATABASE_NAME)) {
+        client.updateQuery("CREATE DATABASE IF NOT EXISTS " + CatalogUtil.denormalizeIdentifier(currentDatabase));
+      }
+      client.selectDatabase(currentDatabase);
+      currentResultFS = currentResultPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
+
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+    testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, "false");
+  }
+
+  protected TajoClient getClient() {
+    return client;
+  }
+
+  public String getCurrentDatabase() {
+    return currentDatabase;
+  }
+
+  private static VerificationState verify(String query) throws TajoException {
+
+    VerificationState state = new VerificationState();
+    QueryContext context = LocalTajoTestingUtility.createDummyContext(conf);
+
+    Expr expr = sqlParser.parse(query);
+
+    verifier.verify(context, state, expr);
+
+    if (state.getErrors().size() > 0) {
+      return state;
+    }
+    LogicalPlan plan = planner.createPlan(context, expr);
+    optimizer.optimize(plan);
+    postVerifier.verify(state, plan);
+
+    return state;
+  }
+
+  public void assertValidSQL(String query) throws IOException {
+    VerificationState state = null;
+    try {
+      state = verify(query);
+      if (state.getErrors().size() > 0) {
+        fail(state.getErrors().get(0).getMessage());
+      }
+    } catch (TajoException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  public void assertValidSQLFromFile(String fileName) throws IOException {
+    Path queryFilePath = getQueryFilePath(fileName);
+    String query = FileUtil.readTextFile(new File(queryFilePath.toUri()));
+    assertValidSQL(query);
+  }
+
+  public void assertInvalidSQL(String query) throws IOException {
+    VerificationState state = null;
+    try {
+      state = verify(query);
+
+      if (state.getErrors().size() == 0) {
+        fail(PreLogicalPlanVerifier.class.getSimpleName() + " cannot catch any verification error: " + query);
+      }
+
+    } catch (TajoException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  public void assertInvalidSQLFromFile(String fileName) throws IOException {
+    Path queryFilePath = getQueryFilePath(fileName);
+    String query = FileUtil.readTextFile(new File(queryFilePath.toUri()));
+    assertInvalidSQL(query);
+  }
+
+  public void assertPlanError(String fileName) throws IOException {
+    Path queryFilePath = getQueryFilePath(fileName);
+    String query = FileUtil.readTextFile(new File(queryFilePath.toUri()));
+    try {
+      verify(query);
+    } catch (TajoException e) {
+      return;
+    }
+    fail("Cannot catch any planning error from: " + query);
+  }
+
+  protected ResultSet executeString(String sql) throws TajoException {
+    return client.executeQueryAndGetResult(sql);
+  }
+
+  /**
+   * It executes the query file and compare the result against the the result file.
+   *
+   * @throws Exception
+   */
+  public void assertQuery() throws Exception {
+    ResultSet res = null;
+    try {
+      res = executeQuery();
+      assertResultSet(res);
+    } finally {
+      if (res != null) {
+        res.close();
+      }
+    }
+  }
+
+  /**
+   * It executes a given query statement and verifies the result against the the result file.
+   *
+   * @param query A query statement
+   * @throws Exception
+   */
+  public void assertQueryStr(String query) throws Exception {
+    ResultSet res = null;
+    try {
+      res = executeString(query);
+      assertResultSet(res);
+    } finally {
+      if (res != null) {
+        res.close();
+      }
+    }
+  }
+
+  /**
+   * Execute a query contained in the file located in src/test/resources/results/<i>ClassName</i>/<i>MethodName</i>.
+   * <i>ClassName</i> and <i>MethodName</i> will be replaced by actual executed class and methods.
+   *
+   * @return ResultSet of query execution.
+   */
+  public ResultSet executeQuery() throws Exception {
+    return executeFile(getMethodName() + ".sql");
+  }
+
+  private volatile Description current;
+
+  @Rule
+  public TestRule watcher = new TestWatcher() {
+    @Override
+    protected void starting(Description description) {
+      QueryTestCaseBase.this.current = description;
+    }
+  };
+
+  @Target({ElementType.METHOD})
+  @Retention(RetentionPolicy.RUNTIME)
+  protected @interface SimpleTest {
+    String[] prepare() default {};
+    QuerySpec[] queries() default {};
+    String[] cleanup() default {};
+  }
+
+  @Target({ElementType.METHOD})
+  @Retention(RetentionPolicy.RUNTIME)
+  protected @interface QuerySpec {
+    String value();
+    boolean override() default false;
+    Option option() default @Option;
+  }
+
+  @Target({ElementType.METHOD})
+  @Retention(RetentionPolicy.RUNTIME)
+  protected @interface Option {
+    boolean withExplain() default false;
+    boolean withExplainGlobal() default false;
+    boolean parameterized() default false;
+    boolean sort() default false;
+  }
+
+  private static class DummyQuerySpec implements QuerySpec {
+    private final String value;
+    private final Option option;
+    public DummyQuerySpec(String value, Option option) {
+      this.value = value;
+      this.option = option;
+    }
+    public Class<? extends Annotation> annotationType() { return QuerySpec.class; }
+    public String value() { return value; }
+    public boolean override() { return option != null; }
+    public Option option() { return option; }
+  }
+
+  private static class DummyOption implements Option {
+    private final boolean explain;
+    private final boolean withExplainGlobal;
+    private final boolean parameterized;
+    private final boolean sort;
+    public DummyOption(boolean explain, boolean withExplainGlobal, boolean parameterized, boolean sort) {
+      this.explain = explain;
+      this.withExplainGlobal = withExplainGlobal;
+      this.parameterized = parameterized;
+      this.sort = sort;
+    }
+    public Class<? extends Annotation> annotationType() { return Option.class; }
+    public boolean withExplain() { return explain;}
+    public boolean withExplainGlobal() { return withExplainGlobal;}
+    public boolean parameterized() { return parameterized;}
+    public boolean sort() { return sort;}
+  }
+
+  protected void runSimpleTests() throws Exception {
+    String methodName = getMethodName();
+    Method method = current.getTestClass().getMethod(methodName);
+    SimpleTest annotation = method.getAnnotation(SimpleTest.class);
+    if (annotation == null) {
+      throw new IllegalStateException("Cannot find test annotation");
+    }
+
+    List<String> prepares = new ArrayList<>(Arrays.asList(annotation.prepare()));
+    QuerySpec[] queries = annotation.queries();
+    Option defaultOption = method.getAnnotation(Option.class);
+    if (defaultOption == null) {
+      defaultOption = new DummyOption(false, false, false, false);
+    }
+
+    boolean fromFile = false;
+    if (queries.length == 0) {
+      Path queryFilePath = getQueryFilePath(getMethodName() + ".sql");
+      List<ParsedResult> parsedResults = SimpleParser.parseScript(FileUtil.readTextFile(new File(queryFilePath.toUri())));
+      int i = 0;
+      for (; i < parsedResults.size() - 1; i++) {
+        prepares.add(parsedResults.get(i).getStatement());
+      }
+      queries = new QuerySpec[] {new DummyQuerySpec(parsedResults.get(i).getHistoryStatement(), null)};
+      fromFile = true;  // do not append query index to result file
+    }
+    
+    try {
+      for (String prepare : prepares) {
+        client.executeQueryAndGetResult(prepare).close();
+      }
+      for (int i = 0; i < queries.length; i++) {
+        QuerySpec spec = queries[i];
+        Option option = spec.override() ? spec.option() : defaultOption;
+        String prefix = "";
+        testingCluster.getConfiguration().set(TajoConf.ConfVars.$TEST_PLAN_SHAPE_FIX_ENABLED.varname, "true");
+        if (option.withExplain()) {// Enable this option to fix the shape of the generated plans.
+          prefix += resultSetToString(executeString("explain " + spec.value()));
+        }
+        if (option.withExplainGlobal()) {
+          // Enable this option to fix the shape of the generated plans.
+          prefix += resultSetToString(executeString("explain global " + spec.value()));
+        }
+
+        // plan test
+        if (prefix.length() > 0) {
+          String planResultName = methodName + (fromFile ? "" : "" + (i + 1)) +
+              ((option.parameterized() && testParameter != null) ? "" + testParameter : "") + ".plan";
+          Path resultPath = StorageUtil.concatPath(currentResultPath, planResultName);
+          if (currentResultFS.exists(resultPath)) {
+            assertEquals("Plan Verification for: " + (i + 1) + " th test",
+                FileUtil.readTextFromStream(currentResultFS.open(resultPath)), prefix);
+          } else if (prefix.length() > 0) {
+            // If there is no result file expected, create gold files for new tests.
+            FileUtil.writeTextToStream(prefix, currentResultFS.create(resultPath));
+            LOG.info("New test output for " + current.getDisplayName() + " is written to " + resultPath);
+            // should be copied to src directory
+          }
+        }
+
+        testingCluster.getConfiguration().set(TajoConf.ConfVars.$TEST_PLAN_SHAPE_FIX_ENABLED.varname, "false");
+        ResultSet result = client.executeQueryAndGetResult(spec.value());
+
+        // result test
+        String fileName = methodName + (fromFile ? "" : "" + (i + 1)) + ".result";
+        Path resultPath = StorageUtil.concatPath(currentResultPath, fileName);
+        if (currentResultFS.exists(resultPath)) {
+          assertEquals("Result Verification for: " + (i + 1) + " th test",
+              FileUtil.readTextFromStream(currentResultFS.open(resultPath)), resultSetToString(result, option.sort()));
+        } else if (!isNull(result)) {
+          // If there is no result file expected, create gold files for new tests.
+          FileUtil.writeTextToStream(resultSetToString(result, option.sort()), currentResultFS.create(resultPath));
+          LOG.info("New test output for " + current.getDisplayName() + " is written to " + resultPath);
+          // should be copied to src directory
+        }
+        result.close();
+      }
+    } finally {
+      for (String cleanup : annotation.cleanup()) {
+        try {
+          client.executeQueryAndGetResult(cleanup).close();
+        } catch (SQLException e) {
+          // ignore
+        }
+      }
+    }
+  }
+
+  private boolean isNull(ResultSet result) throws SQLException {
+    return result.getMetaData().getColumnCount() == 0;
+  }
+
+  protected String getMethodName() {
+    String methodName = name.getMethodName();
+    // In the case of parameter execution name's pattern is methodName[0]
+    if (methodName.endsWith("]")) {
+      methodName = methodName.substring(0, methodName.length() - 3);
+    }
+    return methodName;
+  }
+
+  public ResultSet executeJsonQuery() throws Exception {
+    return executeJsonFile(getMethodName() + ".json");
+  }
+
+  /**
+   * Execute a query contained in the given named file. This methods tries to find the given file within the directory
+   * src/test/resources/results/<i>ClassName</i>.
+   *
+   * @param queryFileName The file name to be used to execute a query.
+   * @return ResultSet of query execution.
+   */
+  public ResultSet executeFile(String queryFileName) throws Exception {
+    Path queryFilePath = getQueryFilePath(queryFileName);
+
+    List<ParsedResult> parsedResults = SimpleParser.parseScript(FileUtil.readTextFile(new File(queryFilePath.toUri())));
+    if (parsedResults.size() > 1) {
+      assertNotNull("This script \"" + queryFileName + "\" includes two or more queries");
+    }
+
+    int idx = 0;
+    for (; idx < parsedResults.size() - 1; idx++) {
+      client.executeQueryAndGetResult(parsedResults.get(idx).getHistoryStatement()).close();
+    }
+
+    ResultSet result = client.executeQueryAndGetResult(parsedResults.get(idx).getHistoryStatement());
+    assertNotNull("Query succeeded test", result);
+    return result;
+  }
+
+  public ResultSet executeJsonFile(String jsonFileName) throws Exception {
+    Path queryFilePath = getQueryFilePath(jsonFileName);
+
+    ResultSet result = client.executeJsonQueryAndGetResult(FileUtil.readTextFile(new File(queryFilePath.toUri())));
+    assertNotNull("Query succeeded test", result);
+    return result;
+  }
+
+  /**
+   * Assert the equivalence between the expected result and an actual query result.
+   * If it isn't it throws an AssertionError.
+   *
+   * @param result Query result to be compared.
+   */
+  public final void assertResultSet(ResultSet result) throws IOException {
+    assertResultSet("Result Verification", result, getMethodName() + ".result");
+  }
+
+  /**
+   * Assert the equivalence between the expected result and an actual query result.
+   * If it isn't it throws an AssertionError.
+   *
+   * @param result Query result to be compared.
+   * @param resultFileName The file name containing the result to be compared
+   */
+  public final void assertResultSet(ResultSet result, String resultFileName) throws IOException {
+    assertResultSet("Result Verification", result, resultFileName);
+  }
+
+  /**
+   * Assert the equivalence between the expected result and an actual query result.
+   * If it isn't it throws an AssertionError with the given message.
+   *
+   * @param message message The message to printed if the assertion is failed.
+   * @param result Query result to be compared.
+   */
+  public final void assertResultSet(String message, ResultSet result, String resultFileName) throws IOException {
+    Path resultFile = getResultFile(resultFileName);
+    try {
+      verifyResultText(message, result, resultFile);
+    } catch (SQLException e) {
+      throw new IOException(e);
+    }
+  }
+
+  public final void assertStrings(String actual) throws IOException {
+    assertStrings(actual, getMethodName() + ".result");
+  }
+
+  public final void assertStrings(String actual, String resultFileName) throws IOException {
+    assertStrings("Result Verification", actual, resultFileName);
+  }
+
+  public final void assertStrings(String message, String actual, String resultFileName) throws IOException {
+    Path resultFile = getResultFile(resultFileName);
+    String expectedResult = FileUtil.readTextFile(new File(resultFile.toUri()));
+    assertEquals(message, expectedResult, actual);
+  }
+
+  /**
+   * Release all resources
+   *
+   * @param resultSet ResultSet
+   */
+  public final void cleanupQuery(ResultSet resultSet) throws IOException {
+    if (resultSet == null) {
+      return;
+    }
+    try {
+      resultSet.close();
+    } catch (SQLException e) {
+      throw new IOException(e);
+    }
+  }
+
+  /**
+   * Assert that the database exists.
+   * @param databaseName The database name to be checked. This name is case sensitive.
+   */
+  public void assertDatabaseExists(String databaseName) throws SQLException {
+    assertTrue(client.existDatabase(databaseName));
+  }
+
+  /**
+   * Assert that the database does not exists.
+   * @param databaseName The database name to be checked. This name is case sensitive.
+   */
+  public void assertDatabaseNotExists(String databaseName) {
+    assertTrue(!client.existDatabase(databaseName));
+  }
+
+  /**
+   * Assert that the table exists.
+   *
+   * @param tableName The table name to be checked. This name is case sensitive.
+   * @throws ServiceException
+   */
+  public void assertTableExists(String tableName) {
+    assertTrue(client.existTable(tableName));
+  }
+
+  /**
+   * Assert that the table does not exist.
+   *
+   * @param tableName The table name to be checked. This name is case sensitive.
+   */
+  public void assertTableNotExists(String tableName) {
+    assertTrue(!client.existTable(tableName));
+  }
+
+  public void assertColumnExists(String tableName,String columnName) throws UndefinedTableException {
+    TableDesc tableDesc = getTableDesc(tableName);
+    assertTrue(tableDesc.getSchema().containsByName(columnName));
+  }
+
+  private TableDesc getTableDesc(String tableName) throws UndefinedTableException {
+    return client.getTableDesc(tableName);
+  }
+
+  public void assertTablePropertyEquals(String tableName, String key, String expectedValue)
+      throws UndefinedTableException {
+
+    TableDesc tableDesc = getTableDesc(tableName);
+    assertEquals(expectedValue, tableDesc.getMeta().getOption(key));
+  }
+
+  public String resultSetToString(ResultSet resultSet) throws SQLException {
+    return resultSetToString(resultSet, false);
+  }
+
+  /**
+   * It transforms a ResultSet instance to rows represented as strings.
+   *
+   * @param resultSet ResultSet that contains a query result
+   * @return String
+   * @throws SQLException
+   */
+  public String resultSetToString(ResultSet resultSet, boolean sort) throws SQLException {
+    StringBuilder sb = new StringBuilder();
+    ResultSetMetaData rsmd = resultSet.getMetaData();
+    int numOfColumns = rsmd.getColumnCount();
+
+    for (int i = 1; i <= numOfColumns; i++) {
+      if (i > 1) sb.append(",");
+      String columnName = rsmd.getColumnName(i);
+      sb.append(columnName);
+    }
+    sb.append("\n-------------------------------\n");
+
+    List<String> results = new ArrayList<String>();
+    while (resultSet.next()) {
+      StringBuilder line = new StringBuilder();
+      for (int i = 1; i <= numOfColumns; i++) {
+        if (i > 1) line.append(",");
+        String columnValue = resultSet.getString(i);
+        if (resultSet.wasNull()) {
+          columnValue = "null";
+        }
+        line.append(columnValue);
+      }
+      results.add(line.toString());
+    }
+    if (sort) {
+      Collections.sort(results);
+    }
+    for (String line : results) {
+      sb.append(line).append('\n');
+    }
+    return sb.toString();
+  }
+
+  private void verifyResultText(String message, ResultSet res, Path resultFile) throws SQLException, IOException {
+    String actualResult = resultSetToString(res);
+    String expectedResult = FileUtil.readTextFile(new File(resultFile.toUri()));
+    assertEquals(message, expectedResult.trim(), actualResult.trim());
+  }
+
+  private Path getQueryFilePath(String fileName) throws IOException {
+    Path queryFilePath = StorageUtil.concatPath(currentQueryPath, fileName);
+    FileSystem fs = currentQueryPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
+    if (!fs.exists(queryFilePath)) {
+      if (namedQueryPath != null) {
+        queryFilePath = StorageUtil.concatPath(namedQueryPath, fileName);
+        fs = namedQueryPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
+        if (!fs.exists(queryFilePath)) {
+          throw new IOException("Cannot find " + fileName + " at " + currentQueryPath + " and " + namedQueryPath);
+        }
+      } else {
+        throw new IOException("Cannot find " + fileName + " at " + currentQueryPath);
+      }
+    }
+    return queryFilePath;
+  }
+
+  protected String getResultContents(String fileName) throws IOException {
+    Path resultFile = getResultFile(getMethodName() + ".result");
+    return FileUtil.readTextFile(new File(resultFile.toUri()));
+  }
+
+  protected Path getResultFile(String fileName) throws IOException {
+    Path resultPath = StorageUtil.concatPath(currentResultPath, fileName);
+    FileSystem fs = currentResultPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
+    assertTrue(resultPath.toString() + " existence check", fs.exists(resultPath));
+    return resultPath;
+  }
+
+  private Path getDataSetFile(String fileName) throws IOException {
+    Path dataFilePath = StorageUtil.concatPath(currentDatasetPath, fileName);
+    FileSystem fs = currentDatasetPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
+    if (!fs.exists(dataFilePath)) {
+      if (namedDatasetPath != null) {
+        dataFilePath = StorageUtil.concatPath(namedDatasetPath, fileName);
+        fs = namedDatasetPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
+        if (!fs.exists(dataFilePath)) {
+          throw new IOException("Cannot find " + fileName + " at " + currentQueryPath + " and " + namedQueryPath);
+        }
+      } else {
+        throw new IOException("Cannot find " + fileName + " at " + currentQueryPath + " and " + namedQueryPath);
+      }
+    }
+    return dataFilePath;
+  }
+
+  public List<String> executeDDL(String ddlFileName, @Nullable String [] args) throws Exception {
+    return executeDDL(ddlFileName, null, true, args);
+  }
+
+  /**
+   *
+   * Execute a data definition language (DDL) template. A general SQL DDL statement can be included in this file. But,
+   * for user-specified table name or exact external table path, you must use some format string to indicate them.
+   * The format string will be replaced by the corresponding arguments.
+   *
+   * The below is predefined format strings:
+   * <ul>
+   *   <li>${table.path} - It is replaced by the absolute file path that <code>dataFileName</code> points. </li>
+   *   <li>${i} - It is replaced by the corresponding element of <code>args</code>. For example, ${0} and ${1} are
+   *   replaced by the first and second elements of <code>args</code> respectively</li>. It uses zero-based index.
+   * </ul>
+   *
+   * Example ddl
+   * <pre>
+   *   CREATE EXTERNAL TABLE ${0} (
+   *     t_timestamp  TIMESTAMP,
+   *     t_date    DATE
+   *   ) USING TEXT LOCATION ${table.path}
+   * </pre>
+   *
+   * @param ddlFileName A file name, containing a data definition statement.
+   * @param dataFileName A file name, containing data rows, which columns have to be separated by vertical bar '|'.
+   *                     This file name is used for replacing some format string indicating an external table location.
+   * @param args A list of arguments, each of which is used to replace corresponding variable which has a form of ${i}.
+   * @return The table names created
+   */
+  public List<String> executeDDL(String ddlFileName, @Nullable String dataFileName, @Nullable String ... args)
+      throws Exception {
+
+    return executeDDL(ddlFileName, dataFileName, true, args);
+  }
+
+  private List<String> executeDDL(String ddlFileName, @Nullable String dataFileName, boolean isLocalTable,
+                                  @Nullable String[] args) throws Exception {
+
+    Path ddlFilePath = getQueryFilePath(ddlFileName);
+
+    String template = FileUtil.readTextFile(new File(ddlFilePath.toUri()));
+    String dataFilePath = null;
+    if (dataFileName != null) {
+      dataFilePath = getDataSetFile(dataFileName).toString();
+    }
+    String compiled = compileTemplate(template, dataFilePath, args);
+
+    List<ParsedResult> parsedResults = SimpleParser.parseScript(compiled);
+    List<String> createdTableNames = new ArrayList<String>();
+
+    for (ParsedResult parsedResult : parsedResults) {
+      // parse a statement
+      Expr expr = sqlParser.parse(parsedResult.getHistoryStatement());
+      assertNotNull(ddlFilePath + " cannot be parsed", expr);
+
+      if (expr.getType() == OpType.CreateTable) {
+        CreateTable createTable = (CreateTable) expr;
+        String tableName = createTable.getTableName();
+        assertTrue("Table [" + tableName + "] creation is failed.", client.updateQuery(parsedResult.getHistoryStatement()));
+
+        TableDesc createdTable = client.getTableDesc(tableName);
+        String createdTableName = createdTable.getName();
+
+        assertTrue("table '" + createdTableName + "' creation check", client.existTable(createdTableName));
+        if (isLocalTable) {
+          createdTableGlobalSet.add(createdTableName);
+          createdTableNames.add(tableName);
+        }
+      } else if (expr.getType() == OpType.DropTable) {
+        DropTable dropTable = (DropTable) expr;
+        String tableName = dropTable.getTableName();
+        assertTrue("table '" + tableName + "' existence check",
+            client.existTable(CatalogUtil.buildFQName(currentDatabase, tableName)));
+        assertTrue("table drop is failed.", client.updateQuery(parsedResult.getHistoryStatement()));
+        assertFalse("table '" + tableName + "' dropped check",
+            client.existTable(CatalogUtil.buildFQName(currentDatabase, tableName)));
+        if (isLocalTable) {
+          createdTableGlobalSet.remove(tableName);
+        }
+      } else if (expr.getType() == OpType.AlterTable) {
+        AlterTable alterTable = (AlterTable) expr;
+        String tableName = alterTable.getTableName();
+        assertTrue("table '" + tableName + "' existence check", client.existTable(tableName));
+        client.updateQuery(compiled);
+        if (isLocalTable) {
+          createdTableGlobalSet.remove(tableName);
+        }
+      } else if (expr.getType() == OpType.CreateIndex) {
+        // TODO: index existence check
+        client.executeQuery(compiled);
+      } else {
+        assertTrue(ddlFilePath + " is not a Create or Drop Table statement", false);
+      }
+    }
+
+    return createdTableNames;
+  }
+
+  /**
+   * Replace format strings by a given parameters.
+   *
+   * @param template
+   * @param dataFileName The data file name to replace <code>${table.path}</code>
+   * @param args The list argument to replace each corresponding format string ${i}. ${i} uses zero-based index.
+   * @return A string compiled
+   */
+  private String compileTemplate(String template, @Nullable String dataFileName, @Nullable String ... args) {
+    String result;
+    if (dataFileName != null) {
+      result = template.replace("${table.path}", "\'" + dataFileName + "'");
+    } else {
+      result = template;
+    }
+
+    if (args != null) {
+      for (int i = 0; i < args.length; i++) {
+        result = result.replace("${" + i + "}", args[i]);
+      }
+    }
+    return result;
+  }
+
+  /**
+   * Reads data file from Test Cluster's HDFS
+   * @param path data parent path
+   * @return data file's contents
+   * @throws Exception
+   */
+  public String getTableFileContents(Path path) throws Exception {
+    FileSystem fs = path.getFileSystem(conf);
+
+    FileStatus[] files = fs.listStatus(path);
+
+    if (files == null || files.length == 0) {
+      return "";
+    }
+
+    StringBuilder sb = new StringBuilder();
+    byte[] buf = new byte[1024];
+
+    for (FileStatus file: files) {
+      if (file.isDirectory()) {
+        sb.append(getTableFileContents(file.getPath()));
+        continue;
+      }
+
+      InputStream in = fs.open(file.getPath());
+      try {
+        while (true) {
+          int readBytes = in.read(buf);
+          if (readBytes <= 0) {
+            break;
+          }
+
+          sb.append(new String(buf, 0, readBytes));
+        }
+      } finally {
+        in.close();
+      }
+    }
+
+    return sb.toString();
+  }
+
+  /**
+   * Reads data file from Test Cluster's HDFS
+   * @param tableName
+   * @return data file's contents
+   * @throws Exception
+   */
+  public String getTableFileContents(String tableName) throws Exception {
+    TableDesc tableDesc = testingCluster.getMaster().getCatalog().getTableDesc(getCurrentDatabase(), tableName);
+    if (tableDesc == null) {
+      return null;
+    }
+
+    Path path = new Path(tableDesc.getUri());
+    return getTableFileContents(path);
+  }
+
+  public List<Path> listTableFiles(String tableName) throws Exception {
+    TableDesc tableDesc = testingCluster.getMaster().getCatalog().getTableDesc(getCurrentDatabase(), tableName);
+    if (tableDesc == null) {
+      return null;
+    }
+
+    Path path = new Path(tableDesc.getUri());
+    FileSystem fs = path.getFileSystem(conf);
+
+    return listFiles(fs, path);
+  }
+
+  private List<Path> listFiles(FileSystem fs, Path path) throws Exception {
+    List<Path> result = new ArrayList<Path>();
+    FileStatus[] files = fs.listStatus(path);
+    if (files == null || files.length == 0) {
+      return result;
+    }
+
+    for (FileStatus eachFile: files) {
+      if (eachFile.isDirectory()) {
+        result.addAll(listFiles(fs, eachFile.getPath()));
+      } else {
+        result.add(eachFile.getPath());
+      }
+    }
+    return result;
+  }
+}


Mime
View raw message