tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jihoon...@apache.org
Subject [1/2] tajo git commit: TAJO-1803: Use in-memory derby as the default catalog for unit tests.
Date Tue, 01 Sep 2015 09:55:54 GMT
Repository: tajo
Updated Branches:
  refs/heads/master 848a8c3d5 -> 77dc88c07


http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-common/src/main/proto/errors.proto
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/proto/errors.proto b/tajo-common/src/main/proto/errors.proto
index b947a95..21746f3 100644
--- a/tajo-common/src/main/proto/errors.proto
+++ b/tajo-common/src/main/proto/errors.proto
@@ -157,6 +157,9 @@ enum ResultCode {
   CAT_UPGRADE_REQUIRED                  = 901;  // Migration
   CAT_CANNOT_CONNECT                    = 902;  // Cannot connect metadata server
 
+  // Catalog store
+  CAT_UNSUPPORTED_CATALOG_STORE         = 905; // Unsupported type of catalog store
+
   // Linked Metadata (LMD)
   LMD_NO_MATCHED_DATATYPE               = 910;  // No matched data type between Tajo and
connector
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-core-tests/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-core-tests/pom.xml b/tajo-core-tests/pom.xml
index a4c0587..a23e420 100644
--- a/tajo-core-tests/pom.xml
+++ b/tajo-core-tests/pom.xml
@@ -55,7 +55,6 @@
         </executions>
         <configuration>
           <excludes>
-            <exclude>derby.log</exclude>
             <exclude>benchmark/**</exclude>
             <exclude>src/test/tpch/**</exclude>
             <exclude>src/test/resources/dataset/**</exclude>
@@ -74,7 +73,7 @@
           <systemProperties>
             <tajo.test>TRUE</tajo.test>
           </systemProperties>
-          <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8</argLine>
+          <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024
-Dderby.stream.error.file=/dev/null</argLine>
         </configuration>
       </plugin>
       <plugin>
@@ -345,7 +344,7 @@
               <forkCount>${maven.fork.count}</forkCount>
               <reuseForks>true</reuseForks>
               <trimStackTrace>false</trimStackTrace>
-              <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8</argLine>
+              <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8
-Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null</argLine>
               <useSystemClassLoader>true</useSystemClassLoader>
               <useManifestOnlyJar>true</useManifestOnlyJar>
               <systemProperties>

http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
index 1705d2b..e835481 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
@@ -90,8 +90,7 @@ public class ExprTestBase {
   public static void setUp() throws Exception {
     util = new TajoTestingCluster();
     conf = util.getConfiguration();
-    util.startCatalogCluster();
-    cat = util.getMiniCatalogCluster().getCatalog();
+    cat = util.startCatalogCluster().getCatalog();
     cat.createTablespace(DEFAULT_TABLESPACE_NAME, "hdfs://localhost:1234/warehouse");
     cat.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
     Map<FunctionSignature, FunctionDesc> map = FunctionLoader.load();

http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
index 02e921a..5382c49 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
@@ -77,8 +77,7 @@ public class TestLogicalPlanner {
   @BeforeClass
   public static void setUp() throws Exception {
     util = new TajoTestingCluster();
-    util.startCatalogCluster();
-    catalog = util.getMiniCatalogCluster().getCatalog();
+    catalog = util.startCatalogCluster().getCatalog();
     catalog.createTablespace(DEFAULT_TABLESPACE_NAME, "hdfs://localhost:1234");
     catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
index 4ae880f..2a6722e 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
@@ -20,12 +20,14 @@ package org.apache.tajo.engine.planner.physical;
 
 import org.apache.hadoop.fs.Path;
 import org.apache.tajo.LocalTajoTestingUtility;
+import org.apache.tajo.SessionVars;
 import org.apache.tajo.TajoConstants;
 import org.apache.tajo.TajoTestingCluster;
 import org.apache.tajo.algebra.Expr;
 import org.apache.tajo.catalog.*;
 import org.apache.tajo.common.TajoDataTypes.Type;
 import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.conf.TajoConf.ConfVars;
 import org.apache.tajo.datum.Datum;
 import org.apache.tajo.datum.DatumFactory;
 import org.apache.tajo.engine.parser.SQLAnalyzer;
@@ -61,7 +63,7 @@ public class TestExternalSortExec {
   private LogicalPlanner planner;
   private Path testDir;
 
-  private final int numTuple = 100000;
+  private final int numTuple = 1000;
   private Random rnd = new Random(System.currentTimeMillis());
 
   private TableDesc employee;
@@ -117,10 +119,14 @@ public class TestExternalSortExec {
 
   @Test
   public final void testNext() throws IOException, TajoException {
+    conf.setIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT, 2);
+    QueryContext queryContext = LocalTajoTestingUtility.createDummyContext(conf);
+    queryContext.setLong(SessionVars.EXTSORT_BUFFER_SIZE, 1024*1024);
+
     FileFragment[] frags = FileTablespace.splitNG(conf, "default.employee", employee.getMeta(),
         new Path(employee.getUri()), Integer.MAX_VALUE);
     Path workDir = new Path(testDir, TestExternalSortExec.class.getName());
-    TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
+    TaskAttemptContext ctx = new TaskAttemptContext(queryContext,
         LocalTajoTestingUtility.newTaskAttemptId(), new FileFragment[] { frags[0] }, workDir);
     ctx.setEnforcer(new Enforcer());
     Expr expr = analyzer.parse(QUERIES[0]);
@@ -140,10 +146,7 @@ public class TestExternalSortExec {
 
       extSort = new ExternalSortExec(ctx, ((MemSortExec)sortExec).getPlan(), scan);
       proj.setChild(extSort);
-    } else {
-      extSort = proj.getChild();
     }
-    extSort.setSortBufferBytesNum(1024*1024);
 
     Tuple tuple;
     Tuple preVal = null;
@@ -183,5 +186,6 @@ public class TestExternalSortExec {
     assertEquals(numTuple, cnt);
     exec.close();
     System.out.println("Sort Time: " + (end - start) + " msc");
+    conf.setIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT, ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT.defaultIntVal);
   }
 }

http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
index 69b36c5..67f1cba 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
@@ -191,7 +191,7 @@ public class TestPhysicalPlanner {
 
     VTuple tuple = new VTuple(scoreSchmea.size());
     int m = 0;
-    for (int i = 1; i <= 40000; i++) {
+    for (int i = 1; i <= 30000; i++) {
       for (int k = 3; k < 5; k++) { // |{3,4}| = 2
         for (int j = 1; j <= 3; j++) { // |{1,2,3}| = 3
           tuple.put(

http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
index d79d292..07424a3 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
@@ -21,6 +21,7 @@ package org.apache.tajo.engine.planner.physical;
 
 import org.apache.hadoop.fs.Path;
 import org.apache.tajo.LocalTajoTestingUtility;
+import org.apache.tajo.SessionVars;
 import org.apache.tajo.TajoConstants;
 import org.apache.tajo.TajoTestingCluster;
 import org.apache.tajo.algebra.Expr;
@@ -28,6 +29,7 @@ import org.apache.tajo.catalog.*;
 import org.apache.tajo.catalog.statistics.TableStats;
 import org.apache.tajo.common.TajoDataTypes;
 import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.conf.TajoConf.ConfVars;
 import org.apache.tajo.datum.Datum;
 import org.apache.tajo.datum.DatumFactory;
 import org.apache.tajo.engine.parser.SQLAnalyzer;
@@ -63,7 +65,7 @@ public class TestProgressExternalSortExec {
   private LogicalPlanner planner;
   private Path testDir;
 
-  private final int numTuple = 100000;
+  private final int numTuple = 5000;
   private Random rnd = new Random(System.currentTimeMillis());
 
   private TableDesc employee;
@@ -123,19 +125,23 @@ public class TestProgressExternalSortExec {
 
   @Test
   public void testExternalSortExecProgressWithMemTableScanner() throws Exception {
-    testProgress(testDataStats.getNumBytes().intValue() * 20);    //multiply 20 for memory
fit
+    testProgress(testDataStats.getNumBytes() * 20);    //multiply 20 for memory fit
   }
 
   @Test
   public void testExternalSortExecProgressWithPairWiseMerger() throws Exception {
-    testProgress(testDataStats.getNumBytes().intValue());
+    testProgress(testDataStats.getNumBytes());
   }
 
-  private void testProgress(int sortBufferBytesNum) throws Exception {
+  private void testProgress(long sortBufferBytesNum) throws Exception {
+    conf.setIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT, 2);
+    QueryContext queryContext = LocalTajoTestingUtility.createDummyContext(conf);
+    queryContext.setLong(SessionVars.EXTSORT_BUFFER_SIZE, sortBufferBytesNum);
+
     FileFragment[] frags = FileTablespace.splitNG(conf, "default.employee", employee.getMeta(),
         new Path(employee.getUri()), Integer.MAX_VALUE);
     Path workDir = new Path(testDir, TestExternalSortExec.class.getName());
-    TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
+    TaskAttemptContext ctx = new TaskAttemptContext(queryContext,
         LocalTajoTestingUtility.newTaskAttemptId(), new FileFragment[] { frags[0] }, workDir);
     ctx.setEnforcer(new Enforcer());
     Expr expr = analyzer.parse(QUERIES[0]);
@@ -153,11 +159,7 @@ public class TestProgressExternalSortExec {
       SeqScanExec scan = sortExec.getChild();
 
       ExternalSortExec extSort = new ExternalSortExec(ctx, ((MemSortExec)sortExec).getPlan(),
scan);
-
-      extSort.setSortBufferBytesNum(sortBufferBytesNum);
       proj.setChild(extSort);
-    } else {
-      ((ExternalSortExec)proj.getChild()).setSortBufferBytesNum(sortBufferBytesNum);
     }
 
     Tuple tuple;
@@ -175,12 +177,13 @@ public class TestProgressExternalSortExec {
     while ((tuple = exec.next()) != null) {
       if (cnt == 0) {
         initProgress = exec.getProgress();
+        System.out.println(initProgress);
         assertTrue(initProgress > 0.5f && initProgress < 1.0f);
       }
 
       if (cnt == testDataStats.getNumRows() / 2) {
         float progress = exec.getProgress();
-
+        System.out.println(progress);
         assertTrue(progress > initProgress);
       }
       curVal = tuple;
@@ -225,5 +228,7 @@ public class TestProgressExternalSortExec {
     assertEquals(cnt, testDataStats.getNumRows().longValue());
     assertEquals(cnt, tableStats.getNumRows().longValue());
     assertEquals(testDataStats.getNumBytes().longValue(), tableStats.getReadBytes().longValue());
+
+    conf.setIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT, ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT.defaultIntVal);
   }
 }

http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestTupleSorter.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestTupleSorter.java
b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestTupleSorter.java
index 4061b25..753eebc 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestTupleSorter.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestTupleSorter.java
@@ -47,8 +47,8 @@ public class TestTupleSorter {
   public final void testSortBench() {
     final int MAX_SORT_KEY = 3;
     final int ITERATION = 10;
-    final int LENGTH = 1000000;
-    final int SAMPLING = 100;
+    final int LENGTH = 5000;
+    final int SAMPLING = 10;
 
     Tuple[] tuples = new Tuple[LENGTH];
     for (int i = 0; i < LENGTH; i++) {

http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
index 3ce7b29..36c2568 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
@@ -19,13 +19,8 @@
 package org.apache.tajo.engine.query;
 
 import com.google.common.collect.Lists;
-import org.apache.tajo.IntegrationTest;
-import org.apache.tajo.QueryId;
-import org.apache.tajo.QueryTestCaseBase;
-import org.apache.tajo.SessionVars;
-import org.apache.tajo.TajoConstants;
+import org.apache.tajo.*;
 import org.apache.tajo.TajoProtos.QueryState;
-import org.apache.tajo.TajoTestingCluster;
 import org.apache.tajo.catalog.CatalogService;
 import org.apache.tajo.catalog.Schema;
 import org.apache.tajo.catalog.TableDesc;

http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-core/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-core/pom.xml b/tajo-core/pom.xml
index 7a817ab..7a66ed0 100644
--- a/tajo-core/pom.xml
+++ b/tajo-core/pom.xml
@@ -67,7 +67,6 @@
         </executions>
         <configuration>
           <excludes>
-            <exclude>derby.log</exclude>
             <exclude>benchmark/**</exclude>
             <exclude>src/main/resources/META-INF/services/*</exclude>
             <exclude>src/main/resources/webapps/static/js/*</exclude>

http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-core/src/main/java/org/apache/tajo/benchmark/BenchmarkSet.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/benchmark/BenchmarkSet.java b/tajo-core/src/main/java/org/apache/tajo/benchmark/BenchmarkSet.java
index 90c95a1..d2f56d3 100644
--- a/tajo-core/src/main/java/org/apache/tajo/benchmark/BenchmarkSet.java
+++ b/tajo-core/src/main/java/org/apache/tajo/benchmark/BenchmarkSet.java
@@ -18,11 +18,10 @@
 
 package org.apache.tajo.benchmark;
 
-import com.google.protobuf.ServiceException;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.tajo.catalog.CatalogConstants;
 import org.apache.tajo.catalog.Schema;
-import org.apache.tajo.catalog.store.MemStore;
+import org.apache.tajo.catalog.store.DerbyStore;
 import org.apache.tajo.client.DummyServiceTracker;
 import org.apache.tajo.client.TajoClient;
 import org.apache.tajo.client.TajoClientImpl;
@@ -58,7 +57,7 @@ public abstract class BenchmarkSet {
       tajo = new TajoClientImpl(serviceTracker, null);
 
     } else {
-      conf.set(CatalogConstants.STORE_CLASS, MemStore.class.getCanonicalName());
+      conf.set(CatalogConstants.STORE_CLASS, DerbyStore.class.getCanonicalName());
       tajo = new TajoClientImpl(ServiceTrackerFactory.get(conf));
     }
   }

http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
index b04bdc4..9847ff6 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
@@ -36,13 +36,16 @@ import org.apache.tajo.engine.planner.global.DataChannel;
 import org.apache.tajo.engine.planner.physical.*;
 import org.apache.tajo.engine.query.QueryContext;
 import org.apache.tajo.exception.TajoInternalError;
+import org.apache.tajo.plan.LogicalPlan;
+import org.apache.tajo.plan.logical.*;
+import org.apache.tajo.plan.serder.LogicalNodeDeserializer;
 import org.apache.tajo.plan.serder.PlanProto.DistinctGroupbyEnforcer;
 import org.apache.tajo.plan.serder.PlanProto.DistinctGroupbyEnforcer.DistinctAggregationAlgorithm;
 import org.apache.tajo.plan.serder.PlanProto.DistinctGroupbyEnforcer.MultipleAggregationStage;
 import org.apache.tajo.plan.serder.PlanProto.DistinctGroupbyEnforcer.SortSpecArray;
-import org.apache.tajo.plan.LogicalPlan;
-import org.apache.tajo.plan.logical.*;
-import org.apache.tajo.plan.serder.LogicalNodeDeserializer;
+import org.apache.tajo.plan.serder.PlanProto.EnforceProperty;
+import org.apache.tajo.plan.serder.PlanProto.SortEnforce;
+import org.apache.tajo.plan.serder.PlanProto.SortedInputEnforce;
 import org.apache.tajo.plan.util.PlannerUtil;
 import org.apache.tajo.storage.FileTablespace;
 import org.apache.tajo.storage.StorageConstants;
@@ -63,12 +66,9 @@ import java.util.Stack;
 import static org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto;
 import static org.apache.tajo.catalog.proto.CatalogProtos.PartitionType;
 import static org.apache.tajo.plan.serder.PlanProto.ColumnPartitionEnforcer.ColumnPartitionAlgorithm;
-import static org.apache.tajo.plan.serder.PlanProto.EnforceProperty;
 import static org.apache.tajo.plan.serder.PlanProto.EnforceProperty.EnforceType;
 import static org.apache.tajo.plan.serder.PlanProto.GroupbyEnforce.GroupbyAlgorithm;
 import static org.apache.tajo.plan.serder.PlanProto.JoinEnforce.JoinAlgorithm;
-import static org.apache.tajo.plan.serder.PlanProto.SortedInputEnforce;
-import static org.apache.tajo.plan.serder.PlanProto.SortEnforce;
 
 public class PhysicalPlannerImpl implements PhysicalPlanner {
   private static final Log LOG = LogFactory.getLog(PhysicalPlannerImpl.class);
@@ -611,7 +611,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
       selectedLeft = rightExec;
       selectedRight = leftExec;
     }
-    LOG.info("Full Outer Join (" + plan.getPID() +") chooses [Hash Join]");
+    LOG.info("Full Outer Join (" + plan.getPID() + ") chooses [Hash Join]");
     return new HashFullOuterJoinExec(context, plan, selectedRight, selectedLeft);
   }
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
index adbafd9..4a9b491 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
@@ -143,11 +143,6 @@ public class ExternalSortExec extends SortExec {
     setChild(child);
   }
 
-  @VisibleForTesting
-  public void setSortBufferBytesNum(int sortBufferBytesNum) {
-    this.sortBufferBytesNum = sortBufferBytesNum;
-  }
-
   public void init() throws IOException {
     inputStats = new TableStats();
     super.init();

http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
index 3ec09aa..dfe8909 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.service.CompositeService;
 import org.apache.hadoop.yarn.event.AsyncDispatcher;
 import org.apache.hadoop.yarn.event.EventHandler;
@@ -38,10 +37,12 @@ import org.apache.tajo.catalog.CatalogServer;
 import org.apache.tajo.catalog.CatalogService;
 import org.apache.tajo.catalog.FunctionDesc;
 import org.apache.tajo.catalog.LocalCatalogWrapper;
-import org.apache.tajo.exception.DuplicateDatabaseException;
+import org.apache.tajo.catalog.store.AbstractDBStore;
+import org.apache.tajo.catalog.store.DerbyStore;
 import org.apache.tajo.conf.TajoConf;
 import org.apache.tajo.conf.TajoConf.ConfVars;
 import org.apache.tajo.engine.function.FunctionLoader;
+import org.apache.tajo.exception.DuplicateDatabaseException;
 import org.apache.tajo.exception.DuplicateTablespaceException;
 import org.apache.tajo.function.FunctionSignature;
 import org.apache.tajo.master.rm.TajoResourceManager;
@@ -396,8 +397,6 @@ public class TajoMaster extends CompositeService {
 
     if (webServer != null) webServer.stop();
 
-    IOUtils.cleanup(LOG, catalogServer);
-
     if (systemMetrics != null) systemMetrics.stop();
 
     if (pauseMonitor != null) pauseMonitor.stop();
@@ -539,6 +538,12 @@ public class TajoMaster extends CompositeService {
         LOG.info("TajoMaster received SIGINT Signal");
         LOG.info("============================================");
         stop();
+
+        // If embedded derby is used as catalog, shutdown it.
+        if (catalogServer.getStoreClassName().equals("org.apache.tajo.catalog.store.DerbyStore")
+            && AbstractDBStore.needShutdown(catalogServer.getStoreUri())) {
+          DerbyStore.shutdown();
+        }
         RpcChannelFactory.shutdownGracefully();
       }
     }

http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
index 09bef44..e3d91bb 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
@@ -439,10 +439,11 @@ public class DDLExecutor {
         throw new DuplicateTableException(alterTable.getNewTableName());
       }
 
+      Path newPath = null;
       if (!desc.isExternal()) { // if the table is the managed table
         Path oldPath = StorageUtil.concatPath(context.getConf().getVar(TajoConf.ConfVars.WAREHOUSE_DIR),
             databaseName, simpleTableName);
-        Path newPath = StorageUtil.concatPath(context.getConf().getVar(TajoConf.ConfVars.WAREHOUSE_DIR),
+        newPath = StorageUtil.concatPath(context.getConf().getVar(TajoConf.ConfVars.WAREHOUSE_DIR),
             databaseName, alterTable.getNewTableName());
         FileSystem fs = oldPath.getFileSystem(context.getConf());
 
@@ -456,7 +457,7 @@ public class DDLExecutor {
         fs.rename(oldPath, newPath);
       }
       catalog.alterTable(CatalogUtil.renameTable(qualifiedName, alterTable.getNewTableName(),
-          AlterTableType.RENAME_TABLE));
+          AlterTableType.RENAME_TABLE, newPath));
       break;
     case RENAME_COLUMN:
       if (ensureColumnExistance(qualifiedName, alterTable.getNewColumnName())) {

http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-jdbc/pom.xml b/tajo-jdbc/pom.xml
index f73a4bd..27cc471 100644
--- a/tajo-jdbc/pom.xml
+++ b/tajo-jdbc/pom.xml
@@ -87,6 +87,16 @@
           </execution>
         </executions>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <configuration>
+          <systemProperties>
+            <tajo.test>TRUE</tajo.test>
+          </systemProperties>
+          <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024
-Dderby.stream.error.file=/dev/null</argLine>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-metrics/src/main/proto/master_metrics.proto
----------------------------------------------------------------------
diff --git a/tajo-metrics/src/main/proto/master_metrics.proto b/tajo-metrics/src/main/proto/master_metrics.proto
index e3dbd12..4723ce7 100644
--- a/tajo-metrics/src/main/proto/master_metrics.proto
+++ b/tajo-metrics/src/main/proto/master_metrics.proto
@@ -49,9 +49,9 @@ enum Query {
   FAILED            = 5;
   KILLED            = 6;
 
-  NUM_PER_HOUR      = 6;
-  MAX_IO_THROUGHPUT = 7;
-  AVG_IO_THROUGHPUT = 8;
+  NUM_PER_HOUR      = 7;
+  MAX_IO_THROUGHPUT = 8;
+  AVG_IO_THROUGHPUT = 9;
 
-  PENDING_QUERIES   = 9;
+  PENDING_QUERIES   = 10;
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
index f70731f..6d40cda 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
@@ -35,18 +35,13 @@ import org.apache.tajo.SessionVars;
 import org.apache.tajo.algebra.*;
 import org.apache.tajo.algebra.WindowSpec;
 import org.apache.tajo.catalog.*;
-import org.apache.tajo.exception.UndefinedColumnException;
-import org.apache.tajo.exception.UndefinedTableException;
+import org.apache.tajo.exception.*;
 import org.apache.tajo.catalog.partition.PartitionMethodDesc;
 import org.apache.tajo.catalog.proto.CatalogProtos;
 import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod;
 import org.apache.tajo.common.TajoDataTypes;
 import org.apache.tajo.conf.TajoConf;
 import org.apache.tajo.datum.NullDatum;
-import org.apache.tajo.exception.ExceptionUtil;
-import org.apache.tajo.exception.TajoException;
-import org.apache.tajo.exception.TajoInternalError;
-import org.apache.tajo.exception.NotImplementedException;
 import org.apache.tajo.plan.LogicalPlan.QueryBlock;
 import org.apache.tajo.plan.algebra.BaseAlgebraVisitor;
 import org.apache.tajo.plan.expr.*;
@@ -2228,11 +2223,19 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
       block.namedExprsMgr.addNamedExprArray(normalizedExprList[i].scalarExprs);
     }
 
-    createIndexNode.setExternal(createIndex.isExternal());
     Collection<RelationNode> relations = block.getRelations();
-    assert relations.size() == 1;
-    createIndexNode.setKeySortSpecs(relations.iterator().next().getLogicalSchema(),
-        annotateSortSpecs(block, referNames, sortSpecs));
+    if (relations.size() > 1) {
+      throw new UnsupportedException("Index on multiple relations");
+    } else if (relations.size() == 0) {
+      throw new TajoInternalError("No relation for indexing");
+    }
+    RelationNode relationNode = relations.iterator().next();
+    if (!(relationNode instanceof ScanNode)) {
+      throw new UnsupportedException("Index on subquery");
+    }
+
+    createIndexNode.setExternal(createIndex.isExternal());
+    createIndexNode.setKeySortSpecs(relationNode.getLogicalSchema(), annotateSortSpecs(block,
referNames, sortSpecs));
     createIndexNode.setIndexMethod(IndexMethod.valueOf(createIndex.getMethodSpec().getName().toUpperCase()));
     if (createIndex.isExternal()) {
       createIndexNode.setIndexPath(new Path(createIndex.getIndexPath()).toUri());

http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathRewriter.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathRewriter.java
b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathRewriter.java
index 33ce4f4..8f3fbe7 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathRewriter.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathRewriter.java
@@ -114,7 +114,7 @@ public class AccessPathRewriter implements LogicalPlanRewriteRule {
 
       if (optimalPath != null && optimalPath.getScanType() == AccessPathInfo.ScanTypeControl.INDEX_SCAN)
{
         IndexScanInfo indexScanInfo = (IndexScanInfo) optimalPath;
-        plan.addHistory("AccessPathRewriter chooses the index scan for " + scanNode.getTableName());
+        plan.addHistory("AccessPathRewriter chooses index scan for " + scanNode.getTableName());
         IndexScanNode indexScanNode = new IndexScanNode(plan.newPID(), scanNode, indexScanInfo.getKeySchema(),
             indexScanInfo.getPredicates(), indexScanInfo.getIndexPath());
         if (stack.empty() || block.getRoot().equals(scanNode)) {

http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-project/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-project/pom.xml b/tajo-project/pom.xml
index f486d7f..7a5ed66 100644
--- a/tajo-project/pom.xml
+++ b/tajo-project/pom.xml
@@ -1111,7 +1111,7 @@
       <dependency>
         <groupId>org.apache.derby</groupId>
         <artifactId>derby</artifactId>
-        <version>10.8.2.2</version>
+        <version>10.11.1.1</version>
       </dependency>
       <dependency>
         <groupId>jline</groupId>

http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-storage/tajo-storage-common/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/pom.xml b/tajo-storage/tajo-storage-common/pom.xml
index 7590c29..afbeead 100644
--- a/tajo-storage/tajo-storage-common/pom.xml
+++ b/tajo-storage/tajo-storage-common/pom.xml
@@ -75,7 +75,7 @@ limitations under the License.
           <systemProperties>
             <tajo.test>TRUE</tajo.test>
           </systemProperties>
-          <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8</argLine>
+          <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024
-Dderby.stream.error.file=/dev/null</argLine>
         </configuration>
       </plugin>
       <plugin>

http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-storage/tajo-storage-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/pom.xml b/tajo-storage/tajo-storage-hbase/pom.xml
index 892ff58..e2d5132 100644
--- a/tajo-storage/tajo-storage-hbase/pom.xml
+++ b/tajo-storage/tajo-storage-hbase/pom.xml
@@ -78,7 +78,7 @@
           <systemProperties>
             <tajo.test>TRUE</tajo.test>
           </systemProperties>
-          <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8</argLine>
+          <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024
-Dderby.stream.error.file=/dev/null</argLine>
         </configuration>
       </plugin>
       <plugin>

http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-storage/tajo-storage-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/pom.xml b/tajo-storage/tajo-storage-hdfs/pom.xml
index b1b0ae3..fe0959f 100644
--- a/tajo-storage/tajo-storage-hdfs/pom.xml
+++ b/tajo-storage/tajo-storage-hdfs/pom.xml
@@ -79,7 +79,7 @@
           <systemProperties>
             <tajo.test>TRUE</tajo.test>
           </systemProperties>
-          <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8</argLine>
+          <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024
-Dderby.stream.error.file=/dev/null</argLine>
         </configuration>
       </plugin>
       <plugin>


Mime
View raw message