Repository: tajo
Updated Branches:
refs/heads/master 848a8c3d5 -> 77dc88c07
http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-common/src/main/proto/errors.proto
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/proto/errors.proto b/tajo-common/src/main/proto/errors.proto
index b947a95..21746f3 100644
--- a/tajo-common/src/main/proto/errors.proto
+++ b/tajo-common/src/main/proto/errors.proto
@@ -157,6 +157,9 @@ enum ResultCode {
CAT_UPGRADE_REQUIRED = 901; // Migration
CAT_CANNOT_CONNECT = 902; // Cannot connect metadata server
+ // Catalog store
+ CAT_UNSUPPORTED_CATALOG_STORE = 905; // Unsupported type of catalog store
+
// Linked Metadata (LMD)
LMD_NO_MATCHED_DATATYPE = 910; // No matched data type between Tajo and
connector
http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-core-tests/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-core-tests/pom.xml b/tajo-core-tests/pom.xml
index a4c0587..a23e420 100644
--- a/tajo-core-tests/pom.xml
+++ b/tajo-core-tests/pom.xml
@@ -55,7 +55,6 @@
</executions>
<configuration>
<excludes>
- <exclude>derby.log</exclude>
<exclude>benchmark/**</exclude>
<exclude>src/test/tpch/**</exclude>
<exclude>src/test/resources/dataset/**</exclude>
@@ -74,7 +73,7 @@
<systemProperties>
<tajo.test>TRUE</tajo.test>
</systemProperties>
- <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8</argLine>
+ <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024
-Dderby.stream.error.file=/dev/null</argLine>
</configuration>
</plugin>
<plugin>
@@ -345,7 +344,7 @@
<forkCount>${maven.fork.count}</forkCount>
<reuseForks>true</reuseForks>
<trimStackTrace>false</trimStackTrace>
- <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8</argLine>
+ <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8
-Dderby.storage.pageSize=1024 -Dderby.stream.error.file=/dev/null</argLine>
<useSystemClassLoader>true</useSystemClassLoader>
<useManifestOnlyJar>true</useManifestOnlyJar>
<systemProperties>
http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
index 1705d2b..e835481 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
@@ -90,8 +90,7 @@ public class ExprTestBase {
public static void setUp() throws Exception {
util = new TajoTestingCluster();
conf = util.getConfiguration();
- util.startCatalogCluster();
- cat = util.getMiniCatalogCluster().getCatalog();
+ cat = util.startCatalogCluster().getCatalog();
cat.createTablespace(DEFAULT_TABLESPACE_NAME, "hdfs://localhost:1234/warehouse");
cat.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
Map<FunctionSignature, FunctionDesc> map = FunctionLoader.load();
http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
index 02e921a..5382c49 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
@@ -77,8 +77,7 @@ public class TestLogicalPlanner {
@BeforeClass
public static void setUp() throws Exception {
util = new TajoTestingCluster();
- util.startCatalogCluster();
- catalog = util.getMiniCatalogCluster().getCatalog();
+ catalog = util.startCatalogCluster().getCatalog();
catalog.createTablespace(DEFAULT_TABLESPACE_NAME, "hdfs://localhost:1234");
catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
index 4ae880f..2a6722e 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
@@ -20,12 +20,14 @@ package org.apache.tajo.engine.planner.physical;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.LocalTajoTestingUtility;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.TajoConstants;
import org.apache.tajo.TajoTestingCluster;
import org.apache.tajo.algebra.Expr;
import org.apache.tajo.catalog.*;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.engine.parser.SQLAnalyzer;
@@ -61,7 +63,7 @@ public class TestExternalSortExec {
private LogicalPlanner planner;
private Path testDir;
- private final int numTuple = 100000;
+ private final int numTuple = 1000;
private Random rnd = new Random(System.currentTimeMillis());
private TableDesc employee;
@@ -117,10 +119,14 @@ public class TestExternalSortExec {
@Test
public final void testNext() throws IOException, TajoException {
+ conf.setIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT, 2);
+ QueryContext queryContext = LocalTajoTestingUtility.createDummyContext(conf);
+ queryContext.setLong(SessionVars.EXTSORT_BUFFER_SIZE, 1024*1024);
+
FileFragment[] frags = FileTablespace.splitNG(conf, "default.employee", employee.getMeta(),
new Path(employee.getUri()), Integer.MAX_VALUE);
Path workDir = new Path(testDir, TestExternalSortExec.class.getName());
- TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
+ TaskAttemptContext ctx = new TaskAttemptContext(queryContext,
LocalTajoTestingUtility.newTaskAttemptId(), new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[0]);
@@ -140,10 +146,7 @@ public class TestExternalSortExec {
extSort = new ExternalSortExec(ctx, ((MemSortExec)sortExec).getPlan(), scan);
proj.setChild(extSort);
- } else {
- extSort = proj.getChild();
}
- extSort.setSortBufferBytesNum(1024*1024);
Tuple tuple;
Tuple preVal = null;
@@ -183,5 +186,6 @@ public class TestExternalSortExec {
assertEquals(numTuple, cnt);
exec.close();
System.out.println("Sort Time: " + (end - start) + " msc");
+ conf.setIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT, ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT.defaultIntVal);
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
index 69b36c5..67f1cba 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
@@ -191,7 +191,7 @@ public class TestPhysicalPlanner {
VTuple tuple = new VTuple(scoreSchmea.size());
int m = 0;
- for (int i = 1; i <= 40000; i++) {
+ for (int i = 1; i <= 30000; i++) {
for (int k = 3; k < 5; k++) { // |{3,4}| = 2
for (int j = 1; j <= 3; j++) { // |{1,2,3}| = 3
tuple.put(
http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
index d79d292..07424a3 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
@@ -21,6 +21,7 @@ package org.apache.tajo.engine.planner.physical;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.LocalTajoTestingUtility;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.TajoConstants;
import org.apache.tajo.TajoTestingCluster;
import org.apache.tajo.algebra.Expr;
@@ -28,6 +29,7 @@ import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.engine.parser.SQLAnalyzer;
@@ -63,7 +65,7 @@ public class TestProgressExternalSortExec {
private LogicalPlanner planner;
private Path testDir;
- private final int numTuple = 100000;
+ private final int numTuple = 5000;
private Random rnd = new Random(System.currentTimeMillis());
private TableDesc employee;
@@ -123,19 +125,23 @@ public class TestProgressExternalSortExec {
@Test
public void testExternalSortExecProgressWithMemTableScanner() throws Exception {
- testProgress(testDataStats.getNumBytes().intValue() * 20); //multiply 20 for memory
fit
+ testProgress(testDataStats.getNumBytes() * 20); //multiply 20 for memory fit
}
@Test
public void testExternalSortExecProgressWithPairWiseMerger() throws Exception {
- testProgress(testDataStats.getNumBytes().intValue());
+ testProgress(testDataStats.getNumBytes());
}
- private void testProgress(int sortBufferBytesNum) throws Exception {
+ private void testProgress(long sortBufferBytesNum) throws Exception {
+ conf.setIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT, 2);
+ QueryContext queryContext = LocalTajoTestingUtility.createDummyContext(conf);
+ queryContext.setLong(SessionVars.EXTSORT_BUFFER_SIZE, sortBufferBytesNum);
+
FileFragment[] frags = FileTablespace.splitNG(conf, "default.employee", employee.getMeta(),
new Path(employee.getUri()), Integer.MAX_VALUE);
Path workDir = new Path(testDir, TestExternalSortExec.class.getName());
- TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
+ TaskAttemptContext ctx = new TaskAttemptContext(queryContext,
LocalTajoTestingUtility.newTaskAttemptId(), new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[0]);
@@ -153,11 +159,7 @@ public class TestProgressExternalSortExec {
SeqScanExec scan = sortExec.getChild();
ExternalSortExec extSort = new ExternalSortExec(ctx, ((MemSortExec)sortExec).getPlan(),
scan);
-
- extSort.setSortBufferBytesNum(sortBufferBytesNum);
proj.setChild(extSort);
- } else {
- ((ExternalSortExec)proj.getChild()).setSortBufferBytesNum(sortBufferBytesNum);
}
Tuple tuple;
@@ -175,12 +177,13 @@ public class TestProgressExternalSortExec {
while ((tuple = exec.next()) != null) {
if (cnt == 0) {
initProgress = exec.getProgress();
+ System.out.println(initProgress);
assertTrue(initProgress > 0.5f && initProgress < 1.0f);
}
if (cnt == testDataStats.getNumRows() / 2) {
float progress = exec.getProgress();
-
+ System.out.println(progress);
assertTrue(progress > initProgress);
}
curVal = tuple;
@@ -225,5 +228,7 @@ public class TestProgressExternalSortExec {
assertEquals(cnt, testDataStats.getNumRows().longValue());
assertEquals(cnt, tableStats.getNumRows().longValue());
assertEquals(testDataStats.getNumBytes().longValue(), tableStats.getReadBytes().longValue());
+
+ conf.setIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT, ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT.defaultIntVal);
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestTupleSorter.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestTupleSorter.java
b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestTupleSorter.java
index 4061b25..753eebc 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestTupleSorter.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestTupleSorter.java
@@ -47,8 +47,8 @@ public class TestTupleSorter {
public final void testSortBench() {
final int MAX_SORT_KEY = 3;
final int ITERATION = 10;
- final int LENGTH = 1000000;
- final int SAMPLING = 100;
+ final int LENGTH = 5000;
+ final int SAMPLING = 10;
Tuple[] tuples = new Tuple[LENGTH];
for (int i = 0; i < LENGTH; i++) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
index 3ce7b29..36c2568 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
@@ -19,13 +19,8 @@
package org.apache.tajo.engine.query;
import com.google.common.collect.Lists;
-import org.apache.tajo.IntegrationTest;
-import org.apache.tajo.QueryId;
-import org.apache.tajo.QueryTestCaseBase;
-import org.apache.tajo.SessionVars;
-import org.apache.tajo.TajoConstants;
+import org.apache.tajo.*;
import org.apache.tajo.TajoProtos.QueryState;
-import org.apache.tajo.TajoTestingCluster;
import org.apache.tajo.catalog.CatalogService;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableDesc;
http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-core/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-core/pom.xml b/tajo-core/pom.xml
index 7a817ab..7a66ed0 100644
--- a/tajo-core/pom.xml
+++ b/tajo-core/pom.xml
@@ -67,7 +67,6 @@
</executions>
<configuration>
<excludes>
- <exclude>derby.log</exclude>
<exclude>benchmark/**</exclude>
<exclude>src/main/resources/META-INF/services/*</exclude>
<exclude>src/main/resources/webapps/static/js/*</exclude>
http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-core/src/main/java/org/apache/tajo/benchmark/BenchmarkSet.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/benchmark/BenchmarkSet.java b/tajo-core/src/main/java/org/apache/tajo/benchmark/BenchmarkSet.java
index 90c95a1..d2f56d3 100644
--- a/tajo-core/src/main/java/org/apache/tajo/benchmark/BenchmarkSet.java
+++ b/tajo-core/src/main/java/org/apache/tajo/benchmark/BenchmarkSet.java
@@ -18,11 +18,10 @@
package org.apache.tajo.benchmark;
-import com.google.protobuf.ServiceException;
import org.apache.hadoop.net.NetUtils;
import org.apache.tajo.catalog.CatalogConstants;
import org.apache.tajo.catalog.Schema;
-import org.apache.tajo.catalog.store.MemStore;
+import org.apache.tajo.catalog.store.DerbyStore;
import org.apache.tajo.client.DummyServiceTracker;
import org.apache.tajo.client.TajoClient;
import org.apache.tajo.client.TajoClientImpl;
@@ -58,7 +57,7 @@ public abstract class BenchmarkSet {
tajo = new TajoClientImpl(serviceTracker, null);
} else {
- conf.set(CatalogConstants.STORE_CLASS, MemStore.class.getCanonicalName());
+ conf.set(CatalogConstants.STORE_CLASS, DerbyStore.class.getCanonicalName());
tajo = new TajoClientImpl(ServiceTrackerFactory.get(conf));
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
index b04bdc4..9847ff6 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
@@ -36,13 +36,16 @@ import org.apache.tajo.engine.planner.global.DataChannel;
import org.apache.tajo.engine.planner.physical.*;
import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.exception.TajoInternalError;
+import org.apache.tajo.plan.LogicalPlan;
+import org.apache.tajo.plan.logical.*;
+import org.apache.tajo.plan.serder.LogicalNodeDeserializer;
import org.apache.tajo.plan.serder.PlanProto.DistinctGroupbyEnforcer;
import org.apache.tajo.plan.serder.PlanProto.DistinctGroupbyEnforcer.DistinctAggregationAlgorithm;
import org.apache.tajo.plan.serder.PlanProto.DistinctGroupbyEnforcer.MultipleAggregationStage;
import org.apache.tajo.plan.serder.PlanProto.DistinctGroupbyEnforcer.SortSpecArray;
-import org.apache.tajo.plan.LogicalPlan;
-import org.apache.tajo.plan.logical.*;
-import org.apache.tajo.plan.serder.LogicalNodeDeserializer;
+import org.apache.tajo.plan.serder.PlanProto.EnforceProperty;
+import org.apache.tajo.plan.serder.PlanProto.SortEnforce;
+import org.apache.tajo.plan.serder.PlanProto.SortedInputEnforce;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.storage.FileTablespace;
import org.apache.tajo.storage.StorageConstants;
@@ -63,12 +66,9 @@ import java.util.Stack;
import static org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto;
import static org.apache.tajo.catalog.proto.CatalogProtos.PartitionType;
import static org.apache.tajo.plan.serder.PlanProto.ColumnPartitionEnforcer.ColumnPartitionAlgorithm;
-import static org.apache.tajo.plan.serder.PlanProto.EnforceProperty;
import static org.apache.tajo.plan.serder.PlanProto.EnforceProperty.EnforceType;
import static org.apache.tajo.plan.serder.PlanProto.GroupbyEnforce.GroupbyAlgorithm;
import static org.apache.tajo.plan.serder.PlanProto.JoinEnforce.JoinAlgorithm;
-import static org.apache.tajo.plan.serder.PlanProto.SortedInputEnforce;
-import static org.apache.tajo.plan.serder.PlanProto.SortEnforce;
public class PhysicalPlannerImpl implements PhysicalPlanner {
private static final Log LOG = LogFactory.getLog(PhysicalPlannerImpl.class);
@@ -611,7 +611,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
selectedLeft = rightExec;
selectedRight = leftExec;
}
- LOG.info("Full Outer Join (" + plan.getPID() +") chooses [Hash Join]");
+ LOG.info("Full Outer Join (" + plan.getPID() + ") chooses [Hash Join]");
return new HashFullOuterJoinExec(context, plan, selectedRight, selectedLeft);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
index adbafd9..4a9b491 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
@@ -143,11 +143,6 @@ public class ExternalSortExec extends SortExec {
setChild(child);
}
- @VisibleForTesting
- public void setSortBufferBytesNum(int sortBufferBytesNum) {
- this.sortBufferBytesNum = sortBufferBytesNum;
- }
-
public void init() throws IOException {
inputStats = new TableStats();
super.init();
http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
index 3ec09aa..dfe8909 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -38,10 +37,12 @@ import org.apache.tajo.catalog.CatalogServer;
import org.apache.tajo.catalog.CatalogService;
import org.apache.tajo.catalog.FunctionDesc;
import org.apache.tajo.catalog.LocalCatalogWrapper;
-import org.apache.tajo.exception.DuplicateDatabaseException;
+import org.apache.tajo.catalog.store.AbstractDBStore;
+import org.apache.tajo.catalog.store.DerbyStore;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.engine.function.FunctionLoader;
+import org.apache.tajo.exception.DuplicateDatabaseException;
import org.apache.tajo.exception.DuplicateTablespaceException;
import org.apache.tajo.function.FunctionSignature;
import org.apache.tajo.master.rm.TajoResourceManager;
@@ -396,8 +397,6 @@ public class TajoMaster extends CompositeService {
if (webServer != null) webServer.stop();
- IOUtils.cleanup(LOG, catalogServer);
-
if (systemMetrics != null) systemMetrics.stop();
if (pauseMonitor != null) pauseMonitor.stop();
@@ -539,6 +538,12 @@ public class TajoMaster extends CompositeService {
LOG.info("TajoMaster received SIGINT Signal");
LOG.info("============================================");
stop();
+
+ // If embedded derby is used as catalog, shutdown it.
+ if (catalogServer.getStoreClassName().equals("org.apache.tajo.catalog.store.DerbyStore")
+ && AbstractDBStore.needShutdown(catalogServer.getStoreUri())) {
+ DerbyStore.shutdown();
+ }
RpcChannelFactory.shutdownGracefully();
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
index 09bef44..e3d91bb 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
@@ -439,10 +439,11 @@ public class DDLExecutor {
throw new DuplicateTableException(alterTable.getNewTableName());
}
+ Path newPath = null;
if (!desc.isExternal()) { // if the table is the managed table
Path oldPath = StorageUtil.concatPath(context.getConf().getVar(TajoConf.ConfVars.WAREHOUSE_DIR),
databaseName, simpleTableName);
- Path newPath = StorageUtil.concatPath(context.getConf().getVar(TajoConf.ConfVars.WAREHOUSE_DIR),
+ newPath = StorageUtil.concatPath(context.getConf().getVar(TajoConf.ConfVars.WAREHOUSE_DIR),
databaseName, alterTable.getNewTableName());
FileSystem fs = oldPath.getFileSystem(context.getConf());
@@ -456,7 +457,7 @@ public class DDLExecutor {
fs.rename(oldPath, newPath);
}
catalog.alterTable(CatalogUtil.renameTable(qualifiedName, alterTable.getNewTableName(),
- AlterTableType.RENAME_TABLE));
+ AlterTableType.RENAME_TABLE, newPath));
break;
case RENAME_COLUMN:
if (ensureColumnExistance(qualifiedName, alterTable.getNewColumnName())) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-jdbc/pom.xml b/tajo-jdbc/pom.xml
index f73a4bd..27cc471 100644
--- a/tajo-jdbc/pom.xml
+++ b/tajo-jdbc/pom.xml
@@ -87,6 +87,16 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <systemProperties>
+ <tajo.test>TRUE</tajo.test>
+ </systemProperties>
+ <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024
-Dderby.stream.error.file=/dev/null</argLine>
+ </configuration>
+ </plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-metrics/src/main/proto/master_metrics.proto
----------------------------------------------------------------------
diff --git a/tajo-metrics/src/main/proto/master_metrics.proto b/tajo-metrics/src/main/proto/master_metrics.proto
index e3dbd12..4723ce7 100644
--- a/tajo-metrics/src/main/proto/master_metrics.proto
+++ b/tajo-metrics/src/main/proto/master_metrics.proto
@@ -49,9 +49,9 @@ enum Query {
FAILED = 5;
KILLED = 6;
- NUM_PER_HOUR = 6;
- MAX_IO_THROUGHPUT = 7;
- AVG_IO_THROUGHPUT = 8;
+ NUM_PER_HOUR = 7;
+ MAX_IO_THROUGHPUT = 8;
+ AVG_IO_THROUGHPUT = 9;
- PENDING_QUERIES = 9;
+ PENDING_QUERIES = 10;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
index f70731f..6d40cda 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
@@ -35,18 +35,13 @@ import org.apache.tajo.SessionVars;
import org.apache.tajo.algebra.*;
import org.apache.tajo.algebra.WindowSpec;
import org.apache.tajo.catalog.*;
-import org.apache.tajo.exception.UndefinedColumnException;
-import org.apache.tajo.exception.UndefinedTableException;
+import org.apache.tajo.exception.*;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.datum.NullDatum;
-import org.apache.tajo.exception.ExceptionUtil;
-import org.apache.tajo.exception.TajoException;
-import org.apache.tajo.exception.TajoInternalError;
-import org.apache.tajo.exception.NotImplementedException;
import org.apache.tajo.plan.LogicalPlan.QueryBlock;
import org.apache.tajo.plan.algebra.BaseAlgebraVisitor;
import org.apache.tajo.plan.expr.*;
@@ -2228,11 +2223,19 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
block.namedExprsMgr.addNamedExprArray(normalizedExprList[i].scalarExprs);
}
- createIndexNode.setExternal(createIndex.isExternal());
Collection<RelationNode> relations = block.getRelations();
- assert relations.size() == 1;
- createIndexNode.setKeySortSpecs(relations.iterator().next().getLogicalSchema(),
- annotateSortSpecs(block, referNames, sortSpecs));
+ if (relations.size() > 1) {
+ throw new UnsupportedException("Index on multiple relations");
+ } else if (relations.size() == 0) {
+ throw new TajoInternalError("No relation for indexing");
+ }
+ RelationNode relationNode = relations.iterator().next();
+ if (!(relationNode instanceof ScanNode)) {
+ throw new UnsupportedException("Index on subquery");
+ }
+
+ createIndexNode.setExternal(createIndex.isExternal());
+ createIndexNode.setKeySortSpecs(relationNode.getLogicalSchema(), annotateSortSpecs(block,
referNames, sortSpecs));
createIndexNode.setIndexMethod(IndexMethod.valueOf(createIndex.getMethodSpec().getName().toUpperCase()));
if (createIndex.isExternal()) {
createIndexNode.setIndexPath(new Path(createIndex.getIndexPath()).toUri());
http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathRewriter.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathRewriter.java
b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathRewriter.java
index 33ce4f4..8f3fbe7 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathRewriter.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathRewriter.java
@@ -114,7 +114,7 @@ public class AccessPathRewriter implements LogicalPlanRewriteRule {
if (optimalPath != null && optimalPath.getScanType() == AccessPathInfo.ScanTypeControl.INDEX_SCAN)
{
IndexScanInfo indexScanInfo = (IndexScanInfo) optimalPath;
- plan.addHistory("AccessPathRewriter chooses the index scan for " + scanNode.getTableName());
+ plan.addHistory("AccessPathRewriter chooses index scan for " + scanNode.getTableName());
IndexScanNode indexScanNode = new IndexScanNode(plan.newPID(), scanNode, indexScanInfo.getKeySchema(),
indexScanInfo.getPredicates(), indexScanInfo.getIndexPath());
if (stack.empty() || block.getRoot().equals(scanNode)) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-project/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-project/pom.xml b/tajo-project/pom.xml
index f486d7f..7a5ed66 100644
--- a/tajo-project/pom.xml
+++ b/tajo-project/pom.xml
@@ -1111,7 +1111,7 @@
<dependency>
<groupId>org.apache.derby</groupId>
<artifactId>derby</artifactId>
- <version>10.8.2.2</version>
+ <version>10.11.1.1</version>
</dependency>
<dependency>
<groupId>jline</groupId>
http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-storage/tajo-storage-common/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/pom.xml b/tajo-storage/tajo-storage-common/pom.xml
index 7590c29..afbeead 100644
--- a/tajo-storage/tajo-storage-common/pom.xml
+++ b/tajo-storage/tajo-storage-common/pom.xml
@@ -75,7 +75,7 @@ limitations under the License.
<systemProperties>
<tajo.test>TRUE</tajo.test>
</systemProperties>
- <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8</argLine>
+ <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024
-Dderby.stream.error.file=/dev/null</argLine>
</configuration>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-storage/tajo-storage-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/pom.xml b/tajo-storage/tajo-storage-hbase/pom.xml
index 892ff58..e2d5132 100644
--- a/tajo-storage/tajo-storage-hbase/pom.xml
+++ b/tajo-storage/tajo-storage-hbase/pom.xml
@@ -78,7 +78,7 @@
<systemProperties>
<tajo.test>TRUE</tajo.test>
</systemProperties>
- <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8</argLine>
+ <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024
-Dderby.stream.error.file=/dev/null</argLine>
</configuration>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/tajo/blob/77dc88c0/tajo-storage/tajo-storage-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/pom.xml b/tajo-storage/tajo-storage-hdfs/pom.xml
index b1b0ae3..fe0959f 100644
--- a/tajo-storage/tajo-storage-hdfs/pom.xml
+++ b/tajo-storage/tajo-storage-hdfs/pom.xml
@@ -79,7 +79,7 @@
<systemProperties>
<tajo.test>TRUE</tajo.test>
</systemProperties>
- <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8</argLine>
+ <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8 -Dderby.storage.pageSize=1024
-Dderby.stream.error.file=/dev/null</argLine>
</configuration>
</plugin>
<plugin>
|