Repository: tajo
Updated Branches:
refs/heads/master 8c50410dc -> 3a30f45c7
http://git-wip-us.apache.org/repos/asf/tajo/blob/3a30f45c/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
index afe0f13..1051b3b 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Writable;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.QueryId;
import org.apache.tajo.TajoIdProtos;
import org.apache.tajo.catalog.CatalogUtil;
@@ -58,41 +59,41 @@ import static org.junit.Assert.assertTrue;
@RunWith(Parameterized.class)
public class TestStorages {
- private TajoConf conf;
- private static String TEST_PATH = "target/test-data/TestStorages";
+ private TajoConf conf;
+ private static String TEST_PATH = "target/test-data/TestStorages";
private static String TEST_PROJECTION_AVRO_SCHEMA =
"{\n" +
- " \"type\": \"record\",\n" +
- " \"namespace\": \"org.apache.tajo\",\n" +
- " \"name\": \"testProjection\",\n" +
- " \"fields\": [\n" +
- " { \"name\": \"id\", \"type\": \"int\" },\n" +
- " { \"name\": \"age\", \"type\": \"long\" },\n" +
- " { \"name\": \"score\", \"type\": \"float\" }\n" +
- " ]\n" +
- "}\n";
+ " \"type\": \"record\",\n" +
+ " \"namespace\": \"org.apache.tajo\",\n" +
+ " \"name\": \"testProjection\",\n" +
+ " \"fields\": [\n" +
+ " { \"name\": \"id\", \"type\": \"int\" },\n" +
+ " { \"name\": \"age\", \"type\": \"long\" },\n" +
+ " { \"name\": \"score\", \"type\": \"float\" }\n" +
+ " ]\n" +
+ "}\n";
private static String TEST_NULL_HANDLING_TYPES_AVRO_SCHEMA =
"{\n" +
- " \"type\": \"record\",\n" +
- " \"namespace\": \"org.apache.tajo\",\n" +
- " \"name\": \"testNullHandlingTypes\",\n" +
- " \"fields\": [\n" +
- " { \"name\": \"col1\", \"type\": [\"null\", \"boolean\"] },\n" +
- " { \"name\": \"col2\", \"type\": [\"null\", \"string\"] },\n" +
- " { \"name\": \"col3\", \"type\": [\"null\", \"int\"] },\n" +
- " { \"name\": \"col4\", \"type\": [\"null\", \"int\"] },\n" +
- " { \"name\": \"col5\", \"type\": [\"null\", \"long\"] },\n" +
- " { \"name\": \"col6\", \"type\": [\"null\", \"float\"] },\n" +
- " { \"name\": \"col7\", \"type\": [\"null\", \"double\"] },\n" +
- " { \"name\": \"col8\", \"type\": [\"null\", \"string\"] },\n" +
- " { \"name\": \"col9\", \"type\": [\"null\", \"bytes\"] },\n" +
- " { \"name\": \"col10\", \"type\": [\"null\", \"bytes\"] },\n" +
- " { \"name\": \"col11\", \"type\": \"null\" },\n" +
- " { \"name\": \"col12\", \"type\": [\"null\", \"bytes\"] }\n" +
- " ]\n" +
- "}\n";
+ " \"type\": \"record\",\n" +
+ " \"namespace\": \"org.apache.tajo\",\n" +
+ " \"name\": \"testNullHandlingTypes\",\n" +
+ " \"fields\": [\n" +
+ " { \"name\": \"col1\", \"type\": [\"null\", \"boolean\"] },\n" +
+ " { \"name\": \"col2\", \"type\": [\"null\", \"string\"] },\n" +
+ " { \"name\": \"col3\", \"type\": [\"null\", \"int\"] },\n" +
+ " { \"name\": \"col4\", \"type\": [\"null\", \"int\"] },\n" +
+ " { \"name\": \"col5\", \"type\": [\"null\", \"long\"] },\n" +
+ " { \"name\": \"col6\", \"type\": [\"null\", \"float\"] },\n" +
+ " { \"name\": \"col7\", \"type\": [\"null\", \"double\"] },\n" +
+ " { \"name\": \"col8\", \"type\": [\"null\", \"string\"] },\n" +
+ " { \"name\": \"col9\", \"type\": [\"null\", \"bytes\"] },\n" +
+ " { \"name\": \"col10\", \"type\": [\"null\", \"bytes\"] },\n" +
+ " { \"name\": \"col11\", \"type\": \"null\" },\n" +
+ " { \"name\": \"col12\", \"type\": [\"null\", \"bytes\"] }\n" +
+ " ]\n" +
+ "}\n";
private static String TEST_MAX_VALUE_AVRO_SCHEMA =
"{\n" +
@@ -112,18 +113,20 @@ public class TestStorages {
private boolean splitable;
private boolean statsable;
private boolean seekable;
+ private boolean internalType;
private Path testDir;
private FileSystem fs;
- public TestStorages(String type, boolean splitable, boolean statsable, boolean seekable)
throws IOException {
+ public TestStorages(String type, boolean splitable, boolean statsable, boolean seekable,
boolean internalType)
+ throws IOException {
this.storeType = type;
this.splitable = splitable;
this.statsable = statsable;
this.seekable = seekable;
-
+ this.internalType = internalType;
conf = new TajoConf();
- if (storeType.equalsIgnoreCase("RCFILE")) {
+ if (storeType.equalsIgnoreCase(BuiltinStorages.RCFILE)) {
conf.setInt(RCFile.RECORD_INTERVAL_CONF_STR, 100);
}
@@ -134,18 +137,19 @@ public class TestStorages {
@Parameterized.Parameters
public static Collection<Object[]> generateParameters() {
return Arrays.asList(new Object[][] {
- //type, splitable, statsable, seekable
- {"RAW", false, true, true},
- {"RCFILE", true, true, false},
- {"PARQUET", false, false, false},
- {"SEQUENCEFILE", true, true, false},
- {"AVRO", false, false, false},
- {"TEXT", true, true, true},
- {"JSON", true, true, false},
+ //type, splitable, statsable, seekable, internalType
+ {BuiltinStorages.RAW, false, true, true, true},
+ {BuiltinStorages.DRAW, false, true, true, true},
+ {BuiltinStorages.RCFILE, true, true, false, false},
+ {BuiltinStorages.PARQUET, false, false, false, false},
+ {BuiltinStorages.SEQUENCE_FILE, true, true, false, false},
+ {BuiltinStorages.AVRO, false, false, false, false},
+ {BuiltinStorages.TEXT, true, true, true, false},
+ {BuiltinStorages.JSON, true, true, false, false},
});
}
- @Test
+ @Test
public void testSplitable() throws IOException {
if (splitable) {
Schema schema = new Schema();
@@ -198,11 +202,11 @@ public class TestStorages {
assertEquals(tupleNum, tupleCnt);
}
- }
+ }
@Test
public void testRCFileSplitable() throws IOException {
- if (storeType.equalsIgnoreCase("StoreType.RCFILE")) {
+ if (storeType.equalsIgnoreCase(BuiltinStorages.RCFILE)) {
Schema schema = new Schema();
schema.addColumn("id", Type.INT4);
schema.addColumn("age", Type.INT8);
@@ -264,9 +268,9 @@ public class TestStorages {
TableMeta meta = CatalogUtil.newTableMeta(storeType);
meta.setOptions(CatalogUtil.newDefaultProperty(storeType));
- if (storeType.equalsIgnoreCase("AVRO")) {
+ if (storeType.equalsIgnoreCase(BuiltinStorages.AVRO)) {
meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL,
- TEST_PROJECTION_AVRO_SCHEMA);
+ TEST_PROJECTION_AVRO_SCHEMA);
}
Path tablePath = new Path(testDir, "testProjection.data");
@@ -309,8 +313,8 @@ public class TestStorages {
assertTrue(tupleCnt + 2 == tuple.getInt8(0));
assertTrue(tupleCnt + 3 == tuple.getFloat4(1));
} else {
- // RAW and ROW always project all fields.
- if (!storeType.equalsIgnoreCase("RAW") && !storeType.equalsIgnoreCase("ROWFILE"))
{
+ // Internal storage always project all fields.
+ if (!internalType) {
assertTrue(tuple.isBlankOrNull(0));
}
assertTrue(tupleCnt + 2 == tuple.getInt8(1));
@@ -320,7 +324,7 @@ public class TestStorages {
@Test
public void testVariousTypes() throws IOException {
- boolean handleProtobuf = !storeType.equalsIgnoreCase("JSON");
+ boolean handleProtobuf = !storeType.equalsIgnoreCase(BuiltinStorages.JSON);
Schema schema = new Schema();
schema.addColumn("col1", Type.BOOLEAN);
@@ -341,7 +345,7 @@ public class TestStorages {
KeyValueSet options = new KeyValueSet();
TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
meta.setOptions(CatalogUtil.newDefaultProperty(storeType));
- if (storeType.equalsIgnoreCase("AVRO")) {
+ if (storeType.equalsIgnoreCase(BuiltinStorages.AVRO)) {
String path = FileUtil.getResourcePath("dataset/testVariousTypes.avsc").toString();
meta.putOption(StorageConstants.AVRO_SCHEMA_URL, path);
}
@@ -393,7 +397,7 @@ public class TestStorages {
@Test
public void testNullHandlingTypes() throws IOException {
- boolean handleProtobuf = !storeType.equalsIgnoreCase("JSON");
+ boolean handleProtobuf = !storeType.equalsIgnoreCase(BuiltinStorages.JSON);
Schema schema = new Schema();
schema.addColumn("col1", Type.BOOLEAN);
@@ -490,7 +494,7 @@ public class TestStorages {
@Test
public void testRCFileTextSerializeDeserialize() throws IOException {
- if(!storeType.equalsIgnoreCase("RCFILE")) return;
+ if(!storeType.equalsIgnoreCase(BuiltinStorages.RCFILE)) return;
Schema schema = new Schema();
schema.addColumn("col1", Type.BOOLEAN);
@@ -559,7 +563,7 @@ public class TestStorages {
@Test
public void testRCFileBinarySerializeDeserialize() throws IOException {
- if(!storeType.equalsIgnoreCase("RCFILE")) return;
+ if(!storeType.equalsIgnoreCase(BuiltinStorages.RCFILE)) return;
Schema schema = new Schema();
schema.addColumn("col1", Type.BOOLEAN);
@@ -628,7 +632,7 @@ public class TestStorages {
@Test
public void testSequenceFileTextSerializeDeserialize() throws IOException {
- if(!storeType.equalsIgnoreCase("SEQUENCEFILE")) return;
+ if(!storeType.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) return;
Schema schema = new Schema();
schema.addColumn("col1", Type.BOOLEAN);
@@ -701,7 +705,7 @@ public class TestStorages {
@Test
public void testSequenceFileBinarySerializeDeserialize() throws IOException {
- if(!storeType.equalsIgnoreCase("SEQUENCEFILE")) return;
+ if(!storeType.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) return;
Schema schema = new Schema();
schema.addColumn("col1", Type.BOOLEAN);
@@ -775,7 +779,7 @@ public class TestStorages {
@Test
public void testTime() throws IOException {
- if (storeType.equalsIgnoreCase("TEXT") || storeType.equalsIgnoreCase("RAW")) {
+ if (storeType.equalsIgnoreCase(BuiltinStorages.TEXT) || internalType) {
Schema schema = new Schema();
schema.addColumn("col1", Type.DATE);
schema.addColumn("col2", Type.TIME);
@@ -869,7 +873,7 @@ public class TestStorages {
long readRows = 0;
for (long offset : offsets) {
scanner = TablespaceManager.getLocalFs().getScanner(meta, schema,
- new FileFragment("table", tablePath, prevOffset, offset - prevOffset), schema);
+ new FileFragment("table", tablePath, prevOffset, offset - prevOffset), schema);
scanner.init();
while (scanner.next() != null) {
@@ -903,11 +907,11 @@ public class TestStorages {
KeyValueSet options = new KeyValueSet();
TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
- if (storeType.equalsIgnoreCase("AVRO")) {
+ if (storeType.equalsIgnoreCase(BuiltinStorages.AVRO)) {
meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_MAX_VALUE_AVRO_SCHEMA);
}
- if (storeType.equalsIgnoreCase("RAW")) {
+ if (storeType.equalsIgnoreCase(BuiltinStorages.RAW)) {
OldStorageManager.clearCache();
/* TAJO-1250 reproduce BufferOverflow of RAWFile */
int headerSize = 4 + 2 + 1; //Integer record length + Short null-flag length + 1 byte
null flags
@@ -948,16 +952,16 @@ public class TestStorages {
scanner.close();
- if (storeType.equalsIgnoreCase("RAW")){
+ if (internalType){
OldStorageManager.clearCache();
}
}
@Test
public void testLessThanSchemaSize() throws IOException {
- /* RAW is internal storage. It must be same with schema size */
- if (storeType.equalsIgnoreCase("RAW") || storeType.equalsIgnoreCase("AVRO")
- || storeType.equalsIgnoreCase("PARQUET")){
+ /* Internal storage must be same with schema size */
+ if (internalType || storeType.equalsIgnoreCase(BuiltinStorages.AVRO)
+ || storeType.equalsIgnoreCase(BuiltinStorages.PARQUET)) {
return;
}
@@ -1020,10 +1024,10 @@ public class TestStorages {
@Test
public final void testInsertFixedCharTypeWithOverSize() throws Exception {
- if (storeType.equalsIgnoreCase("TEXT") == false &&
- storeType.equalsIgnoreCase("SEQUENCEFILE") == false &&
- storeType.equalsIgnoreCase("RCFILE") == false &&
- storeType.equalsIgnoreCase("PARQUET") == false) {
+ if (!storeType.equalsIgnoreCase(BuiltinStorages.TEXT) &&
+ !storeType.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE) &&
+ !storeType.equalsIgnoreCase(BuiltinStorages.RCFILE) &&
+ !storeType.equalsIgnoreCase(BuiltinStorages.PARQUET)) {
return;
}
@@ -1063,4 +1067,4 @@ public class TestStorages {
assertTrue(ok);
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/3a30f45c/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java
b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java
index 46c0d6e..bae81a9 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.SchemaUtil;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.conf.TajoConf;
@@ -40,8 +41,8 @@ import org.apache.tajo.storage.Tuple;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.storage.rawfile.DirectRawFileScanner;
import org.apache.tajo.storage.rawfile.DirectRawFileWriter;
-import org.apache.tajo.tuple.offheap.OffHeapRowBlock;
-import org.apache.tajo.tuple.offheap.RowWriter;
+import org.apache.tajo.tuple.memory.MemoryRowBlock;
+import org.apache.tajo.tuple.memory.RowWriter;
import org.apache.tajo.unit.StorageUnit;
import org.apache.tajo.util.FileUtil;
import org.apache.tajo.util.ProtoUtil;
@@ -147,7 +148,7 @@ public class TestDirectRawFile {
CatalogUtil.newDataType(TajoDataTypes.Type.PROTOBUF, PrimitiveProtos.StringProto.class.getName()));
}
- public FileStatus writeRowBlock(TajoConf conf, TableMeta meta, OffHeapRowBlock rowBlock,
Path outputFile)
+ public FileStatus writeRowBlock(TajoConf conf, TableMeta meta, MemoryRowBlock rowBlock,
Path outputFile)
throws IOException {
DirectRawFileWriter writer = new DirectRawFileWriter(conf, null, schema, meta, outputFile);
writer.init();
@@ -160,7 +161,7 @@ public class TestDirectRawFile {
return status;
}
- public FileStatus writeRowBlock(TajoConf conf, TableMeta meta, OffHeapRowBlock rowBlock)
throws IOException {
+ public FileStatus writeRowBlock(TajoConf conf, TableMeta meta, MemoryRowBlock rowBlock)
throws IOException {
Path outputDir = new Path(testDir, UUID.randomUUID() + "");
outputDir.getFileSystem(conf).mkdirs(outputDir);
Path outputFile = new Path(outputDir, "output.draw");
@@ -171,7 +172,7 @@ public class TestDirectRawFile {
public void testRWForAllTypesWithNextTuple() throws IOException {
int rowNum = 10000;
- OffHeapRowBlock rowBlock = createRowBlock(rowNum);
+ MemoryRowBlock rowBlock = createRowBlock(rowNum);
TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.DRAW);
FileStatus outputFile = writeRowBlock(tajoConf, meta, rowBlock);
@@ -201,7 +202,7 @@ public class TestDirectRawFile {
public void testRepeatedScan() throws IOException {
int rowNum = 2;
- OffHeapRowBlock rowBlock = createRowBlock(rowNum);
+ MemoryRowBlock rowBlock = createRowBlock(rowNum);
TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.DRAW);
FileStatus outputFile = writeRowBlock(tajoConf, meta, rowBlock);
@@ -229,7 +230,7 @@ public class TestDirectRawFile {
public void testReset() throws IOException {
int rowNum = 2;
- OffHeapRowBlock rowBlock = createRowBlock(rowNum);
+ MemoryRowBlock rowBlock = createRowBlock(rowNum);
TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.DRAW);
FileStatus outputFile = writeRowBlock(tajoConf, meta, rowBlock);
@@ -264,11 +265,11 @@ public class TestDirectRawFile {
reader.close();
}
- public static OffHeapRowBlock createRowBlock(int rowNum) {
+ public static MemoryRowBlock createRowBlock(int rowNum) {
long allocateStart = System.currentTimeMillis();
- OffHeapRowBlock rowBlock = new OffHeapRowBlock(schema, StorageUnit.MB * 8);
+ MemoryRowBlock rowBlock = new MemoryRowBlock(SchemaUtil.toDataTypes(schema), StorageUnit.KB
* 128);
long allocatedEnd = System.currentTimeMillis();
- LOG.info(FileUtil.humanReadableByteCount(rowBlock.size(), true) + " bytes allocated "
+ LOG.info(FileUtil.humanReadableByteCount(rowBlock.capacity(), true) + " bytes allocated
"
+ (allocatedEnd - allocateStart) + " msec");
long writeStart = System.currentTimeMillis();
http://git-wip-us.apache.org/repos/asf/tajo/blob/3a30f45c/tajo-storage/tajo-storage-hdfs/src/test/resources/storage-default.xml
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/resources/storage-default.xml b/tajo-storage/tajo-storage-hdfs/src/test/resources/storage-default.xml
index 2de1617..b5a60fe 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/resources/storage-default.xml
+++ b/tajo-storage/tajo-storage-hdfs/src/test/resources/storage-default.xml
@@ -30,15 +30,11 @@
<name>tajo.storage.manager.hdfs.class</name>
<value>org.apache.tajo.storage.FileTablespace</value>
</property>
- <property>
- <name>tajo.storage.manager.hbase.class</name>
- <value>org.apache.tajo.storage.hbase.HBaseTablespace</value>
- </property>
<!--- Registered Scanner Handler -->
<property>
<name>tajo.storage.scanner-handler</name>
- <value>text,json,raw,rcfile,row,parquet,sequencefile,avro</value>
+ <value>text,json,raw,draw,rcfile,row,parquet,orc,sequencefile,avro</value>
</property>
<!--- Fragment Class Configurations -->
@@ -55,6 +51,10 @@
<value>org.apache.tajo.storage.fragment.FileFragment</value>
</property>
<property>
+ <name>tajo.storage.fragment.draw.class</name>
+ <value>org.apache.tajo.storage.fragment.FileFragment</value>
+ </property>
+ <property>
<name>tajo.storage.fragment.rcfile.class</name>
<value>org.apache.tajo.storage.fragment.FileFragment</value>
</property>
@@ -67,6 +67,10 @@
<value>org.apache.tajo.storage.fragment.FileFragment</value>
</property>
<property>
+ <name>tajo.storage.fragment.orc.class</name>
+ <value>org.apache.tajo.storage.fragment.FileFragment</value>
+ </property>
+ <property>
<name>tajo.storage.fragment.sequencefile.class</name>
<value>org.apache.tajo.storage.fragment.FileFragment</value>
</property>
@@ -92,6 +96,11 @@
</property>
<property>
+ <name>tajo.storage.scanner-handler.draw.class</name>
+ <value>org.apache.tajo.storage.rawfile.DirectRawFileScanner</value>
+ </property>
+
+ <property>
<name>tajo.storage.scanner-handler.rcfile.class</name>
<value>org.apache.tajo.storage.rcfile.RCFile$RCFileScanner</value>
</property>
@@ -107,6 +116,11 @@
</property>
<property>
+ <name>tajo.storage.scanner-handler.orc.class</name>
+ <value>org.apache.tajo.storage.orc.ORCScanner</value>
+ </property>
+
+ <property>
<name>tajo.storage.scanner-handler.sequencefile.class</name>
<value>org.apache.tajo.storage.sequencefile.SequenceFileScanner</value>
</property>
@@ -119,7 +133,7 @@
<!--- Appender Handler -->
<property>
<name>tajo.storage.appender-handler</name>
- <value>text,raw,rcfile,row,parquet,sequencefile,avro</value>
+ <value>text,raw,draw,rcfile,row,parquet,sequencefile,avro</value>
</property>
<property>
@@ -138,6 +152,11 @@
</property>
<property>
+ <name>tajo.storage.appender-handler.draw.class</name>
+ <value>org.apache.tajo.storage.rawfile.DirectRawFileWriter</value>
+ </property>
+
+ <property>
<name>tajo.storage.appender-handler.rcfile.class</name>
<value>org.apache.tajo.storage.rcfile.RCFile$RCFileAppender</value>
</property>
@@ -183,4 +202,4 @@
<value>131072</value>
<description>128KB write buffer</description>
</property>
-</configuration>
+</configuration>
\ No newline at end of file
|