tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jihoon...@apache.org
Subject [01/14] tajo git commit: initial work for TAJO-1591.
Date Tue, 12 May 2015 03:14:00 GMT
Repository: tajo
Updated Branches:
  refs/heads/index_support 410ca3188 -> 6f4409629


http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
index 456ea00..7c75973 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
@@ -108,14 +108,14 @@ public class TestStorages {
           "  ]\n" +
           "}\n";
 
-  private StoreType storeType;
+  private String storeType;
   private boolean splitable;
   private boolean statsable;
   private boolean seekable;
   private Path testDir;
   private FileSystem fs;
 
-  public TestStorages(StoreType type, boolean splitable, boolean statsable, boolean seekable)
throws IOException {
+  public TestStorages(String type, boolean splitable, boolean statsable, boolean seekable)
throws IOException {
     this.storeType = type;
     this.splitable = splitable;
     this.statsable = statsable;
@@ -123,7 +123,7 @@ public class TestStorages {
 
     conf = new TajoConf();
 
-    if (storeType == StoreType.RCFILE) {
+    if (storeType.equalsIgnoreCase("RCFILE")) {
       conf.setInt(RCFile.RECORD_INTERVAL_CONF_STR, 100);
     }
 
@@ -135,14 +135,14 @@ public class TestStorages {
   public static Collection<Object[]> generateParameters() {
     return Arrays.asList(new Object[][] {
         //type, splitable, statsable, seekable
-        {StoreType.CSV, true, true, true},
-        {StoreType.RAW, false, true, true},
-        {StoreType.RCFILE, true, true, false},
-        {StoreType.PARQUET, false, false, false},
-        {StoreType.SEQUENCEFILE, true, true, false},
-        {StoreType.AVRO, false, false, false},
-        {StoreType.TEXTFILE, true, true, true},
-        {StoreType.JSON, true, true, false},
+        {"CSV", true, true, true},
+        {"RAW", false, true, true},
+        {"RCFILE", true, true, false},
+        {"PARQUET", false, false, false},
+        {"SEQUENCEFILE", true, true, false},
+        {"AVRO", false, false, false},
+        {"TEXT", true, true, true},
+        {"JSON", true, true, false},
     });
   }
 
@@ -203,7 +203,7 @@ public class TestStorages {
 
   @Test
   public void testRCFileSplitable() throws IOException {
-    if (storeType == StoreType.RCFILE) {
+    if (storeType.equalsIgnoreCase("StoreType.RCFILE")) {
       Schema schema = new Schema();
       schema.addColumn("id", Type.INT4);
       schema.addColumn("age", Type.INT8);
@@ -265,7 +265,7 @@ public class TestStorages {
 
     TableMeta meta = CatalogUtil.newTableMeta(storeType);
     meta.setOptions(CatalogUtil.newPhysicalProperties(storeType));
-    if (storeType == StoreType.AVRO) {
+    if (storeType.equalsIgnoreCase("AVRO")) {
       meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL,
                      TEST_PROJECTION_AVRO_SCHEMA);
     }
@@ -297,11 +297,11 @@ public class TestStorages {
     int tupleCnt = 0;
     Tuple tuple;
     while ((tuple = scanner.next()) != null) {
-      if (storeType == StoreType.RCFILE
-          || storeType == StoreType.CSV
-          || storeType == StoreType.PARQUET
-          || storeType == StoreType.SEQUENCEFILE
-          || storeType == StoreType.AVRO) {
+      if (storeType.equalsIgnoreCase("RCFILE")
+          || storeType.equalsIgnoreCase("CSV")
+          || storeType.equalsIgnoreCase("PARQUET")
+          || storeType.equalsIgnoreCase("SEQUENCEFILE")
+          || storeType.equalsIgnoreCase("AVRO")) {
         assertTrue(tuple.get(0) == null);
       }
       assertTrue(tupleCnt + 2 == tuple.get(1).asInt8());
@@ -315,7 +315,7 @@ public class TestStorages {
 
   @Test
   public void testVariousTypes() throws IOException {
-    boolean handleProtobuf = storeType != StoreType.JSON;
+    boolean handleProtobuf = !storeType.equalsIgnoreCase("JSON");
 
     Schema schema = new Schema();
     schema.addColumn("col1", Type.BOOLEAN);
@@ -336,7 +336,7 @@ public class TestStorages {
     KeyValueSet options = new KeyValueSet();
     TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
     meta.setOptions(CatalogUtil.newPhysicalProperties(storeType));
-    if (storeType == StoreType.AVRO) {
+    if (storeType.equalsIgnoreCase("AVRO")) {
       String path = FileUtil.getResourcePath("dataset/testVariousTypes.avsc").toString();
       meta.putOption(StorageConstants.AVRO_SCHEMA_URL, path);
     }
@@ -387,7 +387,7 @@ public class TestStorages {
 
   @Test
   public void testNullHandlingTypes() throws IOException {
-    boolean handleProtobuf = storeType != StoreType.JSON;
+    boolean handleProtobuf = !storeType.equalsIgnoreCase("JSON");
 
     Schema schema = new Schema();
     schema.addColumn("col1", Type.BOOLEAN);
@@ -413,7 +413,7 @@ public class TestStorages {
     meta.putOption(StorageConstants.RCFILE_NULL, "\\\\N");
     meta.putOption(StorageConstants.RCFILE_SERDE, TextSerializerDeserializer.class.getName());
     meta.putOption(StorageConstants.SEQUENCEFILE_NULL, "\\");
-    if (storeType == StoreType.AVRO) {
+    if (storeType.equalsIgnoreCase("AVRO")) {
       meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL,
                      TEST_NULL_HANDLING_TYPES_AVRO_SCHEMA);
     }
@@ -485,7 +485,7 @@ public class TestStorages {
 
   @Test
   public void testRCFileTextSerializeDeserialize() throws IOException {
-    if(storeType != StoreType.RCFILE) return;
+    if(!storeType.equalsIgnoreCase("RCFILE")) return;
 
     Schema schema = new Schema();
     schema.addColumn("col1", Type.BOOLEAN);
@@ -555,7 +555,7 @@ public class TestStorages {
 
   @Test
   public void testRCFileBinarySerializeDeserialize() throws IOException {
-    if(storeType != StoreType.RCFILE) return;
+    if(!storeType.equalsIgnoreCase("RCFILE")) return;
 
     Schema schema = new Schema();
     schema.addColumn("col1", Type.BOOLEAN);
@@ -625,7 +625,7 @@ public class TestStorages {
 
   @Test
   public void testSequenceFileTextSerializeDeserialize() throws IOException {
-    if(storeType != StoreType.SEQUENCEFILE) return;
+    if(!storeType.equalsIgnoreCase("SEQUENCEFILE")) return;
 
     Schema schema = new Schema();
     schema.addColumn("col1", Type.BOOLEAN);
@@ -699,7 +699,7 @@ public class TestStorages {
 
   @Test
   public void testSequenceFileBinarySerializeDeserialize() throws IOException {
-    if(storeType != StoreType.SEQUENCEFILE) return;
+    if(!storeType.equalsIgnoreCase("SEQUENCEFILE")) return;
 
     Schema schema = new Schema();
     schema.addColumn("col1", Type.BOOLEAN);
@@ -773,7 +773,7 @@ public class TestStorages {
 
   @Test
   public void testTime() throws IOException {
-    if (storeType == StoreType.CSV || storeType == StoreType.RAW) {
+    if (storeType.equalsIgnoreCase("CSV") || storeType.equalsIgnoreCase("RAW")) {
       Schema schema = new Schema();
       schema.addColumn("col1", Type.DATE);
       schema.addColumn("col2", Type.TIME);
@@ -902,11 +902,11 @@ public class TestStorages {
 
     KeyValueSet options = new KeyValueSet();
     TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
-    if (storeType == StoreType.AVRO) {
+    if (storeType.equalsIgnoreCase("AVRO")) {
       meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_MAX_VALUE_AVRO_SCHEMA);
     }
 
-    if (storeType == StoreType.RAW) {
+    if (storeType.equalsIgnoreCase("RAW")) {
       StorageManager.clearCache();
       /* TAJO-1250 reproduce BufferOverflow of RAWFile */
       int headerSize = 4 + 2 + 1; //Integer record length + Short null-flag length + 1 byte
null flags
@@ -948,7 +948,7 @@ public class TestStorages {
     scanner.close();
 
 
-    if (storeType == StoreType.RAW){
+    if (storeType.equalsIgnoreCase("RAW")){
       StorageManager.clearCache();
     }
   }
@@ -956,7 +956,7 @@ public class TestStorages {
   @Test
   public void testLessThanSchemaSize() throws IOException {
     /* RAW is internal storage. It must be same with schema size */
-    if (storeType == StoreType.RAW || storeType == StoreType.AVRO){
+    if (storeType.equalsIgnoreCase("RAW") || storeType.equalsIgnoreCase("AVRO")){
       return;
     }
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java
b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java
index 7b83894..341cc07 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java
@@ -22,7 +22,6 @@ import org.apache.avro.Schema;
 import org.apache.tajo.HttpFileServer;
 import org.apache.tajo.catalog.CatalogUtil;
 import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.catalog.proto.CatalogProtos;
 import org.apache.tajo.conf.TajoConf;
 import org.apache.tajo.storage.StorageConstants;
 import org.apache.tajo.util.FileUtil;
@@ -58,12 +57,12 @@ public class TestAvroUtil {
 
   @Test
   public void testGetSchema() throws IOException, URISyntaxException {
-    TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.AVRO);
+    TableMeta meta = CatalogUtil.newTableMeta("AVRO");
     meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, FileUtil.readTextFile(new File(schemaUrl.getPath())));
     Schema schema = AvroUtil.getAvroSchema(meta, new TajoConf());
     assertEquals(expected, schema);
 
-    meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.AVRO);
+    meta = CatalogUtil.newTableMeta("AVRO");
     meta.putOption(StorageConstants.AVRO_SCHEMA_URL, schemaUrl.getPath());
     schema = AvroUtil.getAvroSchema(meta, new TajoConf());
     assertEquals(expected, schema);
@@ -74,7 +73,7 @@ public class TestAvroUtil {
       InetSocketAddress addr = server.getBindAddress();
 
       String url = "http://127.0.0.1:" + addr.getPort() + schemaUrl.getPath();
-      meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.AVRO);
+      meta = CatalogUtil.newTableMeta("AVRO");
       meta.putOption(StorageConstants.AVRO_SCHEMA_URL, url);
       schema = AvroUtil.getAvroSchema(meta, new TajoConf());
     } finally {

http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java
b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java
index 068f726..114a9cc 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java
@@ -55,9 +55,9 @@ public class TestBSTIndex {
   private static final String TEST_PATH = "target/test-data/TestIndex";
   private Path testDir;
   private FileSystem fs;
-  private StoreType storeType;
+  private String storeType;
 
-  public TestBSTIndex(StoreType type) {
+  public TestBSTIndex(String type) {
     this.storeType = type;
     conf = new TajoConf();
     conf.setVar(TajoConf.ConfVars.ROOT_DIR, TEST_PATH);
@@ -73,9 +73,9 @@ public class TestBSTIndex {
   @Parameterized.Parameters
   public static Collection<Object[]> generateParameters() {
     return Arrays.asList(new Object[][]{
-        {StoreType.CSV},
-        {StoreType.RAW},
-        {StoreType.TEXTFILE}
+        {"CSV"},
+        {"RAW"},
+        {"TEXT"}
     });
   }
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java
b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java
index d7c9f49..cb94353 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java
@@ -73,7 +73,7 @@ public class TestSingleCSVFileBSTIndex {
 
   @Test
   public void testFindValueInSingleCSV() throws IOException {
-    meta = CatalogUtil.newTableMeta(StoreType.CSV);
+    meta = CatalogUtil.newTableMeta("CSV");
 
     Path tablePath = StorageUtil.concatPath(testDir, "testFindValueInSingleCSV", "table.csv");
     fs.mkdirs(tablePath.getParent());
@@ -162,7 +162,7 @@ public class TestSingleCSVFileBSTIndex {
 
   @Test
   public void testFindNextKeyValueInSingleCSV() throws IOException {
-    meta = CatalogUtil.newTableMeta(StoreType.CSV);
+    meta = CatalogUtil.newTableMeta("CSV");
 
     Path tablePath = StorageUtil.concatPath(testDir, "testFindNextKeyValueInSingleCSV",
         "table1.csv");

http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java
b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java
index 70282d9..349da07 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java
@@ -68,7 +68,7 @@ public class TestJsonSerDe {
   public void testVarioutType() throws IOException {
     TajoConf conf = new TajoConf();
 
-    TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.JSON);
+    TableMeta meta = CatalogUtil.newTableMeta("JSON");
     Path tablePath = new Path(getResourcePath("dataset", "TestJsonSerDe"), "testVariousType.json");
     FileSystem fs = FileSystem.getLocal(conf);
     FileStatus status = fs.getFileStatus(tablePath);


Mime
View raw message