Return-Path: X-Original-To: apmail-tajo-commits-archive@minotaur.apache.org Delivered-To: apmail-tajo-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 94C9617ED6 for ; Mon, 11 May 2015 22:30:27 +0000 (UTC) Received: (qmail 43204 invoked by uid 500); 11 May 2015 22:30:27 -0000 Delivered-To: apmail-tajo-commits-archive@tajo.apache.org Received: (qmail 43167 invoked by uid 500); 11 May 2015 22:30:27 -0000 Mailing-List: contact commits-help@tajo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tajo.apache.org Delivered-To: mailing list commits@tajo.apache.org Received: (qmail 43156 invoked by uid 99); 11 May 2015 22:30:27 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 11 May 2015 22:30:27 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 640E4E07EE; Mon, 11 May 2015 22:30:27 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: hyunsik@apache.org To: commits@tajo.apache.org Date: Mon, 11 May 2015 22:30:27 -0000 Message-Id: <47079740d8f2499f86a88c2df4eb8b1d@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/7] tajo git commit: initial work for TAJO-1591. Repository: tajo Updated Branches: refs/heads/master 0d1bf41f3 -> 53ed1c376 http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java index 456ea00..7c75973 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java @@ -108,14 +108,14 @@ public class TestStorages { " ]\n" + "}\n"; - private StoreType storeType; + private String storeType; private boolean splitable; private boolean statsable; private boolean seekable; private Path testDir; private FileSystem fs; - public TestStorages(StoreType type, boolean splitable, boolean statsable, boolean seekable) throws IOException { + public TestStorages(String type, boolean splitable, boolean statsable, boolean seekable) throws IOException { this.storeType = type; this.splitable = splitable; this.statsable = statsable; @@ -123,7 +123,7 @@ public class TestStorages { conf = new TajoConf(); - if (storeType == StoreType.RCFILE) { + if (storeType.equalsIgnoreCase("RCFILE")) { conf.setInt(RCFile.RECORD_INTERVAL_CONF_STR, 100); } @@ -135,14 +135,14 @@ public class TestStorages { public static Collection generateParameters() { return Arrays.asList(new Object[][] { //type, splitable, statsable, seekable - {StoreType.CSV, true, true, true}, - {StoreType.RAW, false, true, true}, - {StoreType.RCFILE, true, true, false}, - {StoreType.PARQUET, false, false, false}, - {StoreType.SEQUENCEFILE, true, true, false}, - {StoreType.AVRO, false, false, false}, - {StoreType.TEXTFILE, true, true, true}, - {StoreType.JSON, true, true, false}, + {"CSV", true, true, true}, + {"RAW", false, true, true}, + {"RCFILE", true, true, false}, + {"PARQUET", false, false, false}, + {"SEQUENCEFILE", true, true, false}, + {"AVRO", false, false, false}, + {"TEXT", true, true, true}, + {"JSON", true, true, false}, }); } @@ -203,7 +203,7 @@ public class TestStorages { @Test public void testRCFileSplitable() throws IOException { - if (storeType == StoreType.RCFILE) { + if (storeType.equalsIgnoreCase("StoreType.RCFILE")) { Schema schema = new Schema(); schema.addColumn("id", Type.INT4); schema.addColumn("age", Type.INT8); @@ -265,7 +265,7 @@ public class TestStorages { TableMeta meta = CatalogUtil.newTableMeta(storeType); meta.setOptions(CatalogUtil.newPhysicalProperties(storeType)); - if (storeType == StoreType.AVRO) { + if (storeType.equalsIgnoreCase("AVRO")) { meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_PROJECTION_AVRO_SCHEMA); } @@ -297,11 +297,11 @@ public class TestStorages { int tupleCnt = 0; Tuple tuple; while ((tuple = scanner.next()) != null) { - if (storeType == StoreType.RCFILE - || storeType == StoreType.CSV - || storeType == StoreType.PARQUET - || storeType == StoreType.SEQUENCEFILE - || storeType == StoreType.AVRO) { + if (storeType.equalsIgnoreCase("RCFILE") + || storeType.equalsIgnoreCase("CSV") + || storeType.equalsIgnoreCase("PARQUET") + || storeType.equalsIgnoreCase("SEQUENCEFILE") + || storeType.equalsIgnoreCase("AVRO")) { assertTrue(tuple.get(0) == null); } assertTrue(tupleCnt + 2 == tuple.get(1).asInt8()); @@ -315,7 +315,7 @@ public class TestStorages { @Test public void testVariousTypes() throws IOException { - boolean handleProtobuf = storeType != StoreType.JSON; + boolean handleProtobuf = !storeType.equalsIgnoreCase("JSON"); Schema schema = new Schema(); schema.addColumn("col1", Type.BOOLEAN); @@ -336,7 +336,7 @@ public class TestStorages { KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(storeType, options); meta.setOptions(CatalogUtil.newPhysicalProperties(storeType)); - if (storeType == StoreType.AVRO) { + if (storeType.equalsIgnoreCase("AVRO")) { String path = FileUtil.getResourcePath("dataset/testVariousTypes.avsc").toString(); meta.putOption(StorageConstants.AVRO_SCHEMA_URL, path); } @@ -387,7 +387,7 @@ public class TestStorages { @Test public void testNullHandlingTypes() throws IOException { - boolean handleProtobuf = storeType != StoreType.JSON; + boolean handleProtobuf = !storeType.equalsIgnoreCase("JSON"); Schema schema = new Schema(); schema.addColumn("col1", Type.BOOLEAN); @@ -413,7 +413,7 @@ public class TestStorages { meta.putOption(StorageConstants.RCFILE_NULL, "\\\\N"); meta.putOption(StorageConstants.RCFILE_SERDE, TextSerializerDeserializer.class.getName()); meta.putOption(StorageConstants.SEQUENCEFILE_NULL, "\\"); - if (storeType == StoreType.AVRO) { + if (storeType.equalsIgnoreCase("AVRO")) { meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_NULL_HANDLING_TYPES_AVRO_SCHEMA); } @@ -485,7 +485,7 @@ public class TestStorages { @Test public void testRCFileTextSerializeDeserialize() throws IOException { - if(storeType != StoreType.RCFILE) return; + if(!storeType.equalsIgnoreCase("RCFILE")) return; Schema schema = new Schema(); schema.addColumn("col1", Type.BOOLEAN); @@ -555,7 +555,7 @@ public class TestStorages { @Test public void testRCFileBinarySerializeDeserialize() throws IOException { - if(storeType != StoreType.RCFILE) return; + if(!storeType.equalsIgnoreCase("RCFILE")) return; Schema schema = new Schema(); schema.addColumn("col1", Type.BOOLEAN); @@ -625,7 +625,7 @@ public class TestStorages { @Test public void testSequenceFileTextSerializeDeserialize() throws IOException { - if(storeType != StoreType.SEQUENCEFILE) return; + if(!storeType.equalsIgnoreCase("SEQUENCEFILE")) return; Schema schema = new Schema(); schema.addColumn("col1", Type.BOOLEAN); @@ -699,7 +699,7 @@ public class TestStorages { @Test public void testSequenceFileBinarySerializeDeserialize() throws IOException { - if(storeType != StoreType.SEQUENCEFILE) return; + if(!storeType.equalsIgnoreCase("SEQUENCEFILE")) return; Schema schema = new Schema(); schema.addColumn("col1", Type.BOOLEAN); @@ -773,7 +773,7 @@ public class TestStorages { @Test public void testTime() throws IOException { - if (storeType == StoreType.CSV || storeType == StoreType.RAW) { + if (storeType.equalsIgnoreCase("CSV") || storeType.equalsIgnoreCase("RAW")) { Schema schema = new Schema(); schema.addColumn("col1", Type.DATE); schema.addColumn("col2", Type.TIME); @@ -902,11 +902,11 @@ public class TestStorages { KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(storeType, options); - if (storeType == StoreType.AVRO) { + if (storeType.equalsIgnoreCase("AVRO")) { meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_MAX_VALUE_AVRO_SCHEMA); } - if (storeType == StoreType.RAW) { + if (storeType.equalsIgnoreCase("RAW")) { StorageManager.clearCache(); /* TAJO-1250 reproduce BufferOverflow of RAWFile */ int headerSize = 4 + 2 + 1; //Integer record length + Short null-flag length + 1 byte null flags @@ -948,7 +948,7 @@ public class TestStorages { scanner.close(); - if (storeType == StoreType.RAW){ + if (storeType.equalsIgnoreCase("RAW")){ StorageManager.clearCache(); } } @@ -956,7 +956,7 @@ public class TestStorages { @Test public void testLessThanSchemaSize() throws IOException { /* RAW is internal storage. It must be same with schema size */ - if (storeType == StoreType.RAW || storeType == StoreType.AVRO){ + if (storeType.equalsIgnoreCase("RAW") || storeType.equalsIgnoreCase("AVRO")){ return; } http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java index 7b83894..341cc07 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java @@ -22,7 +22,6 @@ import org.apache.avro.Schema; import org.apache.tajo.HttpFileServer; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.util.FileUtil; @@ -58,12 +57,12 @@ public class TestAvroUtil { @Test public void testGetSchema() throws IOException, URISyntaxException { - TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.AVRO); + TableMeta meta = CatalogUtil.newTableMeta("AVRO"); meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, FileUtil.readTextFile(new File(schemaUrl.getPath()))); Schema schema = AvroUtil.getAvroSchema(meta, new TajoConf()); assertEquals(expected, schema); - meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.AVRO); + meta = CatalogUtil.newTableMeta("AVRO"); meta.putOption(StorageConstants.AVRO_SCHEMA_URL, schemaUrl.getPath()); schema = AvroUtil.getAvroSchema(meta, new TajoConf()); assertEquals(expected, schema); @@ -74,7 +73,7 @@ public class TestAvroUtil { InetSocketAddress addr = server.getBindAddress(); String url = "http://127.0.0.1:" + addr.getPort() + schemaUrl.getPath(); - meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.AVRO); + meta = CatalogUtil.newTableMeta("AVRO"); meta.putOption(StorageConstants.AVRO_SCHEMA_URL, url); schema = AvroUtil.getAvroSchema(meta, new TajoConf()); } finally { http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java index 068f726..114a9cc 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java @@ -55,9 +55,9 @@ public class TestBSTIndex { private static final String TEST_PATH = "target/test-data/TestIndex"; private Path testDir; private FileSystem fs; - private StoreType storeType; + private String storeType; - public TestBSTIndex(StoreType type) { + public TestBSTIndex(String type) { this.storeType = type; conf = new TajoConf(); conf.setVar(TajoConf.ConfVars.ROOT_DIR, TEST_PATH); @@ -73,9 +73,9 @@ public class TestBSTIndex { @Parameterized.Parameters public static Collection generateParameters() { return Arrays.asList(new Object[][]{ - {StoreType.CSV}, - {StoreType.RAW}, - {StoreType.TEXTFILE} + {"CSV"}, + {"RAW"}, + {"TEXT"} }); } http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java index d7c9f49..cb94353 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java @@ -73,7 +73,7 @@ public class TestSingleCSVFileBSTIndex { @Test public void testFindValueInSingleCSV() throws IOException { - meta = CatalogUtil.newTableMeta(StoreType.CSV); + meta = CatalogUtil.newTableMeta("CSV"); Path tablePath = StorageUtil.concatPath(testDir, "testFindValueInSingleCSV", "table.csv"); fs.mkdirs(tablePath.getParent()); @@ -162,7 +162,7 @@ public class TestSingleCSVFileBSTIndex { @Test public void testFindNextKeyValueInSingleCSV() throws IOException { - meta = CatalogUtil.newTableMeta(StoreType.CSV); + meta = CatalogUtil.newTableMeta("CSV"); Path tablePath = StorageUtil.concatPath(testDir, "testFindNextKeyValueInSingleCSV", "table1.csv"); http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java index 70282d9..349da07 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java @@ -68,7 +68,7 @@ public class TestJsonSerDe { public void testVarioutType() throws IOException { TajoConf conf = new TajoConf(); - TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.JSON); + TableMeta meta = CatalogUtil.newTableMeta("JSON"); Path tablePath = new Path(getResourcePath("dataset", "TestJsonSerDe"), "testVariousType.json"); FileSystem fs = FileSystem.getLocal(conf); FileStatus status = fs.getFileStatus(tablePath);