tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jihoon...@apache.org
Subject [1/2] tajo git commit: Add missing files in the tajo-hcatalog driver
Date Fri, 15 May 2015 01:56:15 GMT
Repository: tajo
Updated Branches:
  refs/heads/branch-0.10.1 19554d801 -> 63c2fb6c5


http://git-wip-us.apache.org/repos/asf/tajo/blob/63c2fb6c/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
new file mode 100644
index 0000000..725f665
--- /dev/null
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
@@ -0,0 +1,402 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.catalog.store;
+
+
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.tajo.catalog.CatalogUtil;
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.TableDesc;
+import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.catalog.partition.PartitionMethodDesc;
+import org.apache.tajo.catalog.proto.CatalogProtos;
+import org.apache.tajo.common.TajoDataTypes;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.storage.StorageConstants;
+import org.apache.tajo.util.CommonTestingUtil;
+import org.apache.tajo.util.KeyValueSet;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+/**
+ * TestHCatalogStore. Test case for
+ * {@link org.apache.tajo.catalog.store.HCatalogStore}
+ */
+
+public class TestHCatalogStore {
+  private static final String DB_NAME = "test_hive";
+  private static final String CUSTOMER = "customer";
+  private static final String NATION = "nation";
+  private static final String REGION = "region";
+  private static final String SUPPLIER = "supplier";
+
+  private static HCatalogStore store;
+  private static Path warehousePath;
+
+  @BeforeClass
+  public static void setUp() throws Exception {
+    Path testPath = CommonTestingUtil.getTestDir();
+    warehousePath = new Path(testPath, "warehouse");
+
+    //create local hiveMeta
+    HiveConf conf = new HiveConf();
+    String jdbcUri = "jdbc:derby:;databaseName="+testPath.toUri().getPath()+"metastore_db;create=true";
+    conf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, warehousePath.toUri().toString());
+    conf.set(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, jdbcUri);
+    conf.set(TajoConf.ConfVars.WAREHOUSE_DIR.varname, warehousePath.toUri().toString());
+
+    // create local HCatalogStore.
+    TajoConf tajoConf = new TajoConf(conf);
+    store = new HCatalogStore(tajoConf);
+    store.createDatabase(DB_NAME, null);
+  }
+
+  @AfterClass
+  public static void tearDown() throws IOException {
+    store.close();
+  }
+
+  @Test
+  public void testTableUsingTextFile() throws Exception {
+    TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, new KeyValueSet());
+
+    org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
+    schema.addColumn("c_custkey", TajoDataTypes.Type.INT4);
+    schema.addColumn("c_name", TajoDataTypes.Type.TEXT);
+    schema.addColumn("c_address", TajoDataTypes.Type.TEXT);
+    schema.addColumn("c_nationkey", TajoDataTypes.Type.INT4);
+    schema.addColumn("c_phone", TajoDataTypes.Type.TEXT);
+    schema.addColumn("c_acctbal", TajoDataTypes.Type.FLOAT8);
+    schema.addColumn("c_mktsegment", TajoDataTypes.Type.TEXT);
+    schema.addColumn("c_comment", TajoDataTypes.Type.TEXT);
+
+    TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, CUSTOMER), schema, meta,
+        new Path(warehousePath, new Path(DB_NAME, CUSTOMER)).toUri());
+    store.createTable(table.getProto());
+    assertTrue(store.existTable(DB_NAME, CUSTOMER));
+
+    TableDesc table1 = new TableDesc(store.getTable(DB_NAME, CUSTOMER));
+    assertEquals(table.getName(), table1.getName());
+    assertEquals(table.getPath(), table1.getPath());
+    assertEquals(table.getSchema().size(), table1.getSchema().size());
+    for (int i = 0; i < table.getSchema().size(); i++) {
+      assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName());
+    }
+
+    assertEquals(StringEscapeUtils.escapeJava(StorageConstants.DEFAULT_FIELD_DELIMITER),
+        table1.getMeta().getOption(StorageConstants.TEXT_DELIMITER));
+    store.dropTable(DB_NAME, CUSTOMER);
+  }
+
+  @Test
+  public void testTableUsingRCFileWithBinarySerde() throws Exception {
+    KeyValueSet options = new KeyValueSet();
+    options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
+    TableMeta meta = new TableMeta(CatalogProtos.StoreType.RCFILE, options);
+
+    org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
+    schema.addColumn("r_regionkey", TajoDataTypes.Type.INT4);
+    schema.addColumn("r_name", TajoDataTypes.Type.TEXT);
+    schema.addColumn("r_comment", TajoDataTypes.Type.TEXT);
+
+    TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta,
+        new Path(warehousePath, new Path(DB_NAME, REGION)).toUri());
+    store.createTable(table.getProto());
+    assertTrue(store.existTable(DB_NAME, REGION));
+
+    TableDesc table1 = new TableDesc(store.getTable(DB_NAME, REGION));
+    assertEquals(table.getName(), table1.getName());
+    assertEquals(table.getPath(), table1.getPath());
+    assertEquals(table.getSchema().size(), table1.getSchema().size());
+    for (int i = 0; i < table.getSchema().size(); i++) {
+      assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName());
+    }
+
+    assertEquals(StorageConstants.DEFAULT_BINARY_SERDE,
+        table1.getMeta().getOption(StorageConstants.RCFILE_SERDE));
+    store.dropTable(DB_NAME, REGION);
+  }
+
+  @Test
+  public void testTableUsingRCFileWithTextSerde() throws Exception {
+    KeyValueSet options = new KeyValueSet();
+    options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
+    TableMeta meta = new TableMeta(CatalogProtos.StoreType.RCFILE, options);
+
+    org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
+    schema.addColumn("r_regionkey", TajoDataTypes.Type.INT4);
+    schema.addColumn("r_name", TajoDataTypes.Type.TEXT);
+    schema.addColumn("r_comment", TajoDataTypes.Type.TEXT);
+
+    TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta,
+        new Path(warehousePath, new Path(DB_NAME, REGION)).toUri());
+    store.createTable(table.getProto());
+    assertTrue(store.existTable(DB_NAME, REGION));
+
+    TableDesc table1 = new TableDesc(store.getTable(DB_NAME, REGION));
+    assertEquals(table.getName(), table1.getName());
+    assertEquals(table.getPath(), table1.getPath());
+    assertEquals(table.getSchema().size(), table1.getSchema().size());
+    for (int i = 0; i < table.getSchema().size(); i++) {
+      assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName());
+    }
+
+    assertEquals(StorageConstants.DEFAULT_TEXT_SERDE, table1.getMeta().getOption(StorageConstants.RCFILE_SERDE));
+    store.dropTable(DB_NAME, REGION);
+  }
+
+  @Test
+  public void testTableWithNullValue() throws Exception {
+    KeyValueSet options = new KeyValueSet();
+    options.set(StorageConstants.TEXT_DELIMITER, StringEscapeUtils.escapeJava("\u0002"));
+    options.set(StorageConstants.TEXT_NULL, StringEscapeUtils.escapeJava("\u0003"));
+    TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, options);
+
+    org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
+    schema.addColumn("s_suppkey", TajoDataTypes.Type.INT4);
+    schema.addColumn("s_name", TajoDataTypes.Type.TEXT);
+    schema.addColumn("s_address", TajoDataTypes.Type.TEXT);
+    schema.addColumn("s_nationkey", TajoDataTypes.Type.INT4);
+    schema.addColumn("s_phone", TajoDataTypes.Type.TEXT);
+    schema.addColumn("s_acctbal", TajoDataTypes.Type.FLOAT8);
+    schema.addColumn("s_comment", TajoDataTypes.Type.TEXT);
+
+    TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, SUPPLIER), schema, meta,
+        new Path(warehousePath, new Path(DB_NAME, SUPPLIER)).toUri());
+
+    store.createTable(table.getProto());
+    assertTrue(store.existTable(DB_NAME, SUPPLIER));
+
+    TableDesc table1 = new TableDesc(store.getTable(DB_NAME, SUPPLIER));
+    assertEquals(table.getName(), table1.getName());
+    assertEquals(table.getPath(), table1.getPath());
+    assertEquals(table.getSchema().size(), table1.getSchema().size());
+    for (int i = 0; i < table.getSchema().size(); i++) {
+      assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName());
+    }
+
+    assertEquals(table.getMeta().getOption(StorageConstants.TEXT_DELIMITER),
+        table1.getMeta().getOption(StorageConstants.TEXT_DELIMITER));
+
+    assertEquals(table.getMeta().getOption(StorageConstants.TEXT_NULL),
+        table1.getMeta().getOption(StorageConstants.TEXT_NULL));
+
+    assertEquals(table1.getMeta().getOption(StorageConstants.TEXT_DELIMITER),
+        StringEscapeUtils.escapeJava("\u0002"));
+
+    assertEquals(table1.getMeta().getOption(StorageConstants.TEXT_NULL),
+        StringEscapeUtils.escapeJava("\u0003"));
+
+    store.dropTable(DB_NAME, SUPPLIER);
+
+  }
+
+  @Test
+  public void testAddTableByPartition() throws Exception {
+    TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, new KeyValueSet());
+
+    org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
+    schema.addColumn("n_name", TajoDataTypes.Type.TEXT);
+    schema.addColumn("n_regionkey", TajoDataTypes.Type.INT4);
+    schema.addColumn("n_comment", TajoDataTypes.Type.TEXT);
+
+
+    TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, NATION), schema, meta,
+        new Path(warehousePath, new Path(DB_NAME, NATION)).toUri());
+
+    org.apache.tajo.catalog.Schema expressionSchema = new org.apache.tajo.catalog.Schema();
+    expressionSchema.addColumn("n_nationkey", TajoDataTypes.Type.INT4);
+
+    PartitionMethodDesc partitions = new PartitionMethodDesc(
+        DB_NAME,
+        NATION,
+        CatalogProtos.PartitionType.COLUMN, expressionSchema.getColumn(0).getQualifiedName(),
expressionSchema);
+    table.setPartitionMethod(partitions);
+
+    store.createTable(table.getProto());
+    assertTrue(store.existTable(DB_NAME, NATION));
+
+    TableDesc table1 = new TableDesc(store.getTable(DB_NAME, NATION));
+    assertEquals(table.getName(), table1.getName());
+    assertEquals(table.getPath(), table1.getPath());
+    assertEquals(table.getSchema().size(), table1.getSchema().size());
+    for (int i = 0; i < table.getSchema().size(); i++) {
+      assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName());
+    }
+
+
+    Schema partitionSchema = table.getPartitionMethod().getExpressionSchema();
+    Schema partitionSchema1 = table1.getPartitionMethod().getExpressionSchema();
+    assertEquals(partitionSchema.size(), partitionSchema1.size());
+    for (int i = 0; i < partitionSchema.size(); i++) {
+      assertEquals(partitionSchema.getColumn(i).getSimpleName(), partitionSchema1.getColumn(i).getSimpleName());
+    }
+
+    store.dropTable(DB_NAME, NATION);
+  }
+
+
+  @Test
+  public void testGetAllTableNames() throws Exception{
+    TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, new KeyValueSet());
+    org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
+    schema.addColumn("n_name", TajoDataTypes.Type.TEXT);
+    schema.addColumn("n_regionkey", TajoDataTypes.Type.INT4);
+    schema.addColumn("n_comment", TajoDataTypes.Type.TEXT);
+
+    String[] tableNames = new String[]{"table1", "table2", "table3"};
+
+    for(String tableName : tableNames){
+      TableDesc table = new TableDesc(CatalogUtil.buildFQName("default", tableName), schema,
meta,
+          new Path(warehousePath, new Path(DB_NAME, tableName)).toUri());
+      store.createTable(table.getProto());
+    }
+
+    List<String> tables = store.getAllTableNames("default");
+    assertEquals(tableNames.length, tables.size());
+
+    for(String tableName : tableNames){
+      assertTrue(tables.contains(tableName));
+    }
+
+    for(String tableName : tableNames){
+      store.dropTable("default", tableName);
+    }
+  }
+
+  @Test
+  public void testDeleteTable() throws Exception {
+    TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, new KeyValueSet());
+    org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
+    schema.addColumn("n_name", TajoDataTypes.Type.TEXT);
+    schema.addColumn("n_regionkey", TajoDataTypes.Type.INT4);
+    schema.addColumn("n_comment", TajoDataTypes.Type.TEXT);
+
+    String tableName = "table1";
+    TableDesc table = new TableDesc(DB_NAME + "." + tableName, schema, meta, warehousePath.toUri());
+    store.createTable(table.getProto());
+    assertTrue(store.existTable(DB_NAME, tableName));
+
+    TableDesc table1 = new TableDesc(store.getTable(DB_NAME, tableName));
+    FileSystem fs = FileSystem.getLocal(new Configuration());
+    assertTrue(fs.exists(new Path(table1.getPath())));
+
+    store.dropTable(DB_NAME, tableName);
+    assertFalse(store.existTable(DB_NAME, tableName));
+    fs.close();
+  }
+
+  @Test
+  public void testTableUsingSequenceFileWithBinarySerde() throws Exception {
+    KeyValueSet options = new KeyValueSet();
+    options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
+    TableMeta meta = new TableMeta(CatalogProtos.StoreType.SEQUENCEFILE, options);
+
+    org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
+    schema.addColumn("r_regionkey", TajoDataTypes.Type.INT4);
+    schema.addColumn("r_name", TajoDataTypes.Type.TEXT);
+    schema.addColumn("r_comment", TajoDataTypes.Type.TEXT);
+
+    TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta,
+        new Path(warehousePath, new Path(DB_NAME, REGION)).toUri());
+    store.createTable(table.getProto());
+    assertTrue(store.existTable(DB_NAME, REGION));
+
+    TableDesc table1 = new TableDesc(store.getTable(DB_NAME, REGION));
+    assertEquals(table.getName(), table1.getName());
+    assertEquals(table.getPath(), table1.getPath());
+    assertEquals(table.getSchema().size(), table1.getSchema().size());
+    for (int i = 0; i < table.getSchema().size(); i++) {
+      assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName());
+    }
+
+    assertEquals(StorageConstants.DEFAULT_BINARY_SERDE,
+        table1.getMeta().getOption(StorageConstants.SEQUENCEFILE_SERDE));
+    store.dropTable(DB_NAME, REGION);
+  }
+
+  @Test
+  public void testTableUsingSequenceFileWithTextSerde() throws Exception {
+    KeyValueSet options = new KeyValueSet();
+    options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
+    TableMeta meta = new TableMeta(CatalogProtos.StoreType.SEQUENCEFILE, options);
+
+    org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
+    schema.addColumn("r_regionkey", TajoDataTypes.Type.INT4);
+    schema.addColumn("r_name", TajoDataTypes.Type.TEXT);
+    schema.addColumn("r_comment", TajoDataTypes.Type.TEXT);
+
+    TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta,
+        new Path(warehousePath, new Path(DB_NAME, REGION)).toUri());
+    store.createTable(table.getProto());
+    assertTrue(store.existTable(DB_NAME, REGION));
+
+    TableDesc table1 = new TableDesc(store.getTable(DB_NAME, REGION));
+    assertEquals(table.getName(), table1.getName());
+    assertEquals(table.getPath(), table1.getPath());
+    assertEquals(table.getSchema().size(), table1.getSchema().size());
+    for (int i = 0; i < table.getSchema().size(); i++) {
+      assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName());
+    }
+
+    assertEquals(StorageConstants.DEFAULT_TEXT_SERDE, table1.getMeta().getOption(StorageConstants.SEQUENCEFILE_SERDE));
+    store.dropTable(DB_NAME, REGION);
+  }
+
+
+  @Test
+  public void testTableUsingParquet() throws Exception {
+    TableMeta meta = new TableMeta(CatalogProtos.StoreType.PARQUET, new KeyValueSet());
+
+    org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
+    schema.addColumn("c_custkey", TajoDataTypes.Type.INT4);
+    schema.addColumn("c_name", TajoDataTypes.Type.TEXT);
+    schema.addColumn("c_address", TajoDataTypes.Type.TEXT);
+    schema.addColumn("c_nationkey", TajoDataTypes.Type.INT4);
+    schema.addColumn("c_phone", TajoDataTypes.Type.TEXT);
+    schema.addColumn("c_acctbal", TajoDataTypes.Type.FLOAT8);
+    schema.addColumn("c_mktsegment", TajoDataTypes.Type.TEXT);
+    schema.addColumn("c_comment", TajoDataTypes.Type.TEXT);
+
+    TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, CUSTOMER), schema, meta,
+        new Path(warehousePath, new Path(DB_NAME, CUSTOMER)).toUri());
+    store.createTable(table.getProto());
+    assertTrue(store.existTable(DB_NAME, CUSTOMER));
+
+    TableDesc table1 = new TableDesc(store.getTable(DB_NAME, CUSTOMER));
+    assertEquals(table.getName(), table1.getName());
+    assertEquals(table.getPath(), table1.getPath());
+    assertEquals(table.getSchema().size(), table1.getSchema().size());
+    for (int i = 0; i < table.getSchema().size(); i++) {
+      assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName());
+    }
+
+    store.dropTable(DB_NAME, CUSTOMER);
+  }
+}


Mime
View raw message