tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From blrun...@apache.org
Subject git commit: TAJO-816: NULL delimiter doesn't apply with HCatalogStore. (jaehwa)
Date Mon, 12 May 2014 15:33:33 GMT
Repository: tajo
Updated Branches:
  refs/heads/branch-0.8.1 db3fa4f3e -> 05cf59539


TAJO-816: NULL delimiter doesn't apply with HCatalogStore. (jaehwa)


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/05cf5953
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/05cf5953
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/05cf5953

Branch: refs/heads/branch-0.8.1
Commit: 05cf595396a24c8fb0f6ae0525740fe5ff44a2e8
Parents: db3fa4f
Author: blrunner <jhjung@gruter.com>
Authored: Tue May 13 00:33:18 2014 +0900
Committer: blrunner <jhjung@gruter.com>
Committed: Tue May 13 00:33:18 2014 +0900

----------------------------------------------------------------------
 CHANGES                                         |  2 ++
 .../tajo/catalog/store/HCatalogStore.java       | 20 ++++++++++++--------
 .../tajo/catalog/store/TestHCatalogStore.java   | 14 ++++++++++----
 3 files changed, 24 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/05cf5953/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 7a6b764..60ab5d7 100644
--- a/CHANGES
+++ b/CHANGES
@@ -16,6 +16,8 @@ Release 0.8.1 - unreleased
 
   BUGS
 
+    TAJO-816: NULL delimiter doesn't apply with HCatalogStore. (jaehwa)
+
     TAJO-795: PlannerUtil::joinJoinKeyForEachTable need to handle theta-join. (jaehwa)
 
     TAJO-792: Insert table error with database name. 

http://git-wip-us.apache.org/repos/asf/tajo/blob/05cf5953/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
index 4880447..96e1fc2 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
@@ -434,7 +434,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore
{
       // If you want to modify table path, you have to modify on Hive cli.
       if (tableDesc.isExternal()) {
         table.setTableType(TableType.EXTERNAL_TABLE.name());
-        table.getParameters().put("EXTERNAL", "TRUE");
+        table.putToParameters("EXTERNAL", "TRUE");
 
         FileSystem fs = tableDesc.getPath().getFileSystem(conf);
         if (fs.isFile(tableDesc.getPath())) {
@@ -477,7 +477,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore
{
         }
 
         if (tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL) != null) {
-          sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_NULL_FORMAT,
+          table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT,
               StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL)));
         }
       } else if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.CSV))
{
@@ -493,14 +493,16 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore
{
         // And hive will un-espace this value again.
         // As a result, user can use right field delimiter.
         // So, we have to un-escape this value.
-        sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT,
+        sd.getSerdeInfo().putToParameters(serdeConstants.SERIALIZATION_FORMAT,
             StringEscapeUtils.unescapeJava(fieldDelimiter));
-        sd.getSerdeInfo().getParameters().put(serdeConstants.FIELD_DELIM,
+        sd.getSerdeInfo().putToParameters(serdeConstants.FIELD_DELIM,
             StringEscapeUtils.unescapeJava(fieldDelimiter));
+        table.getParameters().remove(StorageConstants.CSVFILE_DELIMITER);
 
         if (tableDesc.getMeta().getOption(StorageConstants.CSVFILE_NULL) != null) {
-          sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_NULL_FORMAT,
+          table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT,
               StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.CSVFILE_NULL)));
+          table.getParameters().remove(StorageConstants.CSVFILE_NULL);
         }
       } else if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.SEQUENCEFILE))
{
         String serde = tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_SERDE);
@@ -518,17 +520,19 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore
{
           // And hive will un-espace this value again.
           // As a result, user can use right field delimiter.
           // So, we have to un-escape this value.
-          sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT,
+          sd.getSerdeInfo().putToParameters(serdeConstants.SERIALIZATION_FORMAT,
               StringEscapeUtils.unescapeJava(fieldDelimiter));
-          sd.getSerdeInfo().getParameters().put(serdeConstants.FIELD_DELIM,
+          sd.getSerdeInfo().putToParameters(serdeConstants.FIELD_DELIM,
               StringEscapeUtils.unescapeJava(fieldDelimiter));
+          table.getParameters().remove(StorageConstants.SEQUENCEFILE_DELIMITER);
         } else {
           sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe.class.getName());
         }
 
         if (tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_NULL) != null) {
-          sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_NULL_FORMAT,
+          table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT,
               StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_NULL)));
+          table.getParameters().remove(StorageConstants.SEQUENCEFILE_NULL);
         }
       } else {
         throw new CatalogException(new NotImplementedException(tableDesc.getMeta().getStoreType().name()));

http://git-wip-us.apache.org/repos/asf/tajo/blob/05cf5953/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
index 0cb4520..a0ef9d9 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
@@ -170,11 +170,10 @@ public class TestHCatalogStore {
   @Test
   public void testTableWithNullValue() throws Exception {
     Options options = new Options();
-    options.put(StorageConstants.CSVFILE_DELIMITER, StringEscapeUtils.escapeJava("\u0001"));
-    options.put(StorageConstants.CSVFILE_NULL, StringEscapeUtils.escapeJava("\\N"));
+    options.put(StorageConstants.CSVFILE_DELIMITER, StringEscapeUtils.escapeJava("\u0002"));
+    options.put(StorageConstants.CSVFILE_NULL, StringEscapeUtils.escapeJava("\u0003"));
     TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, options);
 
-
     org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
     schema.addColumn("s_suppkey", TajoDataTypes.Type.INT4);
     schema.addColumn("s_name", TajoDataTypes.Type.TEXT);
@@ -187,7 +186,6 @@ public class TestHCatalogStore {
     TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, SUPPLIER), schema, meta,
         new Path(warehousePath, new Path(DB_NAME, SUPPLIER)));
 
-
     store.createTable(table.getProto());
     assertTrue(store.existTable(DB_NAME, SUPPLIER));
 
@@ -204,7 +202,15 @@ public class TestHCatalogStore {
 
     assertEquals(table.getMeta().getOption(StorageConstants.CSVFILE_NULL),
         table1.getMeta().getOption(StorageConstants.CSVFILE_NULL));
+
+    assertEquals(table1.getMeta().getOption(StorageConstants.CSVFILE_DELIMITER),
+        StringEscapeUtils.escapeJava("\u0002"));
+
+    assertEquals(table1.getMeta().getOption(StorageConstants.CSVFILE_NULL),
+        StringEscapeUtils.escapeJava("\u0003"));
+
     store.dropTable(DB_NAME, SUPPLIER);
+
   }
 
   @Test


Mime
View raw message