tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hyun...@apache.org
Subject [42/48] git commit: TAJO-816: NULL delimiter doesn't apply with HCatalogStore. (jaehwa)
Date Tue, 20 May 2014 18:46:40 GMT
TAJO-816: NULL delimiter doesn't apply with HCatalogStore. (jaehwa)


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/77fdbfb2
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/77fdbfb2
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/77fdbfb2

Branch: refs/heads/window_function
Commit: 77fdbfb2e596c7b1c4d54dff9f4161b9b5378889
Parents: 214301f
Author: blrunner <jhjung@gruter.com>
Authored: Tue May 13 00:32:59 2014 +0900
Committer: blrunner <jhjung@gruter.com>
Committed: Tue May 13 00:32:59 2014 +0900

----------------------------------------------------------------------
 CHANGES                                         |  2 ++
 .../tajo/catalog/store/HCatalogStore.java       | 20 ++++++++++++--------
 .../tajo/catalog/store/TestHCatalogStore.java   | 14 ++++++++++----
 3 files changed, 24 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/77fdbfb2/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index fcd4dbc..75deb0f 100644
--- a/CHANGES
+++ b/CHANGES
@@ -37,6 +37,8 @@ Release 0.9.0 - unreleased
 
   BUG FIXES
 
+    TAJO-816: NULL delimiter doesn't apply with HCatalogStore. (jaehwa)
+
     TAJO-823: Missing INET4 handling in DatumFactory.cast(). (jihoon)
 
     TAJO-803: INSERT INTO without FROM throws ClassCastException.

http://git-wip-us.apache.org/repos/asf/tajo/blob/77fdbfb2/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
index 7a694fa..7924af1 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
@@ -436,7 +436,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore
{
       // If you want to modify table path, you have to modify on Hive cli.
       if (tableDesc.isExternal()) {
         table.setTableType(TableType.EXTERNAL_TABLE.name());
-        table.getParameters().put("EXTERNAL", "TRUE");
+        table.putToParameters("EXTERNAL", "TRUE");
 
         FileSystem fs = tableDesc.getPath().getFileSystem(conf);
         if (fs.isFile(tableDesc.getPath())) {
@@ -479,7 +479,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore
{
         }
 
         if (tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL) != null) {
-          sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_NULL_FORMAT,
+          table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT,
               StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL)));
         }
       } else if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.CSV))
{
@@ -495,14 +495,16 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore
{
         // And hive will un-espace this value again.
         // As a result, user can use right field delimiter.
         // So, we have to un-escape this value.
-        sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT,
+        sd.getSerdeInfo().putToParameters(serdeConstants.SERIALIZATION_FORMAT,
             StringEscapeUtils.unescapeJava(fieldDelimiter));
-        sd.getSerdeInfo().getParameters().put(serdeConstants.FIELD_DELIM,
+        sd.getSerdeInfo().putToParameters(serdeConstants.FIELD_DELIM,
             StringEscapeUtils.unescapeJava(fieldDelimiter));
+        table.getParameters().remove(StorageConstants.CSVFILE_DELIMITER);
 
         if (tableDesc.getMeta().getOption(StorageConstants.CSVFILE_NULL) != null) {
-          sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_NULL_FORMAT,
+          table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT,
               StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.CSVFILE_NULL)));
+          table.getParameters().remove(StorageConstants.CSVFILE_NULL);
         }
       } else if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.SEQUENCEFILE))
{
         String serde = tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_SERDE);
@@ -520,17 +522,19 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore
{
           // And hive will un-espace this value again.
           // As a result, user can use right field delimiter.
           // So, we have to un-escape this value.
-          sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT,
+          sd.getSerdeInfo().putToParameters(serdeConstants.SERIALIZATION_FORMAT,
               StringEscapeUtils.unescapeJava(fieldDelimiter));
-          sd.getSerdeInfo().getParameters().put(serdeConstants.FIELD_DELIM,
+          sd.getSerdeInfo().putToParameters(serdeConstants.FIELD_DELIM,
               StringEscapeUtils.unescapeJava(fieldDelimiter));
+          table.getParameters().remove(StorageConstants.SEQUENCEFILE_DELIMITER);
         } else {
           sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe.class.getName());
         }
 
         if (tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_NULL) != null) {
-          sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_NULL_FORMAT,
+          table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT,
               StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_NULL)));
+          table.getParameters().remove(StorageConstants.SEQUENCEFILE_NULL);
         }
       } else {
         throw new CatalogException(new NotImplementedException(tableDesc.getMeta().getStoreType().name()));

http://git-wip-us.apache.org/repos/asf/tajo/blob/77fdbfb2/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
index a862baf..729184a 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
@@ -174,11 +174,10 @@ public class TestHCatalogStore {
   @Test
   public void testTableWithNullValue() throws Exception {
     KeyValueSet options = new KeyValueSet();
-    options.put(StorageConstants.CSVFILE_DELIMITER, StringEscapeUtils.escapeJava("\u0001"));
-    options.put(StorageConstants.CSVFILE_NULL, StringEscapeUtils.escapeJava("\\N"));
+    options.put(StorageConstants.CSVFILE_DELIMITER, StringEscapeUtils.escapeJava("\u0002"));
+    options.put(StorageConstants.CSVFILE_NULL, StringEscapeUtils.escapeJava("\u0003"));
     TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, options);
 
-
     org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
     schema.addColumn("s_suppkey", TajoDataTypes.Type.INT4);
     schema.addColumn("s_name", TajoDataTypes.Type.TEXT);
@@ -191,7 +190,6 @@ public class TestHCatalogStore {
     TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, SUPPLIER), schema, meta,
         new Path(warehousePath, new Path(DB_NAME, SUPPLIER)));
 
-
     store.createTable(table.getProto());
     assertTrue(store.existTable(DB_NAME, SUPPLIER));
 
@@ -208,7 +206,15 @@ public class TestHCatalogStore {
 
     assertEquals(table.getMeta().getOption(StorageConstants.CSVFILE_NULL),
         table1.getMeta().getOption(StorageConstants.CSVFILE_NULL));
+
+    assertEquals(table1.getMeta().getOption(StorageConstants.CSVFILE_DELIMITER),
+        StringEscapeUtils.escapeJava("\u0002"));
+
+    assertEquals(table1.getMeta().getOption(StorageConstants.CSVFILE_NULL),
+        StringEscapeUtils.escapeJava("\u0003"));
+
     store.dropTable(DB_NAME, SUPPLIER);
+
   }
 
   @Test


Mime
View raw message