incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [3/6] git commit: Handling hive nulls.
Date Tue, 17 Dec 2013 13:57:10 GMT
Handling hive nulls.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/f59b6bbe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/f59b6bbe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/f59b6bbe

Branch: refs/heads/master
Commit: f59b6bbeedd9b993ec08a40b5ae5dc243cddd5bb
Parents: f2ab337
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Tue Dec 17 08:11:01 2013 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Tue Dec 17 08:52:10 2013 -0500

----------------------------------------------------------------------
 .../apache/blur/mapreduce/lib/CsvBlurMapper.java   | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f59b6bbe/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurMapper.java
----------------------------------------------------------------------
diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurMapper.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurMapper.java
index 8557dfc..8f59e31 100644
--- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurMapper.java
+++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurMapper.java
@@ -47,7 +47,7 @@ import com.google.common.base.Splitter;
  */
 public class CsvBlurMapper extends BaseBlurMapper<Writable, Text> {
 
-  private static final String UTF_8 = "UTF-8";
+  public static final String UTF_8 = "UTF-8";
   public static final String BLUR_CSV_AUTO_GENERATE_RECORD_ID_AS_HASH_OF_DATA = "blur.csv.auto.generate.record.id.as.hash.of.data";
   public static final String BLUR_CSV_AUTO_GENERATE_ROW_ID_AS_HASH_OF_DATA = "blur.csv.auto.generate.row.id.as.hash.of.data";
   public static final String BLUR_CSV_FAMILY_PATH_MAPPINGS_FAMILIES = "blur.csv.family.path.mappings.families";
@@ -55,6 +55,7 @@ public class CsvBlurMapper extends BaseBlurMapper<Writable, Text>
{
   public static final String BLUR_CSV_SEPARATOR_BASE64 = "blur.csv.separator.base64";
   public static final String BLUR_CSV_FAMILY_COLUMN_PREFIX = "blur.csv.family.";
   public static final String BLUR_CSV_FAMILIES = "blur.csv.families";
+  public static final String HIVE_NULL = "\\N";
 
   protected Map<String, List<String>> _columnNameMap;
   protected String _separator = Base64.encodeBase64String(",".getBytes());
@@ -440,8 +441,11 @@ public class CsvBlurMapper extends BaseBlurMapper<Writable, Text>
{
     }
 
     for (int i = 0; i < columnNames.size(); i++) {
-      record.addColumn(columnNames.get(i), list.get(i + offset));
-      _columnCounter.increment(1);
+      String val = handleHiveNulls(list.get(i + offset));
+      if (val != null) {
+        record.addColumn(columnNames.get(i), val);
+        _columnCounter.increment(1);
+      }
     }
     _key.set(record.getRowId());
     _mutate.setMutateType(MUTATE_TYPE.REPLACE);
@@ -450,6 +454,13 @@ public class CsvBlurMapper extends BaseBlurMapper<Writable, Text>
{
     context.progress();
   }
 
+  protected String handleHiveNulls(String value) {
+    if (value.equals(HIVE_NULL)) {
+      return null;
+    }
+    return value;
+  }
+
   public void setFamilyFromPath(String familyFromPath) {
     this._familyFromPath = familyFromPath;
   }


Mime
View raw message