incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tra...@apache.org
Subject svn commit: r1391213 - in /incubator/hcatalog/trunk: ./ hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/ src/java/org/apache/hcatalog/common/ src/java/org/apache/hcatalog/data/ src/java/org/apache/hcatalog/data/schema/
Date Thu, 27 Sep 2012 20:55:57 GMT
Author: travis
Date: Thu Sep 27 20:55:56 2012
New Revision: 1391213

URL: http://svn.apache.org/viewvc?rev=1391213&view=rev
Log:
HCATALOG-497 HCatContext should use the jobconf instead of its own conf

Modified:
    incubator/hcatalog/trunk/CHANGES.txt
    incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatLoader.java
    incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatStorer.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatContext.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java

Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1391213&r1=1391212&r2=1391213&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Thu Sep 27 20:55:56 2012
@@ -115,6 +115,8 @@ Trunk (unreleased changes)
   OPTIMIZATIONS
 
   BUG FIXES
+  HCAT-497 HCatContext should use the jobconf instead of its own conf (traviscrawford)
+
   HCAT-494 MultiOutputFormat in 0.23 fails to setAliasConf() correctly. (mithun via toffer)
 
   HCAT-507 e2e harness failing on 0.23 (toffer)

Modified: incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatLoader.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatLoader.java?rev=1391213&r1=1391212&r2=1391213&view=diff
==============================================================================
--- incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatLoader.java
(original)
+++ incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatLoader.java
Thu Sep 27 20:55:56 2012
@@ -82,6 +82,8 @@ public class HCatLoader extends HCatBase
 
     @Override
     public void setLocation(String location, Job job) throws IOException {
+        HCatContext.setupHCatContext(job.getConfiguration()).getConf().get()
+            .setBoolean(HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION, true);
 
         UDFContext udfContext = UDFContext.getUDFContext();
         Properties udfProps = udfContext.getUDFProperties(this.getClass(),
@@ -185,9 +187,8 @@ public class HCatLoader extends HCatBase
 
     @Override
     public ResourceSchema getSchema(String location, Job job) throws IOException {
-        HCatContext.getInstance().mergeConf(job.getConfiguration());
-        HCatContext.getInstance().getConf().setBoolean(
-            HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION, true);
+        HCatContext.setupHCatContext(job.getConfiguration()).getConf().get()
+            .setBoolean(HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION, true);
 
         Table table = phutil.getTable(location,
             hcatServerUri != null ? hcatServerUri : PigHCatUtil.getHCatServerUri(job),

Modified: incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatStorer.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatStorer.java?rev=1391213&r1=1391212&r2=1391213&view=diff
==============================================================================
--- incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatStorer.java
(original)
+++ incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatStorer.java
Thu Sep 27 20:55:56 2012
@@ -77,9 +77,8 @@ public class HCatStorer extends HCatBase
 
     @Override
     public void setStoreLocation(String location, Job job) throws IOException {
-        HCatContext.getInstance().mergeConf(job.getConfiguration());
-        HCatContext.getInstance().getConf().setBoolean(
-            HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION, false);
+        HCatContext.setupHCatContext(job.getConfiguration()).getConf().get()
+            .setBoolean(HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION, false);
 
         Configuration config = job.getConfiguration();
         config.set(INNER_SIGNATURE, INNER_SIGNATURE_PREFIX + "_" + sign);

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatContext.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatContext.java?rev=1391213&r1=1391212&r2=1391213&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatContext.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatContext.java Thu Sep
27 20:55:56 2012
@@ -18,37 +18,54 @@
 
 package org.apache.hcatalog.common;
 
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
 import org.apache.hadoop.conf.Configuration;
 
 import java.util.Map;
 
 /**
- * HCatContext provides global access to configuration data.
+ * HCatContext provides global access to configuration data. It uses a reference to the
+ * job configuration so that settings are automatically passed to the backend by the
+ * MR framework.
  */
 public class HCatContext {
 
     private static final HCatContext hCatContext = new HCatContext();
 
-    private final Configuration conf;
+    private Configuration conf = null;
 
     private HCatContext() {
-        conf = new Configuration();
     }
 
-    public static HCatContext getInstance() {
+    /**
+     * Setup the HCatContext as a reference to the given configuration. Keys
+     * exclusive to an existing config are set in the new conf.
+     */
+    public static synchronized HCatContext setupHCatContext(Configuration newConf) {
+        Preconditions.checkNotNull(newConf, "HCatContext must not have a null conf.");
+
+        if (hCatContext.conf == null) {
+            hCatContext.conf = newConf;
+            return hCatContext;
+        }
+
+        if (hCatContext.conf != newConf) {
+            for (Map.Entry<String, String> entry : hCatContext.conf) {
+                if (newConf.get(entry.getKey()) == null) {
+                    newConf.set(entry.getKey(), entry.getValue());
+                }
+            }
+            hCatContext.conf = newConf;
+        }
         return hCatContext;
     }
 
-    public Configuration getConf() {
-        return conf;
+    public static HCatContext getInstance() {
+        return hCatContext;
     }
 
-    /**
-     * Merge the given configuration into the HCatContext conf, overwriting any existing
keys.
-     */
-    public void mergeConf(Configuration conf) {
-        for (Map.Entry<String, String> entry : conf) {
-            this.conf.set(entry.getKey(), entry.getValue());
-        }
+    public Optional<Configuration> getConf() {
+        return Optional.fromNullable(conf);
     }
 }

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java?rev=1391213&r1=1391212&r2=1391213&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java Thu Sep
27 20:55:56 2012
@@ -188,24 +188,7 @@ public class HCatRecordSerDe implements 
 
         Object res;
         if (fieldObjectInspector.getCategory() == Category.PRIMITIVE) {
-            if (field != null && field instanceof Boolean &&
-                HCatContext.getInstance().getConf().getBoolean(
-                    HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER,
-                    HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER_DEFAULT)) {
-                res = ((Boolean) field) ? 1 : 0;
-            } else if (field != null && field instanceof Short &&
-                HCatContext.getInstance().getConf().getBoolean(
-                    HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION,
-                    HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION_DEFAULT)) {
-                res = new Integer((Short) field);
-            } else if (field != null && field instanceof Byte &&
-                HCatContext.getInstance().getConf().getBoolean(
-                    HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION,
-                    HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION_DEFAULT)) {
-                res = new Integer((Byte) field);
-            } else {
-                res = ((PrimitiveObjectInspector) fieldObjectInspector).getPrimitiveJavaObject(field);
-            }
+            res = serializePrimitiveField(field, fieldObjectInspector);
         } else if (fieldObjectInspector.getCategory() == Category.STRUCT) {
             res = serializeStruct(field, (StructObjectInspector) fieldObjectInspector);
         } else if (fieldObjectInspector.getCategory() == Category.LIST) {
@@ -279,6 +262,32 @@ public class HCatRecordSerDe implements 
         }
     }
 
+    private static Object serializePrimitiveField(Object field,
+            ObjectInspector fieldObjectInspector) {
+
+        if (field != null && HCatContext.getInstance().getConf().isPresent()) {
+            Configuration conf = HCatContext.getInstance().getConf().get();
+
+            if (field instanceof Boolean &&
+                conf.getBoolean(
+                    HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER,
+                    HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER_DEFAULT)) {
+                return ((Boolean) field) ? 1 : 0;
+            } else if (field instanceof Short &&
+                conf.getBoolean(
+                    HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION,
+                    HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION_DEFAULT)) {
+                return new Integer((Short) field);
+            } else if (field instanceof Byte &&
+                conf.getBoolean(
+                    HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION,
+                    HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION_DEFAULT)) {
+                return new Integer((Byte) field);
+            }
+        }
+
+        return ((PrimitiveObjectInspector) fieldObjectInspector).getPrimitiveJavaObject(field);
+    }
 
     /**
      * Return an object inspector that can read through the object

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java?rev=1391213&r1=1391212&r2=1391213&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java
(original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java
Thu Sep 27 20:55:56 2012
@@ -139,14 +139,17 @@ public class HCatSchemaUtils {
     private static Type getPrimitiveHType(TypeInfo basePrimitiveTypeInfo) {
         switch (((PrimitiveTypeInfo) basePrimitiveTypeInfo).getPrimitiveCategory()) {
         case BOOLEAN:
-            return HCatContext.getInstance().getConf().getBoolean(
-                HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER,
-                HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER_DEFAULT) ?
+            return (HCatContext.getInstance().getConf().isPresent() &&
+                HCatContext.getInstance().getConf().get().getBoolean(
+                    HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER,
+                    HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER_DEFAULT)) ?
                 Type.INT : Type.BOOLEAN;
         case BYTE:
-            return HCatContext.getInstance().getConf().getBoolean(
-                HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION,
-                HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION_DEFAULT) ? Type.INT : Type.TINYINT;
+            return (HCatContext.getInstance().getConf().isPresent() &&
+                HCatContext.getInstance().getConf().get().getBoolean(
+                    HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION,
+                    HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION_DEFAULT)) ?
+                Type.INT : Type.TINYINT;
         case DOUBLE:
             return Type.DOUBLE;
         case FLOAT:
@@ -156,9 +159,10 @@ public class HCatSchemaUtils {
         case LONG:
             return Type.BIGINT;
         case SHORT:
-            return HCatContext.getInstance().getConf().getBoolean(
-                HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION,
-                HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION_DEFAULT) ?
+            return (HCatContext.getInstance().getConf().isPresent() &&
+                HCatContext.getInstance().getConf().get().getBoolean(
+                    HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION,
+                    HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION_DEFAULT)) ?
                 Type.INT : Type.SMALLINT;
         case STRING:
             return Type.STRING;



Mime
View raw message