giraph-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apre...@apache.org
Subject svn commit: r1396441 - in /giraph/trunk: CHANGELOG giraph-formats-contrib/pom.xml giraph-formats-contrib/src/main/java/org/apache/giraph/io/hcatalog/HiveGiraphRunner.java
Date Wed, 10 Oct 2012 01:08:10 GMT
Author: apresta
Date: Wed Oct 10 01:08:10 2012
New Revision: 1396441

URL: http://svn.apache.org/viewvc?rev=1396441&view=rev
Log:
GIRAPH-361: Hive output partition parsing is broken (nitay via apresta)

Modified:
    giraph/trunk/CHANGELOG
    giraph/trunk/giraph-formats-contrib/pom.xml
    giraph/trunk/giraph-formats-contrib/src/main/java/org/apache/giraph/io/hcatalog/HiveGiraphRunner.java

Modified: giraph/trunk/CHANGELOG
URL: http://svn.apache.org/viewvc/giraph/trunk/CHANGELOG?rev=1396441&r1=1396440&r2=1396441&view=diff
==============================================================================
--- giraph/trunk/CHANGELOG (original)
+++ giraph/trunk/CHANGELOG Wed Oct 10 01:08:10 2012
@@ -2,6 +2,8 @@ Giraph Change Log
 
 Release 0.2.0 - unreleased
 
+  GIRAPH-361: Hive output partition parsing is broken (nitay via apresta)
+
   GIRAPH-360: Keep track of the task id in ChannelRotater to send
   requests without knowing the worker id upfront (aching via
   ekoontz)

Modified: giraph/trunk/giraph-formats-contrib/pom.xml
URL: http://svn.apache.org/viewvc/giraph/trunk/giraph-formats-contrib/pom.xml?rev=1396441&r1=1396440&r2=1396441&view=diff
==============================================================================
--- giraph/trunk/giraph-formats-contrib/pom.xml (original)
+++ giraph/trunk/giraph-formats-contrib/pom.xml Wed Oct 10 01:08:10 2012
@@ -130,6 +130,11 @@ under the License.
     </repositories>
     <dependencies>
         <dependency>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+          <version>12.0</version>
+        </dependency>
+        <dependency>
           <groupId>org.apache.hcatalog</groupId>
           <artifactId>hcatalog-core</artifactId>
           <!-- TODO: Use 0.5.0 stable release -->

Modified: giraph/trunk/giraph-formats-contrib/src/main/java/org/apache/giraph/io/hcatalog/HiveGiraphRunner.java
URL: http://svn.apache.org/viewvc/giraph/trunk/giraph-formats-contrib/src/main/java/org/apache/giraph/io/hcatalog/HiveGiraphRunner.java?rev=1396441&r1=1396440&r2=1396441&view=diff
==============================================================================
--- giraph/trunk/giraph-formats-contrib/src/main/java/org/apache/giraph/io/hcatalog/HiveGiraphRunner.java
(original)
+++ giraph/trunk/giraph-formats-contrib/src/main/java/org/apache/giraph/io/hcatalog/HiveGiraphRunner.java
Wed Oct 10 01:08:10 2012
@@ -23,7 +23,6 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
-import java.util.Map.Entry;
 
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.CommandLineParser;
@@ -43,7 +42,6 @@ import org.apache.hcatalog.mapreduce.Inp
 import org.apache.hcatalog.mapreduce.OutputJobInfo;
 
 import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
 import org.apache.log4j.Logger;
 
 /**
@@ -292,7 +290,7 @@ public class HiveGiraphRunner implements
     inputTableName = cmdln.getOptionValue("inputTable");
     inputTableFilterExpr = cmdln.getOptionValue("inputFilter");
     outputTableName = cmdln.getOptionValue("outputTable");
-    outputTablePartitionValues = parsePartitionValues(cmdln
+    outputTablePartitionValues = HiveUtils.parsePartitionValues(cmdln
                 .getOptionValue("outputPartition"));
     workers = Integer.parseInt(cmdln.getOptionValue("workers"));
     isVerbose = cmdln.hasOption("verbose");
@@ -362,50 +360,6 @@ public class HiveGiraphRunner implements
     }
   }
 
-  // TODO use Hive util class if this is already provided by it
-
-  /**
-  * @param outputTablePartitionString table partition string
-  * @return Map
-  */
-  public static Map<String, String> parsePartitionValues(
-            String outputTablePartitionString) {
-    if (outputTablePartitionString != null) {
-      Map<String, String> partitionValues = Maps.newHashMap();
-      for (String partkeyval : outputTablePartitionString.split(" *, *")) {
-        String[] keyval = partkeyval.split(" *= *", 2);
-        if (keyval.length < 2) {
-          throw new IllegalArgumentException(
-              "Unrecognized partition value format: " +
-               outputTablePartitionString);
-        }
-        partitionValues.put(keyval[0], keyval[1]);
-        return partitionValues;
-      }
-    }
-    return null;
-  }
-
-  /**
-  *
-  * @param outputTablePartitionValues  output table partitions
-  * @return  String partition values
-  */
-  private static String serializePartitionValues(
-            Map<String, String> outputTablePartitionValues) {
-    StringBuilder outputTablePartitionValuesString = new StringBuilder();
-    for (Entry<String, String> partitionValueEntry : outputTablePartitionValues
-                .entrySet()) {
-      if (outputTablePartitionValuesString.length() != 0) {
-        outputTablePartitionValuesString.append(",");
-        outputTablePartitionValuesString
-                    .append(partitionValueEntry.getKey()).append("=")
-                    .append(partitionValueEntry.getValue());
-      }
-    }
-    return outputTablePartitionValuesString.toString();
-  }
-
   @Override
   public final Configuration getConf() {
     return conf;
@@ -458,9 +412,8 @@ public class HiveGiraphRunner implements
     }
     LOG.info(prefix + "-outputTable=" + outputTableName);
     if (outputTablePartitionValues != null) {
-      LOG.info(prefix + "-outputPartition=\""  +
-        serializePartitionValues(outputTablePartitionValues) +
-        "\"");
+      LOG.info(prefix + "-outputPartition=\"" +
+               outputTablePartitionValues + "\"");
     }
     LOG.info(prefix + "-workers=" + workers);
   }



Mime
View raw message