incubator-giraph-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jgho...@apache.org
Subject svn commit: r1203379 - in /incubator/giraph/trunk: ./ src/main/java/org/apache/giraph/lib/ src/test/java/org/apache/giraph/lib/
Date Thu, 17 Nov 2011 21:37:05 GMT
Author: jghoman
Date: Thu Nov 17 21:37:05 2011
New Revision: 1203379

URL: http://svn.apache.org/viewvc?rev=1203379&view=rev
Log:
GIRAPH-92. Need outputformat for just vertex ID and value. 

Added:
    incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/IdWithValueTextOutputFormat.java
    incubator/giraph/trunk/src/test/java/org/apache/giraph/lib/TestIdWithValueTextOutputFormat.java
Modified:
    incubator/giraph/trunk/CHANGELOG
    incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/AdjacencyListTextVertexOutputFormat.java

Modified: incubator/giraph/trunk/CHANGELOG
URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/CHANGELOG?rev=1203379&r1=1203378&r2=1203379&view=diff
==============================================================================
--- incubator/giraph/trunk/CHANGELOG (original)
+++ incubator/giraph/trunk/CHANGELOG Thu Nov 17 21:37:05 2011
@@ -2,6 +2,8 @@ Giraph Change Log
 
 Release 0.70.0 - unreleased
 
+  GIRAPH-92: Need outputformat for just vertex ID and value. (jghoman)
+
   GIRAPH-86: Simplify boolean expressions in ZooKeeperExt::createExt.
   (attilacsordas via jghoman)
 

Modified: incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/AdjacencyListTextVertexOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/AdjacencyListTextVertexOutputFormat.java?rev=1203379&r1=1203378&r2=1203379&view=diff
==============================================================================
--- incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/AdjacencyListTextVertexOutputFormat.java
(original)
+++ incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/AdjacencyListTextVertexOutputFormat.java
Thu Nov 17 21:37:05 2011
@@ -43,7 +43,7 @@ public class AdjacencyListTextVertexOutp
 
   static class AdjacencyListVertexWriter<I extends WritableComparable, V extends
       Writable, E extends Writable> extends TextVertexWriter<I, V, E> {
-    public static final String LINE_TOKENIZE_VALUE = "adj.list.output.delimiter";
+    public static final String LINE_TOKENIZE_VALUE = "output.delimiter";
     public static final String LINE_TOKENIZE_VALUE_DEFAULT = "\t";
 
     private String delimiter;

Added: incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/IdWithValueTextOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/IdWithValueTextOutputFormat.java?rev=1203379&view=auto
==============================================================================
--- incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/IdWithValueTextOutputFormat.java
(added)
+++ incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/IdWithValueTextOutputFormat.java
Thu Nov 17 21:37:05 2011
@@ -0,0 +1,76 @@
+package org.apache.giraph.lib;
+
+
+import org.apache.giraph.graph.BasicVertex;
+import org.apache.giraph.graph.VertexWriter;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+import java.io.IOException;
+
+/**
+ * Write out Vertices' IDs and values, but not their edges nor edges' values.
+ * This is a useful output format when the final value of the vertex is
+ * all that's needed. The boolean configuration parameter reverse.id.and.value
+ * allows reversing the output of id and value.
+ *
+ * @param <I> Vertex index value
+ * @param <V> Vertex value
+ * @param <E> Edge value
+ */
+public class IdWithValueTextOutputFormat <I extends WritableComparable,
+    V extends Writable, E extends Writable> extends TextVertexOutputFormat<I, V, E>{
+
+  static class IdWithValueVertexWriter<I extends WritableComparable, V extends
+      Writable, E extends Writable> extends TextVertexWriter<I, V, E> {
+
+    public static final String LINE_TOKENIZE_VALUE = "output.delimiter";
+    public static final String LINE_TOKENIZE_VALUE_DEFAULT = "\t";
+
+    public static final String REVERSE_ID_AND_VALUE = "reverse.id.and.value";
+    public static final boolean REVERSE_ID_AND_VALUE_DEFAULT = false;
+
+    private String delimiter;
+
+    public IdWithValueVertexWriter(RecordWriter<Text, Text> recordWriter) {
+      super(recordWriter);
+    }
+
+    @Override
+    public void writeVertex(BasicVertex<I, V, E, ?> vertex) throws IOException,
+        InterruptedException {
+      if (delimiter == null) {
+        delimiter = getContext().getConfiguration()
+           .get(LINE_TOKENIZE_VALUE, LINE_TOKENIZE_VALUE_DEFAULT);
+      }
+
+      String first;
+      String second;
+      boolean reverseOutput = getContext().getConfiguration()
+          .getBoolean(REVERSE_ID_AND_VALUE, REVERSE_ID_AND_VALUE_DEFAULT);
+
+      if (reverseOutput) {
+        first = vertex.getVertexValue().toString();
+        second = vertex.getVertexId().toString();
+      } else {
+        first = vertex.getVertexId().toString();
+        second = vertex.getVertexValue().toString();
+      }
+
+      Text line = new Text(first + delimiter + second);
+
+      getRecordWriter().write(line, null);
+    }
+  }
+
+  @Override
+  public VertexWriter<I, V, E> createVertexWriter(TaskAttemptContext context)
+      throws IOException, InterruptedException {
+    return new IdWithValueVertexWriter<I, V, E>
+        (textOutputFormat.getRecordWriter(context));
+  }
+
+}

Added: incubator/giraph/trunk/src/test/java/org/apache/giraph/lib/TestIdWithValueTextOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/src/test/java/org/apache/giraph/lib/TestIdWithValueTextOutputFormat.java?rev=1203379&view=auto
==============================================================================
--- incubator/giraph/trunk/src/test/java/org/apache/giraph/lib/TestIdWithValueTextOutputFormat.java
(added)
+++ incubator/giraph/trunk/src/test/java/org/apache/giraph/lib/TestIdWithValueTextOutputFormat.java
Thu Nov 17 21:37:05 2011
@@ -0,0 +1,70 @@
+package org.apache.giraph.lib;
+
+import junit.framework.TestCase;
+import org.apache.giraph.graph.BasicVertex;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.mockito.Matchers;
+
+import java.io.IOException;
+import java.util.ArrayList;
+
+import static org.apache.giraph.lib.IdWithValueTextOutputFormat.IdWithValueVertexWriter;
+import static org.apache.giraph.lib.IdWithValueTextOutputFormat.IdWithValueVertexWriter.LINE_TOKENIZE_VALUE;
+import static org.apache.giraph.lib.IdWithValueTextOutputFormat.IdWithValueVertexWriter.REVERSE_ID_AND_VALUE;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class TestIdWithValueTextOutputFormat extends TestCase {
+  public void testHappyPath() throws IOException, InterruptedException {
+    Configuration conf = new Configuration();
+    Text expected = new Text("Four Tops\t4.0");
+
+    IdWithValueTestWorker(conf, expected);
+  }
+
+  public void testReverseIdAndValue() throws IOException, InterruptedException {
+    Configuration conf = new Configuration();
+    conf.setBoolean(REVERSE_ID_AND_VALUE, true);
+    Text expected = new Text("4.0\tFour Tops");
+
+    IdWithValueTestWorker(conf, expected);
+  }
+
+  public void testWithDifferentDelimiter()  throws IOException,
+      InterruptedException {
+    Configuration conf = new Configuration();
+    conf.set(LINE_TOKENIZE_VALUE, "blah");
+    Text expected = new Text("Four Topsblah4.0");
+
+    IdWithValueTestWorker(conf, expected);
+  }
+
+  private void IdWithValueTestWorker(Configuration conf, Text expected)
+      throws IOException, InterruptedException {
+    TaskAttemptContext tac = mock(TaskAttemptContext.class);
+    when(tac.getConfiguration()).thenReturn(conf);
+
+    BasicVertex vertex = mock(BasicVertex.class);
+    when(vertex.getVertexId()).thenReturn(new Text("Four Tops"));
+    when(vertex.getVertexValue()).thenReturn(new DoubleWritable(4d));
+
+    // Create empty iterator == no edges
+    when(vertex.iterator()).thenReturn(new ArrayList<Text>().iterator());
+
+    RecordWriter<Text, Text> tw = mock(RecordWriter.class);
+    IdWithValueVertexWriter writer = new IdWithValueVertexWriter(tw);
+    writer.initialize(tac);
+    writer.writeVertex(vertex);
+
+    verify(tw).write(expected, null);
+    verify(vertex, times(0)).iterator();
+    verify(vertex, times(0)).getEdgeValue(Matchers.<WritableComparable>any());
+  }
+}



Mime
View raw message