incubator-hama-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From edwardy...@apache.org
Subject svn commit: r695728 - in /incubator/hama/trunk: ./ src/java/org/apache/hama/ src/java/org/apache/hama/algebra/ src/java/org/apache/hama/io/ src/java/org/apache/hama/mapred/
Date Tue, 16 Sep 2008 05:08:12 GMT
Author: edwardyoon
Date: Mon Sep 15 22:08:12 2008
New Revision: 695728

URL: http://svn.apache.org/viewvc?rev=695728&view=rev
Log: (empty)

Added:
    incubator/hama/trunk/src/java/org/apache/hama/io/VectorUpdate.java
Modified:
    incubator/hama/trunk/CHANGES.txt
    incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java
    incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
    incubator/hama/trunk/src/java/org/apache/hama/algebra/AdditionReduce.java
    incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationReduce.java
    incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixOutputFormat.java
    incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixReduce.java

Modified: incubator/hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/CHANGES.txt?rev=695728&r1=695727&r2=695728&view=diff
==============================================================================
--- incubator/hama/trunk/CHANGES.txt (original)
+++ incubator/hama/trunk/CHANGES.txt Mon Sep 15 22:08:12 2008
@@ -22,7 +22,8 @@
     HAMA-2: The intial donation of Hama from the google project (edwardyoon)
     
   IMPROVEMENTS
-  
+    
+    HAMA-58: Remove duplicated code (edwardyoon)
     HAMA-56: Add setRow(int row, Vector vector) method to matrix inteface (edwardyoon)
     HAMA-39: IO operations should throws an IOException (edwardyoon)
     HAMA-52: Fixture setup (edwardyoon)

Modified: incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java?rev=695728&r1=695727&r2=695728&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java Mon Sep 15 22:08:12
2008
@@ -26,12 +26,12 @@
 import org.apache.hadoop.hbase.MasterNotRunningException;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.io.BatchUpdate;
 import org.apache.hadoop.hbase.io.Cell;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.mapred.JobClient;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.RunningJob;
+import org.apache.hama.io.VectorUpdate;
 import org.apache.hama.util.Numeric;
 import org.apache.log4j.Logger;
 
@@ -58,9 +58,9 @@
    * @param conf configuration object
    */
   public void setConfiguration(HamaConfiguration conf) {
-    config = (HamaConfiguration) conf;
+    this.config = conf;
     try {
-      admin = new HBaseAdmin(config);
+      this.admin = new HBaseAdmin(config);
     } catch (MasterNotRunningException e) {
       LOG.error(e, e);
     }
@@ -71,10 +71,10 @@
    */
   protected void create() {
     try {
-      tableDesc.addFamily(new HColumnDescriptor(Constants.METADATA));
-      tableDesc.addFamily(new HColumnDescriptor(Constants.ATTRIBUTE));
+      this.tableDesc.addFamily(new HColumnDescriptor(Constants.METADATA));
+      this.tableDesc.addFamily(new HColumnDescriptor(Constants.ATTRIBUTE));
       LOG.info("Initializing the matrix storage.");
-      admin.createTable(tableDesc);
+      this.admin.createTable(this.tableDesc);
     } catch (IOException e) {
       LOG.error(e, e);
     }
@@ -116,9 +116,9 @@
 
   /** {@inheritDoc} */
   public void set(int i, int j, double value) throws IOException {
-    BatchUpdate b = new BatchUpdate(Numeric.intToBytes(i));
-    b.put(Numeric.getColumnIndex(j), Numeric.doubleToBytes(value));
-    table.commit(b);
+    VectorUpdate update = new VectorUpdate(i);
+    update.put(j, value);
+    table.commit(update.getBatchUpdate());
   }
 
   /** {@inheritDoc} */
@@ -128,11 +128,11 @@
 
   /** {@inheritDoc} */
   public void setDimension(int rows, int columns) throws IOException {
-    BatchUpdate b = new BatchUpdate(Constants.METADATA);
-    b.put(Constants.METADATA_ROWS, Numeric.intToBytes(rows));
-    b.put(Constants.METADATA_COLUMNS, Numeric.intToBytes(columns));
+    VectorUpdate update = new VectorUpdate(Constants.METADATA);
+    update.put(Constants.METADATA_ROWS, rows);
+    update.put(Constants.METADATA_COLUMNS, columns);
 
-    table.commit(b);
+    table.commit(update.getBatchUpdate());
   }
 
   public String getRowAttribute(int row) throws IOException {
@@ -144,10 +144,9 @@
   }
 
   public void setRowAttribute(int row, String name) throws IOException {
-    BatchUpdate b = new BatchUpdate(Numeric.intToBytes(row));
-    b.put(Constants.ATTRIBUTE + "string", Bytes.toBytes(name));
-
-    table.commit(b);
+    VectorUpdate update = new VectorUpdate(row);
+    update.put(Constants.ATTRIBUTE + "string", name);
+    table.commit(update.getBatchUpdate());
   }
 
   /** {@inheritDoc} */

Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java?rev=695728&r1=695727&r2=695728&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java Mon Sep 15 22:08:12 2008
@@ -20,14 +20,12 @@
 package org.apache.hama;
 
 import java.io.IOException;
-import java.util.Map;
 
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Scanner;
-import org.apache.hadoop.hbase.io.BatchUpdate;
 import org.apache.hadoop.hbase.io.RowResult;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.mapred.JobConf;
@@ -37,6 +35,7 @@
 import org.apache.hama.algebra.MultiplicationReduce;
 import org.apache.hama.io.VectorEntry;
 import org.apache.hama.io.VectorMapWritable;
+import org.apache.hama.io.VectorUpdate;
 import org.apache.hama.mapred.MatrixReduce;
 import org.apache.hama.util.Numeric;
 import org.apache.hama.util.RandomVariable;
@@ -206,14 +205,10 @@
   }
 
   public void setRow(int row, Vector vector) throws IOException {
-    BatchUpdate b = new BatchUpdate(Numeric.intToBytes(row));
-    for (Map.Entry<Integer, VectorEntry> e : ((DenseVector) vector)
-        .getEntries().entrySet()) {
-      b.put(Numeric.getColumnIndex(e.getKey()), Numeric.doubleToBytes(e
-          .getValue().getValue()));
-    }
+    VectorUpdate update = new VectorUpdate(row);
+    update.putAll(((DenseVector) vector).getEntries().entrySet());
 
-    table.commit(b);
+    table.commit(update.getBatchUpdate());
   }
 
   public void setColumn(int column, Vector vector) throws IOException {

Modified: incubator/hama/trunk/src/java/org/apache/hama/algebra/AdditionReduce.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/algebra/AdditionReduce.java?rev=695728&r1=695727&r2=695728&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/algebra/AdditionReduce.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/algebra/AdditionReduce.java Mon Sep 15 22:08:12
2008
@@ -23,29 +23,29 @@
 import java.util.Iterator;
 import java.util.Map;
 
-import org.apache.hadoop.hbase.io.BatchUpdate;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.mapred.OutputCollector;
 import org.apache.hadoop.mapred.Reporter;
 import org.apache.hama.DenseVector;
 import org.apache.hama.io.VectorEntry;
+import org.apache.hama.io.VectorUpdate;
 import org.apache.hama.mapred.MatrixReduce;
-import org.apache.hama.util.Numeric;
 
 public class AdditionReduce extends MatrixReduce<IntWritable, DenseVector> {
 
   @Override
   public void reduce(IntWritable key, Iterator<DenseVector> values,
-      OutputCollector<IntWritable, BatchUpdate> output, Reporter reporter)
+      OutputCollector<IntWritable, VectorUpdate> output, Reporter reporter)
       throws IOException {
 
-    BatchUpdate b = new BatchUpdate(Numeric.intToBytes(key.get()));
+    VectorUpdate update = new VectorUpdate(key.get());
     DenseVector vector = values.next();
+    
     for (Map.Entry<Integer, VectorEntry> f : vector.entrySet()) {
-      b.put(Numeric.getColumnIndex(f.getKey()), Numeric.doubleToBytes(f.getValue().getValue()));
+      update.put(f.getKey(), f.getValue().getValue());
     }
 
-    output.collect(key, b);
+    output.collect(key, update);
   }
 
 }

Modified: incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationReduce.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationReduce.java?rev=695728&r1=695727&r2=695728&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationReduce.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationReduce.java Mon Sep
15 22:08:12 2008
@@ -24,13 +24,12 @@
 import java.util.Iterator;
 import java.util.Map;
 
-import org.apache.hadoop.hbase.io.BatchUpdate;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.mapred.OutputCollector;
 import org.apache.hadoop.mapred.Reporter;
 import org.apache.hama.DenseVector;
+import org.apache.hama.io.VectorUpdate;
 import org.apache.hama.mapred.MatrixReduce;
-import org.apache.hama.util.Numeric;
 import org.apache.log4j.Logger;
 
 public class MultiplicationReduce extends
@@ -39,10 +38,11 @@
 
   @Override
   public void reduce(IntWritable key, Iterator<DenseVector> values,
-      OutputCollector<IntWritable, BatchUpdate> output, Reporter reporter)
+      OutputCollector<IntWritable, VectorUpdate> output, Reporter reporter)
       throws IOException {
 
-    BatchUpdate b = new BatchUpdate(Numeric.intToBytes(key.get()));
+    VectorUpdate update = new VectorUpdate(key.get());
+
     DenseVector sum;
     Map<Integer, Double> buffer = new HashMap<Integer, Double>();
 
@@ -57,13 +57,8 @@
         }
       }
     }
-
-    for (Map.Entry<Integer, Double> f : buffer.entrySet()) {
-      byte[] value = Numeric.doubleToBytes(f.getValue());
-      b.put(Numeric.getColumnIndex(f.getKey()), value);
-    }
-
-    output.collect(key, b);
+    update.putAll(buffer);
+    output.collect(key, update);
   }
 
 }

Added: incubator/hama/trunk/src/java/org/apache/hama/io/VectorUpdate.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/io/VectorUpdate.java?rev=695728&view=auto
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/io/VectorUpdate.java (added)
+++ incubator/hama/trunk/src/java/org/apache/hama/io/VectorUpdate.java Mon Sep 15 22:08:12
2008
@@ -0,0 +1,69 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.io;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.hbase.io.BatchUpdate;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hama.util.Numeric;
+
+public class VectorUpdate {
+  private BatchUpdate batchUpdate;
+
+  public VectorUpdate(int i) {
+    this.batchUpdate = new BatchUpdate(Numeric.intToBytes(i));
+  }
+
+  public VectorUpdate(String metadata) {
+    this.batchUpdate = new BatchUpdate(metadata);
+  }
+
+  public void put(int j, double value) {
+    this.batchUpdate.put(Numeric.getColumnIndex(j), Numeric
+        .doubleToBytes(value));
+  }
+
+  public void put(String column, String val) {
+    this.batchUpdate.put(column, Bytes.toBytes(val));
+  }
+
+  public void put(String metadataRows, int rows) {
+    this.batchUpdate.put(metadataRows, Numeric.intToBytes(rows));
+  }
+
+  public BatchUpdate getBatchUpdate() {
+    return this.batchUpdate;
+  }
+
+  public void putAll(Map<Integer, Double> buffer) {
+    for (Map.Entry<Integer, Double> f : buffer.entrySet()) {
+      put(f.getKey(), f.getValue());
+    }
+  }
+
+  public void putAll(Set<Entry<Integer, VectorEntry>> entrySet) {
+    for (Map.Entry<Integer, VectorEntry> e : entrySet) {
+      put(e.getKey(), e.getValue().getValue());
+    }
+  }
+}

Modified: incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixOutputFormat.java?rev=695728&r1=695727&r2=695728&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixOutputFormat.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixOutputFormat.java Mon Sep 15
22:08:12 2008
@@ -26,7 +26,6 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.io.BatchUpdate;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.mapred.FileAlreadyExistsException;
 import org.apache.hadoop.mapred.FileOutputFormat;
@@ -35,9 +34,10 @@
 import org.apache.hadoop.mapred.RecordWriter;
 import org.apache.hadoop.mapred.Reporter;
 import org.apache.hadoop.util.Progressable;
+import org.apache.hama.io.VectorUpdate;
 
 public class MatrixOutputFormat extends
-    FileOutputFormat<IntWritable, BatchUpdate> {
+    FileOutputFormat<IntWritable, VectorUpdate> {
 
   /** JobConf parameter that specifies the output table */
   public static final String OUTPUT_TABLE = "hama.mapred.output";
@@ -48,7 +48,7 @@
    * and write to an HBase table
    */
   protected static class TableRecordWriter implements
-      RecordWriter<IntWritable, BatchUpdate> {
+      RecordWriter<IntWritable, VectorUpdate> {
     private HTable m_table;
 
     /**
@@ -65,8 +65,8 @@
     }
 
     /** {@inheritDoc} */
-    public void write(IntWritable key, BatchUpdate value) throws IOException {
-      m_table.commit(value);
+    public void write(IntWritable key, VectorUpdate value) throws IOException {
+      m_table.commit(value.getBatchUpdate());
     }
   }
 

Modified: incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixReduce.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixReduce.java?rev=695728&r1=695727&r2=695728&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixReduce.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/mapred/MatrixReduce.java Mon Sep 15 22:08:12
2008
@@ -31,10 +31,11 @@
 import org.apache.hadoop.mapred.OutputCollector;
 import org.apache.hadoop.mapred.Reducer;
 import org.apache.hadoop.mapred.Reporter;
+import org.apache.hama.io.VectorUpdate;
 
 @SuppressWarnings("unchecked")
 public abstract class MatrixReduce<K extends WritableComparable, V extends Writable>
-    extends MapReduceBase implements Reducer<K, V, IntWritable, BatchUpdate> {
+    extends MapReduceBase implements Reducer<K, V, IntWritable, VectorUpdate> {
   /**
    * Use this before submitting a TableReduce job. It will
    * appropriately set up the JobConf.
@@ -61,6 +62,6 @@
    * @throws IOException
    */
   public abstract void reduce(K key, Iterator<V> values,
-    OutputCollector<IntWritable, BatchUpdate> output, Reporter reporter)
+    OutputCollector<IntWritable, VectorUpdate> output, Reporter reporter)
   throws IOException;
 }
\ No newline at end of file



Mime
View raw message