incubator-hama-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From edwardy...@apache.org
Subject svn commit: r742798 - in /incubator/hama/trunk: CHANGES.txt src/java/org/apache/hama/AbstractMatrix.java src/java/org/apache/hama/DenseMatrix.java src/test/org/apache/hama/TestDenseMatrix.java
Date Tue, 10 Feb 2009 00:15:42 GMT
Author: edwardyoon
Date: Tue Feb 10 00:15:41 2009
New Revision: 742798

URL: http://svn.apache.org/viewvc?rev=742798&view=rev
Log: (empty)

Modified:
    incubator/hama/trunk/CHANGES.txt
    incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java
    incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
    incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java

Modified: incubator/hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/CHANGES.txt?rev=742798&r1=742797&r2=742798&view=diff
==============================================================================
--- incubator/hama/trunk/CHANGES.txt (original)
+++ incubator/hama/trunk/CHANGES.txt Tue Feb 10 00:15:41 2009
@@ -4,6 +4,7 @@
 
   NEW FEATURES
   
+    HAMA-100: Implement of set(Matrix B) (edwardyoon)
     HAMA-145: Add privacy policy page (edwardyoon)
     HAMA-83: 2D sqaure blocking for dense matrix multiplication (edwardyoon)
     HAMA-104: Add getNumMap/reduceTasks to HamaConfiguration (edwardyoon)

Modified: incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java?rev=742798&r1=742797&r2=742798&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java Tue Feb 10 00:15:41
2009
@@ -20,6 +20,7 @@
 package org.apache.hama;
 
 import java.io.IOException;
+import java.util.Map;
 
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
@@ -31,9 +32,19 @@
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.io.BatchUpdate;
 import org.apache.hadoop.hbase.io.Cell;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.io.RowResult;
+import org.apache.hadoop.hbase.mapred.IdentityTableReduce;
+import org.apache.hadoop.hbase.mapred.TableMap;
+import org.apache.hadoop.hbase.mapred.TableMapReduceUtil;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
 import org.apache.hama.io.VectorUpdate;
 import org.apache.hama.util.BytesUtil;
+import org.apache.hama.util.JobManager;
 import org.apache.log4j.Logger;
 
 /**
@@ -73,16 +84,16 @@
   protected void create() throws IOException {
     // It should run only when table doesn't exist.
     if (!admin.tableExists(matrixPath)) {
-      this.tableDesc.addFamily(new HColumnDescriptor(
-          Bytes.toBytes(Constants.COLUMN), 3, CompressionType.NONE, false, false,
+      this.tableDesc.addFamily(new HColumnDescriptor(Bytes
+          .toBytes(Constants.COLUMN), 3, CompressionType.NONE, false, false,
           Integer.MAX_VALUE, HConstants.FOREVER, false));
       this.tableDesc.addFamily(new HColumnDescriptor(Constants.ATTRIBUTE));
       this.tableDesc.addFamily(new HColumnDescriptor(Constants.ALIASEFAMILY));
       // It's a temporary data.
-      this.tableDesc.addFamily(new HColumnDescriptor(
-      Bytes.toBytes(Constants.BLOCK), 1, CompressionType.NONE, false, false,
-      Integer.MAX_VALUE, HConstants.FOREVER, false));
-      
+      this.tableDesc.addFamily(new HColumnDescriptor(Bytes
+          .toBytes(Constants.BLOCK), 1, CompressionType.NONE, false, false,
+          Integer.MAX_VALUE, HConstants.FOREVER, false));
+
       LOG.info("Initializing the matrix storage.");
       this.admin.createTable(this.tableDesc);
       LOG.info("Create Matrix " + matrixPath);
@@ -145,6 +156,45 @@
     table.commit(update.getBatchUpdate());
   }
 
+  /**
+   * Just full scan a table.
+   */
+  public static class TableReadMapper extends MapReduceBase implements
+      TableMap<ImmutableBytesWritable, BatchUpdate> {
+
+    @SuppressWarnings("unchecked")
+    public void map(ImmutableBytesWritable key, RowResult value,
+        OutputCollector<ImmutableBytesWritable, BatchUpdate> output,
+        @SuppressWarnings("unused")
+        Reporter reporter) throws IOException {
+
+      BatchUpdate update = new BatchUpdate(key.get());
+      for (Map.Entry<byte[], Cell> e : value.entrySet()) {
+        update.put(e.getKey(), e.getValue().getValue());
+      }
+      output.collect(key, update);
+    }
+  }
+
+  /** {@inheritDoc} */
+  public Matrix set(Matrix B) throws IOException {
+    JobConf jobConf = new JobConf(config);
+    jobConf.setJobName("set MR job : " + this.getPath());
+
+    jobConf.setNumMapTasks(config.getNumMapTasks());
+    jobConf.setNumReduceTasks(config.getNumReduceTasks());
+
+    TableMapReduceUtil.initTableMapJob(B.getPath(), Constants.COLUMN + " "
+        + Constants.ATTRIBUTE + " " + Constants.ALIASEFAMILY + " "
+        + Constants.BLOCK, TableReadMapper.class,
+        ImmutableBytesWritable.class, BatchUpdate.class, jobConf);
+    TableMapReduceUtil.initTableReduceJob(this.getPath(),
+        IdentityTableReduce.class, jobConf);
+
+    JobManager.execute(jobConf);
+    return this;
+  }
+
   public String getRowLabel(int row) throws IOException {
     Cell rows = null;
     rows = table.get(BytesUtil.getRowIndex(row), Bytes

Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java?rev=742798&r1=742797&r2=742798&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java Tue Feb 10 00:15:41 2009
@@ -446,11 +446,6 @@
     return null;
   }
 
-  public Matrix set(Matrix B) throws IOException {
-    // TODO Auto-generated method stub
-    return null;
-  }
-
   public void setRow(int row, Vector vector) throws IOException {
     VectorUpdate update = new VectorUpdate(row);
     update.putAll(((DenseVector) vector).getEntries().entrySet());

Modified: incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java?rev=742798&r1=742797&r2=742798&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java Tue Feb 10 00:15:41
2009
@@ -29,6 +29,7 @@
 
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hama.io.DoubleEntry;
+import org.apache.hama.util.RandomVariable;
 import org.apache.log4j.Logger;
 
 /**
@@ -203,6 +204,17 @@
     }
   }
 
+  public void testSetMatrix() throws IOException {
+    Matrix a = new DenseMatrix(conf);
+    a.set(m1);
+
+    for (int i = 0; i < 5; i++) {
+      int x = RandomVariable.randInt(0, 10);
+      int y = RandomVariable.randInt(0, 10);
+      assertEquals(a.get(x, y), m1.get(x, y));
+    }
+  }
+
   public void testLoadSave() throws IOException {
     String path1 = m1.getPath();
     // save m1 to aliase1



Mime
View raw message