incubator-hama-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From edwardy...@apache.org
Subject svn commit: r671120 [4/4] - in /incubator/hama/trunk: ./ lib/ lib/findbugs/ lib/findbugs/plugin/ lib/jetty-ext/ src/ src/docs/ src/docs/src/ src/docs/src/documentation/ src/docs/src/documentation/classes/ src/docs/src/documentation/content/ src/docs/sr...
Date Tue, 24 Jun 2008 10:36:25 GMT
Added: incubator/hama/trunk/src/java/org/apache/hama/package.html
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/package.html?rev=671120&view=auto
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/package.html (added)
+++ incubator/hama/trunk/src/java/org/apache/hama/package.html Tue Jun 24 03:36:21 2008
@@ -0,0 +1,23 @@
+<html>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<body>
+The Hama package
+</body>
+</html>

Added: incubator/hama/trunk/src/test/org/apache/hama/Benchmarks.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/Benchmarks.java?rev=671120&view=auto
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/Benchmarks.java (added)
+++ incubator/hama/trunk/src/test/org/apache/hama/Benchmarks.java Tue Jun 24 03:36:21 2008
@@ -0,0 +1,5 @@
+package org.apache.hama;
+
+public class Benchmarks {
+
+}

Added: incubator/hama/trunk/src/test/org/apache/hama/HamaTestCase.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/HamaTestCase.java?rev=671120&view=auto
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/HamaTestCase.java (added)
+++ incubator/hama/trunk/src/test/org/apache/hama/HamaTestCase.java Tue Jun 24 03:36:21 2008
@@ -0,0 +1,56 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama;
+
+import org.apache.hadoop.hbase.HBaseClusterTestCase;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+/**
+ * Forming up the miniDfs and miniHbase
+ */
+public class HamaTestCase extends HBaseClusterTestCase {
+  static final Logger LOG = Logger.getLogger(HamaTestCase.class);
+  protected Matrix matrixA;
+  protected Text A = new Text("matrixA");
+  protected int SIZE = 5;
+  
+  /** constructor */
+  public HamaTestCase() {
+    super();
+
+    // TODO: We should remove this "hadoop.log.dir" path to build.xml
+    System.setProperty("hadoop.log.dir", 
+        conf.get("hadoop.log.dir", "./build/test/logs"));
+
+    // Initializing the hbase configuration
+    conf.set("mapred.output.dir", conf.get("hadoop.tmp.dir"));
+
+    conf.setInt("mapred.map.tasks", 10);
+    conf.setInt("mapred.reduce.tasks", 1);
+
+    conf.setInt("hbase.hregion.memcache.flush.size", 1024 * 1024);
+    conf.setInt("hbase.hstore.compactionThreshold", 2);
+    conf.setLong("hbase.hregion.max.filesize", 1024 * 1024);
+    conf.setInt("hbase.master.lease.period", 10 * 1000);
+    conf.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
+    conf.setInt("hbase.client.pause", 10 * 1000);
+  }
+}

Added: incubator/hama/trunk/src/test/org/apache/hama/TestFeatureVector.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/TestFeatureVector.java?rev=671120&view=auto
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/TestFeatureVector.java (added)
+++ incubator/hama/trunk/src/test/org/apache/hama/TestFeatureVector.java Tue Jun 24 03:36:21
2008
@@ -0,0 +1,51 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama;
+
+public class TestFeatureVector extends HamaTestCase {
+
+  /**
+   * Test cosine similarity
+   */
+  public void testCosine() {
+    final double result = 0.6978227007909176;
+    matrixA = new Matrix(conf, A);
+    
+    // TODO : We need setArray(int row, double[] value) to matrix 
+    // e.g. matrixA.setArray(0, new double[] {2,5,1,4});
+    // -- Edward
+    matrixA.set(0, 0, 2);
+    matrixA.set(0, 1, 5);
+    matrixA.set(0, 2, 1);
+    matrixA.set(0, 3, 4);
+
+    matrixA.set(1, 0, 4);
+    matrixA.set(1, 1, 1);
+    matrixA.set(1, 2, 3);
+    matrixA.set(1, 3, 3);
+
+    FeatureVector v1 = matrixA.getRowVector(0);
+    FeatureVector v2 = matrixA.getRowVector(1);
+
+    double cos = v1.getCosine(v2);
+    assertEquals(cos, result);
+    matrixA.clear();
+  }
+}

Added: incubator/hama/trunk/src/test/org/apache/hama/TestMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/TestMatrix.java?rev=671120&view=auto
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/TestMatrix.java (added)
+++ incubator/hama/trunk/src/test/org/apache/hama/TestMatrix.java Tue Jun 24 03:36:21 2008
@@ -0,0 +1,114 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama;
+
+import org.apache.hadoop.hbase.MasterNotRunningException;
+
+/**
+ * Matrix test
+ */
+public class TestMatrix extends HamaTestCase {
+
+  /**
+   * Random matrix creation test
+   */
+  public void testRandomMatrix() {
+    Matrix rand = Matrix.random(conf, SIZE, SIZE);
+    assertTrue(rand.getRowDimension() == SIZE);
+  }
+
+  /**
+   * Matrix Test
+   */
+  public void testMatrix() {
+    assertTrue(matrixCreate());
+
+    for (int i = 0; i < SIZE; i++) {
+      for (int j = 0; j < SIZE; j++) {
+        matrixA.set(i, j, i + j);
+      }
+    }
+    matrixA.setDimension(SIZE, SIZE);
+
+    assertTrue(matrixA.getRowDimension() == SIZE);
+    assertTrue(matrixA.getColumnDimension() == SIZE);
+
+    for (int i = 0; i < SIZE; i++) {
+      for (int j = 0; j < SIZE; j++) {
+        assertTrue((i + j) == matrixA.get(i, j));
+      }
+    }
+    matrixClose();
+  }
+
+  /**
+   * Constant matrix test
+   */
+  public void testConstantMatrix() {
+    constantMatrixCreate();
+    
+    for (int i = 0; i < SIZE; i++) {
+      for (int j = 0; j < SIZE; j++) {
+        assertTrue(0.5 == matrixA.get(i, j));
+      }
+    }
+    
+    matrixClose();
+  }
+
+  /**
+   * Object clear
+   */
+  public void matrixClose() {
+    matrixA.clear();
+    matrixA.close();
+  }
+
+  /**
+   * Matrix create
+   * 
+   * @return <code>true</code> if the matrix space was initialized.
+   */
+  public boolean matrixCreate() {
+    matrixA = new Matrix(conf, A);
+    try {
+      return matrixA.admin.tableExists(A);
+    } catch (MasterNotRunningException e) {
+      e.printStackTrace();
+      return false;
+    }
+  }
+
+  /**
+   * Matrix create
+   * 
+   * @return <code>true</code> if the matrix space was initialized.
+   */
+  public boolean constantMatrixCreate() {
+    matrixA = new Matrix(conf, SIZE, SIZE, 0.5);
+    try {
+      return matrixA.admin.tableExists(A);
+    } catch (MasterNotRunningException e) {
+      e.printStackTrace();
+      return false;
+    }
+  }
+  
+}

Added: incubator/hama/trunk/src/test/org/apache/hama/TestRandomVariable.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/TestRandomVariable.java?rev=671120&view=auto
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/TestRandomVariable.java (added)
+++ incubator/hama/trunk/src/test/org/apache/hama/TestRandomVariable.java Tue Jun 24 03:36:21
2008
@@ -0,0 +1,71 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama;
+
+import junit.framework.TestCase;
+
+/**
+ * Random variable generation test
+ */
+public class TestRandomVariable extends TestCase {
+  final static int TEST_COUNT = 50;
+
+  /**
+   * Random object test
+   * 
+   * @throws Exception
+   */
+  public void testRand() throws Exception {
+    for (int i = 0; i < TEST_COUNT; i++) {
+      double result = RandomVariable.rand();
+      assertTrue(result >= 0.0d && result <= 1.0);
+    }
+  }
+
+  /**
+   * Random integer test
+   * 
+   * @throws Exception
+   */
+  public void testRandInt() throws Exception {
+    final int min = 122;
+    final int max = 561;
+
+    for (int i = 0; i < TEST_COUNT; i++) {
+      int result = RandomVariable.randInt(min, max);
+      assertTrue(result >= min && result <= max);
+    }
+  }
+
+  /**
+   * Uniform test
+   * 
+   * @throws Exception
+   */
+  public void testUniform() throws Exception {
+    final double min = 1.0d;
+    final double max = 3.0d;
+
+    for (int i = 0; i < TEST_COUNT; i++) {
+      double result = RandomVariable.uniform(min, max);
+      assertTrue(result >= min && result <= max);
+    }
+  }
+}

Added: incubator/hama/trunk/src/test/org/apache/hama/mapred/TestMatrixMapReduce.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/mapred/TestMatrixMapReduce.java?rev=671120&view=auto
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/mapred/TestMatrixMapReduce.java (added)
+++ incubator/hama/trunk/src/test/org/apache/hama/mapred/TestMatrixMapReduce.java Tue Jun
24 03:36:21 2008
@@ -0,0 +1,118 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.mapred;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HStoreKey;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.io.MapWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hama.HamaTestCase;
+import org.apache.hama.Matrix;
+import org.apache.hama.algebra.AdditionSubtractionReduce;
+import org.apache.log4j.Logger;
+
+/**
+ * Test Matrix Map/Reduce job
+ */
+public class TestMatrixMapReduce extends HamaTestCase {
+  static final Logger LOG = Logger.getLogger(TestMatrixMapReduce.class);
+  private Matrix a;
+  private Matrix b;
+  private Matrix c;
+
+  /** constructor */
+  public TestMatrixMapReduce() {
+    super();
+  }
+
+  public static class AdditionMap extends MatrixMap<Text, MapWritable> {
+    protected Matrix matrix_b;
+    public static final String MATRIX_B = "hama.addition.matrix.b";
+
+    public void configure(JobConf job) {
+      Text b_name = new Text(job.get(MATRIX_B, ""));
+      matrix_b = new Matrix(new HBaseConfiguration(), b_name);
+    }
+
+    public static void initJob(String matrix_a, String matrix_b,
+        Class<AdditionMap> map, JobConf jobConf) {
+      initJob(matrix_a, map, jobConf);
+      jobConf.set(MATRIX_B, matrix_b);
+    }
+
+    @Override
+    public void map(HStoreKey key, MapWritable value,
+        OutputCollector<Text, MapWritable> output, Reporter reporter)
+        throws IOException {
+      Text tKey = key.getRow();
+      MapWritable sum = new MapWritable();
+
+      for (Map.Entry<Writable, Writable> e : value.entrySet()) {
+        double a = getDouble(e.getValue());
+        double b = matrix_b.get(Integer.parseInt(tKey.toString()),
+            getIndex((Text) e.getKey()));
+        byte[] result = null;
+        result = toBytes(a + b);
+        sum.put(e.getKey(), new ImmutableBytesWritable(result));
+      }
+      output.collect(tKey, sum);
+    }
+  }
+
+  public void testMatrixMapReduce() throws IOException {
+    a = Matrix.random(conf, SIZE, SIZE);
+    b = Matrix.random(conf, SIZE, SIZE);
+    c = new Matrix(conf, new Text("matrixC"));
+    miniMRJob();
+  }
+
+  public void miniMRJob() throws IOException {
+      JobConf jobConf = new JobConf(conf, TestMatrixMapReduce.class);
+      jobConf.setJobName("test MR job");
+      jobConf.setInputFormat(MatrixInputFormat.class);
+      jobConf.setOutputFormat(MatrixOutputFormat.class);
+      AdditionMap.initJob(a.getName(), b.getName(), AdditionMap.class, jobConf);
+      AdditionSubtractionReduce.initJob("matrixC",
+          AdditionSubtractionReduce.class, jobConf);
+
+      jobConf.setNumMapTasks(1);
+      jobConf.setNumReduceTasks(1);
+
+      JobClient.runJob(jobConf);
+      
+      assertEquals(c.getRowDimension(), SIZE);
+      assertEquals(c.getColumnDimension(), SIZE);
+      
+      for(int i = 0; i < c.getRowDimension(); i++) {
+        for(int j = 0; j < c.getColumnDimension(); j++) {
+          assertEquals(c.get(i, j), (a.get(i, j) + b.get(i, j))); 
+        }
+      }
+  }
+}



Mime
View raw message