incubator-hama-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From edwardy...@apache.org
Subject svn commit: r706484 - in /incubator/hama/trunk: ./ src/java/org/apache/hama/ src/java/org/apache/hama/shell/ src/java/org/apache/hama/shell/execution/ src/java/org/apache/hama/shell/parser/expression/ src/java/org/apache/hama/util/
Date Tue, 21 Oct 2008 01:04:31 GMT
Author: edwardyoon
Date: Mon Oct 20 18:04:30 2008
New Revision: 706484

URL: http://svn.apache.org/viewvc?rev=706484&view=rev
Log:
clear the matrices in hbase after quiting the shell

Modified:
    incubator/hama/trunk/CHANGES.txt
    incubator/hama/trunk/src/java/org/apache/hama/Constants.java
    incubator/hama/trunk/src/java/org/apache/hama/shell/HamaShell.java
    incubator/hama/trunk/src/java/org/apache/hama/shell/HamaShellEnv.java
    incubator/hama/trunk/src/java/org/apache/hama/shell/execution/AlgebraOperator.java
    incubator/hama/trunk/src/java/org/apache/hama/shell/parser/expression/HamaExpressionParser.jjt
    incubator/hama/trunk/src/java/org/apache/hama/util/RandomVariable.java

Modified: incubator/hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/CHANGES.txt?rev=706484&r1=706483&r2=706484&view=diff
==============================================================================
--- incubator/hama/trunk/CHANGES.txt (original)
+++ incubator/hama/trunk/CHANGES.txt Mon Oct 20 18:04:30 2008
@@ -28,6 +28,7 @@
     
   IMPROVEMENTS
     
+    HAMA-77: clear the matrices in hbase after quiting the shell (samuel via edwardyoon)
     HAMA-28: Implement Vector.add() method (edwardyoon)
     HAMA-78: Separate Interface and Implementation for HamaAdmin (edwardyoon)
     HAMA-76: Remove load() since it duplicated with constructor (edwardyoon)

Modified: incubator/hama/trunk/src/java/org/apache/hama/Constants.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/Constants.java?rev=706484&r1=706483&r2=706484&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/Constants.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/Constants.java Mon Oct 20 18:04:30 2008
@@ -1,76 +1,81 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama;
-
-/**
- * Some constants used in the Hama
- */
-public class Constants {
-  
-  /** 
-   * Meta-columnFamily to store the matrix-info 
-   */
-  public final static String METADATA = "metadata";
-  
-  /**
-   * Column index & attributes
-   */
-  public final static String CINDEX = "cIndex";
-  
-  /**
-   * The attribute column family
-   */
-  public final static String ATTRIBUTE = "attribute:";
-  
-  /** 
-   * The number of the matrix rows 
-   */
-  public final static String METADATA_ROWS = "attribute:rows";
-  
-  /** 
-   * The number of the matrix columns 
-   */
-  public final static String METADATA_COLUMNS = "attribute:columns";
-  
-  /** 
-   * The type of the matrix
-   */
-  public final static String METADATA_TYPE = "attribute:type";
-  
-  /** 
-   * Default columnFamily name 
-   */
-  public final static String COLUMN = "column:";
-
-  /** 
-   * Temporary random matrices name prefix 
-   */
-  public final static String RANDOM = "rand";
-  
-  /**
-   * Admin table name
-   */
-  public final static String ADMINTABLE = "admin.table";
-
-  /**
-   * Matrix path columnFamily
-   */
-  public static final String PATHCOLUMN = "path:";
-}
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama;
+
+/**
+ * Some constants used in the Hama
+ */
+public class Constants {
+  
+  /** 
+   * Meta-columnFamily to store the matrix-info 
+   */
+  public final static String METADATA = "metadata";
+  
+  /**
+   * Column index & attributes
+   */
+  public final static String CINDEX = "cIndex";
+  
+  /**
+   * The attribute column family
+   */
+  public final static String ATTRIBUTE = "attribute:";
+  
+  /** 
+   * The number of the matrix rows 
+   */
+  public final static String METADATA_ROWS = "attribute:rows";
+  
+  /** 
+   * The number of the matrix columns 
+   */
+  public final static String METADATA_COLUMNS = "attribute:columns";
+  
+  /** 
+   * The type of the matrix
+   */
+  public final static String METADATA_TYPE = "attribute:type";
+  
+  /** 
+   * Default columnFamily name 
+   */
+  public final static String COLUMN = "column:";
+
+  /** 
+   * Temporary random matrices name prefix 
+   */
+  public final static String RANDOM = "rand";
+  
+  /**
+   * Admin table name
+   */
+  public final static String ADMINTABLE = "admin.table";
+
+  /**
+   * Matrix path columnFamily
+   */
+  public static final String PATHCOLUMN = "path:";
+  
+  /**
+   * Temporary Aliase name prefix in Hama Shell
+   */
+  public static final String RANDOMALIASE = "_";
+}

Modified: incubator/hama/trunk/src/java/org/apache/hama/shell/HamaShell.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/shell/HamaShell.java?rev=706484&r1=706483&r2=706484&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/shell/HamaShell.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/shell/HamaShell.java Mon Oct 20 18:04:30
2008
@@ -85,9 +85,11 @@
   }
 
   public static void main(String[] args) {
+    HamaShellEnv shellEnv = new HamaShellEnv();
     HamaShell shell = new HamaShell(new BufferedReader(new InputStreamReader(
-        System.in)), new HamaShellEnv(), new HamaConfiguration());
+        System.in)), shellEnv, new HamaConfiguration());
     shell.run();
+    shellEnv.clearAliases();
   }
 
 }

Modified: incubator/hama/trunk/src/java/org/apache/hama/shell/HamaShellEnv.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/shell/HamaShellEnv.java?rev=706484&r1=706483&r2=706484&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/shell/HamaShellEnv.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/shell/HamaShellEnv.java Mon Oct 20 18:04:30
2008
@@ -17,8 +17,13 @@
  */
 package org.apache.hama.shell;
 
+import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
+
+import org.apache.hama.Matrix;
+import org.apache.hama.util.RandomVariable;
+import org.apache.log4j.Logger;
 /**
  * HamaShellEnv.
  * 
@@ -32,7 +37,8 @@
 // so that we can use these information to rearrange the expressions
 // and make the hama shell executed more efficiently and parallel.
 public class HamaShellEnv {
-
+  
+  static final Logger LOG = Logger.getLogger(HamaShellEnv.class);
   Map<String, Object> aliases = new HashMap<String, Object>();
   
   public static final int DEFAULT_MAP_NUM = 2;
@@ -57,4 +63,34 @@
     return aliases;
   }
   
+  /**
+   * Get the random aliase name from hama shell.
+   * make sure the random aliase name doesn't exist in hama shell before.
+   * @return random aliase name
+   */
+  public String getRandomAliaseName() {
+    String rName = RandomVariable.randAliaseName();
+    while(aliases.containsKey(rName)) {
+      rName = RandomVariable.randAliaseName();
+    }
+    return rName;
+  }
+    
+  /**
+   * Clear all the aliases in the hama shell.
+   */
+  public void clearAliases() {
+    for(Object obj : aliases.values()) {
+      if(obj == null)
+        continue;
+      if(obj instanceof Matrix) {
+        try {
+          ((Matrix)obj).close();
+        } catch (IOException e) {
+          LOG.info("Matrix close : " + e.getMessage());
+        }
+      }
+    }
+  }
+
 }

Modified: incubator/hama/trunk/src/java/org/apache/hama/shell/execution/AlgebraOperator.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/shell/execution/AlgebraOperator.java?rev=706484&r1=706483&r2=706484&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/shell/execution/AlgebraOperator.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/shell/execution/AlgebraOperator.java Mon
Oct 20 18:04:30 2008
@@ -22,6 +22,7 @@
 import java.util.List;
 
 import org.apache.hama.Matrix;
+import org.apache.hama.shell.HamaShellEnv;
 
 public class AlgebraOperator {
 
@@ -32,6 +33,9 @@
   Object operand;
   List<Entry> otherOperands;
 
+  // HamaShellEnv to keep all the temporary matrices generated 
+  // during the execution of hama expression in shell
+  HamaShellEnv env;
   /**
    * 
    * An Entry Class to store all the Operators and other Operands.
@@ -55,9 +59,10 @@
     }
   }
 
-  public AlgebraOperator(Object operand) {
+  public AlgebraOperator(Object operand, HamaShellEnv env) {
     this.operand = operand;
     this.otherOperands = null;
+    this.env = env;
   }
 
   public void addOpAndOperand(String op, AlgebraOperator otherOperand) {
@@ -183,7 +188,10 @@
           } else
             throw new AlgebraOpException(
                 "Uncognized Operator. It should be '+' '-' '*'.");
-
+      
+          // for "m" will be replace by a new matrix in next loop.
+          // so we need to record it.
+          env.setAliase(env.getRandomAliaseName(), m);
         }
 
         return m;
@@ -209,10 +217,21 @@
     }
 
     // Now we can do multiplication.
+    // All the matrices in the multiplication has been recorded as
+    // Aliase or Temp Aliase in Hama Shell Env. so we just need to
+    // keep all the temp matrix in Hama Shell Env during the matrix
+    // multiplication.
     Matrix matrix = matrices.get(0);
+    if(matrices.size() == 1)
+      return matrix;
+
     for (int i = 1; i < matrices.size(); i++) {
       Matrix m = matrices.get(i);
       matrix = matrix.mult(m);
+      // keep the temp matrix in Hama shell env for cleaning
+      // TODO: the last matrix may be keeped by an Aliase an a TempAliase.
+      //       we need some method to tell them from each other.
+      env.setAliase(env.getRandomAliaseName(), matrix);
     }
 
     return matrix;

Modified: incubator/hama/trunk/src/java/org/apache/hama/shell/parser/expression/HamaExpressionParser.jjt
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/shell/parser/expression/HamaExpressionParser.jjt?rev=706484&r1=706483&r2=706484&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/shell/parser/expression/HamaExpressionParser.jjt
(original)
+++ incubator/hama/trunk/src/java/org/apache/hama/shell/parser/expression/HamaExpressionParser.jjt
Mon Oct 20 18:04:30 2008
@@ -221,7 +221,7 @@
 /** An Additive Operation. */
 AlgebraOperator AdditiveOp() : {AlgebraOperator op, op1, op2; Token t;}
 {
-  ( op1=MultiplicativeOp() { op = new AlgebraOperator(op1); } ( ( t=<ADDOP> op2=MultiplicativeOp()
){ op.addOpAndOperand(t.image, op2); } )* )
+  ( op1=MultiplicativeOp() { op = new AlgebraOperator(op1, shellEnv); } ( ( t=<ADDOP>
op2=MultiplicativeOp() ){ op.addOpAndOperand(t.image, op2); } )* )
   { 
   	return op; 
   }
@@ -230,7 +230,7 @@
 /** A Multiplicative Operation. */
 AlgebraOperator MultiplicativeOp() : {AlgebraOperator op, op1, op2; Token t;}
 {
-  ( op1=UnaryOp() { op = new AlgebraOperator(op1); } ( ( t=<MULOP> op2=UnaryOp() ){op.addOpAndOperand(t.image,
op2);} )* )
+  ( op1=UnaryOp() { op = new AlgebraOperator(op1, shellEnv); } ( ( t=<MULOP> op2=UnaryOp()
){op.addOpAndOperand(t.image, op2);} )* )
   {
   	return op;
   }
@@ -242,7 +242,7 @@
   (
   ( "(" op=AdditiveOp() ")" ) 
   | 
-  b = AliaseName() { op = new AlgebraOperator(b); }
+  b = AliaseName() { op = new AlgebraOperator(b, shellEnv); }
   )
   {
   	return op;

Modified: incubator/hama/trunk/src/java/org/apache/hama/util/RandomVariable.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/util/RandomVariable.java?rev=706484&r1=706483&r2=706484&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/util/RandomVariable.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/util/RandomVariable.java Mon Oct 20 18:04:30
2008
@@ -51,18 +51,39 @@
   }
 
   /**
-   * Generate a random name.
+   * Generate a random matrix name.
    * 
-   * @return random name
+   * @return random matrix name
    */
   public static String randMatrixName() {
-    String rName = Constants.RANDOM;
-    for (int i = 1; i <= 5; i++) {
+    return randString(Constants.RANDOM, 5);
+  }
+  
+  /**
+   * Generate a random aliase name.
+   * 
+   * @return random aliase name
+   */
+  public static String randAliaseName() {
+    return randString(Constants.RANDOMALIASE, 5);
+  }
+  
+  /**
+    * Generate a random string using the specified prefix and a fixed length. 
+    * @param prefix
+    *        the specified string prefix.
+    * @param length
+    *        the length of the string to be appended.
+    * @return random string.
+    */
+  public static String randString(String prefix, int length) {
+    StringBuilder result = new StringBuilder(prefix);
+    for (int i = 0; i < length; i++) {
       char ch = (char) ((Math.random() * 26) + 97);
-      rName += ch;
+      result.append(ch);
     }
-
-    return rName;
+      
+    return result.toString();
   }
 
   /**



Mime
View raw message