hama-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From edwardy...@apache.org
Subject svn commit: r1526783 - in /hama/trunk/examples/src: main/java/org/apache/hama/examples/NeuralNetwork.java test/java/org/apache/hama/examples/NeuralNetworkTest.java
Date Fri, 27 Sep 2013 03:34:50 GMT
Author: edwardyoon
Date: Fri Sep 27 03:34:49 2013
New Revision: 1526783

URL: http://svn.apache.org/r1526783
Log:
Minor fix

Modified:
    hama/trunk/examples/src/main/java/org/apache/hama/examples/NeuralNetwork.java
    hama/trunk/examples/src/test/java/org/apache/hama/examples/NeuralNetworkTest.java

Modified: hama/trunk/examples/src/main/java/org/apache/hama/examples/NeuralNetwork.java
URL: http://svn.apache.org/viewvc/hama/trunk/examples/src/main/java/org/apache/hama/examples/NeuralNetwork.java?rev=1526783&r1=1526782&r2=1526783&view=diff
==============================================================================
--- hama/trunk/examples/src/main/java/org/apache/hama/examples/NeuralNetwork.java (original)
+++ hama/trunk/examples/src/main/java/org/apache/hama/examples/NeuralNetwork.java Fri Sep
27 03:34:49 2013
@@ -22,7 +22,6 @@ import java.io.BufferedWriter;
 import java.io.InputStreamReader;
 import java.io.OutputStreamWriter;
 import java.net.URI;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -36,10 +35,9 @@ import org.apache.hama.ml.math.FunctionF
 
 /**
  * 
- *
  */
 public class NeuralNetwork {
-  
+
   public static void main(String[] args) throws Exception {
     if (args.length < 3) {
       printUsage();
@@ -51,24 +49,27 @@ public class NeuralNetwork {
         printUsage();
         return;
       }
-      
+
       String modelPath = args[1];
       String featureDataPath = args[2];
       String resultDataPath = args[3];
-      
+
       SmallLayeredNeuralNetwork ann = new SmallLayeredNeuralNetwork(modelPath);
-      
+
       // process data in streaming approach
-      FileSystem fs = FileSystem.get(new URI(featureDataPath), new Configuration());
-      BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(new Path(featureDataPath))));
+      FileSystem fs = FileSystem.get(new URI(featureDataPath),
+          new Configuration());
+      BufferedReader br = new BufferedReader(new InputStreamReader(
+          fs.open(new Path(featureDataPath))));
       Path outputPath = new Path(resultDataPath);
       if (fs.exists(outputPath)) {
         fs.delete(outputPath, true);
       }
-      BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fs.create(outputPath)));
-      
+      BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(
+          fs.create(outputPath)));
+
       String line = null;
-      
+
       while ((line = br.readLine()) != null) {
         if (line.trim().length() == 0) {
           continue;
@@ -86,18 +87,16 @@ public class NeuralNetwork {
           sb.append(arrResult[i]);
           if (i != arrResult.length - 1) {
             sb.append(",");
-          }
-          else {
+          } else {
             sb.append("\n");
           }
         }
         bw.write(sb.toString());
       }
-      
+
       br.close();
       bw.close();
-    }
-    else if (mode.equals("train")) {
+    } else if (mode.equals("train")) {
       if (args.length < 5) {
         printUsage();
         return;
@@ -105,22 +104,23 @@ public class NeuralNetwork {
 
       String trainingDataPath = args[1];
       String trainedModelPath = args[2];
-      
+
       int featureDimension = Integer.parseInt(args[3]);
       int labelDimension = Integer.parseInt(args[4]);
-      
+
       int iteration = 1000;
       double learningRate = 0.4;
       double momemtumWeight = 0.2;
       double regularizationWeight = 0.01;
-      
+
       // parse parameters
       if (args.length >= 6) {
         try {
           iteration = Integer.parseInt(args[5]);
           System.out.printf("Iteration: %d\n", iteration);
         } catch (NumberFormatException e) {
-          System.err.println("MAX_ITERATION format invalid. It should be a positive number.");
+          System.err
+              .println("MAX_ITERATION format invalid. It should be a positive number.");
           return;
         }
       }
@@ -129,7 +129,8 @@ public class NeuralNetwork {
           learningRate = Double.parseDouble(args[6]);
           System.out.printf("Learning rate: %f\n", learningRate);
         } catch (NumberFormatException e) {
-          System.err.println("LEARNING_RATE format invalid. It should be a positive double
in range (0, 1.0)");
+          System.err
+              .println("LEARNING_RATE format invalid. It should be a positive double in range
(0, 1.0)");
           return;
         }
       }
@@ -138,21 +139,24 @@ public class NeuralNetwork {
           momemtumWeight = Double.parseDouble(args[7]);
           System.out.printf("Momemtum weight: %f\n", momemtumWeight);
         } catch (NumberFormatException e) {
-          System.err.println("MOMEMTUM_WEIGHT format invalid. It should be a positive double
in range (0, 1.0)");
+          System.err
+              .println("MOMEMTUM_WEIGHT format invalid. It should be a positive double in
range (0, 1.0)");
           return;
         }
       }
       if (args.length >= 9) {
         try {
           regularizationWeight = Double.parseDouble(args[8]);
-          System.out.printf("Regularization weight: %f\n", regularizationWeight);
+          System.out
+              .printf("Regularization weight: %f\n", regularizationWeight);
         } catch (NumberFormatException e) {
-          System.err.println("REGULARIZATION_WEIGHT format invalid. It should be a positive
double in range (0, 1.0)");
+          System.err
+              .println("REGULARIZATION_WEIGHT format invalid. It should be a positive double
in range (0, 1.0)");
           return;
         }
       }
-      
-      //  train the model
+
+      // train the model
       SmallLayeredNeuralNetwork ann = new SmallLayeredNeuralNetwork();
       ann.setLearningRate(learningRate);
       ann.setMomemtumWeight(momemtumWeight);
@@ -161,7 +165,8 @@ public class NeuralNetwork {
           FunctionFactory.createDoubleFunction("Sigmoid"));
       ann.addLayer(featureDimension, false,
           FunctionFactory.createDoubleFunction("Sigmoid"));
-      ann.addLayer(labelDimension, true, FunctionFactory.createDoubleFunction("Sigmoid"));
+      ann.addLayer(labelDimension, true,
+          FunctionFactory.createDoubleFunction("Sigmoid"));
       ann.setCostFunction(FunctionFactory
           .createDoubleDoubleFunction("CrossEntropy"));
       ann.setModelPath(trainedModelPath);
@@ -173,25 +178,38 @@ public class NeuralNetwork {
       trainingParameters.put("convergence.check.interval", "1000");
       ann.train(new Path(trainingDataPath), trainingParameters);
     }
-    
+
   }
-  
+
   private static void printUsage() {
-    System.out.println("USAGE: <MODE> <INPUT_PATH> <OUTPUT_PATH> <MODEL_PATH>|<FEATURE_DIMENSION>
<LABEL_DIMENSION> [<MAX_ITERATION> <LEARNING_RATE> <MOMEMTUM_WEIGHT>
<REGULARIZATION_WEIGHT>]");
-    System.out.println("\tMODE\t- train: train the model with given training data.");
-    System.out.println("\t\t- evaluate: obtain the result by feeding the features to the
neural network.");
-    System.out.println("\tINPUT_PATH\tin 'train' mode, it is the path of the training data;
in 'evaluate' mode, it is the path of the to be evaluated data that lacks the label.");
-    System.out.println("\tOUTPUT_PATH\tin 'train' mode, it is where the trained model is
stored; in 'evaluate' mode, it is where the labeled data is stored.");
+    System.out
+        .println("USAGE: <MODE> <INPUT_PATH> <OUTPUT_PATH> <MODEL_PATH>|<FEATURE_DIMENSION>
<LABEL_DIMENSION> [<MAX_ITERATION> <LEARNING_RATE> <MOMEMTUM_WEIGHT>
<REGULARIZATION_WEIGHT>]");
+    System.out
+        .println("\tMODE\t- train: train the model with given training data.");
+    System.out
+        .println("\t\t- evaluate: obtain the result by feeding the features to the neural
network.");
+    System.out
+        .println("\tINPUT_PATH\tin 'train' mode, it is the path of the training data; in
'evaluate' mode, it is the path of the to be evaluated data that lacks the label.");
+    System.out
+        .println("\tOUTPUT_PATH\tin 'train' mode, it is where the trained model is stored;
in 'evaluate' mode, it is where the labeled data is stored.");
     System.out.println("\n\tConditional Parameters:");
-    System.out.println("\tMODEL_PATH\tonly required in 'evaluate' mode. It specifies where
to load the trained neural network model.");
-    System.out.println("\tMAX_ITERATION\tonly used in 'train' mode. It specifies how many
iterations for the neural network to run. Default is 0.01.");
-    System.out.println("\tLEARNING_RATE\tonly used to 'train' mode. It specifies the degree
of aggregation for learning, usually in range (0, 1.0). Default is 0.1.");
-    System.out.println("\tMOMEMTUM_WEIGHT\tonly used to 'train' mode. It specifies the weight
of momemtum. Default is 0.");
-    System.out.println("\tREGULARIZATION_WEIGHT\tonly required in 'train' model. It specifies
the weight of reqularization.");
+    System.out
+        .println("\tMODEL_PATH\tonly required in 'evaluate' mode. It specifies where to load
the trained neural network model.");
+    System.out
+        .println("\tMAX_ITERATION\tonly used in 'train' mode. It specifies how many iterations
for the neural network to run. Default is 0.01.");
+    System.out
+        .println("\tLEARNING_RATE\tonly used to 'train' mode. It specifies the degree of
aggregation for learning, usually in range (0, 1.0). Default is 0.1.");
+    System.out
+        .println("\tMOMEMTUM_WEIGHT\tonly used to 'train' mode. It specifies the weight of
momemtum. Default is 0.");
+    System.out
+        .println("\tREGULARIZATION_WEIGHT\tonly required in 'train' model. It specifies the
weight of reqularization.");
     System.out.println("\nExample:");
-    System.out.println("Train a neural network with default setting:\n\tneuralnets train
hdfs://localhost:30002/training_data hdfs://localhost:30002/model 8 1");
-    System.out.println("Train a neural network by specify learning rate as 0.1, momemtum
rate as 0.2, and regularization weight as 0.01:\n\tneuralnets.train hdfs://localhost:30002/training_data
hdfs://localhost:30002/model 0.1 0.2 0.01");
-    System.out.println("Label the data with trained model:\n\tneuralnets evaluate hdfs://localhost:30002/unlabeled_data
hdfs://localhost:30002/result hdfs://localhost:30002/model");
+    System.out
+        .println("Train a neural network with default setting:\n\tneuralnets train hdfs://localhost:30002/training_data
hdfs://localhost:30002/model 8 1");
+    System.out
+        .println("Train a neural network by specify learning rate as 0.1, momemtum rate as
0.2, and regularization weight as 0.01:\n\tneuralnets.train hdfs://localhost:30002/training_data
hdfs://localhost:30002/model 0.1 0.2 0.01");
+    System.out
+        .println("Label the data with trained model:\n\tneuralnets evaluate hdfs://localhost:30002/unlabeled_data
hdfs://localhost:30002/result hdfs://localhost:30002/model");
   }
 
 }

Modified: hama/trunk/examples/src/test/java/org/apache/hama/examples/NeuralNetworkTest.java
URL: http://svn.apache.org/viewvc/hama/trunk/examples/src/test/java/org/apache/hama/examples/NeuralNetworkTest.java?rev=1526783&r1=1526782&r2=1526783&view=diff
==============================================================================
--- hama/trunk/examples/src/test/java/org/apache/hama/examples/NeuralNetworkTest.java (original)
+++ hama/trunk/examples/src/test/java/org/apache/hama/examples/NeuralNetworkTest.java Fri
Sep 27 03:34:49 2013
@@ -23,34 +23,42 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
+import junit.framework.TestCase;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.SequenceFile;
+import org.apache.hama.HamaConfiguration;
 import org.apache.hama.ml.math.DenseDoubleVector;
 import org.apache.hama.ml.writable.VectorWritable;
-import org.junit.Test;
 
 /**
  * Test the functionality of NeuralNetwork Example.
  * 
  */
-public class NeuralNetworkTest {
+public class NeuralNetworkTest extends TestCase {
+  private Configuration conf = new HamaConfiguration();
+  private FileSystem fs;
+  private String MODEL_PATH = "/tmp/neuralnets.model";
+  private String RESULT_PATH = "/tmp/neuralnets.txt";
+  private String SEQTRAIN_DATA = "/tmp/test-neuralnets.data";
+  
+  @Override
+  protected void setUp() throws Exception {
+    super.setUp();
+    fs = FileSystem.get(conf);
+  }
 
-  @Test
-  public void testNeuralnetsLabeling() {
-    this.testNeuralNetworkTraining();
+  public void testNeuralnetsLabeling() throws IOException {
+    this.neuralNetworkTraining();
 
     String dataPath = "src/test/resources/neuralnets_classification_test.txt";
-    String modelPath = "/tmp/neuralnets.model";
-    String resultPath = "/tmp/neuralnets.txt";
     String mode = "label";
-    Configuration conf = new Configuration();
     try {
-      FileSystem fs = FileSystem.get(conf);
       NeuralNetwork
-          .main(new String[] { mode, modelPath, dataPath, resultPath });
+          .main(new String[] { mode, MODEL_PATH, dataPath, RESULT_PATH });
 
       // compare results with ground-truth
       BufferedReader groundTruthReader = new BufferedReader(new FileReader(
@@ -61,8 +69,9 @@ public class NeuralNetworkTest {
         groundTruthList.add(Double.parseDouble(line));
       }
       groundTruthReader.close();
-      
-      BufferedReader resultReader = new BufferedReader(new FileReader(resultPath));
+
+      BufferedReader resultReader = new BufferedReader(new FileReader(
+          RESULT_PATH));
       List<Double> resultList = new ArrayList<Double>();
       while ((line = resultReader.readLine()) != null) {
         resultList.add(Double.parseDouble(line));
@@ -78,22 +87,23 @@ public class NeuralNetworkTest {
         }
       }
       System.out.printf("Precision: %f\n", correct / total);
-      fs.delete(new Path(resultPath), true);
-      fs.delete(new Path(modelPath), true);
 
     } catch (Exception e) {
       e.printStackTrace();
+    } finally {
+      fs.delete(new Path(RESULT_PATH), true);
+      fs.delete(new Path(MODEL_PATH), true);
+      fs.delete(new Path(SEQTRAIN_DATA), true);
     }
   }
-  
-  private void testNeuralNetworkTraining() {
+
+  private void neuralNetworkTraining() {
     String mode = "train";
     String strTrainingDataPath = "src/test/resources/neuralnets_classification_training.txt";
-    String strSequenceTrainingDataPath = "/tmp/test-neuralnets.data";
     int featureDimension = 8;
     int labelDimension = 1;
 
-    Path sequenceTrainingDataPath = new Path(strSequenceTrainingDataPath);
+    Path sequenceTrainingDataPath = new Path(SEQTRAIN_DATA);
     Configuration conf = new Configuration();
     FileSystem fs;
     try {
@@ -119,13 +129,9 @@ public class NeuralNetworkTest {
       e1.printStackTrace();
     }
 
-    String modelPath = "/tmp/neuralnets.model";
     try {
-      NeuralNetwork.main(new String[] { mode, strSequenceTrainingDataPath,
-          modelPath, "" + featureDimension, "" + labelDimension });
-      fs = FileSystem.get(conf);
-      fs.delete(new Path(strSequenceTrainingDataPath), true);
-      fs.delete(new Path(modelPath), true);
+      NeuralNetwork.main(new String[] { mode, SEQTRAIN_DATA,
+          MODEL_PATH, "" + featureDimension, "" + labelDimension });
     } catch (Exception e) {
       e.printStackTrace();
     }



Mime
View raw message