mrql-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From fega...@apache.org
Subject git commit: MRQL-9: Make BSP mode compatible with Hama 0.7.0
Date Fri, 03 May 2013 22:42:50 GMT
Updated Branches:
  refs/heads/master 71a77cdfe -> 0868e7110


MRQL-9: Make BSP mode compatible with Hama 0.7.0


Project: http://git-wip-us.apache.org/repos/asf/incubator-mrql/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-mrql/commit/0868e711
Tree: http://git-wip-us.apache.org/repos/asf/incubator-mrql/tree/0868e711
Diff: http://git-wip-us.apache.org/repos/asf/incubator-mrql/diff/0868e711

Branch: refs/heads/master
Commit: 0868e71100c9cc47942acc7f205a5e947311d23f
Parents: 71a77cd
Author: fegaras <fegaras@cse.uta.edu>
Authored: Fri May 3 17:42:38 2013 -0500
Committer: fegaras <fegaras@cse.uta.edu>
Committed: Fri May 3 17:42:38 2013 -0500

----------------------------------------------------------------------
 Makefile             |    6 +++---
 build.xml            |   14 +++++++++-----
 conf/mrql-env.sh     |    8 +++++---
 src/BSP/BSPPlan.java |   14 +++++---------
 src/Config.java      |    1 +
 5 files changed, 23 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/0868e711/Makefile
----------------------------------------------------------------------
diff --git a/Makefile b/Makefile
index c9be632..1009d22 100644
--- a/Makefile
+++ b/Makefile
@@ -20,7 +20,7 @@
 #--------------------------------------------------------------------------------
 #
 # Makefile for MRQL
-# Requires: jflex, cup  (they can be install as Linux packages)
+# Requires: jflex  (it can be installed as a Linux package)
 #
 #--------------------------------------------------------------------------------
 
@@ -35,7 +35,7 @@ JAVAC = ${JAVA_HOME}/bin/javac -g:none -d classes
 JAVA = ${JAVA_HOME}/bin/java
 JAR = ${JAVA_HOME}/bin/jar
 JFLEX = jflex --quiet --nobak
-CUP = cup -nosummary
+CUP = ${JAVA} -jar ${CUP_JAR} -nosummary
 GEN = ${JAVA} Gen.Main
 
 sources := src/*.java
@@ -59,7 +59,7 @@ common: clean_build mrql_parser json_parser
 
 clean_build:
 	@rm -rf classes tmp
-	@mkdir -p classes tmp
+	@mkdir -p classes tmp tests/results tests/results/memory tests/results/hadoop tests/results/bsp
 
 mrql_parser:
 	@${JFLEX} src/mrql.lex -d tmp

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/0868e711/build.xml
----------------------------------------------------------------------
diff --git a/build.xml b/build.xml
index b7045d1..680d7d2 100644
--- a/build.xml
+++ b/build.xml
@@ -19,7 +19,7 @@
 <!--
 
  Ant build file for MRQL
- Requires: jflex, cup  (they can be install as Linux packages)
+ Requires: jflex  (it can be installed as a Linux package)
 
 -->
 
@@ -104,6 +104,10 @@
     <delete dir="tmp"/>
     <mkdir dir="classes"/>
     <mkdir dir="tmp"/>
+    <mkdir dir="tests/results"/>
+    <mkdir dir="tests/results/memory"/>
+    <mkdir dir="tests/results/hadoop"/>
+    <mkdir dir="tests/results/bsp"/>
   </target>
 
   <target name="mrql_parser"
@@ -120,11 +124,11 @@
       <arg line="-o"/>
       <arg file="tmp/mrql.cup"/>
     </java>
-    <exec executable="cup">
+    <java jar="${CUP_JAR}" fork='true'>
       <arg line="-nosummary"/>
       <arg line="-parser MRQLParser"/>
       <arg file="tmp/mrql.cup"/>
-    </exec>
+    </java>
     <move todir="tmp">
       <filelist dir=".">
 	<file name="sym.java"/>
@@ -141,12 +145,12 @@
       <arg line="-d"/>
       <arg file="tmp"/>
     </exec>
-    <exec executable="cup">
+    <java jar="${CUP_JAR}" fork='true'>
       <arg line="-nosummary"/>
       <arg line="-parser JSONParser"/>
       <arg line="-symbols jsym"/>
       <arg file="src/JSON.cup"/>
-    </exec>
+    </java>
     <move todir="tmp">
       <filelist dir=".">
 	<file name="jsym.java"/>

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/0868e711/conf/mrql-env.sh
----------------------------------------------------------------------
diff --git a/conf/mrql-env.sh b/conf/mrql-env.sh
index be8b3e1..53bb0da 100644
--- a/conf/mrql-env.sh
+++ b/conf/mrql-env.sh
@@ -47,10 +47,12 @@ HAMA_CONF=${MRQL_HOME}/conf/conf-hama
 CUP_JAR=/usr/share/java/cup.jar
 
 # Hadoop libraries are from a Hadoop tarball
-MRQL_CLASSPATH=classes:${HAMA_JAR}:${HADOOP_HOME}/hadoop-core-${HADOOP_VERSION}.jar:${HADOOP_HOME}/lib/commons-cli-1.2.jar
+HADOOP_JARS=${HADOOP_HOME}/hadoop-core-${HADOOP_VERSION}.jar:${HADOOP_HOME}/lib/commons-cli-1.2.jar
 
 # Hadoop libraries are from a YARN tarball
-# MRQL_CLASSPATH=classes:$HAMA_JAR:$HADOOP_HOME/share/hadoop/common/hadoop-common-$HADOOP_VERSION.jar:$HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-client-core-$HADOOP_VERSION.jar:$HADOOP_HOME/share/hadoop/hdfs/hadoop-hdfs-$HADOOP_VERSION.jar:$HADOOP_HOME/share/hadoop/common/lib/hadoop-annotations-$HADOOP_VERSION.jar:$HADOOP_HOME/share/hadoop/common/lib/commons-cli-1.2.jar
+#HADOOP_JARS=${HADOOP_HOME}/share/hadoop/common/hadoop-common-$HADOOP_VERSION.jar:${HADOOP_HOME}/share/hadoop/mapreduce/hadoop-mapreduce-client-core-$HADOOP_VERSION.jar:${HADOOP_HOME}/share/hadoop/hdfs/hadoop-hdfs-$HADOOP_VERSION.jar:${HADOOP_HOME}/share/hadoop/common/lib/hadoop-annotations-$HADOOP_VERSION.jar:${HADOOP_HOME}/share/hadoop/common/lib/commons-cli-1.2.jar
 
 # Hadoop libraries are from a Cloudera package
-#MRQL_CLASSPATH=classes:$HAMA_JAR:/usr/lib/hadoop-0.20/hadoop-core.jar:/usr/lib/hadoop-0.20/lib/commons-cli-1.2.jar
+#HADOOP_JARS=/usr/lib/hadoop-0.20/hadoop-core.jar:/usr/lib/hadoop-0.20/lib/commons-cli-1.2.jar
+
+MRQL_CLASSPATH=classes:${HAMA_JAR}:${HADOOP_JARS}

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/0868e711/src/BSP/BSPPlan.java
----------------------------------------------------------------------
diff --git a/src/BSP/BSPPlan.java b/src/BSP/BSPPlan.java
index 4c22c41..8168789 100644
--- a/src/BSP/BSPPlan.java
+++ b/src/BSP/BSPPlan.java
@@ -57,8 +57,6 @@ final public class BSPPlan extends Plan {
 	Bag msg_cache;
 	// the cache that holds all local data in memory
 	Tuple local_cache;
-	// syncronization point for local_cache
-	final static Integer cache_sync = new Integer(0);
 
 	/** shuffle values to BSP peers based on uniform hashing on key */
 	private static String shuffle ( MRData key ) {
@@ -76,7 +74,7 @@ final public class BSPPlan extends Plan {
 	    try {
 		// this case is only used for checking the exit condition of repeat/closure
 		boolean exit = mr_exit.get();
-		if (!exit)
+		if (!exit && all_peers.length > 1)
 		    // this peer is not ready to exit, so no peer should exit
 		    for ( String p: this_peer.getAllPeerNames() )
 			this_peer.send(p,new MRContainer(more_supersteps));
@@ -227,12 +225,10 @@ final public class BSPPlan extends Plan {
 		};
 		pair.set(0,msg_cache);
 		pair.set(1,state);
-		synchronized (cache_sync) {
-		    this_peer = peer;
-		    cache = local_cache;
-		    // evaluate one superstep
-		    result = (Tuple)superstep_fnc.eval(pair);
-		};
+		this_peer = peer;
+		cache = local_cache;
+		// evaluate one superstep
+		result = (Tuple)superstep_fnc.eval(pair);
 		Bag msgs = (Bag)result.get(0);
 		exit = ((MR_bool)result.get(2)).get();
 		state = result.get(1);

http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/0868e711/src/Config.java
----------------------------------------------------------------------
diff --git a/src/Config.java b/src/Config.java
index be06206..00c1993 100644
--- a/src/Config.java
+++ b/src/Config.java
@@ -225,6 +225,7 @@ final public class Config {
 	    } else if (args[i].equals("-trace_execution")) {
 		trace_execution = true;
 		trace_exp_execution = true;
+		compile_functional_arguments = false;
 		i++;
 	    } else if (args[i].equals("-methods")) {
 		System.out.print("\nImported methods: ");


Mime
View raw message