Updated Branches:
refs/heads/master 71a77cdfe -> 0868e7110
MRQL-9: Make BSP mode compatible with Hama 0.7.0
Project: http://git-wip-us.apache.org/repos/asf/incubator-mrql/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-mrql/commit/0868e711
Tree: http://git-wip-us.apache.org/repos/asf/incubator-mrql/tree/0868e711
Diff: http://git-wip-us.apache.org/repos/asf/incubator-mrql/diff/0868e711
Branch: refs/heads/master
Commit: 0868e71100c9cc47942acc7f205a5e947311d23f
Parents: 71a77cd
Author: fegaras <fegaras@cse.uta.edu>
Authored: Fri May 3 17:42:38 2013 -0500
Committer: fegaras <fegaras@cse.uta.edu>
Committed: Fri May 3 17:42:38 2013 -0500
----------------------------------------------------------------------
Makefile | 6 +++---
build.xml | 14 +++++++++-----
conf/mrql-env.sh | 8 +++++---
src/BSP/BSPPlan.java | 14 +++++---------
src/Config.java | 1 +
5 files changed, 23 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/0868e711/Makefile
----------------------------------------------------------------------
diff --git a/Makefile b/Makefile
index c9be632..1009d22 100644
--- a/Makefile
+++ b/Makefile
@@ -20,7 +20,7 @@
#--------------------------------------------------------------------------------
#
# Makefile for MRQL
-# Requires: jflex, cup (they can be install as Linux packages)
+# Requires: jflex (it can be installed as a Linux package)
#
#--------------------------------------------------------------------------------
@@ -35,7 +35,7 @@ JAVAC = ${JAVA_HOME}/bin/javac -g:none -d classes
JAVA = ${JAVA_HOME}/bin/java
JAR = ${JAVA_HOME}/bin/jar
JFLEX = jflex --quiet --nobak
-CUP = cup -nosummary
+CUP = ${JAVA} -jar ${CUP_JAR} -nosummary
GEN = ${JAVA} Gen.Main
sources := src/*.java
@@ -59,7 +59,7 @@ common: clean_build mrql_parser json_parser
clean_build:
@rm -rf classes tmp
- @mkdir -p classes tmp
+ @mkdir -p classes tmp tests/results tests/results/memory tests/results/hadoop tests/results/bsp
mrql_parser:
@${JFLEX} src/mrql.lex -d tmp
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/0868e711/build.xml
----------------------------------------------------------------------
diff --git a/build.xml b/build.xml
index b7045d1..680d7d2 100644
--- a/build.xml
+++ b/build.xml
@@ -19,7 +19,7 @@
<!--
Ant build file for MRQL
- Requires: jflex, cup (they can be install as Linux packages)
+ Requires: jflex (it can be installed as a Linux package)
-->
@@ -104,6 +104,10 @@
<delete dir="tmp"/>
<mkdir dir="classes"/>
<mkdir dir="tmp"/>
+ <mkdir dir="tests/results"/>
+ <mkdir dir="tests/results/memory"/>
+ <mkdir dir="tests/results/hadoop"/>
+ <mkdir dir="tests/results/bsp"/>
</target>
<target name="mrql_parser"
@@ -120,11 +124,11 @@
<arg line="-o"/>
<arg file="tmp/mrql.cup"/>
</java>
- <exec executable="cup">
+ <java jar="${CUP_JAR}" fork='true'>
<arg line="-nosummary"/>
<arg line="-parser MRQLParser"/>
<arg file="tmp/mrql.cup"/>
- </exec>
+ </java>
<move todir="tmp">
<filelist dir=".">
<file name="sym.java"/>
@@ -141,12 +145,12 @@
<arg line="-d"/>
<arg file="tmp"/>
</exec>
- <exec executable="cup">
+ <java jar="${CUP_JAR}" fork='true'>
<arg line="-nosummary"/>
<arg line="-parser JSONParser"/>
<arg line="-symbols jsym"/>
<arg file="src/JSON.cup"/>
- </exec>
+ </java>
<move todir="tmp">
<filelist dir=".">
<file name="jsym.java"/>
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/0868e711/conf/mrql-env.sh
----------------------------------------------------------------------
diff --git a/conf/mrql-env.sh b/conf/mrql-env.sh
index be8b3e1..53bb0da 100644
--- a/conf/mrql-env.sh
+++ b/conf/mrql-env.sh
@@ -47,10 +47,12 @@ HAMA_CONF=${MRQL_HOME}/conf/conf-hama
CUP_JAR=/usr/share/java/cup.jar
# Hadoop libraries are from a Hadoop tarball
-MRQL_CLASSPATH=classes:${HAMA_JAR}:${HADOOP_HOME}/hadoop-core-${HADOOP_VERSION}.jar:${HADOOP_HOME}/lib/commons-cli-1.2.jar
+HADOOP_JARS=${HADOOP_HOME}/hadoop-core-${HADOOP_VERSION}.jar:${HADOOP_HOME}/lib/commons-cli-1.2.jar
# Hadoop libraries are from a YARN tarball
-# MRQL_CLASSPATH=classes:$HAMA_JAR:$HADOOP_HOME/share/hadoop/common/hadoop-common-$HADOOP_VERSION.jar:$HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-client-core-$HADOOP_VERSION.jar:$HADOOP_HOME/share/hadoop/hdfs/hadoop-hdfs-$HADOOP_VERSION.jar:$HADOOP_HOME/share/hadoop/common/lib/hadoop-annotations-$HADOOP_VERSION.jar:$HADOOP_HOME/share/hadoop/common/lib/commons-cli-1.2.jar
+#HADOOP_JARS=${HADOOP_HOME}/share/hadoop/common/hadoop-common-$HADOOP_VERSION.jar:${HADOOP_HOME}/share/hadoop/mapreduce/hadoop-mapreduce-client-core-$HADOOP_VERSION.jar:${HADOOP_HOME}/share/hadoop/hdfs/hadoop-hdfs-$HADOOP_VERSION.jar:${HADOOP_HOME}/share/hadoop/common/lib/hadoop-annotations-$HADOOP_VERSION.jar:${HADOOP_HOME}/share/hadoop/common/lib/commons-cli-1.2.jar
# Hadoop libraries are from a Cloudera package
-#MRQL_CLASSPATH=classes:$HAMA_JAR:/usr/lib/hadoop-0.20/hadoop-core.jar:/usr/lib/hadoop-0.20/lib/commons-cli-1.2.jar
+#HADOOP_JARS=/usr/lib/hadoop-0.20/hadoop-core.jar:/usr/lib/hadoop-0.20/lib/commons-cli-1.2.jar
+
+MRQL_CLASSPATH=classes:${HAMA_JAR}:${HADOOP_JARS}
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/0868e711/src/BSP/BSPPlan.java
----------------------------------------------------------------------
diff --git a/src/BSP/BSPPlan.java b/src/BSP/BSPPlan.java
index 4c22c41..8168789 100644
--- a/src/BSP/BSPPlan.java
+++ b/src/BSP/BSPPlan.java
@@ -57,8 +57,6 @@ final public class BSPPlan extends Plan {
Bag msg_cache;
// the cache that holds all local data in memory
Tuple local_cache;
- // syncronization point for local_cache
- final static Integer cache_sync = new Integer(0);
/** shuffle values to BSP peers based on uniform hashing on key */
private static String shuffle ( MRData key ) {
@@ -76,7 +74,7 @@ final public class BSPPlan extends Plan {
try {
// this case is only used for checking the exit condition of repeat/closure
boolean exit = mr_exit.get();
- if (!exit)
+ if (!exit && all_peers.length > 1)
// this peer is not ready to exit, so no peer should exit
for ( String p: this_peer.getAllPeerNames() )
this_peer.send(p,new MRContainer(more_supersteps));
@@ -227,12 +225,10 @@ final public class BSPPlan extends Plan {
};
pair.set(0,msg_cache);
pair.set(1,state);
- synchronized (cache_sync) {
- this_peer = peer;
- cache = local_cache;
- // evaluate one superstep
- result = (Tuple)superstep_fnc.eval(pair);
- };
+ this_peer = peer;
+ cache = local_cache;
+ // evaluate one superstep
+ result = (Tuple)superstep_fnc.eval(pair);
Bag msgs = (Bag)result.get(0);
exit = ((MR_bool)result.get(2)).get();
state = result.get(1);
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/0868e711/src/Config.java
----------------------------------------------------------------------
diff --git a/src/Config.java b/src/Config.java
index be06206..00c1993 100644
--- a/src/Config.java
+++ b/src/Config.java
@@ -225,6 +225,7 @@ final public class Config {
} else if (args[i].equals("-trace_execution")) {
trace_execution = true;
trace_exp_execution = true;
+ compile_functional_arguments = false;
i++;
} else if (args[i].equals("-methods")) {
System.out.print("\nImported methods: ");
|