pirk-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jacobwil...@apache.org
Subject incubator-pirk git commit: PIRK-80 Upgrade to Spark 2.0.0 - closes apache/incubator-pirk#113
Date Thu, 27 Oct 2016 14:22:00 GMT
Repository: incubator-pirk
Updated Branches:
  refs/heads/master 4abbefea7 -> bed244fe2


PIRK-80 Upgrade to Spark 2.0.0 - closes apache/incubator-pirk#113


Project: http://git-wip-us.apache.org/repos/asf/incubator-pirk/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-pirk/commit/bed244fe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-pirk/tree/bed244fe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-pirk/diff/bed244fe

Branch: refs/heads/master
Commit: bed244fe2facda1fc1eda1cb60e65a9d43b4d29d
Parents: 4abbefe
Author: eawilliams <eawilliams@apache.org>
Authored: Thu Oct 27 10:21:46 2016 -0400
Committer: Jacob Wilder <jacobwilder@apache.org>
Committed: Thu Oct 27 10:21:46 2016 -0400

----------------------------------------------------------------------
 pom.xml                                                         | 2 +-
 .../org/apache/pirk/responder/wideskies/spark/EncRowCalc.java   | 5 +++--
 .../responder/wideskies/spark/EncRowCalcPrecomputedCache.java   | 5 +++--
 .../pirk/responder/wideskies/spark/ExpKeyFilenameMap.java       | 4 ++--
 .../pirk/responder/wideskies/spark/ExpTableGenerator.java       | 5 +++--
 5 files changed, 12 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/bed244fe/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 80e9a2b..88dff10 100644
--- a/pom.xml
+++ b/pom.xml
@@ -86,7 +86,7 @@
         <junit.version>4.12</junit.version>
         <log4j.configuration>log4j2.xml</log4j.configuration>
         <hadoop.version>2.7.3</hadoop.version>
-        <spark.version>1.6.1</spark.version>
+        <spark.version>2.0.0</spark.version>
         <elasticsearch.version>2.3.4</elasticsearch.version>
         <storm.version>1.0.1</storm.version>
         <kafka.version>0.9.0.1</kafka.version>

http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/bed244fe/src/main/java/org/apache/pirk/responder/wideskies/spark/EncRowCalc.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/responder/wideskies/spark/EncRowCalc.java b/src/main/java/org/apache/pirk/responder/wideskies/spark/EncRowCalc.java
index f5a591e..e0b3b25 100644
--- a/src/main/java/org/apache/pirk/responder/wideskies/spark/EncRowCalc.java
+++ b/src/main/java/org/apache/pirk/responder/wideskies/spark/EncRowCalc.java
@@ -21,6 +21,7 @@ package org.apache.pirk.responder.wideskies.spark;
 import java.io.IOException;
 import java.math.BigInteger;
 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
 
 import org.apache.hadoop.conf.Configuration;
@@ -71,7 +72,7 @@ public class EncRowCalc implements PairFlatMapFunction<Tuple2<Integer,Iterable<L
   }
 
   @Override
-  public Iterable<Tuple2<Long,BigInteger>> call(Tuple2<Integer,Iterable<List<BigInteger>>>
hashDocTuple) throws Exception
+  public Iterator<Tuple2<Long,BigInteger>> call(Tuple2<Integer,Iterable<List<BigInteger>>>
hashDocTuple) throws Exception
   {
     List<Tuple2<Long,BigInteger>> returnPairs = new ArrayList<>();
 
@@ -98,6 +99,6 @@ public class EncRowCalc implements PairFlatMapFunction<Tuple2<Integer,Iterable<L
 
     returnPairs.addAll(encRowValues);
 
-    return returnPairs;
+    return returnPairs.iterator();
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/bed244fe/src/main/java/org/apache/pirk/responder/wideskies/spark/EncRowCalcPrecomputedCache.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/responder/wideskies/spark/EncRowCalcPrecomputedCache.java
b/src/main/java/org/apache/pirk/responder/wideskies/spark/EncRowCalcPrecomputedCache.java
index 8147ff6..5c1de76 100644
--- a/src/main/java/org/apache/pirk/responder/wideskies/spark/EncRowCalcPrecomputedCache.java
+++ b/src/main/java/org/apache/pirk/responder/wideskies/spark/EncRowCalcPrecomputedCache.java
@@ -21,6 +21,7 @@ package org.apache.pirk.responder.wideskies.spark;
 import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 
 import org.apache.pirk.query.wideskies.Query;
@@ -65,7 +66,7 @@ public class EncRowCalcPrecomputedCache
   }
 
   @Override
-  public Iterable<Tuple2<Long,BigInteger>> call(Tuple2<Integer,Tuple2<Iterable<Tuple2<Integer,BigInteger>>,Iterable<List<BigInteger>>>>
hashDocTuple)
+  public Iterator<Tuple2<Long, BigInteger>> call(Tuple2<Integer,Tuple2<Iterable<Tuple2<Integer,BigInteger>>,Iterable<List<BigInteger>>>>
hashDocTuple)
       throws Exception
   {
     List<Tuple2<Long,BigInteger>> returnPairs = new ArrayList<>();
@@ -95,6 +96,6 @@ public class EncRowCalcPrecomputedCache
 
     returnPairs.addAll(encRowValues);
 
-    return returnPairs;
+    return returnPairs.iterator();
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/bed244fe/src/main/java/org/apache/pirk/responder/wideskies/spark/ExpKeyFilenameMap.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/responder/wideskies/spark/ExpKeyFilenameMap.java
b/src/main/java/org/apache/pirk/responder/wideskies/spark/ExpKeyFilenameMap.java
index 09b1e52..03c80f4 100644
--- a/src/main/java/org/apache/pirk/responder/wideskies/spark/ExpKeyFilenameMap.java
+++ b/src/main/java/org/apache/pirk/responder/wideskies/spark/ExpKeyFilenameMap.java
@@ -52,7 +52,7 @@ public class ExpKeyFilenameMap implements PairFlatMapFunction<Iterator<Tuple2<In
   }
 
   @Override
-  public Iterable<Tuple2<Integer,String>> call(Iterator<Tuple2<Integer,Iterable<Tuple2<Integer,BigInteger>>>>
iter) throws Exception
+  public Iterator<Tuple2<Integer, String>> call(Iterator<Tuple2<Integer,Iterable<Tuple2<Integer,BigInteger>>>>
iter) throws Exception
   {
     List<Tuple2<Integer,String>> keyFileList = new ArrayList<>();
 
@@ -87,6 +87,6 @@ public class ExpKeyFilenameMap implements PairFlatMapFunction<Iterator<Tuple2<In
     }
     bw.close();
 
-    return keyFileList;
+    return keyFileList.iterator();
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/bed244fe/src/main/java/org/apache/pirk/responder/wideskies/spark/ExpTableGenerator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/responder/wideskies/spark/ExpTableGenerator.java
b/src/main/java/org/apache/pirk/responder/wideskies/spark/ExpTableGenerator.java
index 4c54935..d14e36a 100644
--- a/src/main/java/org/apache/pirk/responder/wideskies/spark/ExpTableGenerator.java
+++ b/src/main/java/org/apache/pirk/responder/wideskies/spark/ExpTableGenerator.java
@@ -20,6 +20,7 @@ package org.apache.pirk.responder.wideskies.spark;
 
 import java.math.BigInteger;
 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
 
 import org.apache.pirk.encryption.ModPowAbstraction;
@@ -51,7 +52,7 @@ public class ExpTableGenerator implements PairFlatMapFunction<Integer,Integer,Tu
   }
 
   @Override
-  public Iterable<Tuple2<Integer,Tuple2<Integer,BigInteger>>> call(Integer
queryHashKey) throws Exception
+  public Iterator<Tuple2<Integer, Tuple2<Integer, BigInteger>>> call(Integer
queryHashKey) throws Exception
   {
     // queryHashKey -> <<power>,<element^power mod N^2>>
     List<Tuple2<Integer,Tuple2<Integer,BigInteger>>> modExp = new ArrayList<>();
@@ -64,6 +65,6 @@ public class ExpTableGenerator implements PairFlatMapFunction<Integer,Integer,Tu
       modExp.add(new Tuple2<>(queryHashKey, modPowTuple));
     }
 
-    return modExp;
+    return modExp.iterator();
   }
 }


Mime
View raw message