cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From brandonwilli...@apache.org
Subject [12/17] git commit: Fix word count
Date Fri, 18 Jul 2014 17:36:48 GMT
Fix word count

Patch by  Ala' Alkhaldi, reviewed by brandonwilliams for CASSANDRA-7200


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/8fd8832a
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/8fd8832a
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/8fd8832a

Branch: refs/heads/trunk
Commit: 8fd8832a8723ef5e663834def078ea0afc7ff8d6
Parents: 2215a4f
Author: Brandon Williams <brandonwilliams@apache.org>
Authored: Fri Jul 18 12:35:57 2014 -0500
Committer: Brandon Williams <brandonwilliams@apache.org>
Committed: Fri Jul 18 12:35:57 2014 -0500

----------------------------------------------------------------------
 examples/hadoop_cql3_word_count/README.txt      |  6 +++--
 .../conf/log4j.properties                       |  2 +-
 .../src/WordCountSetup.java                     |  5 +++-
 examples/hadoop_word_count/README.txt           |  6 +++--
 .../hadoop_word_count/conf/log4j.properties     |  2 +-
 examples/hadoop_word_count/src/WordCount.java   |  5 +++-
 .../src/WordCountCounters.java                  |  2 ++
 .../hadoop_word_count/src/WordCountSetup.java   | 25 +++++++++++++++++++-
 8 files changed, 44 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/8fd8832a/examples/hadoop_cql3_word_count/README.txt
----------------------------------------------------------------------
diff --git a/examples/hadoop_cql3_word_count/README.txt b/examples/hadoop_cql3_word_count/README.txt
index f984b02..e01632b 100644
--- a/examples/hadoop_cql3_word_count/README.txt
+++ b/examples/hadoop_cql3_word_count/README.txt
@@ -36,8 +36,10 @@ Read the code in src/ for more details.
 The word_count_counters example sums the counter columns for a row. The output
 is written to a text file in /tmp/word_count_counters.
 
-*If you want to point wordcount at a real cluster, modify the seed
-and listenaddress settings accordingly.
+*It is recommended to turn off vnodes when running Cassandra with hadoop. 
+This is done by setting "num_tokens: 1" in cassandra.yaml. If you want to
+point wordcount at a real cluster, modify the seed and listenaddress 
+settings accordingly.
 
 
 Troubleshooting

http://git-wip-us.apache.org/repos/asf/cassandra/blob/8fd8832a/examples/hadoop_cql3_word_count/conf/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/hadoop_cql3_word_count/conf/log4j.properties b/examples/hadoop_cql3_word_count/conf/log4j.properties
index 70f7657..508d60f 100644
--- a/examples/hadoop_cql3_word_count/conf/log4j.properties
+++ b/examples/hadoop_cql3_word_count/conf/log4j.properties
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-log4j.rootLogger=DEBUG,stdout,F
+log4j.rootLogger=INFO,stdout,F
 
 #stdout
 log4j.appender.stdout=org.apache.log4j.ConsoleAppender

http://git-wip-us.apache.org/repos/asf/cassandra/blob/8fd8832a/examples/hadoop_cql3_word_count/src/WordCountSetup.java
----------------------------------------------------------------------
diff --git a/examples/hadoop_cql3_word_count/src/WordCountSetup.java b/examples/hadoop_cql3_word_count/src/WordCountSetup.java
index ebf7485..e514d63 100644
--- a/examples/hadoop_cql3_word_count/src/WordCountSetup.java
+++ b/examples/hadoop_cql3_word_count/src/WordCountSetup.java
@@ -70,7 +70,10 @@ public class WordCountSetup
 
             client.execute_cql3_query(ByteBufferUtil.bytes(query), Compression.NONE, ConsistencyLevel.ONE);
 
-            int magnitude = client.describe_ring(WordCount.KEYSPACE).size();
+            String verifyQuery = "select count(*) from system.peers";
+            CqlResult result = client.execute_cql3_query(ByteBufferUtil.bytes(verifyQuery),
Compression.NONE, ConsistencyLevel.ONE);
+
+            long magnitude = ByteBufferUtil.toLong(result.rows.get(0).columns.get(0).value);
             try
             {
                 Thread.sleep(1000 * magnitude);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/8fd8832a/examples/hadoop_word_count/README.txt
----------------------------------------------------------------------
diff --git a/examples/hadoop_word_count/README.txt b/examples/hadoop_word_count/README.txt
index cf8a344..9a93908 100644
--- a/examples/hadoop_word_count/README.txt
+++ b/examples/hadoop_word_count/README.txt
@@ -37,8 +37,10 @@ Read the code in src/ for more details.
 The word_count_counters example sums the counter columns for a row. The output
 is written to a text file in /tmp/word_count_counters.
 
-*If you want to point wordcount at a real cluster, modify the seed
-and listenaddress settings accordingly.
+*It is recommended to turn off vnodes when running Cassandra with hadoop.
+This is done by setting "num_tokens: 1" in cassandra.yaml. If you want to
+point wordcount at a real cluster, modify the seed and listenaddress
+settings accordingly.
 
 
 Troubleshooting

http://git-wip-us.apache.org/repos/asf/cassandra/blob/8fd8832a/examples/hadoop_word_count/conf/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/hadoop_word_count/conf/log4j.properties b/examples/hadoop_word_count/conf/log4j.properties
index 70f7657..508d60f 100644
--- a/examples/hadoop_word_count/conf/log4j.properties
+++ b/examples/hadoop_word_count/conf/log4j.properties
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-log4j.rootLogger=DEBUG,stdout,F
+log4j.rootLogger=INFO,stdout,F
 
 #stdout
 log4j.appender.stdout=org.apache.log4j.ConsoleAppender

http://git-wip-us.apache.org/repos/asf/cassandra/blob/8fd8832a/examples/hadoop_word_count/src/WordCount.java
----------------------------------------------------------------------
diff --git a/examples/hadoop_word_count/src/WordCount.java b/examples/hadoop_word_count/src/WordCount.java
index ce9565d..f6bca77 100644
--- a/examples/hadoop_word_count/src/WordCount.java
+++ b/examples/hadoop_word_count/src/WordCount.java
@@ -18,8 +18,11 @@
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.*;
 
 import org.apache.cassandra.db.Cell;
+import org.apache.cassandra.thrift.*;
+import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.hadoop.ColumnFamilyOutputFormat;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -83,7 +86,7 @@ public class WordCount extends Configured implements Tool
         {
             for (Cell cell : columns.values())
             {
-                String name  = ByteBufferUtil.string(cell.name());
+                String name  = ByteBufferUtil.string(cell.name().toByteBuffer());
                 String value = null;
                 
                 if (name.contains("int"))

http://git-wip-us.apache.org/repos/asf/cassandra/blob/8fd8832a/examples/hadoop_word_count/src/WordCountCounters.java
----------------------------------------------------------------------
diff --git a/examples/hadoop_word_count/src/WordCountCounters.java b/examples/hadoop_word_count/src/WordCountCounters.java
index 4ea063b..39fb778 100644
--- a/examples/hadoop_word_count/src/WordCountCounters.java
+++ b/examples/hadoop_word_count/src/WordCountCounters.java
@@ -18,8 +18,10 @@
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.SortedMap;
 
 import org.apache.cassandra.db.Cell;
+import org.apache.cassandra.thrift.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/8fd8832a/examples/hadoop_word_count/src/WordCountSetup.java
----------------------------------------------------------------------
diff --git a/examples/hadoop_word_count/src/WordCountSetup.java b/examples/hadoop_word_count/src/WordCountSetup.java
index e222327..0ef5341 100644
--- a/examples/hadoop_word_count/src/WordCountSetup.java
+++ b/examples/hadoop_word_count/src/WordCountSetup.java
@@ -176,10 +176,33 @@ public class WordCountSetup
         KsDef ksDef = new KsDef(WordCount.KEYSPACE, "org.apache.cassandra.locator.SimpleStrategy",
cfDefList);
         ksDef.putToStrategy_options("replication_factor", "1");
         client.system_add_keyspace(ksDef);
-        int magnitude = client.describe_ring(WordCount.KEYSPACE).size();
+
+        int magnitude = getNumberOfHosts(client);
         Uninterruptibles.sleepUninterruptibly(magnitude, TimeUnit.SECONDS);
     }
 
+    private static int getNumberOfHosts(Cassandra.Iface client)
+            throws InvalidRequestException, UnavailableException, TimedOutException, TException
+    {
+        client.set_keyspace("system");
+        SlicePredicate predicate = new SlicePredicate();
+        SliceRange sliceRange = new SliceRange();
+        sliceRange.setStart(new byte[0]);
+        sliceRange.setFinish(new byte[0]);
+        predicate.setSlice_range(sliceRange);
+
+        KeyRange keyrRange = new KeyRange();
+        keyrRange.setStart_key(new byte[0]);
+        keyrRange.setEnd_key(new byte[0]);
+        //keyrRange.setCount(100);
+
+        ColumnParent parent = new ColumnParent("peers");
+
+        List<KeySlice> ls = client.get_range_slices(parent, predicate, keyrRange, ConsistencyLevel.ONE);
+
+        return ls.size();
+    }
+
     private static Cassandra.Iface createConnection() throws TTransportException
     {
         if (System.getProperty("cassandra.host") == null || System.getProperty("cassandra.port")
== null)


Mime
View raw message