cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jo...@apache.org
Subject svn commit: r915376 - in /incubator/cassandra/trunk: ./ interface/ interface/thrift/gen-java/org/apache/cassandra/thrift/ src/java/org/apache/cassandra/hadoop/
Date Tue, 23 Feb 2010 15:17:43 GMT
Author: johan
Date: Tue Feb 23 15:17:42 2010
New Revision: 915376

URL: http://svn.apache.org/viewvc?rev=915376&view=rev
Log:
Merge with 0.6 branch

Modified:
    incubator/cassandra/trunk/   (props changed)
    incubator/cassandra/trunk/NEWS.txt
    incubator/cassandra/trunk/interface/cassandra.thrift
    incubator/cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
  (props changed)
    incubator/cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
  (props changed)
    incubator/cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
  (props changed)
    incubator/cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
  (props changed)
    incubator/cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
  (props changed)
    incubator/cassandra/trunk/src/java/org/apache/cassandra/hadoop/ColumnFamilyInputFormat.java

Propchange: incubator/cassandra/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Feb 23 15:17:42 2010
@@ -1,5 +1,5 @@
 /incubator/cassandra/branches/cassandra-0.3:774578-796573
 /incubator/cassandra/branches/cassandra-0.4:810145-834239,834349-834350
 /incubator/cassandra/branches/cassandra-0.5:888872-910752
-/incubator/cassandra/branches/cassandra-0.6:911237-912672
+/incubator/cassandra/branches/cassandra-0.6:911237-915366
 /incubator/cassandra/trunk:749219-888871

Modified: incubator/cassandra/trunk/NEWS.txt
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/NEWS.txt?rev=915376&r1=915375&r2=915376&view=diff
==============================================================================
--- incubator/cassandra/trunk/NEWS.txt (original)
+++ incubator/cassandra/trunk/NEWS.txt Tue Feb 23 15:17:42 2010
@@ -11,8 +11,11 @@
 
 Features
 --------
-    - row caching added; configure with the RowsCached attribute
-      in ColumnFamily definition
+    - row caching: configure with the RowsCached attribute in
+      ColumnFamily definition
+    - Hadoop map/reduce support see contrib/word_count for an example
+    - experimental authentication support, described under
+      Authenticator in storage.conf
 
 Configuraton
 ------------
@@ -27,12 +30,16 @@
 
 Thrift API
 ----------
-    - removed deprecated get_key_range Thrift method
+    - removed deprecated get_key_range method
+    - added batch_mutate meethod
+    - deprecated multiget and batch_insert methods in favor of
+      multiget_slice and batch_mutate, respectively
 
 JMX metrics
 -----------
     - read and write statistics are reported as lifetime totals,
-      instead of averages over the last minute.
+      instead of averages over the last minute.  average-since-last
+      requested are also available for convenience.
     - cache hit rate statistics are now available from JMX under
       org.apache.cassandra.db.Caches
     - compaction JMX metrics are moved to
@@ -48,10 +55,14 @@
       shipped in lib/. If you are installing from binary distribution then
       you will need to install Ant (http://ant.apache.org/), and invoke
       `ant ivy-retrieve' in order to download needed dependencies.
+    - 0.6 network traffic is not compatible with earlier versions.  You
+      will need to shut down all your nodes at once, upgrade, then restart.
+
 
 
 0.5.0
 =====
+
 0. The commitlog format has changed (but sstable format has not). 
    When upgrading from 0.4, empty the commitlog either by running 
    bin/nodeprobe flush on each machine and waiting for the flush to finish,
@@ -82,21 +93,27 @@
      - Added get_range_slice, deprecating get_key_range
 
 
+
 0.4.2
 =====
+
 1. Improve default garbage collector options significantly --
    throughput will be 30% higher or more.
 
 
+
 0.4.1
 =====
+
 1. SnapshotBeforeCompaction configuration option allows snapshotting
    before each compaction, which allows rolling back to any version
    of the data.
 
 
+
 0.4.0
 =====
+
 1. On-disk data format has changed to allow billions of keys/rows per
    node instead of only millions.  The new format is incompatible with 0.3;
    see 0.3 notes below for how to import data from a 0.3 install.
@@ -117,8 +134,10 @@
 6. Added commitlog fsync; see "<CommitLogSync>" in configuration.
 
 
+
 0.3.0
 =====
+
 1. With enough and large enough keys in a ColumnFamily, Cassandra will
    run out of memory trying to perform compactions (data file merges).
    The size of what is stored in memory is (S + 16) * (N + M) where S

Modified: incubator/cassandra/trunk/interface/cassandra.thrift
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/interface/cassandra.thrift?rev=915376&r1=915375&r2=915376&view=diff
==============================================================================
--- incubator/cassandra/trunk/interface/cassandra.thrift (original)
+++ incubator/cassandra/trunk/interface/cassandra.thrift Tue Feb 23 15:17:42 2010
@@ -318,6 +318,7 @@
     Perform a get for column_path in parallel on the given list<string> keys. The return
value maps keys to the
     ColumnOrSuperColumn found. If no value corresponding to a key is present, the key will
still be in the map, but both
     the column and super_column references of the ColumnOrSuperColumn object it maps to will
be null.  
+    @deprecated; use multiget_slice
   */
   map<string,ColumnOrSuperColumn> multiget(1:required string keyspace, 
                                            2:required list<string> keys, 

Propchange: incubator/cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Feb 23 15:17:42 2010
@@ -1,6 +1,6 @@
 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/Cassandra.java:774578-796573
 /incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/Cassandra.java:810145-834239,834349-834350
 /incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/Cassandra.java:888872-903502
-/incubator/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:911237-912672
+/incubator/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:911237-915366
 /incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/Cassandra.java:749219-768588
 /incubator/cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:749219-904544

Propchange: incubator/cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Feb 23 15:17:42 2010
@@ -1,7 +1,7 @@
 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/column_t.java:774578-792198
 /incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/Column.java:810145-834239,834349-834350
 /incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/Column.java:888872-903502
-/incubator/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:911237-912672
+/incubator/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:911237-915366
 /incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/Column.java:749219-794428
 /incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/column_t.java:749219-768588
 /incubator/cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:749219-904544

Propchange: incubator/cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Feb 23 15:17:42 2010
@@ -1,6 +1,6 @@
 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:774578-796573
 /incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:810145-834239,834349-834350
 /incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:888872-903502
-/incubator/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:911237-912672
+/incubator/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:911237-915366
 /incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:749219-768588
 /incubator/cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:749219-904544

Propchange: incubator/cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Feb 23 15:17:42 2010
@@ -1,5 +1,5 @@
 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:774578-796573
 /incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:810145-834239,834349-834350
 /incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:888872-903502
-/incubator/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:911237-912672
+/incubator/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:911237-915366
 /incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:749219-768588

Propchange: incubator/cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Feb 23 15:17:42 2010
@@ -1,7 +1,7 @@
 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/superColumn_t.java:774578-792198
 /incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/SuperColumn.java:810145-834239,834349-834350
 /incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/SuperColumn.java:888872-903502
-/incubator/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:911237-912672
+/incubator/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:911237-915366
 /incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/SuperColumn.java:749219-794428
 /incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/superColumn_t.java:749219-768588
 /incubator/cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:749219-904544

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/hadoop/ColumnFamilyInputFormat.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/hadoop/ColumnFamilyInputFormat.java?rev=915376&r1=915375&r2=915376&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/hadoop/ColumnFamilyInputFormat.java
(original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/hadoop/ColumnFamilyInputFormat.java
Tue Feb 23 15:17:42 2010
@@ -32,6 +32,7 @@
     private static final String KEYSPACE_CONFIG = "cassandra.input.keyspace";
     private static final String COLUMNFAMILY_CONFIG = "cassandra.input.columnfamily";
     private static final String PREDICATE_CONFIG = "cassandra.input.predicate";
+    private static final String INPUT_SPLIT_SIZE_CONFIG = "cassandra.input.split.size";
 
     private static final Logger logger = Logger.getLogger(StorageService.class);
 
@@ -62,6 +63,19 @@
         conf.set(COLUMNFAMILY_CONFIG, columnFamily);
     }
 
+    /**
+     * Set the size of the input split.
+     * This affects the number of maps created, if the number is too small
+     * the overhead of each map will take up the bulk of the job time.
+     *  
+     * @param job Job you are about to run.
+     * @param splitsize Size of the input split
+     */
+    public static void setInputSplitSize(Job job, int splitsize)
+    {
+        job.getConfiguration().setInt(INPUT_SPLIT_SIZE_CONFIG, splitsize);
+    }
+    
     public static void setSlicePredicate(Job job, SlicePredicate predicate)
     {
         Configuration conf = job.getConfiguration();
@@ -122,13 +136,15 @@
         // cannonical ranges and nodes holding replicas
         List<TokenRange> masterRangeNodes = getRangeMap();
 
+        int splitsize = context.getConfiguration().getInt(INPUT_SPLIT_SIZE_CONFIG, 16384);
+        
         // cannonical ranges, split into pieces:
         // for each range, pick a live owner and ask it to compute bite-sized splits
         // TODO parallelize this thread-per-range
         Map<TokenRange, List<String>> splitRanges = new HashMap<TokenRange,
List<String>>();
         for (TokenRange range : masterRangeNodes)
         {
-            splitRanges.put(range, getSubSplits(range));
+            splitRanges.put(range, getSubSplits(range, splitsize));
         }
 
         // turn the sub-ranges into InputSplits
@@ -143,7 +159,7 @@
             for ( ; i < tokens.size(); i++)
             {
                 ColumnFamilySplit split = new ColumnFamilySplit(keyspace, columnFamily, predicate,
tokens.get(i - 1), tokens.get(i), endpoints);
-                logger.info("adding " + split);
+                logger.debug("adding " + split);
                 splits.add(split);
             }
         }
@@ -152,7 +168,7 @@
         return splits;
     }
 
-    private List<String> getSubSplits(TokenRange range) throws IOException
+    private List<String> getSubSplits(TokenRange range, int splitsize) throws IOException
     {
         // TODO handle failure of range replicas & retry
         TSocket socket = new TSocket(range.endpoints.get(0),
@@ -170,7 +186,7 @@
         List<String> splits;
         try
         {
-            splits = client.describe_splits(range.start_token, range.end_token, 128); //
TODO make split size configurable
+            splits = client.describe_splits(range.start_token, range.end_token, splitsize);
         }
         catch (TException e)
         {



Mime
View raw message