cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbel...@apache.org
Subject svn commit: r942768 - in /cassandra/trunk: ./ interface/thrift/gen-java/org/apache/cassandra/thrift/ src/java/org/apache/cassandra/io/sstable/ test/unit/org/apache/cassandra/io/sstable/
Date Mon, 10 May 2010 14:59:14 GMT
Author: jbellis
Date: Mon May 10 14:59:13 2010
New Revision: 942768

URL: http://svn.apache.org/viewvc?rev=942768&view=rev
Log:
merge from 0.6

Modified:
    cassandra/trunk/   (props changed)
    cassandra/trunk/CHANGES.txt
    cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java 
 (props changed)
    cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java   (props
changed)
    cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
  (props changed)
    cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
  (props changed)
    cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
  (props changed)
    cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java
    cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
    cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java

Propchange: cassandra/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 10 14:59:13 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6:922689-942206
+/cassandra/branches/cassandra-0.6:922689-942757
 /incubator/cassandra/branches/cassandra-0.3:774578-796573
 /incubator/cassandra/branches/cassandra-0.4:810145-834239,834349-834350
 /incubator/cassandra/branches/cassandra-0.5:888872-915439

Modified: cassandra/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=942768&r1=942767&r2=942768&view=diff
==============================================================================
--- cassandra/trunk/CHANGES.txt (original)
+++ cassandra/trunk/CHANGES.txt Mon May 10 14:59:13 2010
@@ -36,6 +36,7 @@ dev
    (CASSANDRA-1060)
  * disable GCInspector on non-Sun JVMs (CASSANDRA-1061)
  * fix tombstone handling in sstable rows with no other data (CASSANDRA-1063)
+ * fix size of row in spanned index entries (CASSANDRA-1056)
 
 
 0.6.1

Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 10 14:59:13 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-942206
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-942757
 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/Cassandra.java:774578-796573
 /incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/Cassandra.java:810145-834239,834349-834350
 /incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/Cassandra.java:888872-903502

Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 10 14:59:13 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-942206
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-942757
 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/column_t.java:774578-792198
 /incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/Column.java:810145-834239,834349-834350
 /incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/Column.java:888872-903502

Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 10 14:59:13 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-942206
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-942757
 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:774578-796573
 /incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:810145-834239,834349-834350
 /incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:888872-903502

Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 10 14:59:13 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-942206
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-942757
 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:774578-796573
 /incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:810145-834239,834349-834350
 /incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:888872-903502

Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 10 14:59:13 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-942206
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-942757
 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/superColumn_t.java:774578-792198
 /incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/SuperColumn.java:810145-834239,834349-834350
 /incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/SuperColumn.java:888872-903502

Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java?rev=942768&r1=942767&r2=942768&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java Mon May 10
14:59:13 2010
@@ -41,11 +41,11 @@ public class IndexSummary
     private int keysWritten = 0;
     private long lastIndexPosition;
 
-    public void maybeAddEntry(DecoratedKey decoratedKey, long dataPosition, long dataSize,
long indexPosition, long nextIndexPosition)
+    public void maybeAddEntry(DecoratedKey decoratedKey, long dataPosition, long rowSize,
long indexPosition, long nextIndexPosition)
     {
         boolean spannedIndexEntry = DatabaseDescriptor.getIndexAccessMode() == Config.DiskAccessMode.mmap
                                     && RowIndexedReader.bufferIndex(indexPosition)
!= RowIndexedReader.bufferIndex(nextIndexPosition);
-        if (keysWritten++ % INDEX_INTERVAL == 0 || spannedIndexEntry)
+        if ((keysWritten++ % INDEX_INTERVAL == 0) || spannedIndexEntry)
         {
             if (indexPositions == null)
             {
@@ -61,13 +61,18 @@ public class IndexSummary
                     spannedIndexDataPositions = new HashMap<KeyPosition, SSTable.PositionSize>();
                     spannedIndexPositions = new HashMap<Long, KeyPosition>();
                 }
-                spannedIndexDataPositions.put(info, new SSTable.PositionSize(dataPosition,
dataSize));
+                spannedIndexDataPositions.put(info, new SSTable.PositionSize(dataPosition,
rowSize));
                 spannedIndexPositions.put(info.indexPosition, info);
             }
         }
         lastIndexPosition = indexPosition;
     }
 
+    public Map<KeyPosition, SSTable.PositionSize> getSpannedIndexDataPositions()
+    {
+        return spannedIndexDataPositions;
+    }
+
     public List<KeyPosition> getIndexPositions()
     {
         return indexPositions;

Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java?rev=942768&r1=942767&r2=942768&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java Mon May 10
14:59:13 2010
@@ -59,7 +59,6 @@ public class SSTableWriter extends SSTab
 {
     private static Logger logger = LoggerFactory.getLogger(SSTableWriter.class);
 
-    private long keysWritten;
     private BufferedRandomAccessFile dataFile;
     private BufferedRandomAccessFile indexFile;
     private DecoratedKey lastWrittenKey;
@@ -90,7 +89,7 @@ public class SSTableWriter extends SSTab
         return (lastWrittenKey == null) ? 0 : dataFile.getFilePointer();
     }
 
-    private void afterAppend(DecoratedKey decoratedKey, long dataPosition, int dataSize)
throws IOException
+    private void afterAppend(DecoratedKey decoratedKey, long dataPosition) throws IOException
     {
         byte[] diskKey = partitioner.convertToDiskFormat(decoratedKey);
         bf.add(diskKey);
@@ -103,7 +102,8 @@ public class SSTableWriter extends SSTab
         if (logger.isTraceEnabled())
             logger.trace("wrote index of " + decoratedKey + " at " + indexPosition);
 
-        indexSummary.maybeAddEntry(decoratedKey, dataPosition, dataSize, indexPosition, indexFile.getFilePointer());
+        int rowSize = (int)(dataFile.getFilePointer() - dataPosition);
+        indexSummary.maybeAddEntry(decoratedKey, dataPosition, rowSize, indexPosition, indexFile.getFilePointer());
     }
 
     // TODO make this take a DataOutputStream and wrap the byte[] version to combine them
@@ -115,7 +115,7 @@ public class SSTableWriter extends SSTab
         assert length > 0;
         dataFile.writeInt(length);
         dataFile.write(buffer.getData(), 0, length);
-        afterAppend(decoratedKey, currentPosition, length);
+        afterAppend(decoratedKey, currentPosition);
     }
 
     public void append(DecoratedKey decoratedKey, byte[] value) throws IOException
@@ -125,7 +125,7 @@ public class SSTableWriter extends SSTab
         assert value.length > 0;
         dataFile.writeInt(value.length);
         dataFile.write(value);
-        afterAppend(decoratedKey, currentPosition, value.length);
+        afterAppend(decoratedKey, currentPosition);
     }
 
     public SSTableReader closeAndOpenReader() throws IOException

Modified: cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java?rev=942768&r1=942767&r2=942768&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java (original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java Mon May
10 14:59:13 2010
@@ -2,6 +2,7 @@ package org.apache.cassandra.io.sstable;
 
 import java.io.IOException;
 import java.util.concurrent.ExecutionException;
+import java.util.Map;
 
 import org.junit.Test;
 
@@ -9,22 +10,26 @@ import org.apache.cassandra.CleanupHelpe
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.*;
 import org.apache.cassandra.db.filter.QueryPath;
+import org.apache.cassandra.io.util.BufferedRandomAccessFile;
 import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.FBUtilities;
 
 import org.apache.cassandra.Util;
 
+import static org.junit.Assert.assertEquals;
+
 public class SSTableReaderTest extends CleanupHelper
 {
     @Test
     public void testSpannedIndexPositions() throws IOException, ExecutionException, InterruptedException
     {
-        RowIndexedReader.BUFFER_SIZE = 40;
+        RowIndexedReader.BUFFER_SIZE = 40; // each index entry is ~11 bytes, so this will
generate lots of spanned entries
 
         Table table = Table.open("Keyspace1");
         ColumnFamilyStore store = table.getColumnFamilyStore("Standard1");
 
-        // insert a bunch of data
+        // insert a bunch of data and compact to a single sstable
         CompactionManager.instance.disableAutoCompaction();
         for (int j = 0; j < 100; j += 2)
         {
@@ -52,5 +57,22 @@ public class SSTableReaderTest extends C
             DecoratedKey dk = Util.dk(String.valueOf(j));
             assert sstable.getPosition(dk) == null;
         }
+
+        // check positionsize information
+        assert sstable.indexSummary.getSpannedIndexDataPositions().entrySet().size() >
0;
+        for (Map.Entry<IndexSummary.KeyPosition, SSTable.PositionSize> entry : sstable.indexSummary.getSpannedIndexDataPositions().entrySet())
+        {
+            IndexSummary.KeyPosition kp = entry.getKey();
+            SSTable.PositionSize info = entry.getValue();
+
+            long nextIndexPosition = kp.indexPosition + 2 + StorageService.getPartitioner().convertToDiskFormat(kp.key).length
+ 8;
+            BufferedRandomAccessFile indexFile = new BufferedRandomAccessFile(sstable.indexFilename(),
"r");
+            indexFile.seek(nextIndexPosition);
+            String nextKey = indexFile.readUTF();
+
+            BufferedRandomAccessFile file = new BufferedRandomAccessFile(sstable.getFilename(),
"r");
+            file.seek(info.position + info.size);
+            assertEquals(nextKey, file.readUTF());
+        }
     }
 }



Mime
View raw message