cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s...@apache.org
Subject [2/6] cassandra git commit: Fix current position calculation in OldFormatDeserializer
Date Wed, 24 May 2017 18:13:47 GMT
Fix current position calculation in OldFormatDeserializer

Patch by Sam Tunnicliffe; reviewed by Jeff Jirsa for CASSANDRA-13525

This patch also adds a unit test for CASSANDRA-13236.


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/62092e45
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/62092e45
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/62092e45

Branch: refs/heads/cassandra-3.11
Commit: 62092e45c8bbb75ac9f680188b3746913602507b
Parents: 3872580
Author: Sam Tunnicliffe <sam@beobal.com>
Authored: Wed May 10 18:10:51 2017 +0100
Committer: Sam Tunnicliffe <sam@beobal.com>
Committed: Wed May 24 19:01:25 2017 +0100

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 .../cassandra/db/UnfilteredDeserializer.java    |  28 +++++++++++--
 ...es-legacy_ka_indexed-ka-1-CompressionInfo.db | Bin 0 -> 91 bytes
 ...legacy_tables-legacy_ka_indexed-ka-1-Data.db | Bin 0 -> 196203 bytes
 ...cy_tables-legacy_ka_indexed-ka-1-Digest.sha1 |   1 +
 ...gacy_tables-legacy_ka_indexed-ka-1-Filter.db | Bin 0 -> 16 bytes
 ...egacy_tables-legacy_ka_indexed-ka-1-Index.db | Bin 0 -> 332 bytes
 ..._tables-legacy_ka_indexed-ka-1-Statistics.db | Bin 0 -> 4450 bytes
 ...acy_tables-legacy_ka_indexed-ka-1-Summary.db | Bin 0 -> 92 bytes
 ...legacy_tables-legacy_ka_indexed-ka-1-TOC.txt |   8 ++++
 ...cy_ka_indexed_static-ka-1-CompressionInfo.db | Bin 0 -> 91 bytes
 ...tables-legacy_ka_indexed_static-ka-1-Data.db | Bin 0 -> 196269 bytes
 ...es-legacy_ka_indexed_static-ka-1-Digest.sha1 |   1 +
 ...bles-legacy_ka_indexed_static-ka-1-Filter.db | Bin 0 -> 16 bytes
 ...ables-legacy_ka_indexed_static-ka-1-Index.db | Bin 0 -> 332 bytes
 ...-legacy_ka_indexed_static-ka-1-Statistics.db | Bin 0 -> 4446 bytes
 ...les-legacy_ka_indexed_static-ka-1-Summary.db | Bin 0 -> 92 bytes
 ...tables-legacy_ka_indexed_static-ka-1-TOC.txt |   8 ++++
 .../cassandra/io/sstable/LegacySSTableTest.java |  39 +++++++++++++++++++
 19 files changed, 83 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 98a1808..fe6ab09 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 3.0.14
+ * ReverseIndexedReader may drop rows during 2.1 to 3.0 upgrade (CASSANDRA-13525)
  * Fix repair process violating start/end token limits for small ranges (CASSANDRA-13052)
  * Add storage port options to sstableloader (CASSANDRA-13518)
  * Properly handle quoted index names in cqlsh DESCRIBE output (CASSANDRA-12847)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java b/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java
index 92690e1..ea65633 100644
--- a/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java
+++ b/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java
@@ -256,6 +256,10 @@ public abstract class UnfilteredDeserializer
         // The position in the input after the last data consumption (readNext/skipNext).
         private long lastConsumedPosition;
 
+        // Tracks the size of the last LegacyAtom read from disk, because this needs to be
accounted
+        // for when marking lastConsumedPosition after readNext/skipNext
+        private long bytesReadForNextAtom;
+
         private OldFormatDeserializer(CFMetaData metadata,
                                       DataInputPlus in,
                                       SerializationHelper helper,
@@ -272,7 +276,10 @@ public abstract class UnfilteredDeserializer
         {
             try
             {
-                return LegacyLayout.readLegacyAtom(metadata, in, readAllAsDynamic);
+                long pos = currentPosition();
+                LegacyLayout.LegacyAtom atom =  LegacyLayout.readLegacyAtom(metadata, in,
readAllAsDynamic);
+                bytesReadForNextAtom = currentPosition() - pos;
+                return atom;
             }
             catch (IOException e)
             {
@@ -359,7 +366,7 @@ public abstract class UnfilteredDeserializer
                 throw new IllegalStateException();
             Unfiltered toReturn = next;
             next = null;
-            lastConsumedPosition = currentPosition();
+            lastConsumedPosition = currentPosition() - bytesReadForNextAtom();
             return toReturn;
         }
 
@@ -368,7 +375,7 @@ public abstract class UnfilteredDeserializer
             if (!hasNext())
                 throw new UnsupportedOperationException();
             next = null;
-            lastConsumedPosition = currentPosition();
+            lastConsumedPosition = currentPosition() - bytesReadForNextAtom();
         }
 
         public long bytesReadForUnconsumedData()
@@ -379,6 +386,21 @@ public abstract class UnfilteredDeserializer
             return currentPosition() - lastConsumedPosition;
         }
 
+        // Reading/skipping an Unfiltered consumes LegacyAtoms from the underlying legacy
atom iterator
+        // e.g. hasNext() -> iterator.hasNext() -> iterator.readRow() -> atoms.next()
+        // The stop condition of the loop which groups legacy atoms into rows causes that
AtomIterator
+        // to read in the first atom which doesn't belong in the row. So by that point, our
position
+        // is actually past the end of the next Unfiltered. To compensate, we record the
size of
+        // the last LegacyAtom read and subtract it from the current position when we calculate
lastConsumedPosition.
+        // If we don't, then when reading an indexed block, we can over correct and may think
that we've
+        // exhausted the block before we actually have.
+        private long bytesReadForNextAtom()
+        {
+            // If we've read anything at all then we will have recorded this in bytesReadForNextAtom,
+            // but being extra careful here just incase this method is called before any
reads happen.
+            return iterator.atoms.next == null ? 0 : bytesReadForNextAtom;
+        }
+
         public void clearState()
         {
             next = null;

http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-CompressionInfo.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-CompressionInfo.db
b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-CompressionInfo.db
new file mode 100644
index 0000000..ecd3ddb
Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-CompressionInfo.db
differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Data.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Data.db
b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Data.db
new file mode 100644
index 0000000..d1e4e2f
Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Data.db
differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Digest.sha1
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Digest.sha1
b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Digest.sha1
new file mode 100644
index 0000000..bce117c
--- /dev/null
+++ b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Digest.sha1
@@ -0,0 +1 @@
+76435450
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Filter.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Filter.db
b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Filter.db
new file mode 100644
index 0000000..00a88b4
Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Filter.db
differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Index.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Index.db
b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Index.db
new file mode 100644
index 0000000..9ba4894
Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Index.db
differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Statistics.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Statistics.db
b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Statistics.db
new file mode 100644
index 0000000..a57d32b
Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Statistics.db
differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Summary.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Summary.db
b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Summary.db
new file mode 100644
index 0000000..d60d8f4
Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Summary.db
differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-TOC.txt
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-TOC.txt
b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-TOC.txt
new file mode 100644
index 0000000..25fc863
--- /dev/null
+++ b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-TOC.txt
@@ -0,0 +1,8 @@
+Data.db
+Summary.db
+TOC.txt
+Statistics.db
+Digest.sha1
+Filter.db
+Index.db
+CompressionInfo.db

http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-CompressionInfo.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-CompressionInfo.db
b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-CompressionInfo.db
new file mode 100644
index 0000000..09c4cfa
Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-CompressionInfo.db
differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Data.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Data.db
b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Data.db
new file mode 100644
index 0000000..40ee3c6
Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Data.db
differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Digest.sha1
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Digest.sha1
b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Digest.sha1
new file mode 100644
index 0000000..55ac08c
--- /dev/null
+++ b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Digest.sha1
@@ -0,0 +1 @@
+3851004816
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Filter.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Filter.db
b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Filter.db
new file mode 100644
index 0000000..00a88b4
Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Filter.db
differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Index.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Index.db
b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Index.db
new file mode 100644
index 0000000..fb6ceed
Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Index.db
differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Statistics.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Statistics.db
b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Statistics.db
new file mode 100644
index 0000000..b08f500
Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Statistics.db
differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Summary.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Summary.db
b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Summary.db
new file mode 100644
index 0000000..d60d8f4
Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Summary.db
differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-TOC.txt
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-TOC.txt
b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-TOC.txt
new file mode 100644
index 0000000..6865eca
--- /dev/null
+++ b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-TOC.txt
@@ -0,0 +1,8 @@
+CompressionInfo.db
+Summary.db
+Data.db
+Index.db
+Statistics.db
+TOC.txt
+Digest.sha1
+Filter.db

http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
index cefee52..f4f6e85 100644
--- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
@@ -146,6 +146,45 @@ public class LegacySSTableTest
             verifyReads(legacyVersion);
         }
     }
+    @Test
+    public void testReverseIterationOfLegacyIndexedSSTable() throws Exception
+    {
+        // During upgrades from 2.1 to 3.0, reverse queries can drop rows before upgradesstables
is completed
+        QueryProcessor.executeInternal("CREATE TABLE legacy_tables.legacy_ka_indexed (" +
+                                       "  p int," +
+                                       "  c int," +
+                                       "  v1 int," +
+                                       "  v2 int," +
+                                       "  PRIMARY KEY(p, c)" +
+                                       ")");
+        loadLegacyTable("legacy_%s_indexed%s", "ka", "");
+        UntypedResultSet rs = QueryProcessor.executeInternal("SELECT * " +
+                                                             "FROM legacy_tables.legacy_ka_indexed
" +
+                                                             "WHERE p=1 " +
+                                                             "ORDER BY c DESC");
+        Assert.assertEquals(5000, rs.size());
+    }
+
+    @Test
+    public void testReadingLegacyIndexedSSTableWithStaticColumns() throws Exception
+    {
+        // During upgrades from 2.1 to 3.0, reading from tables with static columns errors
before upgradesstables
+        // is completed
+        QueryProcessor.executeInternal("CREATE TABLE legacy_tables.legacy_ka_indexed_static
(" +
+                                       "  p int," +
+                                       "  c int," +
+                                       "  v1 int," +
+                                       "  v2 int," +
+                                       "  s1 int static," +
+                                       "  s2 int static," +
+                                       "  PRIMARY KEY(p, c)" +
+                                       ")");
+        loadLegacyTable("legacy_%s_indexed_static%s", "ka", "");
+        UntypedResultSet rs = QueryProcessor.executeInternal("SELECT * " +
+                                                             "FROM legacy_tables.legacy_ka_indexed_static
" +
+                                                             "WHERE p=1 ");
+        Assert.assertEquals(5000, rs.size());
+    }
 
     private void streamLegacyTables(String legacyVersion) throws Exception
     {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org


Mime
View raw message