cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bened...@apache.org
Subject [12/12] cassandra git commit: Merge branch 'cassandra-2.2' into trunk
Date Tue, 07 Jul 2015 15:46:17 GMT
Merge branch 'cassandra-2.2' into trunk

Conflicts:
	CHANGES.txt
	src/java/org/apache/cassandra/db/compaction/Scrubber.java
	src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/a8bb75a7
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/a8bb75a7
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/a8bb75a7

Branch: refs/heads/trunk
Commit: a8bb75a7e1a09ca05ceb8d566f2c9a88d3122c27
Parents: db68d1c ebe18bb
Author: Benedict Elliott Smith <benedict@apache.org>
Authored: Tue Jul 7 16:45:02 2015 +0100
Committer: Benedict Elliott Smith <benedict@apache.org>
Committed: Tue Jul 7 16:45:02 2015 +0100

----------------------------------------------------------------------
 CHANGES.txt                                     |  3 ++
 .../db/compaction/CompactionManager.java        | 15 ++++--
 .../cassandra/db/compaction/Scrubber.java       | 42 ++++++++++++-----
 .../io/sstable/format/SSTableReader.java        | 48 ++++++++++++++------
 .../io/sstable/format/big/BigTableReader.java   |  3 ++
 .../cassandra/tools/StandaloneScrubber.java     |  7 +--
 .../unit/org/apache/cassandra/db/ScrubTest.java | 22 +++++++++
 7 files changed, 106 insertions(+), 34 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/a8bb75a7/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 53beb26,a863ad8..9dee57d
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,19 -1,7 +1,22 @@@
 -2.2.0-rc3
 +3.0
 + * Storage engine refactor (CASSANDRA-8099, 9743)
 + * Update Guava to 18.0 (CASSANDRA-9653)
 + * Bloom filter false positive ratio is not honoured (CASSANDRA-8413)
 + * New option for cassandra-stress to leave a ratio of columns null (CASSANDRA-9522)
 + * Change hinted_handoff_enabled yaml setting, JMX (CASSANDRA-9035)
 + * Add algorithmic token allocation (CASSANDRA-7032)
 + * Add nodetool command to replay batchlog (CASSANDRA-9547)
 + * Make file buffer cache independent of paths being read (CASSANDRA-8897)
 + * Remove deprecated legacy Hadoop code (CASSANDRA-9353)
 + * Decommissioned nodes will not rejoin the cluster (CASSANDRA-8801)
 + * Change gossip stabilization to use endpoit size (CASSANDRA-9401)
 + * Change default garbage collector to G1 (CASSANDRA-7486)
 + * Populate TokenMetadata early during startup (CASSANDRA-9317)
 + * undeprecate cache recentHitRate (CASSANDRA-6591)
 + * Add support for selectively varint encoding fields (CASSANDRA-9499)
+ Merged from 2.0:
+  * Scrub (recover) sstables even when -Index.db is missing (CASSANDRA-9591)
+ 
  
  2.2.0-rc2
   * Re-enable memory-mapped I/O on Windows (CASSANDRA-9658)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/a8bb75a7/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index a6c3d8c,4c94fa0..e3e9b03
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@@ -331,7 -332,7 +331,14 @@@ public class CompactionManager implemen
          }
      }
  
--    public AllSSTableOpStatus performScrub(final ColumnFamilyStore cfs, final boolean skipCorrupted,
final boolean checkData) throws InterruptedException, ExecutionException
++    public AllSSTableOpStatus performScrub(final ColumnFamilyStore cfs, final boolean skipCorrupted,
final boolean checkData)
++    throws InterruptedException, ExecutionException
++    {
++        return performScrub(cfs, skipCorrupted, checkData, false);
++    }
++
++    public AllSSTableOpStatus performScrub(final ColumnFamilyStore cfs, final boolean skipCorrupted,
final boolean checkData, final boolean offline)
++    throws InterruptedException, ExecutionException
      {
          return parallelAllSSTableOperation(cfs, new OneSSTableOperation()
          {
@@@ -344,7 -345,7 +351,7 @@@
              @Override
              public void execute(LifecycleTransaction input) throws IOException
              {
--                scrubOne(cfs, input, skipCorrupted, checkData);
++                scrubOne(cfs, input, skipCorrupted, checkData, offline);
              }
          }, OperationType.SCRUB);
      }
@@@ -691,11 -691,11 +698,11 @@@
          }
      }
  
--    private void scrubOne(ColumnFamilyStore cfs, LifecycleTransaction modifier, boolean
skipCorrupted, boolean checkData) throws IOException
++    private void scrubOne(ColumnFamilyStore cfs, LifecycleTransaction modifier, boolean
skipCorrupted, boolean checkData, boolean offline) throws IOException
      {
          CompactionInfo.Holder scrubInfo = null;
  
--        try (Scrubber scrubber = new Scrubber(cfs, modifier, skipCorrupted, false, checkData))
++        try (Scrubber scrubber = new Scrubber(cfs, modifier, skipCorrupted, offline, checkData))
          {
              scrubInfo = scrubber.getScrubInfo();
              metrics.beginCompaction(scrubInfo);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/a8bb75a7/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/Scrubber.java
index 562d681,5a0b354..94f3af7
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@@ -108,10 -104,23 +108,19 @@@ public class Scrubber implements Closea
          if (destination == null)
              throw new IOException("disk full");
  
 -        // If we run scrub offline, we should never purge tombstone, as we cannot know if
other sstable have data that the tombstone deletes.
 -        this.controller = isOffline
 -                        ? new ScrubController(cfs)
 -                        : new CompactionController(cfs, Collections.singleton(sstable),
CompactionManager.getDefaultGcBefore(cfs));
          this.isCommutative = cfs.metadata.isCounter();
+ 
+         boolean hasIndexFile = (new File(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX))).exists();
          this.isIndex = cfs.isIndex();
+         if (!hasIndexFile)
+         {
+             // if there's any corruption in the -Data.db then rows can't be skipped over.
but it's worth a shot.
+             outputHandler.warn("Missing component: " + sstable.descriptor.filenameFor(Component.PRIMARY_INDEX));
+         }
          this.checkData = checkData && !this.isIndex; //LocalByPartitionerType does
not support validation
-         this.expectedBloomFilterSize = Math.max(cfs.metadata.getMinIndexInterval(), (int)(SSTableReader.getApproximateKeyCount(toScrub)));
+         this.expectedBloomFilterSize = Math.max(
+             cfs.metadata.getMinIndexInterval(),
+             hasIndexFile ? (int)(SSTableReader.getApproximateKeyCount(toScrub)) : 0);
  
          // loop through each row, deserializing to check for damage.
          // we'll also loop through the index at the same time, using the position from the
index to recover if the
@@@ -135,13 -143,13 +148,14 @@@
      public void scrub()
      {
          outputHandler.output(String.format("Scrubbing %s (%s bytes)", sstable, dataFile.length()));
 +        int nowInSec = FBUtilities.nowInSeconds();
-         try (SSTableRewriter writer = new SSTableRewriter(cfs, transaction, sstable.maxDataAge,
isOffline);)
+         try (SSTableRewriter writer = new SSTableRewriter(cfs, transaction, sstable.maxDataAge,
isOffline))
          {
-             nextIndexKey = ByteBufferUtil.readWithShortLength(indexFile);
+             nextIndexKey = indexAvailable() ? ByteBufferUtil.readWithShortLength(indexFile)
: null;
+             if (indexAvailable())
              {
                  // throw away variable so we don't have a side effect in the assert
 -                long firstRowPositionFromIndex = rowIndexEntrySerializer.deserialize(indexFile,
sstable.descriptor.version).position;
 +                long firstRowPositionFromIndex = rowIndexEntrySerializer.deserialize(indexFile).position;
                  assert firstRowPositionFromIndex == 0 : firstRowPositionFromIndex;
              }
  
@@@ -194,29 -202,29 +208,29 @@@
                      if (currentIndexKey != null && !key.getKey().equals(currentIndexKey))
                      {
                          throw new IOError(new IOException(String.format("Key from data file
(%s) does not match key from index file (%s)",
 -                                ByteBufferUtil.bytesToHex(key.getKey()), ByteBufferUtil.bytesToHex(currentIndexKey))));
 +                                //ByteBufferUtil.bytesToHex(key.getKey()), ByteBufferUtil.bytesToHex(currentIndexKey))));
 +                                "_too big_", ByteBufferUtil.bytesToHex(currentIndexKey))));
                      }
  
-                     if (dataSizeFromIndex > dataFile.length())
+                     if (indexFile != null && dataSizeFromIndex > dataFile.length())
                          throw new IOError(new IOException("Impossible row size (greater
than file length): " + dataSizeFromIndex));
  
-                     if (dataStart != dataStartFromIndex)
+                     if (indexFile != null && dataStart != dataStartFromIndex)
                          outputHandler.warn(String.format("Data file row position %d differs
from index file row position %d", dataStart, dataStartFromIndex));
  
 -                    SSTableIdentityIterator atoms = new SSTableIdentityIterator(sstable,
dataFile, key, checkData);
 -
 -                    if (prevKey != null && prevKey.compareTo(key) > 0)
 +                    try (UnfilteredRowIterator iterator = withValidation(new SSTableIdentityIterator(sstable,
dataFile, key), dataFile.getPath()))
                      {
 -                        saveOutOfOrderRow(prevKey, key, atoms);
 -                        continue;
 -                    }
 +                        if (prevKey != null && prevKey.compareTo(key) > 0)
 +                        {
 +                            saveOutOfOrderRow(prevKey, key, iterator);
 +                            continue;
 +                        }
  
 -                    @SuppressWarnings("resource")
 -                    AbstractCompactedRow compactedRow = new LazilyCompactedRow(controller,
Collections.singletonList(atoms));
 -                    if (writer.tryAppend(compactedRow) == null)
 -                        emptyRows++;
 -                    else
 -                        goodRows++;
 +                        if (writer.tryAppend(iterator) == null)
 +                            emptyRows++;
 +                        else
 +                            goodRows++;
 +                    }
  
                      prevKey = key;
                  }
@@@ -318,10 -330,11 +332,11 @@@
          currentRowPositionFromIndex = nextRowPositionFromIndex;
          try
          {
-             nextIndexKey = indexFile.isEOF() ? null : ByteBufferUtil.readWithShortLength(indexFile);
-             nextRowPositionFromIndex = indexFile.isEOF()
-                                      ? dataFile.length()
-                                      : rowIndexEntrySerializer.deserialize(indexFile).position;
+             nextIndexKey = !indexAvailable() ? null : ByteBufferUtil.readWithShortLength(indexFile);
+ 
+             nextRowPositionFromIndex = !indexAvailable()
+                     ? dataFile.length()
 -                    : rowIndexEntrySerializer.deserialize(indexFile, sstable.descriptor.version).position;
++                    : rowIndexEntrySerializer.deserialize(indexFile).position;
          }
          catch (Throwable th)
          {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/a8bb75a7/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
index ab357a5,247d181..c326881
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
@@@ -430,17 -421,12 +430,17 @@@ public abstract class SSTableReader ext
      {
          // Minimum components without which we can't do anything
          assert components.contains(Component.DATA) : "Data component is missing for sstable
" + descriptor;
-         assert components.contains(Component.PRIMARY_INDEX) : "Primary index component is
missing for sstable " + descriptor;
+         assert !validate || components.contains(Component.PRIMARY_INDEX) : "Primary index
component is missing for sstable " + descriptor;
  
 -        Map<MetadataType, MetadataComponent> sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor,
 -                                                                                       
                       EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS));
 +        // For the 3.0+ sstable format, the (misnomed) stats component hold the serialization
header which we need to deserialize the sstable content
 +        assert !descriptor.version.storeRows() || components.contains(Component.STATS) :
"Stats component is missing for sstable " + descriptor;
 +
 +        EnumSet<MetadataType> types = EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS,
MetadataType.HEADER);
 +        Map<MetadataType, MetadataComponent> sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor,
types);
          ValidationMetadata validationMetadata = (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION);
          StatsMetadata statsMetadata = (StatsMetadata) sstableMetadata.get(MetadataType.STATS);
 +        SerializationHeader.Component header = (SerializationHeader.Component) sstableMetadata.get(MetadataType.HEADER);
 +        assert !descriptor.version.storeRows() || header != null;
  
          // Check if sstable is created using same partitioner.
          // Partitioner can be null, which indicates older version of sstable or no stats
available.

http://git-wip-us.apache.org/repos/asf/cassandra/blob/a8bb75a7/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java
index 7a7b913,3f375e7..b539c79
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java
@@@ -168,6 -167,6 +168,9 @@@ public class BigTableReader extends SST
  
          int effectiveInterval = indexSummary.getEffectiveIndexIntervalAfterIndex(sampledIndex);
  
++        if (ifile == null)
++            return null;
++
          // scan the on-disk index, starting at the nearest sampled position.
          // The check against IndexInterval is to be exit the loop in the EQ case when the
key looked for is not present
          // (bloom filter false positive). But note that for non-EQ cases, we might need
to check the first key of the

http://git-wip-us.apache.org/repos/asf/cassandra/blob/a8bb75a7/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/tools/StandaloneScrubber.java
index c71caab,8982e2f..be73cf7
--- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
@@@ -119,6 -119,6 +119,7 @@@ public class StandaloneScrubbe
                  {
                      try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB,
sstable))
                      {
++                        txn.obsoleteOriginals(); // make sure originals are deleted and
avoid NPE if index is missing, CASSANDRA-9591
                          try (Scrubber scrubber = new Scrubber(cfs, txn, options.skipCorrupted,
handler, true, !options.noValidate))
                          {
                              scrubber.scrub();
@@@ -131,10 -131,10 +132,6 @@@
                                  throw t;
                              }
                          }
--
--                        // Remove the sstable (it's been copied by scrub and snapshotted)
--                        sstable.markObsolete(null);
--                        sstable.selfRef().release();
                      }
                      catch (Exception e)
                      {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/a8bb75a7/test/unit/org/apache/cassandra/db/ScrubTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/ScrubTest.java
index c728f0d,b233e1f..2574847
--- a/test/unit/org/apache/cassandra/db/ScrubTest.java
+++ b/test/unit/org/apache/cassandra/db/ScrubTest.java
@@@ -29,15 -31,15 +29,16 @@@ import java.io.IOError
  import java.io.IOException;
  import java.io.RandomAccessFile;
  
 -import org.apache.cassandra.config.KSMetaData;
 +import org.apache.cassandra.Util;
 +import org.apache.cassandra.UpdateBuilder;
  import org.apache.cassandra.cql3.QueryProcessor;
 -import org.apache.cassandra.db.composites.CellNameType;
 +import org.apache.cassandra.db.partitions.*;
  import org.apache.cassandra.db.marshal.BytesType;
 -import org.apache.cassandra.db.marshal.CounterColumnType;
  import org.apache.cassandra.db.marshal.UUIDType;
 -import org.apache.cassandra.exceptions.ConfigurationException;
  import org.apache.cassandra.db.compaction.OperationType;
  import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
++import org.apache.cassandra.db.rows.Row;
 +import org.apache.cassandra.exceptions.ConfigurationException;
  import org.apache.cassandra.exceptions.RequestExecutionException;
  import org.apache.cassandra.io.compress.CompressionMetadata;
  import org.apache.cassandra.io.sstable.format.SSTableFormat;
@@@ -280,6 -328,31 +281,27 @@@ public class ScrubTes
      }
  
      @Test
+     public void testScrubNoIndex() throws IOException, ExecutionException, InterruptedException,
ConfigurationException
+     {
+         CompactionManager.instance.disableAutoCompaction();
+         Keyspace keyspace = Keyspace.open(KEYSPACE);
+         ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+         cfs.clearUnsafe();
+ 
 -        List<Row> rows;
 -
+         // insert data and verify we get it back w/ range query
+         fillCF(cfs, 10);
 -        rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
 -        assertEquals(10, rows.size());
++        assertOrderedAll(cfs, 10);
+ 
+         for (SSTableReader sstable : cfs.getSSTables())
+             new File(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX)).delete();
+ 
 -        CompactionManager.instance.performScrub(cfs, false, true);
++        CompactionManager.instance.performScrub(cfs, false, true, true);
+ 
+         // check data is still there
 -        rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
 -        assertEquals(10, rows.size());
++        assertOrderedAll(cfs, 10);
+     }
+ 
+     @Test
      public void testScrubOutOfOrder() throws Exception
      {
          CompactionManager.instance.disableAutoCompaction();


Mime
View raw message