cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From marc...@apache.org
Subject [6/6] cassandra git commit: Merge branch 'cassandra-3.11' into trunk
Date Mon, 28 Aug 2017 13:42:46 GMT
Merge branch 'cassandra-3.11' into trunk


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/326f3a7c
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/326f3a7c
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/326f3a7c

Branch: refs/heads/trunk
Commit: 326f3a7c7d8f20d0389d9cc036b0bb32d37462be
Parents: 68c2d2e 809f3b3
Author: Marcus Eriksson <marcuse@apache.org>
Authored: Mon Aug 28 15:40:11 2017 +0200
Committer: Marcus Eriksson <marcuse@apache.org>
Committed: Mon Aug 28 15:41:04 2017 +0200

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../apache/cassandra/db/ColumnFamilyStore.java  | 19 ++++--
 .../cassandra/io/sstable/SSTableLoader.java     |  4 +-
 .../io/sstable/format/SSTableReader.java        | 66 +++++++++++++-------
 .../unit/org/apache/cassandra/db/ScrubTest.java |  2 +-
 5 files changed, 62 insertions(+), 30 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/326f3a7c/CHANGES.txt
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/326f3a7c/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 73e7db6,23d0b8e..5aecc9d
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@@ -62,8 -60,10 +62,11 @@@ import org.apache.cassandra.exceptions.
  import org.apache.cassandra.index.SecondaryIndexManager;
  import org.apache.cassandra.index.internal.CassandraIndex;
  import org.apache.cassandra.index.transactions.UpdateTransaction;
+ import org.apache.cassandra.io.FSError;
++import org.apache.cassandra.io.FSReadError;
  import org.apache.cassandra.io.FSWriteError;
  import org.apache.cassandra.io.sstable.Component;
+ import org.apache.cassandra.io.sstable.CorruptSSTableException;
  import org.apache.cassandra.io.sstable.Descriptor;
  import org.apache.cassandra.io.sstable.SSTableMultiWriter;
  import org.apache.cassandra.io.sstable.format.*;
@@@ -781,11 -767,24 +785,18 @@@ public class ColumnFamilyStore implemen
              {
                  reader = SSTableReader.open(newDescriptor, entry.getValue(), metadata);
              }
-             catch (IOException e)
+             catch (CorruptSSTableException ex)
+             {
+                 FileUtils.handleCorruptSSTable(ex);
+                 logger.error("Corrupt sstable {}; skipping table", entry, ex);
+                 continue;
+             }
+             catch (FSError ex)
              {
-                 SSTableReader.logOpenException(entry.getKey(), e);
+                 FileUtils.handleFSError(ex);
+                 logger.error("Cannot read sstable {}; file system error, skipping table",
entry, ex);
                  continue;
              }
 -            catch (IOException ex)
 -            {
 -                FileUtils.handleCorruptSSTable(new CorruptSSTableException(ex, entry.getKey().filenameFor(Component.DATA)));
 -                logger.error("Cannot read sstable {}; other IO error, skipping table", entry,
ex);
 -                continue;
 -            }
              newSSTables.add(reader);
          }
  
@@@ -1912,7 -1906,7 +1923,7 @@@
                  }
              }
          }
--        catch (IOException | RuntimeException e)
++        catch (FSReadError | RuntimeException e)
          {
              // In case one of the snapshot sstables fails to open,
              // we must release the references to the ones we opened so far

http://git-wip-us.apache.org/repos/asf/cassandra/blob/326f3a7c/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/SSTableLoader.java
index dc56520,043f6fa..69e5c85
--- a/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java
@@@ -25,7 -25,7 +25,8 @@@ import java.util.*
  import com.google.common.collect.HashMultimap;
  import com.google.common.collect.Multimap;
  
 -import org.apache.cassandra.config.CFMetaData;
++import org.apache.cassandra.io.FSError;
 +import org.apache.cassandra.schema.TableMetadataRef;
  import org.apache.cassandra.db.Directories;
  import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
  import org.apache.cassandra.dht.Range;
@@@ -138,8 -138,8 +139,9 @@@ public class SSTableLoader implements S
                                                // to conserve heap space when bulk loading
                                                sstable.releaseSummary();
                                            }
--                                          catch (IOException e)
++                                          catch (FSError e)
                                            {
++                                              // todo: should we really continue if we can't
open all sstables?
                                                outputHandler.output(String.format("Skipping
file %s, error opening it: %s", name, e.getMessage()));
                                            }
                                            return false;

http://git-wip-us.apache.org/repos/asf/cassandra/blob/326f3a7c/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
index bb1eb0d,03af2bb..2801e36
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
@@@ -344,9 -352,9 +344,9 @@@ public abstract class SSTableReader ext
          return base;
      }
  
--    public static SSTableReader open(Descriptor descriptor) throws IOException
++    public static SSTableReader open(Descriptor descriptor)
      {
 -        CFMetaData metadata;
 +        TableMetadataRef metadata;
          if (descriptor.cfname.contains(SECONDARY_INDEX_NAME_SEPARATOR))
          {
              int i = descriptor.cfname.indexOf(SECONDARY_INDEX_NAME_SEPARATOR);
@@@ -362,24 -374,24 +362,24 @@@
          return open(descriptor, metadata);
      }
  
-     public static SSTableReader open(Descriptor desc, TableMetadataRef metadata) throws
IOException
 -    public static SSTableReader open(Descriptor desc, CFMetaData metadata) throws IOException
++    public static SSTableReader open(Descriptor desc, TableMetadataRef metadata)
      {
          return open(desc, componentsFor(desc), metadata);
      }
  
-     public static SSTableReader open(Descriptor descriptor, Set<Component> components,
TableMetadataRef metadata) throws IOException
 -    public static SSTableReader open(Descriptor descriptor, Set<Component> components,
CFMetaData metadata) throws IOException
++    public static SSTableReader open(Descriptor descriptor, Set<Component> components,
TableMetadataRef metadata)
      {
          return open(descriptor, components, metadata, true, true);
      }
  
      // use only for offline or "Standalone" operations
--    public static SSTableReader openNoValidation(Descriptor descriptor, Set<Component>
components, ColumnFamilyStore cfs) throws IOException
++    public static SSTableReader openNoValidation(Descriptor descriptor, Set<Component>
components, ColumnFamilyStore cfs)
      {
          return open(descriptor, components, cfs.metadata, false, false); // do not track
hotness
      }
  
      // use only for offline or "Standalone" operations
-     public static SSTableReader openNoValidation(Descriptor descriptor, TableMetadataRef
metadata) throws IOException
 -    public static SSTableReader openNoValidation(Descriptor descriptor, CFMetaData metadata)
throws IOException
++    public static SSTableReader openNoValidation(Descriptor descriptor, TableMetadataRef
metadata)
      {
          return open(descriptor, componentsFor(descriptor), metadata, false, false); // do
not track hotness
      }
@@@ -393,14 -405,14 +393,22 @@@
       * @return opened SSTableReader
       * @throws IOException
       */
-     public static SSTableReader openForBatch(Descriptor descriptor, Set<Component>
components, TableMetadataRef metadata) throws IOException
 -    public static SSTableReader openForBatch(Descriptor descriptor, Set<Component>
components, CFMetaData metadata) throws IOException
++    public static SSTableReader openForBatch(Descriptor descriptor, Set<Component>
components, TableMetadataRef metadata)
      {
          // Minimum components without which we can't do anything
          assert components.contains(Component.DATA) : "Data component is missing for sstable
" + descriptor;
          assert components.contains(Component.PRIMARY_INDEX) : "Primary index component is
missing for sstable " + descriptor;
  
          EnumSet<MetadataType> types = EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS,
MetadataType.HEADER);
--        Map<MetadataType, MetadataComponent> sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor,
types);
++        Map<MetadataType, MetadataComponent> sstableMetadata;
++        try
++        {
++             sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor,
types);
++        }
++        catch (IOException e)
++        {
++            throw new CorruptSSTableException(e, descriptor.filenameFor(Component.STATS));
++        }
  
          ValidationMetadata validationMetadata = (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION);
          StatsMetadata statsMetadata = (StatsMetadata) sstableMetadata.get(MetadataType.STATS);
@@@ -435,7 -447,7 +443,16 @@@
                                                       .withChunkCache(ChunkCache.instance))
          {
              if (!sstable.loadSummary())
--                sstable.buildSummary(false, false, Downsampling.BASE_SAMPLING_LEVEL);
++            {
++                try
++                {
++                    sstable.buildSummary(false, false, Downsampling.BASE_SAMPLING_LEVEL);
++                }
++                catch(IOException e)
++                {
++                    throw new CorruptSSTableException(e, sstable.getFilename());
++                }
++            }
              long indexFileLength = new File(descriptor.filenameFor(Component.PRIMARY_INDEX)).length();
              int dataBufferSize = sstable.optimizationStrategy.bufferSize(statsMetadata.estimatedPartitionSize.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile()));
              int indexBufferSize = sstable.optimizationStrategy.bufferSize(indexFileLength
/ sstable.indexSummary.size());
@@@ -448,20 -460,29 +465,29 @@@
      }
  
      public static SSTableReader open(Descriptor descriptor,
 -                                      Set<Component> components,
 -                                      CFMetaData metadata,
 -                                      boolean validate,
 -                                      boolean trackHotness) throws IOException
 +                                     Set<Component> components,
 +                                     TableMetadataRef metadata,
 +                                     boolean validate,
-                                      boolean trackHotness) throws IOException
++                                     boolean trackHotness)
      {
          // Minimum components without which we can't do anything
          assert components.contains(Component.DATA) : "Data component is missing for sstable
" + descriptor;
          assert !validate || components.contains(Component.PRIMARY_INDEX) : "Primary index
component is missing for sstable " + descriptor;
  
          // For the 3.0+ sstable format, the (misnomed) stats component hold the serialization
header which we need to deserialize the sstable content
 -        assert !descriptor.version.storeRows() || components.contains(Component.STATS) :
"Stats component is missing for sstable " + descriptor;
 +        assert components.contains(Component.STATS) : "Stats component is missing for sstable
" + descriptor;
  
          EnumSet<MetadataType> types = EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS,
MetadataType.HEADER);
-         Map<MetadataType, MetadataComponent> sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor,
types);
+ 
+         Map<MetadataType, MetadataComponent> sstableMetadata;
+         try
+         {
+             sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor,
types);
+         }
+         catch (IOException e)
+         {
+             throw new CorruptSSTableException(e, descriptor.filenameFor(Component.STATS));
+         }
          ValidationMetadata validationMetadata = (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION);
          StatsMetadata statsMetadata = (StatsMetadata) sstableMetadata.get(MetadataType.STATS);
          SerializationHeader.Component header = (SerializationHeader.Component) sstableMetadata.get(MetadataType.HEADER);
@@@ -511,16 -537,16 +542,8 @@@
          }
      }
  
--    public static void logOpenException(Descriptor descriptor, IOException e)
--    {
--        if (e instanceof FileNotFoundException)
--            logger.error("Missing sstable component in {}; skipped because of {}", descriptor,
e.getMessage());
--        else
--            logger.error("Corrupt sstable {}; skipped", descriptor, e);
--    }
--
      public static Collection<SSTableReader> openAll(Set<Map.Entry<Descriptor,
Set<Component>>> entries,
 -                                                    final CFMetaData metadata)
 +                                                    final TableMetadataRef metadata)
      {
          final Collection<SSTableReader> sstables = new LinkedBlockingQueue<>();
  
@@@ -548,11 -574,12 +571,6 @@@
                          logger.error("Cannot read sstable {}; file system error, skipping
table", entry, ex);
                          return;
                      }
--                    catch (IOException ex)
--                    {
 -                        FileUtils.handleCorruptSSTable(new CorruptSSTableException(ex, entry.getKey().filenameFor(Component.DATA)));
--                        logger.error("Cannot read sstable {}; other IO error, skipping table",
entry, ex);
--                        return;
--                    }
                      sstables.add(sstable);
                  }
              };

http://git-wip-us.apache.org/repos/asf/cassandra/blob/326f3a7c/test/unit/org/apache/cassandra/db/ScrubTest.java
----------------------------------------------------------------------


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org


Mime
View raw message