cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xe...@apache.org
Subject [4/4] git commit: merge with 1.2
Date Thu, 21 Mar 2013 22:17:42 GMT
merge with 1.2


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/480a1a8f
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/480a1a8f
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/480a1a8f

Branch: refs/heads/trunk
Commit: 480a1a8f5acfd6e93c789f43305d9506d41da3e3
Parents: 207cdf3 27ed655
Author: Pavel Yaskevich <xedin@apache.org>
Authored: Thu Mar 21 15:16:39 2013 -0700
Committer: Pavel Yaskevich <xedin@apache.org>
Committed: Thu Mar 21 15:16:39 2013 -0700

----------------------------------------------------------------------
 CHANGES.txt                                        |    3 +
 conf/cassandra-env.sh                              |    1 +
 conf/cassandra.yaml                                |    6 +
 src/java/org/apache/cassandra/config/Config.java   |    3 +
 .../cassandra/config/DatabaseDescriptor.java       |    5 +
 .../cassandra/db/commitlog/CommitLogReplayer.java  |    2 +-
 .../db/compaction/AbstractCompactionStrategy.java  |    2 +-
 .../cassandra/db/compaction/CompactionManager.java |    2 +-
 .../cassandra/db/compaction/CompactionTask.java    |    5 +-
 .../db/compaction/LeveledCompactionStrategy.java   |    6 +-
 .../apache/cassandra/db/compaction/Scrubber.java   |    4 +-
 .../io/compress/CompressedRandomAccessReader.java  |   10 +-
 .../apache/cassandra/io/sstable/KeyIterator.java   |    2 +-
 .../io/sstable/SSTableBoundedScanner.java          |    4 +-
 .../apache/cassandra/io/sstable/SSTableReader.java |  134 +++++++++++++--
 .../cassandra/io/sstable/SSTableScanner.java       |   10 +-
 .../cassandra/io/util/RandomAccessReader.java      |   54 +-----
 .../apache/cassandra/streaming/FileStreamTask.java |    2 +-
 .../compress/CompressedFileStreamTask.java         |    2 +-
 .../org/apache/cassandra/tools/SSTableExport.java  |    4 +-
 src/java/org/apache/cassandra/utils/CLibrary.java  |   21 +++
 .../unit/org/apache/cassandra/db/KeyCacheTest.java |    2 +-
 .../compress/CompressedRandomAccessReaderTest.java |    8 +-
 .../apache/cassandra/io/sstable/SSTableTest.java   |    4 +-
 .../apache/cassandra/io/sstable/SSTableUtils.java  |    4 +-
 .../io/util/BufferedRandomAccessFileTest.java      |    4 +-
 26 files changed, 207 insertions(+), 97 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/480a1a8f/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index fb96cbd,e309eae..fccdc7f
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -28,9 -5,10 +28,11 @@@
   * Improve asynchronous hint delivery (CASSANDRA-5179)
   * Fix Guava dependency version (12.0 -> 13.0.1) for Maven (CASSANDRA-5364)
   * Validate that provided CQL3 collection value are < 64K (CASSANDRA-5355)
+  * Change Kernel Page Cache skipping into row preheating (disabled by default)
+    (CASSANDRA-4937)
  
  
 +
  1.2.3
   * add check for sstable overlap within a level on startup (CASSANDRA-5327)
   * replace ipv6 colons in jmx object names (CASSANDRA-5298, 5328)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/480a1a8f/conf/cassandra-env.sh
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/480a1a8f/conf/cassandra.yaml
----------------------------------------------------------------------
diff --cc conf/cassandra.yaml
index cab2e91,178487d..abd4e9a
--- a/conf/cassandra.yaml
+++ b/conf/cassandra.yaml
@@@ -659,4 -683,10 +659,10 @@@ internode_compression: al
  # Disabling it will result in larger (but fewer) network packets being sent,
  # reducing overhead from the TCP protocol itself, at the cost of increasing
  # latency if you block for cross-datacenter responses.
 -inter_dc_tcp_nodelay: true
 +inter_dc_tcp_nodelay: false
+ 
+ # Enable or disable kernel page cache preheating from contents of the key cache after compaction.
+ # When enabled it would preheat only first "page" (4KB) of each row to optimize
+ # for sequential access. Note: This could be harmful for fat rows, see CASSANDRA-4937
+ # for further details on that topic.
+ preheat_kernel_page_cache: false

http://git-wip-us.apache.org/repos/asf/cassandra/blob/480a1a8f/src/java/org/apache/cassandra/config/Config.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/config/Config.java
index 368c5d6,212147a..8235d70
--- a/src/java/org/apache/cassandra/config/Config.java
+++ b/src/java/org/apache/cassandra/config/Config.java
@@@ -160,12 -163,13 +160,15 @@@ public class Confi
      public volatile int row_cache_save_period = 0;
      public int row_cache_keys_to_save = Integer.MAX_VALUE;
      public String row_cache_provider = SerializingCacheProvider.class.getSimpleName();
 +    public String memory_allocator = NativeAllocator.class.getSimpleName();
      public boolean populate_io_cache_on_flush = false;
  
 -    public boolean inter_dc_tcp_nodelay = true;
 +    public boolean inter_dc_tcp_nodelay = false;
 +
 +    private static boolean isClientMode = false;
+ 
+     public boolean preheat_kernel_page_cache = false;
+ 
 -    private static boolean loadYaml = true;
      private static boolean outboundBindAny = false;
  
      public static boolean getOutboundBindAny()

http://git-wip-us.apache.org/repos/asf/cassandra/blob/480a1a8f/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/480a1a8f/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/480a1a8f/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/480a1a8f/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index ca0e945,0fe3a7a..b2633cb
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@@ -466,9 -568,9 +466,9 @@@ public class CompactionManager implemen
              if (compactionFileLocation == null)
                  throw new IOException("disk full");
  
-             SSTableScanner scanner = sstable.getDirectScanner();
+             SSTableScanner scanner = sstable.getScanner();
              long rowsRead = 0;
 -            List<IColumn> indexedColumnsInRow = null;
 +            List<Column> indexedColumnsInRow = null;
  
              CleanupInfo ci = new CleanupInfo(sstable, scanner);
              metrics.beginCompaction(ci);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/480a1a8f/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/480a1a8f/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
index d081542,d916c48..c777972
--- a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
@@@ -195,39 -206,15 +195,39 @@@ public class LeveledCompactionStrategy 
          public LeveledScanner(Collection<SSTableReader> sstables, Range<Token>
range)
          {
              this.range = range;
 -            this.sstables = new ArrayList<SSTableReader>(sstables);
 +
 +            // add only sstables that intersect our range, and estimate how much data that
involves
 +            this.sstables = new ArrayList<SSTableReader>(sstables.size());
 +            long length = 0;
 +            for (SSTableReader sstable : sstables)
 +            {
 +                this.sstables.add(sstable);
 +                long estimatedKeys = sstable.estimatedKeys();
 +                double estKeysInRangeRatio = 1.0;
 +
 +                if (estimatedKeys > 0 && range != null)
 +                    estKeysInRangeRatio = ((double) sstable.estimatedKeysForRanges(Collections.singleton(range)))
/ estimatedKeys;
 +
 +                length += sstable.uncompressedLength() * estKeysInRangeRatio;
 +            }
 +
 +            totalLength = length;
              Collections.sort(this.sstables, SSTable.sstableComparator);
              sstableIterator = this.sstables.iterator();
 +            assert sstableIterator.hasNext(); // caller should check intersecting first
-             currentScanner = sstableIterator.next().getDirectScanner(range);
+             currentScanner = sstableIterator.next().getScanner(range);
 +        }
  
 -            long length = 0;
 +        public static List<SSTableReader> intersecting(Collection<SSTableReader>
sstables, Range<Token> range)
 +        {
 +            ArrayList<SSTableReader> filtered = new ArrayList<SSTableReader>();
              for (SSTableReader sstable : sstables)
 -                length += sstable.uncompressedLength();
 -            totalLength = length;
 +            {
 +                Range<Token> sstableRange = new Range<Token>(sstable.first.getToken(),
sstable.last.getToken(), sstable.partitioner);
 +                if (range == null || sstableRange.intersects(range))
 +                    filtered.add(sstable);
 +            }
 +            return filtered;
          }
  
          protected OnDiskAtomIterator computeNext()

http://git-wip-us.apache.org/repos/asf/cassandra/blob/480a1a8f/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/480a1a8f/src/java/org/apache/cassandra/io/sstable/SSTableBoundedScanner.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/SSTableBoundedScanner.java
index 0e31896,d5bec82..af6a654
--- a/src/java/org/apache/cassandra/io/sstable/SSTableBoundedScanner.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableBoundedScanner.java
@@@ -35,10 -35,11 +35,10 @@@ public class SSTableBoundedScanner exte
      private final Iterator<Pair<Long, Long>> rangeIterator;
      private Pair<Long, Long> currentRange;
  
-     SSTableBoundedScanner(SSTableReader sstable, boolean skipCache, Range<Token> range)
 -    SSTableBoundedScanner(SSTableReader sstable, Iterator<Pair<Long, Long>>
rangeIterator)
++    SSTableBoundedScanner(SSTableReader sstable, Range<Token> range)
      {
-         super(sstable, skipCache);
+         super(sstable);
 -        this.rangeIterator = rangeIterator;
 -        assert rangeIterator.hasNext(); // use EmptyCompactionScanner otherwise
 +        this.rangeIterator = sstable.getPositionsForRanges(Collections.singletonList(range)).iterator();
          currentRange = rangeIterator.next();
          dfile.seek(currentRange.left);
      }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/480a1a8f/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/SSTableReader.java
index bbef4ec,dbc45d8..180ff16
--- a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
@@@ -968,12 -994,15 +1000,12 @@@ public class SSTableReader extends SSTa
      * @param range the range of keys to cover
      * @return A Scanner for seeking over the rows of the SSTable.
      */
-     public ICompactionScanner getDirectScanner(Range<Token> range)
+     public ICompactionScanner getScanner(Range<Token> range)
      {
          if (range == null)
-             return getDirectScanner();
+             return getScanner();
  
-         return new SSTableBoundedScanner(this, true, range);
 -        Iterator<Pair<Long, Long>> rangeIterator = getPositionsForRanges(Collections.singletonList(range)).iterator();
 -        return rangeIterator.hasNext()
 -               ? new SSTableBoundedScanner(this, rangeIterator)
 -               : new EmptyCompactionScanner(getFilename());
++        return new SSTableBoundedScanner(this, range);
      }
  
      public FileDataInput getFileDataInput(long position)
@@@ -1125,42 -1154,16 +1157,42 @@@
          return sstableMetadata.ancestors;
      }
  
 +    public int getSSTableLevel()
 +    {
 +        return sstableMetadata.sstableLevel;
 +    }
 +
 +    /**
 +     * Reloads the sstable metadata from disk.
 +     *
 +     * Called after level is changed on sstable, for example if the sstable is dropped to
L0
 +     *
 +     * Might be possible to remove in future versions
 +     *
 +     * @throws IOException
 +     */
 +    public void reloadSSTableMetadata() throws IOException
 +    {
 +        this.sstableMetadata = components.contains(Component.STATS)
 +                             ? SSTableMetadata.serializer.deserialize(descriptor)
 +                             : SSTableMetadata.createDefaultInstance();
 +    }
 +
 +    public SSTableMetadata getSSTableMetadata()
 +    {
 +        return sstableMetadata;
 +    }
 +
-     public RandomAccessReader openDataReader(boolean skipIOCache)
+     public RandomAccessReader openDataReader()
      {
          return compression
-                ? CompressedRandomAccessReader.open(getFilename(), getCompressionMetadata(),
skipIOCache)
-                : RandomAccessReader.open(new File(getFilename()), skipIOCache);
+                ? CompressedRandomAccessReader.open(getFilename(), getCompressionMetadata())
+                : RandomAccessReader.open(new File(getFilename()));
      }
  
-     public RandomAccessReader openIndexReader(boolean skipIOCache)
+     public RandomAccessReader openIndexReader()
      {
-         return RandomAccessReader.open(new File(getIndexFilename()), skipIOCache);
+         return RandomAccessReader.open(new File(getIndexFilename()));
      }
  
      /**

http://git-wip-us.apache.org/repos/asf/cassandra/blob/480a1a8f/src/java/org/apache/cassandra/streaming/FileStreamTask.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/streaming/FileStreamTask.java
index 4f40162,979b2e1..8dbed9d
--- a/src/java/org/apache/cassandra/streaming/FileStreamTask.java
+++ b/src/java/org/apache/cassandra/streaming/FileStreamTask.java
@@@ -145,12 -139,7 +145,12 @@@ public class FileStreamTask extends Wra
              return;
  
          // try to skip kernel page cache if possible
-         RandomAccessReader file = RandomAccessReader.open(new File(header.file.getFilename()),
true);
+         RandomAccessReader file = RandomAccessReader.open(new File(header.file.getFilename()));
 +        Descriptor desc = Descriptor.fromFilename(header.file.getFilename());
 +        ChecksumValidator metadata = null;
 +        if (new File(desc.filenameFor(Component.CRC)).exists())
 +            metadata = DataIntegrityMetadata.checksumValidator(desc);
 +        transferBuffer = metadata == null ? new byte[DEFAULT_CHUNK_SIZE] : new byte[metadata.chunkSize];
  
          // setting up data compression stream
          compressedoutput = new LZFOutputStream(output);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/480a1a8f/src/java/org/apache/cassandra/tools/SSTableExport.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/480a1a8f/test/unit/org/apache/cassandra/db/KeyCacheTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/KeyCacheTest.java
index 5af8aca,b05a607..7f57aae
--- a/test/unit/org/apache/cassandra/db/KeyCacheTest.java
+++ b/test/unit/org/apache/cassandra/db/KeyCacheTest.java
@@@ -128,12 -130,13 +128,12 @@@ public class KeyCacheTest extends Schem
                                                         false,
                                                         10));
  
-         assert CacheService.instance.keyCache.size() == 2;
+         assertEquals(2, CacheService.instance.keyCache.size());
  
          Util.compactAll(cfs).get();
 -        keyCacheSize = CacheService.instance.keyCache.size();
          // after compaction cache should have entries for
          // new SSTables, if we had 2 keys in cache previously it should become 4
 -        assertEquals(4, keyCacheSize);
 +        assertEquals(4, CacheService.instance.keyCache.size());
  
          // re-read same keys to verify that key cache didn't grow further
          cfs.getColumnFamily(QueryFilter.getSliceFilter(key1,

http://git-wip-us.apache.org/repos/asf/cassandra/blob/480a1a8f/test/unit/org/apache/cassandra/io/sstable/SSTableTest.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/480a1a8f/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java
----------------------------------------------------------------------


Mime
View raw message