cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From adelap...@apache.org
Subject [1/2] cassandra git commit: Improve secondary index (re)build failure and concurrency handling
Date Fri, 23 Jun 2017 10:02:10 GMT
Repository: cassandra
Updated Branches:
  refs/heads/trunk a1c6a623a -> 679c31718
Updated Tags:  refs/tags/0.8.10-tentative [created] 038b8f212
  refs/tags/1.0.10-tentative [created] b2ca7f821
  refs/tags/1.0.11-tentative [created] 4f0237acd
  refs/tags/1.0.12-tentative [created] 373e5b0d5
  refs/tags/1.0.7-tentative [created] 6a1ed6205
  refs/tags/1.0.8-tentative [created] fe6980eb7
  refs/tags/1.0.9-tentative [created] 3fd0fb6aa
  refs/tags/1.1.0-beta1-tentative [created] 271630d58
  refs/tags/1.1.0-beta2-tentative [created] 643d18af2
  refs/tags/1.1.0-rc1-tentative [created] cbbf54910
  refs/tags/1.1.0-tentative [created] c67153282
  refs/tags/1.1.1-tentative [created] a788a23ca
  refs/tags/1.1.10-tentative [created] 684994215
  refs/tags/1.1.11-tentative [created] d939a0c95
  refs/tags/1.1.12-tentative [created] 2dd73d171
  refs/tags/1.1.2-tentative [created] b94d8d40f
  refs/tags/1.1.3-tentative [created] 94bcc6a6f
  refs/tags/1.1.4-tentative [created] 94e46ff95
  refs/tags/1.1.6-tentative [created] a0900f3d3
  refs/tags/1.1.7-tentative [created] 2f4b6d8c6
  refs/tags/1.1.8-tentative [created] 4885bfccf
  refs/tags/1.1.9-tentative [created] 7eb47c50c
  refs/tags/1.2.0-beta1-tentative [created] 60bf68caa
  refs/tags/1.2.0-beta2-tentative [created] f04bebfa7
  refs/tags/1.2.0-beta3-tentative [created] b86f75dcd
  refs/tags/1.2.0-rc1-tentative [created] d791e0b3f
  refs/tags/1.2.0-rc2-tentative [created] cfe51fb4f
  refs/tags/1.2.0-tentative [created] 69337a436
  refs/tags/1.2.1-tentative [created] 8540974db
  refs/tags/1.2.10-tentative [created] 937536363
  refs/tags/1.2.2-tentative [created] 068b53dd5
  refs/tags/1.2.3-tentative [created] f07804e0d
  refs/tags/1.2.4-tentative [created] 2e96d0711
  refs/tags/1.2.5-tentative [created] 7d4380d66
  refs/tags/1.2.6-tentative [created] a6ca5d496
  refs/tags/1.2.7-tentative [created] ce4e4b9b5
  refs/tags/1.2.8 [created] dd105e6ff
  refs/tags/1.2.8-tentative [created] 0291d6960
  refs/tags/1.2.9-tentative [created] 6164d011e
  refs/tags/2.0.0-beta1-tentative [created] fcdb39384
  refs/tags/2.0.0-beta2-tentative [created] e0eacd281
  refs/tags/2.0.0-rc1-tentative [created] e8ae6720e
  refs/tags/2.0.0-rc2-tentative [created] 3e516a328
  refs/tags/2.0.0-tentative [created] 03045ca22
  refs/tags/2.0.1-tentative [created] eb96db6c1
  refs/tags/3.0.13-tentative [created] 91661ec29
  refs/tags/cassandra-1.2.0 [created] 95395aadf
  refs/tags/list [created] 03b528aa2
  refs/tags/trunk/3881 [created] 602e383d6
  refs/tags/trunk/4120 [created] 087f902d6


http://git-wip-us.apache.org/repos/asf/cassandra/blob/679c3171/test/unit/org/apache/cassandra/index/SecondaryIndexManagerTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/index/SecondaryIndexManagerTest.java b/test/unit/org/apache/cassandra/index/SecondaryIndexManagerTest.java
new file mode 100644
index 0000000..9af7072
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/SecondaryIndexManagerTest.java
@@ -0,0 +1,721 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index;
+
+import java.io.FileNotFoundException;
+import java.net.SocketException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.junit.After;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.notifications.SSTableAddedNotification;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.KillerForTests;
+import org.apache.cassandra.utils.concurrent.Refs;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class SecondaryIndexManagerTest extends CQLTester
+{
+
+    private static final String builtIndexesQuery = String.format("SELECT * FROM %s.\"%s\"",
+                                                                  SchemaConstants.SYSTEM_KEYSPACE_NAME,
+                                                                  SystemKeyspace.BUILT_INDEXES);
+
+    @After
+    public void after()
+    {
+        TestingIndex.clear();
+    }
+
+    @Test
+    public void creatingIndexMarksTheIndexAsBuilt() throws Throwable
+    {
+        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY
(a, b))");
+        String indexName = createIndex("CREATE INDEX ON %s(c)");
+
+        waitForIndex(KEYSPACE, tableName, indexName);
+        assertMarkedAsBuilt(indexName);
+    }
+
+    @Test
+    public void rebuildingIndexMarksTheIndexAsBuilt() throws Throwable
+    {
+        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY
(a, b))");
+        String indexName = createIndex("CREATE INDEX ON %s(c)");
+
+        waitForIndex(KEYSPACE, tableName, indexName);
+        assertMarkedAsBuilt(indexName);
+
+        assertTrue(tryRebuild(indexName, false));
+        assertMarkedAsBuilt(indexName);
+    }
+
+    @Test
+    public void recreatingIndexMarksTheIndexAsBuilt() throws Throwable
+    {
+        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY
(a, b))");
+        String indexName = createIndex("CREATE INDEX ON %s(c)");
+
+        waitForIndex(KEYSPACE, tableName, indexName);
+        assertMarkedAsBuilt(indexName);
+
+        // drop the index and verify that it has been removed from the built indexes table
+        dropIndex("DROP INDEX %s." + indexName);
+        assertNotMarkedAsBuilt();
+
+        // create the index again and verify that it's added to the built indexes table
+        createIndex(String.format("CREATE INDEX %s ON %%s(c)", indexName));
+        waitForIndex(KEYSPACE, tableName, indexName);
+        assertMarkedAsBuilt(indexName);
+    }
+
+    @Test
+    public void addingSSTablesMarksTheIndexAsBuilt() throws Throwable
+    {
+        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY
(a, b))");
+        String indexName = createIndex("CREATE INDEX ON %s(c)");
+
+        waitForIndex(KEYSPACE, tableName, indexName);
+        assertMarkedAsBuilt(indexName);
+
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        cfs.indexManager.markAllIndexesRemoved();
+        assertNotMarkedAsBuilt();
+
+        try (Refs<SSTableReader> sstables = Refs.ref(cfs.getSSTables(SSTableSet.CANONICAL)))
+        {
+            cfs.indexManager.handleNotification(new SSTableAddedNotification(sstables, null),
cfs.getTracker());
+            assertMarkedAsBuilt(indexName);
+        }
+    }
+
+    @Test
+    public void cannotRebuildWhileInitializationIsInProgress() throws Throwable
+    {
+        // create an index which blocks on creation
+        TestingIndex.blockCreate();
+        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY
(a, b))");
+        String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING
'%s'", TestingIndex.class.getName()));
+
+        // try to rebuild the index before the index creation task has finished
+        assertFalse(tryRebuild(indexName, false));
+        assertNotMarkedAsBuilt();
+
+        // check that the index is marked as built when the creation finishes
+        TestingIndex.unblockCreate();
+        waitForIndex(KEYSPACE, tableName, indexName);
+        assertMarkedAsBuilt(indexName);
+
+        // now verify you can rebuild
+        assertTrue(tryRebuild(indexName, false));
+        assertMarkedAsBuilt(indexName);
+    }
+
+    @Test
+    public void cannotRebuildWhileAnotherRebuildIsInProgress() throws Throwable
+    {
+        final String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY
KEY (a, b))");
+        final String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c)
USING '%s'", TestingIndex.class.getName()));
+        final AtomicBoolean error = new AtomicBoolean();
+
+        // wait for index initialization and verify it's built:
+        waitForIndex(KEYSPACE, tableName, indexName);
+        assertMarkedAsBuilt(indexName);
+
+        // rebuild the index in another thread, but make it block:
+        TestingIndex.blockBuild();
+        Thread asyncBuild = new Thread()
+        {
+
+            @Override
+            public void run()
+            {
+                try
+                {
+                    tryRebuild(indexName, false);
+                }
+                catch (Throwable ex)
+                {
+                    error.set(true);
+                }
+            }
+        };
+        asyncBuild.start();
+
+        // wait for the rebuild to block, so that we can proceed unblocking all further operations:
+        TestingIndex.waitBlockedOnBuild();
+
+        // do not block further builds:
+        TestingIndex.shouldBlockBuild = false;
+
+        // verify rebuilding the index before the previous index build task has finished
fails
+        assertFalse(tryRebuild(indexName, false));
+        assertNotMarkedAsBuilt();
+
+        // check that the index is marked as built when the build finishes
+        TestingIndex.unblockBuild();
+        asyncBuild.join();
+        assertMarkedAsBuilt(indexName);
+
+        // now verify you can rebuild
+        assertTrue(tryRebuild(indexName, false));
+        assertMarkedAsBuilt(indexName);
+    }
+
+    @Test
+    public void cannotRebuildWhileAnSSTableBuildIsInProgress() throws Throwable
+    {
+        final String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY
KEY (a, b))");
+        final String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c)
USING '%s'", TestingIndex.class.getName()));
+        final AtomicBoolean error = new AtomicBoolean();
+
+        // wait for index initialization and verify it's built:
+        waitForIndex(KEYSPACE, tableName, indexName);
+        assertMarkedAsBuilt(indexName);
+
+        // add sstables in another thread, but make it block:
+        TestingIndex.blockBuild();
+        Thread asyncBuild = new Thread()
+        {
+
+            @Override
+            public void run()
+            {
+                ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+                try (Refs<SSTableReader> sstables = Refs.ref(cfs.getSSTables(SSTableSet.CANONICAL)))
+                {
+                    cfs.indexManager.handleNotification(new SSTableAddedNotification(sstables,
null), cfs.getTracker());
+                }
+                catch (Throwable ex)
+                {
+                    error.set(true);
+                }
+            }
+        };
+        asyncBuild.start();
+
+        // wait for the build to block, so that we can proceed unblocking all further operations:
+        TestingIndex.waitBlockedOnBuild();
+
+        // do not block further builds:
+        TestingIndex.shouldBlockBuild = false;
+
+        // verify rebuilding the index before the previous index build task has finished
fails
+        assertFalse(tryRebuild(indexName, false));
+        assertNotMarkedAsBuilt();
+
+        // check that the index is marked as built when the build finishes
+        TestingIndex.unblockBuild();
+        asyncBuild.join();
+        assertMarkedAsBuilt(indexName);
+
+        // now verify you can rebuild
+        assertTrue(tryRebuild(indexName, false));
+        assertMarkedAsBuilt(indexName);
+    }
+
+    @Test
+    public void addingSSTableWhileRebuildIsInProgress() throws Throwable
+    {
+        final String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY
KEY (a, b))");
+        final String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c)
USING '%s'", TestingIndex.class.getName()));
+        final AtomicBoolean error = new AtomicBoolean();
+
+        // wait for index initialization and verify it's built:
+        waitForIndex(KEYSPACE, tableName, indexName);
+        assertMarkedAsBuilt(indexName);
+
+        // rebuild the index in another thread, but make it block:
+        TestingIndex.blockBuild();
+        Thread asyncBuild = new Thread()
+        {
+
+            @Override
+            public void run()
+            {
+                try
+                {
+                    tryRebuild(indexName, false);
+                }
+                catch (Throwable ex)
+                {
+                    error.set(true);
+                }
+            }
+        };
+        asyncBuild.start();
+
+        // wait for the rebuild to block, so that we can proceed unblocking all further operations:
+        TestingIndex.waitBlockedOnBuild();
+
+        // do not block further builds:
+        TestingIndex.shouldBlockBuild = false;
+
+        // try adding sstables and verify they are built but the index is not marked as built
because of the pending build:
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        try (Refs<SSTableReader> sstables = Refs.ref(cfs.getSSTables(SSTableSet.CANONICAL)))
+        {
+            cfs.indexManager.handleNotification(new SSTableAddedNotification(sstables, null),
cfs.getTracker());
+            assertNotMarkedAsBuilt();
+        }
+
+        // unblock the pending build:
+        TestingIndex.unblockBuild();
+        asyncBuild.join();
+
+        // verify the index is now built:
+        assertMarkedAsBuilt(indexName);
+        assertFalse(error.get());
+    }
+
+    @Test
+    public void addingSSTableWithBuildFailureWhileRebuildIsInProgress() throws Throwable
+    {
+        final String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY
KEY (a, b))");
+        final String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c)
USING '%s'", TestingIndex.class.getName()));
+        final AtomicBoolean error = new AtomicBoolean();
+
+        // wait for index initialization and verify it's built:
+        waitForIndex(KEYSPACE, tableName, indexName);
+        assertMarkedAsBuilt(indexName);
+
+        // rebuild the index in another thread, but make it block:
+        TestingIndex.blockBuild();
+        Thread asyncBuild = new Thread()
+        {
+
+            @Override
+            public void run()
+            {
+                try
+                {
+                    tryRebuild(indexName, false);
+                }
+                catch (Throwable ex)
+                {
+                    error.set(true);
+                }
+            }
+        };
+        asyncBuild.start();
+
+        // wait for the rebuild to block, so that we can proceed unblocking all further operations:
+        TestingIndex.waitBlockedOnBuild();
+
+        // do not block further builds:
+        TestingIndex.shouldBlockBuild = false;
+
+        // try adding sstables but make the build fail:
+        TestingIndex.shouldFailBuild = true;
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        try (Refs<SSTableReader> sstables = Refs.ref(cfs.getSSTables(SSTableSet.CANONICAL)))
+        {
+            cfs.indexManager.handleNotification(new SSTableAddedNotification(sstables, null),
cfs.getTracker());
+            fail("Should have failed!");
+        }
+        catch (Throwable ex)
+        {
+            assertTrue(ex.getMessage().contains("configured to fail"));
+        }
+
+        // disable failures:
+        TestingIndex.shouldFailBuild = false;
+
+        // unblock the pending build:
+        TestingIndex.unblockBuild();
+        asyncBuild.join();
+
+        // verify the index is *not* built due to the failing sstable build:
+        assertNotMarkedAsBuilt();
+        assertFalse(error.get());
+    }
+
+    @Test
+    public void rebuildWithFailure() throws Throwable
+    {
+        final String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY
KEY (a, b))");
+        final String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c)
USING '%s'", TestingIndex.class.getName()));
+        waitForIndex(KEYSPACE, tableName, indexName);
+
+        // Rebuild the index with failure and verify it is not marked as built
+        TestingIndex.shouldFailBuild = true;
+        try
+        {
+            ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+            cfs.indexManager.rebuildIndexesBlocking(Collections.singleton(indexName));
+            fail("Should have failed!");
+        }
+        catch (Throwable ex)
+        {
+            assertTrue(ex.getMessage().contains("configured to fail"));
+        }
+        assertNotMarkedAsBuilt();
+    }
+
+    @Test
+    public void initializingIndexNotQueryable() throws Throwable
+    {
+        TestingIndex.blockCreate();
+        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY
(a, b))");
+        String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING
'%s'", TestingIndex.class.getName()));
+
+        // the index shouldn't be queryable while the initialization hasn't finished
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        Index index = cfs.indexManager.getIndexByName(indexName);
+        assertFalse(cfs.indexManager.isIndexQueryable(index));
+
+        // the index should be queryable once the initialization has finished
+        TestingIndex.unblockCreate();
+        waitForIndex(KEYSPACE, tableName, indexName);
+        assertTrue(cfs.indexManager.isIndexQueryable(index));
+    }
+
+    @Test
+    public void initializingIndexNotQueryableAfterPartialRebuild() throws Throwable
+    {
+        TestingIndex.blockCreate();
+        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY
(a, b))");
+        String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING
'%s'", TestingIndex.class.getName()));
+
+        // the index shouldn't be queryable while the initialization hasn't finished
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        Index index = cfs.indexManager.getIndexByName(indexName);
+        assertFalse(cfs.indexManager.isIndexQueryable(index));
+
+        // a failing partial build doesn't set the index as queryable
+        TestingIndex.shouldFailBuild = true;
+        try
+        {
+            cfs.indexManager.handleNotification(new SSTableAddedNotification(cfs.getLiveSSTables(),
null), this);
+            fail("Should have failed!");
+        }
+        catch (Throwable ex)
+        {
+            assertTrue(ex.getMessage().contains("configured to fail"));
+        }
+        assertFalse(cfs.indexManager.isIndexQueryable(index));
+
+        // a successful partial build doesn't set the index as queryable
+        TestingIndex.shouldFailBuild = false;
+        cfs.indexManager.handleNotification(new SSTableAddedNotification(cfs.getLiveSSTables(),
null), this);
+        assertFalse(cfs.indexManager.isIndexQueryable(index));
+
+        // the index should be queryable once the initialization has finished
+        TestingIndex.unblockCreate();
+        waitForIndex(KEYSPACE, tableName, indexName);
+        assertTrue(cfs.indexManager.isIndexQueryable(index));
+    }
+
+    @Test
+    public void indexWithfailedInitializationIsQueryableAfterFullRebuild() throws Throwable
+    {
+        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+
+        TestingIndex.shouldFailCreate = true;
+        String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING
'%s'", TestingIndex.class.getName()));
+
+        tryRebuild(indexName, true);
+        TestingIndex.shouldFailCreate = false;
+
+        // a successfull full rebuild should set the index as queryable
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        Index index = cfs.indexManager.getIndexByName(indexName);
+        cfs.indexManager.rebuildIndexesBlocking(Collections.singleton(indexName));
+        assertTrue(cfs.indexManager.isIndexQueryable(index));
+    }
+
+    @Test
+    public void indexWithfailedInitializationIsNotQueryableAfterPartialRebuild() throws Throwable
+    {
+        TestingIndex.shouldFailCreate = true;
+        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+        String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING
'%s'", TestingIndex.class.getName()));
+        TestingIndex.shouldFailCreate = false;
+
+        // the index shouldn't be queryable after the failed initialization
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        Index index = cfs.indexManager.getIndexByName(indexName);
+        assertFalse(cfs.indexManager.isIndexQueryable(index));
+
+        // a successful partial build doesn't set the index as queryable
+        cfs.indexManager.handleNotification(new SSTableAddedNotification(cfs.getLiveSSTables(),
null), this);
+        assertFalse(cfs.indexManager.isIndexQueryable(index));
+    }
+
+    @Test
+    public void handleJVMStablityOnFailedCreate() throws Throwable
+    {
+        handleJVMStablityOnFailedCreate(new SocketException("Should not fail"), false);
+        handleJVMStablityOnFailedCreate(new FileNotFoundException("Should not fail"), false);
+        handleJVMStablityOnFailedCreate(new SocketException("Too many open files"), true);
+        handleJVMStablityOnFailedCreate(new FileNotFoundException("Too many open files"),
true);
+        handleJVMStablityOnFailedCreate(new RuntimeException("Should not fail"), false);
+    }
+
+    private void handleJVMStablityOnFailedCreate(Throwable throwable, boolean shouldKillJVM)
throws Throwable
+    {
+        KillerForTests killerForTests = new KillerForTests();
+        JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+
+        try
+        {
+            TestingIndex.shouldFailCreate = true;
+            TestingIndex.failedCreateThrowable = throwable;
+
+            createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+            String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING
'%s'", TestingIndex.class.getName()));
+            tryRebuild(indexName, true);
+            fail("Should have failed!");
+        }
+        catch (Throwable t)
+        {
+            assertEquals(shouldKillJVM, killerForTests.wasKilled());
+        }
+        finally
+        {
+            JVMStabilityInspector.replaceKiller(originalKiller);
+            TestingIndex.shouldFailCreate = false;
+            TestingIndex.failedCreateThrowable = null;
+        }
+    }
+
+    @Test
+    public void handleJVMStablityOnFailedRebuild() throws Throwable
+    {
+        handleJVMStablityOnFailedRebuild(new SocketException("Should not fail"), false);
+        handleJVMStablityOnFailedRebuild(new FileNotFoundException("Should not fail"), false);
+        handleJVMStablityOnFailedRebuild(new SocketException("Too many open files"), true);
+        handleJVMStablityOnFailedRebuild(new FileNotFoundException("Too many open files"),
true);
+        handleJVMStablityOnFailedRebuild(new RuntimeException("Should not fail"), false);
+    }
+
+    private void handleJVMStablityOnFailedRebuild(Throwable throwable, boolean shouldKillJVM)
throws Throwable
+    {
+        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY
(a, b))");
+        String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING
'%s'", TestingIndex.class.getName()));
+        waitForIndex(KEYSPACE, tableName, indexName);
+
+        KillerForTests killerForTests = new KillerForTests();
+        JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+
+        try
+        {
+            TestingIndex.shouldFailBuild = true;
+            TestingIndex.failedBuildTrowable = throwable;
+
+            getCurrentColumnFamilyStore().indexManager.rebuildIndexesBlocking(Collections.singleton(indexName));
+            fail("Should have failed!");
+        }
+        catch (Throwable t)
+        {
+            assertEquals(shouldKillJVM, killerForTests.wasKilled());
+        }
+        finally
+        {
+            JVMStabilityInspector.replaceKiller(originalKiller);
+            TestingIndex.shouldFailBuild = false;
+            TestingIndex.failedBuildTrowable = null;
+        }
+    }
+
+    private void assertMarkedAsBuilt(String indexName) throws Throwable
+    {
+        assertRows(execute(builtIndexesQuery), row(KEYSPACE, indexName));
+    }
+
+    private void assertNotMarkedAsBuilt() throws Throwable
+    {
+        assertEmpty(execute(builtIndexesQuery));
+    }
+
+    private boolean tryRebuild(String indexName, boolean wait) throws Throwable
+    {
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        boolean done = false;
+        do
+        {
+            try
+            {
+                cfs.indexManager.rebuildIndexesBlocking(Collections.singleton(indexName));
+                done = true;
+            }
+            catch (IllegalStateException e)
+            {
+                assertTrue(e.getMessage().contains("currently in progress"));
+            }
+            Thread.sleep(500);
+        }
+        while (!done && wait);
+
+        return done;
+    }
+
+    public static class TestingIndex extends StubIndex
+    {
+        private static volatile CountDownLatch createLatch;
+        private static volatile CountDownLatch buildLatch;
+        private static volatile CountDownLatch createWaitLatch;
+        private static volatile CountDownLatch buildWaitLatch;
+        public static volatile boolean shouldBlockCreate = false;
+        public static volatile boolean shouldBlockBuild = false;
+        public static volatile boolean shouldFailCreate = false;
+        public static volatile boolean shouldFailBuild = false;
+        public static volatile Throwable failedCreateThrowable;
+        public static volatile Throwable failedBuildTrowable;
+
+        public TestingIndex(ColumnFamilyStore baseCfs, IndexMetadata metadata)
+        {
+            super(baseCfs, metadata);
+        }
+
+        public static void blockCreate()
+        {
+            shouldBlockCreate = true;
+            createLatch = new CountDownLatch(1);
+            createWaitLatch = new CountDownLatch(1);
+        }
+
+        public static void blockBuild()
+        {
+            shouldBlockBuild = true;
+            buildLatch = new CountDownLatch(1);
+            buildWaitLatch = new CountDownLatch(1);
+        }
+
+        public static void unblockCreate()
+        {
+            createLatch.countDown();
+        }
+
+        public static void unblockBuild()
+        {
+            buildLatch.countDown();
+        }
+
+        public static void waitBlockedOnCreate() throws InterruptedException
+        {
+            createWaitLatch.await();
+        }
+
+        public static void waitBlockedOnBuild() throws InterruptedException
+        {
+            buildWaitLatch.await();
+        }
+
+        public static void clear()
+        {
+            createLatch = null;
+            createWaitLatch = null;
+            buildLatch = null;
+            buildWaitLatch = null;
+            shouldBlockCreate = false;
+            shouldBlockBuild = false;
+            shouldFailBuild = false;
+            failedCreateThrowable = null;
+            failedBuildTrowable = null;
+        }
+
+        public Callable<?> getInitializationTask()
+        {
+            return () ->
+            {
+                if (shouldBlockCreate && createLatch != null)
+                {
+                    createWaitLatch.countDown();
+                    createLatch.await();
+                }
+
+                if (shouldFailCreate)
+                {
+                    throw failedCreateThrowable == null
+                          ? new IllegalStateException("Index is configured to fail.")
+                          : new RuntimeException(failedCreateThrowable);
+                }
+
+                return null;
+            };
+        }
+
+        public IndexBuildingSupport getBuildTaskSupport()
+        {
+            return new CollatedViewIndexBuildingSupport()
+            {
+                public SecondaryIndexBuilder getIndexBuildTask(ColumnFamilyStore cfs, Set<Index>
indexes, Collection<SSTableReader> sstables)
+                {
+                    try
+                    {
+                        if (shouldBlockBuild && buildLatch != null)
+                        {
+                            buildWaitLatch.countDown();
+                            buildLatch.await();
+                        }
+                        final SecondaryIndexBuilder builder = super.getIndexBuildTask(cfs,
indexes, sstables);
+                        return new SecondaryIndexBuilder()
+                        {
+
+                            @Override
+                            public void build()
+                            {
+                                if (shouldFailBuild)
+                                {
+                                    throw failedBuildTrowable == null
+                                          ? new IllegalStateException("Index is configured
to fail.")
+                                          : new RuntimeException(failedBuildTrowable);
+                                }
+                                builder.build();
+                            }
+
+                            @Override
+                            public CompactionInfo getCompactionInfo()
+                            {
+                                return builder.getCompactionInfo();
+                            }
+                        };
+                    }
+                    catch (InterruptedException ex)
+                    {
+                        throw new RuntimeException(ex);
+                    }
+                }
+            };
+        }
+
+        public boolean shouldBuildBlocking()
+        {
+            return true;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/cassandra/blob/679c3171/test/unit/org/apache/cassandra/index/internal/CassandraIndexTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/index/internal/CassandraIndexTest.java b/test/unit/org/apache/cassandra/index/internal/CassandraIndexTest.java
index 8b9402b..c0ca2ba 100644
--- a/test/unit/org/apache/cassandra/index/internal/CassandraIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/internal/CassandraIndexTest.java
@@ -523,6 +523,7 @@ public class CassandraIndexTest extends CQLTester
                    row(KEYSPACE, indexName));
     }
 
+
     // this is slightly annoying, but we cannot read rows from the methods in Util as
     // ReadCommand#executeInternal uses metadata retrieved via the tableId, which the index
     // CFS inherits from the base CFS. This has the 'wrong' partitioner (the index table

http://git-wip-us.apache.org/repos/asf/cassandra/blob/679c3171/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java b/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java
index 4194ec5..028bbd0 100644
--- a/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java
+++ b/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java
@@ -632,7 +632,6 @@ public class CustomCassandraIndex implements Index
                             baseCfs.metadata.keyspace,
                             baseCfs.metadata.name,
                             metadata.name);
-                baseCfs.indexManager.markIndexBuilt(metadata.name);
                 return;
             }
 
@@ -646,7 +645,6 @@ public class CustomCassandraIndex implements Index
             Future<?> future = CompactionManager.instance.submitIndexBuild(builder);
             FBUtilities.waitOnFuture(future);
             indexCfs.forceBlockingFlush();
-            baseCfs.indexManager.markIndexBuilt(metadata.name);
         }
         logger.info("Index build of {} complete", metadata.name);
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/679c3171/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
index 7f2f22b..d622725 100644
--- a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
@@ -32,11 +32,14 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.index.Index;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.*;
 import org.apache.cassandra.cql3.Term;
 import org.apache.cassandra.cql3.statements.IndexTarget;
 import org.apache.cassandra.db.*;
@@ -69,6 +72,7 @@ import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Pair;
 
 import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.Uninterruptibles;
 
 import junit.framework.Assert;
@@ -815,14 +819,14 @@ public class SASIIndexTest
         Assert.assertTrue(rows.toString(), rows.isEmpty());
 
         // now let's trigger index rebuild and check if we got the data back
-        store.indexManager.buildIndexBlocking(store.indexManager.getIndexByName(store.name
+ "_data_output_id"));
+        store.indexManager.rebuildIndexesBlocking(Sets.newHashSet(store.name + "_data_output_id"));
 
         rows = getIndexed(store, 10, buildExpression(dataOutputId, Operator.LIKE_CONTAINS,
UTF8Type.instance.decompose("a")));
         Assert.assertTrue(rows.toString(), Arrays.equals(new String[] { "key1", "key2" },
rows.toArray(new String[rows.size()])));
 
         // also let's try to build an index for column which has no data to make sure that
doesn't fail
-        store.indexManager.buildIndexBlocking(store.indexManager.getIndexByName(store.name
+ "_first_name"));
-        store.indexManager.buildIndexBlocking(store.indexManager.getIndexByName(store.name
+ "_data_output_id"));
+        store.indexManager.rebuildIndexesBlocking(Sets.newHashSet(store.name + "_first_name"));
+        store.indexManager.rebuildIndexesBlocking(Sets.newHashSet(store.name + "_data_output_id"));
 
         rows = getIndexed(store, 10, buildExpression(dataOutputId, Operator.LIKE_CONTAINS,
UTF8Type.instance.decompose("a")));
         Assert.assertTrue(rows.toString(), Arrays.equals(new String[] { "key1", "key2" },
rows.toArray(new String[rows.size()])));

http://git-wip-us.apache.org/repos/asf/cassandra/blob/679c3171/tools/stress/src/org/apache/cassandra/stress/CompactionStress.java
----------------------------------------------------------------------
diff --git a/tools/stress/src/org/apache/cassandra/stress/CompactionStress.java b/tools/stress/src/org/apache/cassandra/stress/CompactionStress.java
index 1860fef..1b63b5a 100644
--- a/tools/stress/src/org/apache/cassandra/stress/CompactionStress.java
+++ b/tools/stress/src/org/apache/cassandra/stress/CompactionStress.java
@@ -143,6 +143,10 @@ public abstract class CompactionStress implements Runnable
 
             cfs.disableAutoCompaction();
 
+            // We want to add the SSTables without firing their indexing by any eventual
unsupported 2i
+            if (cfs.indexManager.hasIndexes())
+                throw new IllegalStateException("CompactionStress does not support secondary
indexes");
+
             //Register with cfs
             cfs.addSSTables(sstables);
         }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org


Mime
View raw message