asterixdb-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Murtadha Hubail (Code Review)" <do-not-re...@asterixdb.incubator.apache.org>
Subject Change in asterixdb[master]: [ASTERIXDB-2130][TX] Prevent Dropping Index Pending Txn Comp...
Date Thu, 09 Nov 2017 18:06:39 GMT
Murtadha Hubail has submitted this change and it was merged.

Change subject: [ASTERIXDB-2130][TX] Prevent Dropping Index Pending Txn Completion
......................................................................


[ASTERIXDB-2130][TX] Prevent Dropping Index Pending Txn Completion

- user model changes: no
- storage format changes: no
- interface changes: yes
  Added resource id to ITransactionOperationTracker
  before/after txn operations.

Details:
 - Currently, an index could be evicted/dropped while a transaction
   waiting for its completion (commit/rollback). This change prevents
   that by incrementing the reference counter of the indexes registered
   in the transaction.
 - Add test case.

Change-Id: If6a938cbb5a9c3b7f5cc59505c07ae45b3425223
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2132
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mblow@apache.org>
---
M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java
M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/IndexDropOperatorNodePushableTest.java
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionOperationTracker.java
M asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/TransactionContext.java
5 files changed, 78 insertions(+), 6 deletions(-)

Approvals:
  Anon. E. Moose #1000171: 
  Jenkins: Verified; No violations found; ; Verified
  Michael Blow: Looks good to me, approved



diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java
index ac5e606..6bff50d 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java
@@ -264,6 +264,7 @@
                 tupleAppender.write(insertOp, true);
             }
             insertOp.close();
+            nc.getTransactionManager().completedTransaction(txnCtx, DatasetId.NULL, -1, true);
             searchAndAssertCount(nc, ctx, dataset, storageManager, TOTAL_NUM_OF_RECORDS);
             // rollback the last disk component
             lsmAccessor = lsmBtree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/IndexDropOperatorNodePushableTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/IndexDropOperatorNodePushableTest.java
index 2bac49e..7ced87d 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/IndexDropOperatorNodePushableTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/IndexDropOperatorNodePushableTest.java
@@ -30,17 +30,28 @@
 
 import org.apache.asterix.app.bootstrap.TestNodeController;
 import org.apache.asterix.common.config.DatasetConfig;
+import org.apache.asterix.common.dataflow.ICcApplicationContext;
 import org.apache.asterix.file.StorageComponentProvider;
+import org.apache.asterix.metadata.MetadataManager;
+import org.apache.asterix.metadata.MetadataTransactionContext;
+import org.apache.asterix.metadata.bootstrap.MetadataBuiltinEntities;
+import org.apache.asterix.metadata.declared.MetadataProvider;
 import org.apache.asterix.metadata.entities.Dataset;
 import org.apache.asterix.metadata.entities.Index;
 import org.apache.asterix.metadata.entities.InternalDatasetDetails;
+import org.apache.asterix.metadata.utils.SplitsAndConstraintsUtil;
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.asterix.om.types.BuiltinType;
 import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.test.common.TestExecutor;
 import org.apache.asterix.test.common.TestHelper;
+import org.apache.asterix.test.runtime.ExecutionTestUtil;
+import org.apache.asterix.testframework.context.TestCaseContext;
 import org.apache.hyracks.api.context.IHyracksTaskContext;
 import org.apache.hyracks.api.exceptions.ErrorCode;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileSplit;
+import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
 import org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper;
 import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory;
 import org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorNodePushable;
@@ -64,6 +75,7 @@
     private static final String DATA_TYPE_NAME = "DUMMY";
     private static final String NODE_GROUP_NAME = "DEFAULT";
     private final AtomicBoolean dropFailed = new AtomicBoolean(false);
+    private final TestExecutor testExecutor = new TestExecutor();
 
     @Before
     public void setUp() throws Exception {
@@ -77,6 +89,12 @@
         TestHelper.deleteExistingInstanceFiles();
     }
 
+    /**
+     * Tests dropping a dataset using different
+     * drop options
+     *
+     * @throws Exception
+     */
     @Test
     public void dropOptionsTest() throws Exception {
         TestNodeController nc = new TestNodeController(null, false);
@@ -107,6 +125,47 @@
         }
     }
 
+    /**
+     * Tests dropping an index whose dataset has no active
+     * operations
+     *
+     * @throws Exception
+     */
+    @Test
+    public void dropIndexInUseTest() throws Exception {
+        TestNodeController nc = new TestNodeController(null, false);
+        try {
+            nc.init();
+            String datasetName = "ds";
+            String indexName = "fooIdx";
+            // create dataset and index
+            final TestCaseContext.OutputFormat format = TestCaseContext.OutputFormat.CLEAN_JSON;
+            testExecutor.executeSqlppUpdateOrDdl("CREATE TYPE KeyType AS { id: int, foo:
int };", format);
+            testExecutor.executeSqlppUpdateOrDdl("CREATE DATASET " + datasetName + "(KeyType)
PRIMARY KEY id;", format);
+            testExecutor.executeSqlppUpdateOrDdl("CREATE INDEX " + indexName + " on " + datasetName
+ "(foo)", format);
+            final MetadataTransactionContext mdTxn = MetadataManager.INSTANCE.beginTransaction();
+            ICcApplicationContext appCtx =
+                    (ICcApplicationContext) ExecutionTestUtil.integrationUtil.getClusterControllerService()
+                            .getApplicationContext();
+            MetadataProvider metadataProver = new MetadataProvider(appCtx, null);
+            metadataProver.setMetadataTxnContext(mdTxn);
+            final String defaultDv = MetadataBuiltinEntities.DEFAULT_DATAVERSE.getDataverseName();
+            final Dataset dataset = MetadataManager.INSTANCE.getDataset(mdTxn, defaultDv,
datasetName);
+            MetadataManager.INSTANCE.commitTransaction(mdTxn);
+            FileSplit[] splits = SplitsAndConstraintsUtil
+                    .getIndexSplits(appCtx.getClusterStateManager(), dataset, indexName,
Arrays.asList("asterix_nc1"));
+            final ConstantFileSplitProvider constantFileSplitProvider =
+                    new ConstantFileSplitProvider(Arrays.copyOfRange(splits, 0, 1));
+            IndexDataflowHelperFactory helperFactory =
+                    new IndexDataflowHelperFactory(nc.getStorageManager(), constantFileSplitProvider);
+            IHyracksTaskContext ctx = nc.createTestContext(true);
+            IIndexDataflowHelper dataflowHelper = helperFactory.create(ctx.getJobletContext().getServiceContext(),
0);
+            dropInUse(ctx, helperFactory, dataflowHelper);
+        } finally {
+            nc.deInit();
+        }
+    }
+
     private void dropInUse(IHyracksTaskContext ctx, IndexDataflowHelperFactory helperFactory,
             IIndexDataflowHelper dataflowHelper) throws Exception {
         dropFailed.set(false);
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java
index 2068192..9f57981 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java
@@ -63,12 +63,18 @@
     }
 
     @Override
-    public void beforeTransaction() {
+    public void beforeTransaction(long resourceId) {
+        /*
+         * Increment dataset and index ref count to prevent them
+         * from being evicted/dropped until the transaction completes
+         */
         dsInfo.touch();
+        dsInfo.getIndexes().get(resourceId).touch();
     }
 
     @Override
-    public void afterTransaction() {
+    public void afterTransaction(long resourceId) {
         dsInfo.untouch();
+        dsInfo.getIndexes().get(resourceId).untouch();
     }
 }
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionOperationTracker.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionOperationTracker.java
index e6aeec5..301801b 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionOperationTracker.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionOperationTracker.java
@@ -25,12 +25,18 @@
     /**
      * Called before a transaction performs any operations on
      * {@link org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex}
+     * with resource id {@code resourceId}
+     *
+     * @param resourceId
      */
-    void beforeTransaction();
+    void beforeTransaction(long resourceId);
 
     /**
      * Called after a transaction completes its operations on
      * {@link org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex}
+     * with resource id {@code resourceId}
+     *
+     * @param resourceId
      */
-    void afterTransaction();
+    void afterTransaction(long resourceId);
 }
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/TransactionContext.java
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/TransactionContext.java
index 4846b6e..3159e6b 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/TransactionContext.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/TransactionContext.java
@@ -122,7 +122,7 @@
                 final ITransactionOperationTracker txnOpTracker =
                         (ITransactionOperationTracker) index.getOperationTracker();
                 indexMap.put(resourceId, txnOpTracker);
-                txnOpTracker.beforeTransaction();
+                txnOpTracker.beforeTransaction(resourceId);
             }
         }
     }
@@ -258,7 +258,7 @@
             }
         } finally {
             synchronized (indexMap) {
-                indexMap.values().forEach(ITransactionOperationTracker::afterTransaction);
+                indexMap.forEach((resource, opTracker) -> opTracker.afterTransaction(resource));
             }
         }
     }

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/2132
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: merged
Gerrit-Change-Id: If6a938cbb5a9c3b7f5cc59505c07ae45b3425223
Gerrit-PatchSet: 6
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <mhubail@apache.org>
Gerrit-Reviewer: Anon. E. Moose #1000171
Gerrit-Reviewer: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mblow@apache.org>
Gerrit-Reviewer: Murtadha Hubail <mhubail@apache.org>
Gerrit-Reviewer: Till Westmann <tillw@apache.org>
Gerrit-Reviewer: abdullah alamoudi <bamousaa@gmail.com>

Mime
View raw message