asterixdb-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Murtadha Hubail (Code Review)" <do-not-re...@asterixdb.incubator.apache.org>
Subject Change in asterixdb[master]: ASTERIXDB-1841: Fix Inverted Index Drop Operator
Date Mon, 20 Mar 2017 19:31:31 GMT
Murtadha Hubail has uploaded a new change for review.

  https://asterix-gerrit.ics.uci.edu/1600

Change subject: ASTERIXDB-1841: Fix Inverted Index Drop Operator
......................................................................

ASTERIXDB-1841: Fix Inverted Index Drop Operator

Change-Id: I2c0f52770fd791ab9293c661de810c0b3b9ba8fb
---
M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
A asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.1.script.aql
A asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.2.ddl.aql
A asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.3.script.aql
A asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.4.ddl.aql
A asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.5.script.aql
M asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java
A asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryTreeIndexOperationsHelper.java
A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDropOperatorDescriptor.java
15 files changed, 313 insertions(+), 107 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/00/1600/1

diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index e64cf14..c4868fc 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -1052,7 +1052,7 @@
                 bActiveTxn = true;
                 metadataProvider.setMetadataTxnContext(mdTxnCtx);
                 try {
-                    JobSpecification jobSpec = IndexUtil.buildDropSecondaryIndexJobSpec(index,
metadataProvider, ds);
+                    JobSpecification jobSpec = IndexUtil.buildDropIndexJobSpec(index, metadataProvider,
ds);
                     MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
                     bActiveTxn = false;
                     JobUtils.runJob(hcc, jobSpec, true);
@@ -1201,7 +1201,7 @@
                             MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName,
datasetName);
                     for (int k = 0; k < indexes.size(); k++) {
                         if (indexes.get(k).isSecondaryIndex()) {
-                            jobsToExecute.add(IndexUtil.buildDropSecondaryIndexJobSpec(indexes.get(k),
+                            jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(indexes.get(k),
                                     metadataProvider, datasets.get(j)));
                         }
                     }
@@ -1217,7 +1217,7 @@
                             jobsToExecute.add(ExternalIndexingOperations.buildDropFilesIndexJobSpec(metadataProvider,
                                     datasets.get(j)));
                         } else {
-                            jobsToExecute.add(IndexUtil.buildDropSecondaryIndexJobSpec(indexes.get(k),
+                            jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(indexes.get(k),
                                     metadataProvider, datasets.get(j)));
                         }
                     }
@@ -1390,7 +1390,7 @@
                     MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx.getValue(), dataverseName,
datasetName);
             for (int j = 0; j < indexes.size(); j++) {
                 if (indexes.get(j).isSecondaryIndex()) {
-                    jobsToExecute.add(IndexUtil.buildDropSecondaryIndexJobSpec(indexes.get(j),
metadataProvider, ds));
+                    jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(indexes.get(j), metadataProvider,
ds));
                 }
             }
             Index primaryIndex =
@@ -1429,7 +1429,7 @@
                     MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx.getValue(), dataverseName,
datasetName);
             for (int j = 0; j < indexes.size(); j++) {
                 if (ExternalIndexingOperations.isFileIndex(indexes.get(j))) {
-                    jobsToExecute.add(IndexUtil.buildDropSecondaryIndexJobSpec(indexes.get(j),
metadataProvider, ds));
+                    jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(indexes.get(j), metadataProvider,
ds));
                 } else {
                     jobsToExecute.add(ExternalIndexingOperations.buildDropFilesIndexJobSpec(metadataProvider,
ds));
                 }
@@ -1519,7 +1519,7 @@
                     }
                 }
                 // #. prepare a job to drop the index in NC.
-                jobsToExecute.add(IndexUtil.buildDropSecondaryIndexJobSpec(index, metadataProvider,
ds));
+                jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(index, metadataProvider,
ds));
 
                 // #. mark PendingDropOp on the existing index
                 MetadataManager.INSTANCE.dropIndex(mdTxnCtx, dataverseName, datasetName,
indexName);
@@ -1560,7 +1560,7 @@
                     throw new AlgebricksException("Dropping a dataset's files index is not
allowed.");
                 }
                 // #. prepare a job to drop the index in NC.
-                jobsToExecute.add(IndexUtil.buildDropSecondaryIndexJobSpec(index, metadataProvider,
ds));
+                jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(index, metadataProvider,
ds));
                 List<Index> datasetIndexes =
                         MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName,
datasetName);
                 if (datasetIndexes.size() == 2) {
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.1.script.aql
b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.1.script.aql
new file mode 100644
index 0000000..cc46136
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.1.script.aql
@@ -0,0 +1 @@
+create_and_start.sh
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.2.ddl.aql
b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.2.ddl.aql
new file mode 100644
index 0000000..60deb2d
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.2.ddl.aql
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+* Description  : Create different types of secondary indexes then drop them after restart.
+* Expected Res : Success
+* Date         : 20th March 2017
+*/
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+create type Name as open {
+first : string,
+last : string
+}
+
+create type Person as open {
+name : Name
+}
+
+create type TestType as open {
+id : int32,
+name : string,
+locn : point,
+zip : string,
+person : Person
+}
+
+create dataset t1(TestType) primary key id;
+create index rtree_index_point on t1(locn) type rtree;
+create index rtree_index_point_open on t1(open_locn:point?) type rtree enforced;
+create index keyWD_indx on t1(name) type keyword;
+create index keyWD_indx_open on t1(nickname:string?) type keyword enforced;
+create index secndIndx on t1(zip);
+create index nested on t1(person.name.first);
+create index secndIndx_open on t1(address:string?) enforced;
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.3.script.aql
b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.3.script.aql
new file mode 100644
index 0000000..3ba1dc0
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.3.script.aql
@@ -0,0 +1 @@
+stop_and_start.sh
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.4.ddl.aql
b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.4.ddl.aql
new file mode 100644
index 0000000..42a361e
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.4.ddl.aql
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+* Description  : Create different types of secondary indexes then drop them after restart.
+* Expected Res : Success
+* Date         : 20th March 2017
+*/
+use dataverse test;
+drop index t1.rtree_index_point;
+drop index t1.rtree_index_point_open;
+drop index t1.keyWD_indx;
+drop index t1.keyWD_indx_open;
+drop index t1.secndIndx;
+drop index t1.nested;
+drop index t1.secndIndx_open;
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.5.script.aql
b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.5.script.aql
new file mode 100644
index 0000000..10e1a51
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.5.script.aql
@@ -0,0 +1 @@
+stop_and_delete.sh
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml b/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
index a9644b0..07e2d6d 100644
--- a/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
@@ -34,6 +34,13 @@
       </compilation-unit>
     </test-case>
   </test-group>
+  <test-group name="dml_after_restart">
+    <test-case FilePath="dml_after_restart">
+      <compilation-unit name="multiple_secondary_indices">
+        <output-dir compare="Text">multiple_secondary_indices</output-dir>
+      </compilation-unit>
+    </test-case>
+  </test-group>
   <test-group name="recover_after_abort">
 
     <test-case FilePath="recover_after_abort">
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
index ad75b6b..bf2749d 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
@@ -295,7 +295,7 @@
                     MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx.getValue(), dataverseName,
datasetName);
             for (int j = 0; j < indexes.size(); j++) {
                 if (indexes.get(j).isSecondaryIndex()) {
-                    jobsToExecute.add(IndexUtil.dropJob(indexes.get(j), metadataProvider,
this));
+                    jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(indexes.get(j), metadataProvider,
this));
                 }
             }
             Index primaryIndex =
@@ -334,7 +334,7 @@
                     MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx.getValue(), dataverseName,
datasetName);
             for (int j = 0; j < indexes.size(); j++) {
                 if (ExternalIndexingOperations.isFileIndex(indexes.get(j))) {
-                    jobsToExecute.add(IndexUtil.dropJob(indexes.get(j), metadataProvider,
this));
+                    jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(indexes.get(j), metadataProvider,
this));
                 } else {
                     jobsToExecute.add(DatasetUtil.buildDropFilesIndexJobSpec(metadataProvider,
this));
                 }
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
index 701d0d6..a5df766 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
@@ -18,11 +18,8 @@
  */
 package org.apache.asterix.metadata.utils;
 
-import java.util.List;
-import java.util.Map;
-
+import com.google.common.collect.Lists;
 import org.apache.asterix.common.config.OptimizationConfUtil;
-import org.apache.asterix.common.context.IStorageComponentProvider;
 import org.apache.asterix.common.exceptions.CompilationException;
 import org.apache.asterix.common.exceptions.ErrorCode;
 import org.apache.asterix.external.indexing.ExternalFile;
@@ -30,18 +27,13 @@
 import org.apache.asterix.metadata.entities.Dataset;
 import org.apache.asterix.metadata.entities.Index;
 import org.apache.asterix.om.types.ARecordType;
-import org.apache.asterix.runtime.utils.RuntimeUtils;
-import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.algebricks.common.utils.Pair;
 import org.apache.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
 import org.apache.hyracks.api.dataflow.value.ITypeTraits;
 import org.apache.hyracks.api.job.JobSpecification;
-import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
-import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
-import org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorDescriptor;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
+
+import java.util.List;
 
 public class IndexUtil {
 
@@ -97,27 +89,24 @@
         return empty;
     }
 
-    public static JobSpecification dropJob(Index index, MetadataProvider metadataProvider,
Dataset dataset)
+    public static JobSpecification buildDropIndexJobSpec(Index index, MetadataProvider metadataProvider,
+            Dataset dataset)
             throws AlgebricksException {
-        JobSpecification spec = RuntimeUtils.createJobSpecification();
-        IStorageComponentProvider storageComponentProvider = metadataProvider.getStorageComponentProvider();
-        Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint
=
-                metadataProvider.getSplitProviderAndConstraints(dataset, index.getIndexName());
-        Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo =
-                DatasetUtil.getMergePolicyFactory(dataset, metadataProvider.getMetadataTxnContext());
         ARecordType recordType =
                 (ARecordType) metadataProvider.findType(dataset.getItemTypeDataverseName(),
dataset.getItemTypeName());
         ARecordType metaType = DatasetUtil.getMetaType(metadataProvider, dataset);
-        IIndexDataflowHelperFactory dataflowHelperFactory = dataset.getIndexDataflowHelperFactory(metadataProvider,
-                index, recordType, metaType, compactionInfo.first, compactionInfo.second);
-        IndexDropOperatorDescriptor btreeDrop =
-                new IndexDropOperatorDescriptor(spec, storageComponentProvider.getStorageManager(),
-                        storageComponentProvider.getIndexLifecycleManagerProvider(), splitsAndConstraint.first,
-                        dataflowHelperFactory, storageComponentProvider.getMetadataPageManagerFactory());
-        AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, btreeDrop,
-                splitsAndConstraint.second);
-        spec.addRoot(btreeDrop);
-        return spec;
+        ARecordType enforcedType = null;
+        ARecordType enforcedMetaType = null;
+        if (index.isEnforcingKeyFileds()) {
+            Pair<ARecordType, ARecordType> enforcedTypes =
+                    TypeUtil.createEnforcedType(recordType, metaType, Lists.newArrayList(index));
+            enforcedType = enforcedTypes.first;
+            enforcedMetaType = enforcedTypes.second;
+        }
+        SecondaryIndexOperationsHelper secondaryIndexHelper =
+                SecondaryIndexOperationsHelper.createIndexOperationsHelper(dataset, index,
metadataProvider,
+                        physicalOptimizationConfig, recordType, metaType, enforcedType, enforcedMetaType);
+        return secondaryIndexHelper.buildDropJobSpec();
     }
 
     public static JobSpecification buildSecondaryIndexCreationJobSpec(Dataset dataset, Index
index,
@@ -146,31 +135,6 @@
                         physicalOptimizationConfig, recType, metaType, enforcedType, enforcedMetaType);
         secondaryIndexHelper.setExternalFiles(files);
         return secondaryIndexHelper.buildLoadingJobSpec();
-    }
-
-    public static JobSpecification buildDropSecondaryIndexJobSpec(Index index, MetadataProvider
metadataProvider,
-            Dataset dataset) throws AlgebricksException {
-        JobSpecification spec = RuntimeUtils.createJobSpecification();
-        IStorageComponentProvider storageComponentProvider = metadataProvider.getStorageComponentProvider();
-        Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint
=
-                metadataProvider.getSplitProviderAndConstraints(dataset, index.getIndexName());
-        Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo =
-                DatasetUtil.getMergePolicyFactory(dataset, metadataProvider.getMetadataTxnContext());
-        ARecordType recordType =
-                (ARecordType) metadataProvider.findType(dataset.getItemTypeDataverseName(),
dataset.getItemTypeName());
-        ARecordType metaType = DatasetUtil.getMetaType(metadataProvider, dataset);
-        IIndexDataflowHelperFactory dataflowHelperFactory = dataset.getIndexDataflowHelperFactory(metadataProvider,
-                index, recordType, metaType, compactionInfo.first, compactionInfo.second);
-        // The index drop operation should be persistent regardless of temp datasets or permanent
dataset.
-        IndexDropOperatorDescriptor btreeDrop =
-                new IndexDropOperatorDescriptor(spec, storageComponentProvider.getStorageManager(),
-                        storageComponentProvider.getIndexLifecycleManagerProvider(), splitsAndConstraint.first,
-                        dataflowHelperFactory, storageComponentProvider.getMetadataPageManagerFactory());
-        AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, btreeDrop,
-                splitsAndConstraint.second);
-        spec.addRoot(btreeDrop);
-
-        return spec;
     }
 
     public static JobSpecification buildSecondaryIndexCompactJobSpec(Dataset dataset, Index
index, ARecordType recType,
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
index 63368c7..59068c8 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
@@ -66,7 +66,7 @@
 import org.apache.hyracks.storage.common.file.ILocalResourceFactoryProvider;
 import org.apache.hyracks.storage.common.file.LocalResource;
 
-public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelper {
+public class SecondaryBTreeOperationsHelper extends SecondaryTreeIndexOperationsHelper {
 
     protected SecondaryBTreeOperationsHelper(Dataset dataset, Index index, PhysicalOptimizationConfig
physOptConf,
             IPropertiesProvider propertiesProvider, MetadataProvider metadataProvider, ARecordType
recType,
@@ -234,28 +234,6 @@
     @Override
     protected int getNumSecondaryKeys() {
         return index.getKeyFieldNames().size();
-    }
-
-    @Override
-    public JobSpecification buildCompactJobSpec() throws AlgebricksException {
-        JobSpecification spec = RuntimeUtils.createJobSpecification();
-        LSMTreeIndexCompactOperatorDescriptor compactOp;
-        IIndexDataflowHelperFactory dataflowHelperFactory = dataset.getIndexDataflowHelperFactory(metadataProvider,
-                index, itemType, metaType, mergePolicyFactory, mergePolicyFactoryProperties);
-        IStorageComponentProvider storageComponentProvider = metadataProvider.getStorageComponentProvider();
-        compactOp =
-                new LSMTreeIndexCompactOperatorDescriptor(spec, storageComponentProvider.getStorageManager(),
-                        storageComponentProvider.getIndexLifecycleManagerProvider(), secondaryFileSplitProvider,
-                        secondaryTypeTraits, secondaryComparatorFactories, secondaryBloomFilterKeyFields,
-                        dataflowHelperFactory,
-                        dataset.getModificationCallbackFactory(storageComponentProvider,
index, null,
-                                IndexOperation.FULL_MERGE, null),
-                        storageComponentProvider.getMetadataPageManagerFactory());
-        AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, compactOp,
-                secondaryPartitionConstraint);
-        spec.addRoot(compactOp);
-        spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
-        return spec;
     }
 
     @Override
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
index d731603..efb3315 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
@@ -185,6 +185,8 @@
 
     public abstract JobSpecification buildCompactJobSpec() throws AlgebricksException;
 
+    public abstract JobSpecification buildDropJobSpec() throws AlgebricksException;
+
     protected void init() throws AlgebricksException {
         payloadSerde = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(itemType);
         metaSerde =
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
index b86004a..5ab36c1 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
@@ -36,6 +36,7 @@
 import org.apache.asterix.runtime.utils.RuntimeUtils;
 import org.apache.asterix.transaction.management.resource.LSMInvertedIndexLocalResourceMetadataFactory;
 import org.apache.asterix.transaction.management.resource.PersistentLocalResourceFactoryProvider;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.algebricks.common.utils.Pair;
@@ -57,17 +58,22 @@
 import org.apache.hyracks.dataflow.common.data.marshalling.ShortSerializerDeserializer;
 import org.apache.hyracks.dataflow.std.base.AbstractOperatorDescriptor;
 import org.apache.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
+import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
 import org.apache.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
 import org.apache.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
 import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
 import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
 import org.apache.hyracks.storage.am.lsm.invertedindex.dataflow.BinaryTokenizerOperatorDescriptor;
 import org.apache.hyracks.storage.am.lsm.invertedindex.dataflow.LSMInvertedIndexBulkLoadOperatorDescriptor;
 import org.apache.hyracks.storage.am.lsm.invertedindex.dataflow.LSMInvertedIndexCompactOperator;
 import org.apache.hyracks.storage.am.lsm.invertedindex.dataflow.LSMInvertedIndexCreateOperatorDescriptor;
+import org.apache.hyracks.storage.am.lsm.invertedindex.dataflow.LSMInvertedIndexDropOperatorDescriptor;
 import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
 import org.apache.hyracks.storage.common.file.ILocalResourceFactoryProvider;
 import org.apache.hyracks.storage.common.file.LocalResource;
+
+import java.util.Map;
 
 public class SecondaryInvertedIndexOperationsHelper extends SecondaryIndexOperationsHelper
{
 
@@ -376,4 +382,27 @@
         spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
         return spec;
     }
+
+    @Override
+    public JobSpecification buildDropJobSpec() throws AlgebricksException {
+        JobSpecification spec = RuntimeUtils.createJobSpecification();
+        IStorageComponentProvider storageComponentProvider = metadataProvider.getStorageComponentProvider();
+        Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint
=
+                metadataProvider.getSplitProviderAndConstraints(dataset, index.getIndexName());
+        Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo =
+                DatasetUtil.getMergePolicyFactory(dataset, metadataProvider.getMetadataTxnContext());
+        ARecordType recordType =
+                (ARecordType) metadataProvider.findType(dataset.getItemTypeDataverseName(),
dataset.getItemTypeName());
+        ARecordType metaType = DatasetUtil.getMetaType(metadataProvider, dataset);
+        IIndexDataflowHelperFactory dataflowHelperFactory = dataset.getIndexDataflowHelperFactory(metadataProvider,
+                index, recordType, metaType, compactionInfo.first, compactionInfo.second);
+        LSMInvertedIndexDropOperatorDescriptor invIdxDrop =
+                new LSMInvertedIndexDropOperatorDescriptor(spec, storageComponentProvider.getStorageManager(),
+                        storageComponentProvider.getIndexLifecycleManagerProvider(), splitsAndConstraint.first,
+                        dataflowHelperFactory, storageComponentProvider.getMetadataPageManagerFactory());
+        AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, invIdxDrop,
+                splitsAndConstraint.second);
+        spec.addRoot(invIdxDrop);
+        return spec;
+    }
 }
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java
index 460b635..93a88e2 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java
@@ -72,7 +72,7 @@
 import org.apache.hyracks.storage.common.file.LocalResource;
 
 @SuppressWarnings("rawtypes")
-public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelper {
+public class SecondaryRTreeOperationsHelper extends SecondaryTreeIndexOperationsHelper {
 
     protected IPrimitiveValueProviderFactory[] valueProviderFactories;
     protected int numNestedSecondaryKeyFields;
@@ -429,26 +429,5 @@
             }
         }
         return fieldPermutation;
-    }
-
-    @Override
-    public JobSpecification buildCompactJobSpec() throws AsterixException, AlgebricksException
{
-        JobSpecification spec = RuntimeUtils.createJobSpecification();
-        IIndexDataflowHelperFactory indexDataflowHelperFactory = dataset.getIndexDataflowHelperFactory(
-                metadataProvider, index, itemType, metaType, mergePolicyFactory, mergePolicyFactoryProperties);
-        LSMTreeIndexCompactOperatorDescriptor compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
-                metadataProvider.getStorageComponentProvider().getStorageManager(),
-                metadataProvider.getStorageComponentProvider().getIndexLifecycleManagerProvider(),
-                secondaryFileSplitProvider, secondaryTypeTraits, secondaryComparatorFactories,
-                secondaryBloomFilterKeyFields, indexDataflowHelperFactory,
-                dataset.getModificationCallbackFactory(metadataProvider.getStorageComponentProvider(),
index, null,
-                        IndexOperation.FULL_MERGE, null),
-                metadataProvider.getStorageComponentProvider().getMetadataPageManagerFactory());
-
-        AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, compactOp,
-                secondaryPartitionConstraint);
-        spec.addRoot(compactOp);
-        spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
-        return spec;
     }
 }
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryTreeIndexOperationsHelper.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryTreeIndexOperationsHelper.java
new file mode 100644
index 0000000..2b025c9
--- /dev/null
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryTreeIndexOperationsHelper.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.metadata.utils;
+
+import org.apache.asterix.common.config.IPropertiesProvider;
+import org.apache.asterix.common.context.IStorageComponentProvider;
+import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.runtime.utils.RuntimeUtils;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.algebricks.core.jobgen.impl.ConnectorPolicyAssignmentPolicy;
+import org.apache.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
+import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
+import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
+import org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorDescriptor;
+import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
+import org.apache.hyracks.storage.am.lsm.common.dataflow.LSMTreeIndexCompactOperatorDescriptor;
+
+import java.util.Map;
+
+public abstract class SecondaryTreeIndexOperationsHelper extends SecondaryIndexOperationsHelper
{
+
+    protected SecondaryTreeIndexOperationsHelper(Dataset dataset,
+            Index index,
+            PhysicalOptimizationConfig physOptConf,
+            IPropertiesProvider propertiesProvider,
+            MetadataProvider metadataProvider,
+            ARecordType recType, ARecordType metaType,
+            ARecordType enforcedType,
+            ARecordType enforcedMetaType) {
+        super(dataset, index, physOptConf, propertiesProvider, metadataProvider, recType,
metaType, enforcedType,
+                enforcedMetaType);
+    }
+
+    @Override
+    public JobSpecification buildDropJobSpec() throws AlgebricksException {
+        JobSpecification spec = RuntimeUtils.createJobSpecification();
+        IStorageComponentProvider storageComponentProvider = metadataProvider.getStorageComponentProvider();
+        Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint
=
+                metadataProvider.getSplitProviderAndConstraints(dataset, index.getIndexName());
+        Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo =
+                DatasetUtil.getMergePolicyFactory(dataset, metadataProvider.getMetadataTxnContext());
+        ARecordType recordType =
+                (ARecordType) metadataProvider.findType(dataset.getItemTypeDataverseName(),
dataset.getItemTypeName());
+        ARecordType metaType = DatasetUtil.getMetaType(metadataProvider, dataset);
+        IIndexDataflowHelperFactory dataflowHelperFactory = dataset.getIndexDataflowHelperFactory(metadataProvider,
+                index, recordType, metaType, compactionInfo.first, compactionInfo.second);
+        // The index drop operation should be persistent regardless of temp datasets or permanent
dataset.
+        IndexDropOperatorDescriptor btreeDrop =
+                new IndexDropOperatorDescriptor(spec, storageComponentProvider.getStorageManager(),
+                        storageComponentProvider.getIndexLifecycleManagerProvider(), splitsAndConstraint.first,
+                        dataflowHelperFactory, storageComponentProvider.getMetadataPageManagerFactory());
+        AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, btreeDrop,
+                splitsAndConstraint.second);
+        spec.addRoot(btreeDrop);
+        return spec;
+    }
+
+    @Override
+    public JobSpecification buildCompactJobSpec() throws AlgebricksException {
+        JobSpecification spec = RuntimeUtils.createJobSpecification();
+        IIndexDataflowHelperFactory indexDataflowHelperFactory = dataset.getIndexDataflowHelperFactory(
+                metadataProvider, index, itemType, metaType, mergePolicyFactory, mergePolicyFactoryProperties);
+        LSMTreeIndexCompactOperatorDescriptor compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
+                metadataProvider.getStorageComponentProvider().getStorageManager(),
+                metadataProvider.getStorageComponentProvider().getIndexLifecycleManagerProvider(),
+                secondaryFileSplitProvider, secondaryTypeTraits, secondaryComparatorFactories,
+                secondaryBloomFilterKeyFields, indexDataflowHelperFactory,
+                dataset.getModificationCallbackFactory(metadataProvider.getStorageComponentProvider(),
index, null,
+                        IndexOperation.FULL_MERGE, null),
+                metadataProvider.getStorageComponentProvider().getMetadataPageManagerFactory());
+
+        AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, compactOp,
+                secondaryPartitionConstraint);
+        spec.addRoot(compactOp);
+        spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
+        return spec;
+    }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDropOperatorDescriptor.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDropOperatorDescriptor.java
new file mode 100644
index 0000000..fbbd088
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDropOperatorDescriptor.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.invertedindex.dataflow;
+
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
+import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
+import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
+import org.apache.hyracks.storage.am.common.api.IPageManagerFactory;
+import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
+import org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorNodePushable;
+import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
+import org.apache.hyracks.storage.common.IStorageManager;
+import org.apache.hyracks.storage.common.file.NoOpLocalResourceFactoryProvider;
+
+public class LSMInvertedIndexDropOperatorDescriptor extends AbstractLSMInvertedIndexOperatorDescriptor
{
+
+    private static final long serialVersionUID = 1L;
+
+    public LSMInvertedIndexDropOperatorDescriptor(IOperatorDescriptorRegistry spec, IStorageManager
storageManager,
+            IIndexLifecycleManagerProvider lifecycleManagerProvider, IFileSplitProvider fileSplitProvider,
+            IIndexDataflowHelperFactory dataflowHelperFactory, IPageManagerFactory pageManagerFactory)
{
+        super(spec, 0, 0, null, storageManager, fileSplitProvider, lifecycleManagerProvider,
new ITypeTraits[] {
+                        null },
+                new IBinaryComparatorFactory[] { null }, new ITypeTraits[] {
+                        null }, new IBinaryComparatorFactory[] { null }, null,
+                dataflowHelperFactory, null, false, false, null, NoOpLocalResourceFactoryProvider.INSTANCE,
+                NoOpOperationCallbackFactory.INSTANCE, NoOpOperationCallbackFactory.INSTANCE,
pageManagerFactory);
+    }
+
+    @Override
+    public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
+            IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions)
throws HyracksDataException {
+        return new IndexDropOperatorNodePushable(this, ctx, partition);
+    }
+}

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1600
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I2c0f52770fd791ab9293c661de810c0b3b9ba8fb
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <hubailmor@gmail.com>


Mime
View raw message