asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ima...@apache.org
Subject [09/10] incubator-asterixdb git commit: Temp dataset support: 1. DDLs for creating a temporary dataset 2. Garbage collection for temporary dataset 3. Reading, inserting, and deleting data from (to) a temporary dataset is locking-free (except metadata loc
Date Wed, 20 May 2015 01:25:50 GMT
Temp dataset support:
1. DDLs for creating a temporary dataset
2. Garbage collection for temporary dataset
3. Reading, inserting, and deleting data from (to) a temporary dataset is locking-free (except metadata locks) and logging-free (except flush/merge/job-commit logs).

Change-Id: Id0960acd1b4dfbfa7685dba227634572ca50b126
Reviewed-on: https://asterix-gerrit.ics.uci.edu/241
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Young-Seok Kim <kisskys@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/commit/ccd67fe8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/tree/ccd67fe8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/diff/ccd67fe8

Branch: refs/heads/master
Commit: ccd67fe832b1396a9d637eb34212e68f5d39ff19
Parents: f61bdb8
Author: buyingyi <buyingyi@gmail.com>
Authored: Tue May 19 11:53:39 2015 -0700
Committer: Ian Maxon <imaxon@uci.edu>
Committed: Tue May 19 17:24:27 2015 -0700

----------------------------------------------------------------------
 .../operators/physical/CommitPOperator.java     |  14 +-
 .../operators/physical/CommitRuntime.java       |  43 ++-
 .../physical/CommitRuntimeFactory.java          |   8 +-
 .../physical/InvertedIndexPOperator.java        |  15 +-
 .../rules/ReplaceSinkOpWithCommitOpRule.java    |   3 +-
 .../ics/asterix/api/common/APIFramework.java    |  18 +-
 .../asterix/aql/translator/AqlTranslator.java   |  24 +-
 .../uci/ics/asterix/file/DatasetOperations.java |  21 +-
 .../file/ExternalIndexingOperations.java        |  32 +-
 .../uci/ics/asterix/file/IndexOperations.java   |   7 +-
 .../file/SecondaryBTreeOperationsHelper.java    |  16 +-
 .../file/SecondaryIndexOperationsHelper.java    |  24 +-
 .../SecondaryInvertedIndexOperationsHelper.java |   6 +-
 .../file/SecondaryRTreeOperationsHelper.java    |  17 +-
 .../bootstrap/NCApplicationEntryPoint.java      |  27 +-
 .../basic/temp_dataset/temp_dataset.1.ddl.aql   |  34 +++
 .../temp_dataset/temp_dataset.2.update.aql      |   6 +
 .../basic/temp_dataset/temp_dataset.3.query.aql |  11 +
 .../basic/temp_dataset/temp_dataset.1.adm       |   1 +
 .../src/test/resources/metadata/testsuite.xml   |   5 +
 ...ete-from-loaded-dataset-with-index.1.ddl.aql |  34 +++
 ...-from-loaded-dataset-with-index.2.update.aql |  13 +
 ...ete-from-loaded-dataset-with-index.3.ddl.aql |  12 +
 ...-from-loaded-dataset-with-index.4.update.aql |  10 +
 ...e-from-loaded-dataset-with-index.5.query.aql |  13 +
 .../delete-from-loaded-dataset.1.ddl.aql        |  34 +++
 .../delete-from-loaded-dataset.2.update.aql     |  15 +
 .../delete-from-loaded-dataset.3.query.aql      |  12 +
 .../drop-empty-secondary-indexes.1.ddl.aql      |  57 ++++
 .../drop-empty-secondary-indexes.2.update.aql   |   7 +
 .../drop-empty-secondary-indexes.3.query.aql    |  10 +
 .../drop-index/drop-index.1.ddl.aql             |  33 ++
 .../drop-index/drop-index.2.update.aql          |  14 +
 .../drop-index/drop-index.3.ddl.aql             |  17 ++
 .../drop-index/drop-index.4.query.aql           |  12 +
 .../empty-load-with-index.1.ddl.aql             |  34 +++
 .../empty-load-with-index.2.update.aql          |  15 +
 .../empty-load-with-index.3.query.aql           |  12 +
 ...insert-and-scan-dataset-with-index.1.ddl.aql |  21 ++
 ...ert-and-scan-dataset-with-index.2.update.aql |  24 ++
 ...sert-and-scan-dataset-with-index.3.query.aql |  13 +
 .../insert-and-scan-dataset.1.ddl.aql           |  18 ++
 .../insert-and-scan-dataset.2.update.aql        |  21 ++
 .../insert-and-scan-dataset.3.query.aql         |  14 +
 .../q01_pricing_summary_report_nt.1.ddl.aql     |  33 ++
 .../q01_pricing_summary_report_nt.2.update.aql  |  12 +
 .../q01_pricing_summary_report_nt.3.query.aql   |  28 ++
 .../scan-delete-rtree-secondary-index.1.ddl.aql |  26 ++
 ...an-delete-rtree-secondary-index.2.update.aql |  12 +
 .../scan-delete-rtree-secondary-index.3.ddl.aql |  10 +
 ...an-delete-rtree-secondary-index.4.update.aql |  10 +
 ...can-delete-rtree-secondary-index.5.query.aql |  12 +
 .../scan-insert-persistent-to-temp.1.ddl.aql    |  35 +++
 .../scan-insert-persistent-to-temp.2.update.aql |  17 ++
 .../scan-insert-persistent-to-temp.3.ddl.aql    |  12 +
 .../scan-insert-persistent-to-temp.4.update.aql |  18 ++
 .../scan-insert-persistent-to-temp.5.query.aql  |  13 +
 .../scan-insert-rtree-secondary-index.1.ddl.aql |  35 +++
 ...an-insert-rtree-secondary-index.2.update.aql |  17 ++
 .../scan-insert-rtree-secondary-index.3.ddl.aql |  12 +
 ...an-insert-rtree-secondary-index.4.update.aql |  18 ++
 ...can-insert-rtree-secondary-index.5.query.aql |  13 +
 .../scan-insert-temp-to-persistent.1.ddl.aql    |  35 +++
 .../scan-insert-temp-to-persistent.2.update.aql |  17 ++
 .../scan-insert-temp-to-persistent.3.ddl.aql    |  12 +
 .../scan-insert-temp-to-persistent.4.update.aql |  18 ++
 .../scan-insert-temp-to-persistent.5.query.aql  |  13 +
 .../compact-dataset-and-its-indexes.1.adm       | 168 +++++++++++
 .../delete-from-loaded-dataset-with-index.1.adm | 168 +++++++++++
 .../delete-from-loaded-dataset.1.adm            |  26 ++
 .../drop-empty-secondary-indexes.1.adm          |   1 +
 .../temp-dataset/drop-index/drop-index.1.adm    |   2 +
 .../empty-load-with-index.1.adm                 |   2 +
 .../insert-and-scan-dataset-with-index.1.adm    | 241 +++++++++++++++
 .../insert-and-scan-dataset.1.adm               |  21 ++
 .../q01_pricing_summary_report_nt.1.adm         |   5 +
 .../scan-delete-rtree-secondary-index.1.adm     |   2 +
 .../scan-insert-persistent-to-temp.1.adm        |   4 +
 .../scan-insert-rtree-secondary-index.1.adm     |   4 +
 .../scan-insert-persistent-to-temp.1.adm        |   4 +
 .../src/test/resources/runtimets/testsuite.xml  |  62 ++++
 .../aql/expression/ExternalDetailsDecl.java     |   5 +
 .../asterix/aql/expression/FeedDetailsDecl.java |   2 +-
 .../aql/expression/IDatasetDetailsDecl.java     |   2 +
 .../aql/expression/InternalDetailsDecl.java     |  10 +-
 asterix-aql/src/main/javacc/AQL.html            | 300 ++++++++++---------
 asterix-aql/src/main/javacc/AQL.jj              |  10 +-
 .../asterix/common/config/DatasetConfig.java    |  12 +-
 .../common/context/DatasetLifecycleManager.java |   3 +-
 .../context/PrimaryIndexOperationTracker.java   |  11 +-
 .../uci/ics/asterix/test/aql/TestsUtils.java    |  18 +-
 .../dataflow/HDFSLookupAdapterFactory.java      |  11 +-
 .../primary_index_only.1.script.aql             |   1 +
 .../primary_index_only.2.ddl.aql                |  41 +++
 .../primary_index_only.3.update.aql             |  11 +
 .../primary_index_only.4.txneu.aql              |  22 ++
 .../primary_index_only.5.txnqbc.aql             |  10 +
 .../primary_index_only.6.script.aql             |   1 +
 .../primary_index_only.7.script.aql             |   1 +
 .../primary_index_only.8.txnqar.aql             |  10 +
 .../primary_index_only.9.script.aql             |   1 +
 ...ry_plus_default_secondary_index.1.script.aql |   1 +
 ...imary_plus_default_secondary_index.2.ddl.aql |  53 ++++
 ...ry_plus_default_secondary_index.3.update.aql |  11 +
 ...ary_plus_default_secondary_index.4.txneu.aql |  25 ++
 ...ry_plus_default_secondary_index.5.txnqbc.aql |  10 +
 ...ry_plus_default_secondary_index.6.script.aql |   1 +
 ...ry_plus_default_secondary_index.7.script.aql |   1 +
 ...ry_plus_default_secondary_index.8.txnqar.aql |  10 +
 ...ry_plus_default_secondary_index.9.script.aql |   1 +
 ...ry_plus_keyword_secondary_index.1.script.aql |   1 +
 ...imary_plus_keyword_secondary_index.2.ddl.aql |  54 ++++
 ...ry_plus_keyword_secondary_index.3.update.aql |  12 +
 ...ary_plus_keyword_secondary_index.4.txneu.aql |  25 ++
 ...ry_plus_keyword_secondary_index.5.txnqbc.aql |   5 +
 ...ry_plus_keyword_secondary_index.6.script.aql |   1 +
 ...ry_plus_keyword_secondary_index.7.script.aql |   1 +
 ...ry_plus_keyword_secondary_index.8.txnqar.aql |   5 +
 ...ry_plus_keyword_secondary_index.9.script.aql |   1 +
 ...plus_multiple_secondary_indices.1.script.aql |   1 +
 ...ry_plus_multiple_secondary_indices.2.ddl.aql |  60 ++++
 ...plus_multiple_secondary_indices.3.update.aql |  12 +
 ..._plus_multiple_secondary_indices.4.txneu.aql |  25 ++
 ...plus_multiple_secondary_indices.5.txnqbc.aql |  12 +
 ...plus_multiple_secondary_indices.6.script.aql |   1 +
 ...plus_multiple_secondary_indices.7.script.aql |   1 +
 ...plus_multiple_secondary_indices.8.txnqar.aql |   5 +
 ...plus_multiple_secondary_indices.9.script.aql |   1 +
 .../primary_plus_ngram_index.1.script.aql       |   1 +
 .../primary_plus_ngram_index.2.ddl.aql          |  53 ++++
 .../primary_plus_ngram_index.3.update.aql       |  11 +
 .../primary_plus_ngram_index.4.txneu.aql        |  25 ++
 .../primary_plus_ngram_index.5.txnqbc.aql       |  11 +
 .../primary_plus_ngram_index.6.script.aql       |   1 +
 .../primary_plus_ngram_index.7.script.aql       |   1 +
 .../primary_plus_ngram_index.8.txnqar.aql       |  11 +
 .../primary_plus_ngram_index.9.script.aql       |   1 +
 .../primary_plus_rtree_index.1.script.aql       |   1 +
 .../primary_plus_rtree_index.2.ddl.aql          |  54 ++++
 .../primary_plus_rtree_index.3.update.aql       |  12 +
 .../primary_plus_rtree_index.4.txneu.aql        |  25 ++
 .../primary_plus_rtree_index.5.txnqbc.aql       |   5 +
 .../primary_plus_rtree_index.6.script.aql       |   1 +
 .../primary_plus_rtree_index.7.script.aql       |   1 +
 .../primary_plus_rtree_index.8.txnqar.aql       |   5 +
 .../primary_plus_rtree_index.9.script.aql       |   1 +
 .../dataset_recovery.1.script.aql               |   1 +
 .../dataset_recovery.2.ddl.aql                  |  19 ++
 .../dataset_recovery.3.script.aql               |   1 +
 .../dataset_recovery.4.script.aql               |   1 +
 .../dataset_recovery.5.errddl.aql               |   2 +
 .../dataset_recovery.6.script.aql               |   1 +
 .../delete_after_recovery.1.script.aql          |   1 +
 .../delete_after_recovery.2.ddl.aql             |  41 +++
 .../delete_after_recovery.3.update.aql          |  11 +
 .../delete_after_recovery.4.script.aql          |   1 +
 .../delete_after_recovery.5.script.aql          |   1 +
 .../delete_after_recovery.6.update.aql          |   5 +
 .../delete_after_recovery.7.query.aql           |  11 +
 .../delete_after_recovery.8.script.aql          |   1 +
 .../insert_after_recovery.1.script.aql          |   1 +
 .../insert_after_recovery.2.ddl.aql             |  41 +++
 .../insert_after_recovery.3.update.aql          |  11 +
 .../insert_after_recovery.4.script.aql          |   1 +
 .../insert_after_recovery.5.script.aql          |   1 +
 .../insert_after_recovery.6.update.aql          |  15 +
 .../insert_after_recovery.7.query.aql           |  11 +
 .../insert_after_recovery.8.script.aql          |   1 +
 .../load_after_recovery.1.script.aql            |   1 +
 .../load_after_recovery.2.ddl.aql               |  41 +++
 .../load_after_recovery.3.script.aql            |   1 +
 .../load_after_recovery.4.script.aql            |   1 +
 .../load_after_recovery.5.update.aql            |  11 +
 .../load_after_recovery.6.query.aql             |  10 +
 .../load_after_recovery.7.script.aql            |   1 +
 .../secondary_index_recovery.1.script.aql       |   1 +
 .../secondary_index_recovery.2.ddl.aql          |  21 ++
 .../secondary_index_recovery.3.script.aql       |   1 +
 .../secondary_index_recovery.4.script.aql       |   1 +
 .../secondary_index_recovery.5.errddl.aql       |   2 +
 .../secondary_index_recovery.6.script.aql       |   1 +
 .../delete_after_recovery.1.adm                 |   2 +
 .../insert_after_recovery.1.adm                 |   2 +
 .../load_after_recovery.1.adm                   |   2 +
 .../temp_primary_index_only/create_and_start.sh |   1 +
 .../temp_primary_index_only/kill_cc_and_nc.sh   |   1 +
 .../temp_primary_index_only/stop_and_delete.sh  |   3 +
 .../temp_primary_index_only/stop_and_start.sh   |   3 +
 .../create_and_start.sh                         |   1 +
 .../kill_cc_and_nc.sh                           |   1 +
 .../stop_and_delete.sh                          |   3 +
 .../stop_and_start.sh                           |   3 +
 .../create_and_start.sh                         |   1 +
 .../kill_cc_and_nc.sh                           |   1 +
 .../stop_and_delete.sh                          |   3 +
 .../stop_and_start.sh                           |   3 +
 .../create_and_start.sh                         |   1 +
 .../kill_cc_and_nc.sh                           |   1 +
 .../stop_and_delete.sh                          |   3 +
 .../stop_and_start.sh                           |   3 +
 .../create_and_start.sh                         |   1 +
 .../kill_cc_and_nc.sh                           |   1 +
 .../stop_and_delete.sh                          |   3 +
 .../stop_and_start.sh                           |   3 +
 .../create_and_start.sh                         |   1 +
 .../kill_cc_and_nc.sh                           |   1 +
 .../stop_and_delete.sh                          |   3 +
 .../stop_and_start.sh                           |   3 +
 .../temp_dataset_recovery/create_and_start.sh   |   1 +
 .../temp_dataset_recovery/kill_cc_and_nc.sh     |   1 +
 .../temp_dataset_recovery/stop_and_delete.sh    |   3 +
 .../temp_dataset_recovery/stop_and_start.sh     |   3 +
 .../create_and_start.sh                         |   1 +
 .../kill_cc_and_nc.sh                           |   1 +
 .../stop_and_delete.sh                          |   3 +
 .../stop_and_start.sh                           |   3 +
 .../create_and_start.sh                         |   1 +
 .../kill_cc_and_nc.sh                           |   1 +
 .../stop_and_delete.sh                          |   3 +
 .../stop_and_start.sh                           |   3 +
 .../create_and_start.sh                         |   1 +
 .../temp_load_after_recovery/kill_cc_and_nc.sh  |   1 +
 .../temp_load_after_recovery/stop_and_delete.sh |   3 +
 .../temp_load_after_recovery/stop_and_start.sh  |   3 +
 .../create_and_start.sh                         |   1 +
 .../kill_cc_and_nc.sh                           |   1 +
 .../stop_and_delete.sh                          |   3 +
 .../stop_and_start.sh                           |   3 +
 .../test/resources/transactionts/testsuite.xml  |  95 +++++-
 .../ics/asterix/metadata/GarbageCollector.java  |  39 +++
 .../ics/asterix/metadata/IDatasetDetails.java   |  25 ++
 .../uci/ics/asterix/metadata/MetadataCache.java | 103 +++++--
 .../ics/asterix/metadata/MetadataManager.java   | 111 +++++--
 .../metadata/MetadataTransactionContext.java    |   3 +-
 .../asterix/metadata/api/IMetadataManager.java  |  24 +-
 .../metadata/bootstrap/MetadataBootstrap.java   |   8 +-
 .../metadata/declared/AqlMetadataProvider.java  | 136 +++++----
 .../entities/ExternalDatasetDetails.java        |  12 +
 .../entities/InternalDatasetDetails.java        |  17 +-
 .../DatasetTupleTranslator.java                 |  11 +-
 .../ExternalLoopkupOperatorDiscriptor.java      |   9 +-
 ...tasetIndexModificationOperationCallback.java |  51 ++++
 ...dexModificationOperationCallbackFactory.java |  67 +++++
 ...dexModificationOperationCallbackFactory.java |  67 +++++
 .../ExternalBTreeLocalResourceMetadata.java     |  19 +-
 ...rnalBTreeWithBuddyLocalResourceMetadata.java |  21 +-
 .../ExternalRTreeLocalResourceMetadata.java     |  23 +-
 .../resource/LSMBTreeLocalResourceMetadata.java |  24 +-
 .../LSMInvertedIndexLocalResourceMetadata.java  |  12 +-
 .../resource/LSMRTreeLocalResourceMetadata.java |  24 +-
 .../management/service/locking/LockManager.java |  31 +-
 .../management/service/logging/LogPage.java     |   4 +-
 .../service/transaction/TransactionContext.java |  35 ++-
 253 files changed, 3832 insertions(+), 478 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/ccd67fe8/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitPOperator.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitPOperator.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitPOperator.java
index 18c0793..ee71851 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitPOperator.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitPOperator.java
@@ -18,6 +18,7 @@ package edu.uci.ics.asterix.algebra.operators.physical;
 import java.util.List;
 
 import edu.uci.ics.asterix.common.transactions.JobId;
+import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
 import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
@@ -29,7 +30,6 @@ import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSc
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AbstractPhysicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
 import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
 import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
 import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
@@ -39,14 +39,11 @@ public class CommitPOperator extends AbstractPhysicalOperator {
     private final List<LogicalVariable> primaryKeyLogicalVars;
     private final JobId jobId;
     private final int datasetId;
-    private final boolean isWriteTransaction;
 
-    public CommitPOperator(JobId jobId, int datasetId, List<LogicalVariable> primaryKeyLogicalVars,
-            boolean isWriteTransaction) {
+    public CommitPOperator(JobId jobId, int datasetId, List<LogicalVariable> primaryKeyLogicalVars) {
         this.jobId = jobId;
         this.datasetId = datasetId;
         this.primaryKeyLogicalVars = primaryKeyLogicalVars;
-        this.isWriteTransaction = isWriteTransaction;
     }
 
     @Override
@@ -69,7 +66,7 @@ public class CommitPOperator extends AbstractPhysicalOperator {
     public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context)
             throws AlgebricksException {
         AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-        deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
+        deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
     }
 
     @Override
@@ -80,7 +77,10 @@ public class CommitPOperator extends AbstractPhysicalOperator {
         RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema,
                 context);
         int[] primaryKeyFields = JobGenHelper.variablesToFieldIndexes(primaryKeyLogicalVars, inputSchemas[0]);
-        CommitRuntimeFactory runtime = new CommitRuntimeFactory(jobId, datasetId, primaryKeyFields, isWriteTransaction);
+
+        AqlMetadataProvider metadataProvider = (AqlMetadataProvider) context.getMetadataProvider();
+        CommitRuntimeFactory runtime = new CommitRuntimeFactory(jobId, datasetId, primaryKeyFields,
+                metadataProvider.isTemporaryDatasetWriteJob(), metadataProvider.isWriteTransaction());
         builder.contributeMicroOperator(op, runtime, recDesc);
         ILogicalOperator src = op.getInputs().get(0).getValue();
         builder.contributeGraphEdge(src, 0, op, 0);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/ccd67fe8/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitRuntime.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitRuntime.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitRuntime.java
index af83b83..bed18fe 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitRuntime.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitRuntime.java
@@ -44,17 +44,17 @@ public class CommitRuntime implements IPushRuntime {
     private final JobId jobId;
     private final int datasetId;
     private final int[] primaryKeyFields;
+    private final boolean isTemporaryDatasetWriteJob;
     private final boolean isWriteTransaction;
     private final long[] longHashes;
     private final LogRecord logRecord;
 
     private ITransactionContext transactionContext;
-    private RecordDescriptor inputRecordDesc;
     private FrameTupleAccessor frameTupleAccessor;
-    private FrameTupleReference frameTupleReference;
+    private final FrameTupleReference frameTupleReference;
 
     public CommitRuntime(IHyracksTaskContext ctx, JobId jobId, int datasetId, int[] primaryKeyFields,
-            boolean isWriteTransaction) {
+            boolean isTemporaryDatasetWriteJob, boolean isWriteTransaction) {
         this.hyracksTaskCtx = ctx;
         IAsterixAppRuntimeContext runtimeCtx = (IAsterixAppRuntimeContext) ctx.getJobletContext()
                 .getApplicationContext().getApplicationObject();
@@ -64,6 +64,7 @@ public class CommitRuntime implements IPushRuntime {
         this.datasetId = datasetId;
         this.primaryKeyFields = primaryKeyFields;
         this.frameTupleReference = new FrameTupleReference();
+        this.isTemporaryDatasetWriteJob = isTemporaryDatasetWriteJob;
         this.isWriteTransaction = isWriteTransaction;
         this.longHashes = new long[2];
         this.logRecord = new LogRecord();
@@ -85,14 +86,28 @@ public class CommitRuntime implements IPushRuntime {
         frameTupleAccessor.reset(buffer);
         int nTuple = frameTupleAccessor.getTupleCount();
         for (int t = 0; t < nTuple; t++) {
-            frameTupleReference.reset(frameTupleAccessor, t);
-            pkHash = computePrimaryKeyHashValue(frameTupleReference, primaryKeyFields);
-            logRecord.formEntityCommitLogRecord(transactionContext, datasetId, pkHash, frameTupleReference,
-                    primaryKeyFields);
-            try {
-                logMgr.log(logRecord);
-            } catch (ACIDException e) {
-                throw new HyracksDataException(e);
+            if (isTemporaryDatasetWriteJob) {
+                /**
+                 * This "if branch" is for writes over temporary datasets.
+                 * A temporary dataset does not require any lock and does not generate any write-ahead
+                 * update and commit log but generates flush log and job commit log.
+                 * However, a temporary dataset still MUST guarantee no-steal policy so that this
+                 * notification call should be delivered to PrimaryIndexOptracker and used correctly in order
+                 * to decrement number of active operation count of PrimaryIndexOptracker.
+                 * By maintaining the count correctly and only allowing flushing when the count is 0, it can
+                 * guarantee the no-steal policy for temporary datasets, too.
+                 */
+                transactionContext.notifyOptracker(false);
+            } else {
+                frameTupleReference.reset(frameTupleAccessor, t);
+                pkHash = computePrimaryKeyHashValue(frameTupleReference, primaryKeyFields);
+                logRecord.formEntityCommitLogRecord(transactionContext, datasetId, pkHash, frameTupleReference,
+                        primaryKeyFields);
+                try {
+                    logMgr.log(logRecord);
+                } catch (ACIDException e) {
+                    throw new HyracksDataException(e);
+                }
             }
         }
     }
@@ -104,13 +119,12 @@ public class CommitRuntime implements IPushRuntime {
 
     @Override
     public void fail() throws HyracksDataException {
-        // TODO Auto-generated method stub
 
     }
 
     @Override
     public void close() throws HyracksDataException {
-        // TODO Auto-generated method stub
+
     }
 
     @Override
@@ -120,7 +134,6 @@ public class CommitRuntime implements IPushRuntime {
 
     @Override
     public void setInputRecordDescriptor(int index, RecordDescriptor recordDescriptor) {
-        this.inputRecordDesc = recordDescriptor;
         this.frameTupleAccessor = new FrameTupleAccessor(hyracksTaskCtx.getFrameSize(), recordDescriptor);
     }
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/ccd67fe8/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitRuntimeFactory.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitRuntimeFactory.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitRuntimeFactory.java
index 6776f28..b996630 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitRuntimeFactory.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitRuntimeFactory.java
@@ -28,12 +28,15 @@ public class CommitRuntimeFactory implements IPushRuntimeFactory {
     private final JobId jobId;
     private final int datasetId;
     private final int[] primaryKeyFields;
+    private final boolean isTemporaryDatasetWriteJob;
     private final boolean isWriteTransaction;
 
-    public CommitRuntimeFactory(JobId jobId, int datasetId, int[] primaryKeyFields, boolean isWriteTransaction) {
+    public CommitRuntimeFactory(JobId jobId, int datasetId, int[] primaryKeyFields, boolean isTemporaryDatasetWriteJob,
+            boolean isWriteTransaction) {
         this.jobId = jobId;
         this.datasetId = datasetId;
         this.primaryKeyFields = primaryKeyFields;
+        this.isTemporaryDatasetWriteJob = isTemporaryDatasetWriteJob;
         this.isWriteTransaction = isWriteTransaction;
     }
 
@@ -44,6 +47,7 @@ public class CommitRuntimeFactory implements IPushRuntimeFactory {
 
     @Override
     public IPushRuntime createPushRuntime(IHyracksTaskContext ctx) throws AlgebricksException {
-        return new CommitRuntime(ctx, jobId, datasetId, primaryKeyFields, isWriteTransaction);
+        return new CommitRuntime(ctx, jobId, datasetId, primaryKeyFields, isTemporaryDatasetWriteJob,
+                isWriteTransaction);
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/ccd67fe8/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java
index ff21aff..67be923 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java
@@ -172,10 +172,12 @@ public class InvertedIndexPOperator extends IndexSearchPOperator {
                 throw new AlgebricksException("Only record types can be indexed.");
             }
             ARecordType recordType = (ARecordType) itemType;
-            Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(secondaryKeyTypeEntries.get(0), secondaryKeyFieldEntries.get(0), recordType);
+            Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(secondaryKeyTypeEntries.get(0),
+                    secondaryKeyFieldEntries.get(0), recordType);
             IAType secondaryKeyType = keyPairType.first;
             if (secondaryKeyType == null) {
-                throw new AlgebricksException("Could not find field " + secondaryKeyFieldEntries.get(0) + " in the schema.");
+                throw new AlgebricksException("Could not find field " + secondaryKeyFieldEntries.get(0)
+                        + " in the schema.");
             }
 
             // TODO: For now we assume the type of the generated tokens is the
@@ -236,8 +238,8 @@ public class InvertedIndexPOperator extends IndexSearchPOperator {
 
             IAsterixApplicationContextInfo appContext = (IAsterixApplicationContextInfo) context.getAppContext();
             Pair<IFileSplitProvider, AlgebricksPartitionConstraint> secondarySplitsAndConstraint = metadataProvider
-                    .splitProviderAndPartitionConstraintsForDataset(dataset.getDataverseName(),
-                            datasetName, indexName);
+                    .splitProviderAndPartitionConstraintsForDataset(dataset.getDataverseName(), datasetName, indexName,
+                            dataset.getDatasetDetails().isTemp());
             // TODO: Here we assume there is only one search key field.
             int queryField = keyFields[0];
             // Get tokenizer and search modifier factories.
@@ -250,6 +252,7 @@ public class InvertedIndexPOperator extends IndexSearchPOperator {
             AsterixStorageProperties storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
             Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(
                     dataset, metadataProvider.getMetadataTxnContext());
+            boolean temp = dataset.getDatasetDetails().isTemp();
             if (!isPartitioned) {
                 dataflowHelperFactory = new LSMInvertedIndexDataflowHelperFactory(
                         new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()), compactionInfo.first,
@@ -258,7 +261,7 @@ public class InvertedIndexPOperator extends IndexSearchPOperator {
                         LSMInvertedIndexIOOperationCallbackFactory.INSTANCE,
                         storageProperties.getBloomFilterFalsePositiveRate(), invertedIndexFields, filterTypeTraits,
                         filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps,
-                        invertedIndexFieldsForNonBulkLoadOps);
+                        invertedIndexFieldsForNonBulkLoadOps, !temp);
             } else {
                 dataflowHelperFactory = new PartitionedLSMInvertedIndexDataflowHelperFactory(
                         new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()), compactionInfo.first,
@@ -267,7 +270,7 @@ public class InvertedIndexPOperator extends IndexSearchPOperator {
                         LSMInvertedIndexIOOperationCallbackFactory.INSTANCE,
                         storageProperties.getBloomFilterFalsePositiveRate(), invertedIndexFields, filterTypeTraits,
                         filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps,
-                        invertedIndexFieldsForNonBulkLoadOps);
+                        invertedIndexFieldsForNonBulkLoadOps, !temp);
             }
             LSMInvertedIndexSearchOperatorDescriptor invIndexSearchOp = new LSMInvertedIndexSearchOperatorDescriptor(
                     jobSpec, queryField, appContext.getStorageManagerInterface(), secondarySplitsAndConstraint.first,

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/ccd67fe8/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java
index ef9d6cd..8de761a 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java
@@ -99,8 +99,7 @@ public class ReplaceSinkOpWithCommitOpRule implements IAlgebraicRewriteRule {
 
         //create the logical and physical operator
         CommitOperator commitOperator = new CommitOperator(primaryKeyLogicalVars);
-        CommitPOperator commitPOperator = new CommitPOperator(jobId, datasetId, primaryKeyLogicalVars,
-                mp.isWriteTransaction());
+        CommitPOperator commitPOperator = new CommitPOperator(jobId, datasetId, primaryKeyLogicalVars);
         commitOperator.setPhysicalOperator(commitPOperator);
 
         //create ExtensionOperator and put the commitOperator in it.

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/ccd67fe8/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/APIFramework.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/APIFramework.java b/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/APIFramework.java
index 51fef41..0ce15ed 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/APIFramework.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/APIFramework.java
@@ -72,7 +72,6 @@ import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
 import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IOptimizationContextFactory;
 import edu.uci.ics.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
 import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
-import edu.uci.ics.hyracks.api.job.IJobletEventListenerFactory;
 import edu.uci.ics.hyracks.api.job.JobId;
 import edu.uci.ics.hyracks.api.job.JobSpecification;
 
@@ -156,8 +155,7 @@ public class APIFramework {
     }
 
     public static Pair<Query, Integer> reWriteQuery(List<FunctionDecl> declaredFunctions,
-            AqlMetadataProvider metadataProvider, Query q, SessionConfig conf)
-            throws AsterixException {
+            AqlMetadataProvider metadataProvider, Query q, SessionConfig conf) throws AsterixException {
 
         if (conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_EXPR_TREE)) {
             conf.out().println();
@@ -185,8 +183,8 @@ public class APIFramework {
 
     public static JobSpecification compileQuery(List<FunctionDecl> declaredFunctions,
             AqlMetadataProvider queryMetadataProvider, Query rwQ, int varCounter, String outputDatasetName,
-            SessionConfig conf, ICompiledDmlStatement statement)
-            throws AsterixException, AlgebricksException, JSONException, RemoteException, ACIDException {
+            SessionConfig conf, ICompiledDmlStatement statement) throws AsterixException, AlgebricksException,
+            JSONException, RemoteException, ACIDException {
 
         if (conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_REWRITTEN_EXPR_TREE)) {
             conf.out().println();
@@ -219,7 +217,6 @@ public class APIFramework {
         } else {
             plan = t.translateLoad();
         }
-        boolean isWriteTransaction = queryMetadataProvider.isWriteTransaction();
 
         LogicalOperatorPrettyPrintVisitor pvisitor = new LogicalOperatorPrettyPrintVisitor();
         if (conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_LOGICAL_PLAN)) {
@@ -340,8 +337,8 @@ public class APIFramework {
         builder.setTypeTraitProvider(format.getTypeTraitProvider());
         builder.setNormalizedKeyComputerFactoryProvider(format.getNormalizedKeyComputerFactoryProvider());
 
-        IJobletEventListenerFactory jobEventListenerFactory = new JobEventListenerFactory(asterixJobId,
-                isWriteTransaction);
+        JobEventListenerFactory jobEventListenerFactory = new JobEventListenerFactory(asterixJobId,
+                queryMetadataProvider.isWriteTransaction());
         JobSpecification spec = compiler.createJob(AsterixAppContextInfo.getInstance(), jobEventListenerFactory);
 
         if (conf.is(SessionConfig.OOB_HYRACKS_JOB)) {
@@ -365,7 +362,7 @@ public class APIFramework {
     }
 
     public static void executeJobArray(IHyracksClientConnection hcc, JobSpecification[] specs, PrintWriter out)
-        throws Exception {
+            throws Exception {
         for (int i = 0; i < specs.length; i++) {
             specs[i].setMaxReattempts(0);
             JobId jobId = hcc.startJob(specs[i]);
@@ -378,8 +375,7 @@ public class APIFramework {
 
     }
 
-    public static void executeJobArray(IHyracksClientConnection hcc, Job[] jobs, PrintWriter out)
-            throws Exception {
+    public static void executeJobArray(IHyracksClientConnection hcc, Job[] jobs, PrintWriter out) throws Exception {
         for (int i = 0; i < jobs.length; i++) {
             jobs[i].getJobSpec().setMaxReattempts(0);
             long startTime = System.currentTimeMillis();

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/ccd67fe8/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
index 4c47c7d..e5ddf2b 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
@@ -159,7 +159,6 @@ import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
 import edu.uci.ics.hyracks.api.dataset.IHyracksDataset;
 import edu.uci.ics.hyracks.api.dataset.ResultSetId;
 import edu.uci.ics.hyracks.api.io.FileReference;
-import edu.uci.ics.hyracks.api.job.IJobletEventListenerFactory;
 import edu.uci.ics.hyracks.api.job.JobId;
 import edu.uci.ics.hyracks.api.job.JobSpecification;
 import edu.uci.ics.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
@@ -210,7 +209,7 @@ public class AqlTranslator extends AbstractAqlTranslator {
 
     /**
      * Compiles and submits for execution a list of AQL statements.
-     *
+     * 
      * @param hcc
      *            A Hyracks client connection that is used to submit a jobspec to Hyracks.
      * @param hdc
@@ -478,6 +477,7 @@ public class AqlTranslator extends AbstractAqlTranslator {
         String compactionPolicy = dd.getDatasetDetailsDecl().getCompactionPolicy();
         Map<String, String> compactionPolicyProperties = dd.getDatasetDetailsDecl().getCompactionPolicyProperties();
         boolean defaultCompactionPolicy = (compactionPolicy == null);
+        boolean temp = dd.getDatasetDetailsDecl().isTemp();
 
         MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
         boolean bActiveTxn = true;
@@ -539,7 +539,7 @@ public class AqlTranslator extends AbstractAqlTranslator {
                     datasetDetails = new InternalDatasetDetails(InternalDatasetDetails.FileStructure.BTREE,
                             InternalDatasetDetails.PartitioningStrategy.HASH, partitioningExprs, partitioningExprs,
                             partitioningTypes, ngName, autogenerated, compactionPolicy, compactionPolicyProperties,
-                            filterField);
+                            filterField, temp);
                     break;
                 }
                 case EXTERNAL: {
@@ -2532,8 +2532,10 @@ public class AqlTranslator extends AbstractAqlTranslator {
                 throw new AlgebricksException("Tried to access non-existing dataset: " + datasetNameFrom);
             }
 
+            Dataset datasetFrom = MetadataManager.INSTANCE.getDataset(mdTxnCtx, dataverseNameFrom, datasetNameFrom);
             IFileSplitProvider fromSplits = metadataProvider.splitProviderAndPartitionConstraintsForDataset(
-                    dataverseNameFrom, datasetNameFrom, fromIndex.getIndexName()).first;
+                    dataverseNameFrom, datasetNameFrom, fromIndex.getIndexName(), datasetFrom.getDatasetDetails()
+                            .isTemp()).first;
             StringBuilder fromSplitsPaths = new StringBuilder();
 
             for (FileSplit f : fromSplits.getFileSplits()) {
@@ -2555,8 +2557,9 @@ public class AqlTranslator extends AbstractAqlTranslator {
                 throw new AlgebricksException("Tried to access non-existing dataset: " + datasetNameTo);
             }
 
+            Dataset datasetTo = MetadataManager.INSTANCE.getDataset(mdTxnCtx, dataverseNameTo, datasetNameTo);
             IFileSplitProvider toSplits = metadataProvider.splitProviderAndPartitionConstraintsForDataset(
-                    dataverseNameTo, datasetNameTo, toIndex.getIndexName()).first;
+                    dataverseNameTo, datasetNameTo, toIndex.getIndexName(), datasetTo.getDatasetDetails().isTemp()).first;
             StringBuilder toSplitsPaths = new StringBuilder();
 
             for (FileSplit f : toSplits.getFileSplits()) {
@@ -2573,7 +2576,8 @@ public class AqlTranslator extends AbstractAqlTranslator {
 
                 IDatasetDetailsDecl idd = new InternalDetailsDecl(new Identifier(toDataset.getDatasetDetails()
                         .getNodeGroupName()), toIndex.getKeyFieldNames(), false, toDataset.getDatasetDetails()
-                        .getCompactionPolicy(), toDataset.getDatasetDetails().getCompactionPolicyProperties(), null);
+                        .getCompactionPolicy(), toDataset.getDatasetDetails().getCompactionPolicyProperties(), null,
+                        toDataset.getDatasetDetails().isTemp());
                 DatasetDecl createToDataset = new DatasetDecl(new Identifier(dataverseNameTo),
                         pregelixStmt.getDatasetNameTo(), new Identifier(toDataset.getItemTypeName()),
                         toDataset.getHints(), toDataset.getDatasetType(), idd, false);
@@ -2691,19 +2695,21 @@ public class AqlTranslator extends AbstractAqlTranslator {
                 new IPushRuntimeFactory[] { new EmptyTupleSourceRuntimeFactory() }, rDescs);
 
         edu.uci.ics.asterix.common.transactions.JobId jobId = JobIdFactory.generateJobId();
+        Dataset dataset = MetadataManager.INSTANCE.getDataset(mdTxnCtx, dataverseName, datasetName);
         FlushDatasetOperatorDescriptor flushOperator = new FlushDatasetOperatorDescriptor(spec, jobId,
-                MetadataManager.INSTANCE.getDataset(mdTxnCtx, dataverseName, datasetName).getDatasetId());
+                dataset.getDatasetId());
 
         spec.connect(new OneToOneConnectorDescriptor(spec), emptySource, 0, flushOperator, 0);
 
         Pair<IFileSplitProvider, AlgebricksPartitionConstraint> primarySplitsAndConstraint = metadataProvider
-                .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, indexName);
+                .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, indexName, dataset
+                        .getDatasetDetails().isTemp());
         AlgebricksPartitionConstraint primaryPartitionConstraint = primarySplitsAndConstraint.second;
 
         AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, emptySource,
                 primaryPartitionConstraint);
 
-        IJobletEventListenerFactory jobEventListenerFactory = new JobEventListenerFactory(jobId, false);
+        JobEventListenerFactory jobEventListenerFactory = new JobEventListenerFactory(jobId, true);
         spec.setJobletEventListenerFactory(jobEventListenerFactory);
         runJob(hcc, spec, true);
     }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/ccd67fe8/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
index 7a9e1b2..b2da545 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
@@ -86,6 +86,7 @@ public class DatasetOperations {
         if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
             return JobSpecificationUtils.createJobSpecification();
         }
+        boolean temp = dataset.getDatasetDetails().isTemp();
 
         Dataverse dataverse = MetadataManager.INSTANCE.getDataverse(metadataProvider.getMetadataTxnContext(),
                 dataverseName);
@@ -106,11 +107,13 @@ public class DatasetOperations {
         JobSpecification specPrimary = JobSpecificationUtils.createJobSpecification();
 
         Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadataProvider
-                .splitProviderAndPartitionConstraintsForDataset(dataset.getDataverseName(), datasetName,
-                        datasetName);
+                .splitProviderAndPartitionConstraintsForDataset(dataset.getDataverseName(), datasetName, datasetName,
+                        temp);
         AsterixStorageProperties storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
         Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(dataset,
                 metadataProvider.getMetadataTxnContext());
+
+        // The index drop operation should be persistent regardless of temp datasets or permanent dataset
         IndexDropOperatorDescriptor primaryBtreeDrop = new IndexDropOperatorDescriptor(specPrimary,
                 AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
                 splitsAndConstraint.first, new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(
@@ -118,7 +121,7 @@ public class DatasetOperations {
                         new PrimaryIndexOperationTrackerProvider(dataset.getDatasetId()),
                         AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
                         storageProperties.getBloomFilterFalsePositiveRate(), true, filterTypeTraits,
-                        filterCmpFactories, btreeFields, filterFields));
+                        filterCmpFactories, btreeFields, filterFields, true));
         AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(specPrimary, primaryBtreeDrop,
                 splitsAndConstraint.second);
 
@@ -140,6 +143,7 @@ public class DatasetOperations {
         if (dataset == null) {
             throw new AsterixException("Could not find dataset " + datasetName + " in dataverse " + dataverseName);
         }
+        boolean temp = dataset.getDatasetDetails().isTemp();
         ARecordType itemType = (ARecordType) metadata.findType(dataverseName, dataset.getItemTypeName());
         JobSpecification spec = JobSpecificationUtils.createJobSpecification();
         IBinaryComparatorFactory[] comparatorFactories = DatasetUtils.computeKeysBinaryComparatorFactories(dataset,
@@ -154,7 +158,7 @@ public class DatasetOperations {
         int[] btreeFields = DatasetUtils.createBTreeFieldsWhenThereisAFilter(dataset);
 
         Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadata
-                .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, datasetName);
+                .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, datasetName, temp);
         FileSplit[] fs = splitsAndConstraint.first.getFileSplits();
         StringBuilder sb = new StringBuilder();
         for (int i = 0; i < fs.length; i++) {
@@ -172,6 +176,7 @@ public class DatasetOperations {
         ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
                 localResourceMetadata, LocalResource.LSMBTreeResource);
 
+        // The index create operation should be persistent regardless of temp datasets or permanent dataset
         TreeIndexCreateOperatorDescriptor indexCreateOp = new TreeIndexCreateOperatorDescriptor(spec,
                 AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
                 splitsAndConstraint.first, typeTraits, comparatorFactories, bloomFilterKeyFields,
@@ -180,7 +185,7 @@ public class DatasetOperations {
                                 .getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
                         LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties
                                 .getBloomFilterFalsePositiveRate(), true, filterTypeTraits, filterCmpFactories,
-                        btreeFields, filterFields), localResourceFactoryProvider,
+                        btreeFields, filterFields, true), localResourceFactoryProvider,
                 NoOpOperationCallbackFactory.INSTANCE);
         AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, indexCreateOp,
                 splitsAndConstraint.second);
@@ -205,6 +210,8 @@ public class DatasetOperations {
         if (dataset == null) {
             throw new AsterixException("Could not find dataset " + datasetName + " in dataverse " + dataverseName);
         }
+        boolean temp = dataset.getDatasetDetails().isTemp();
+
         ARecordType itemType = (ARecordType) metadata.findType(dataverseName, dataset.getItemTypeName());
         JobSpecification spec = JobSpecificationUtils.createJobSpecification();
         IBinaryComparatorFactory[] comparatorFactories = DatasetUtils.computeKeysBinaryComparatorFactories(dataset,
@@ -219,7 +226,7 @@ public class DatasetOperations {
         int[] btreeFields = DatasetUtils.createBTreeFieldsWhenThereisAFilter(dataset);
 
         Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadata
-                .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, datasetName);
+                .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, datasetName, temp);
 
         AsterixStorageProperties storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
 
@@ -233,7 +240,7 @@ public class DatasetOperations {
                                 dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
                         LSMBTreeIOOperationCallbackFactory.INSTANCE,
                         storageProperties.getBloomFilterFalsePositiveRate(), true, filterTypeTraits,
-                        filterCmpFactories, btreeFields, filterFields), NoOpOperationCallbackFactory.INSTANCE);
+                        filterCmpFactories, btreeFields, filterFields, !temp), NoOpOperationCallbackFactory.INSTANCE);
         AlgebricksPartitionConstraintHelper
                 .setPartitionConstraintInJobSpec(spec, compactOp, splitsAndConstraint.second);
 

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/ccd67fe8/asterix-app/src/main/java/edu/uci/ics/asterix/file/ExternalIndexingOperations.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/ExternalIndexingOperations.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/ExternalIndexingOperations.java
index ba6357b..a519eaf 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/ExternalIndexingOperations.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/ExternalIndexingOperations.java
@@ -212,7 +212,7 @@ public class ExternalIndexingOperations {
 
     public static FileSystem getFileSystemObject(Map<String, String> map) throws IOException {
         Configuration conf = new Configuration();
-        conf.set("fs.default.name", ((String) map.get(HDFSAdapterFactory.KEY_HDFS_URL)).trim());
+        conf.set("fs.default.name", map.get(HDFSAdapterFactory.KEY_HDFS_URL).trim());
         conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
         return FileSystem.get(conf);
     }
@@ -241,7 +241,7 @@ public class ExternalIndexingOperations {
                 mergePolicyFactory, mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(
                         dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
                 LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties.getBloomFilterFalsePositiveRate(),
-                ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+                ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true);
         ExternalFilesIndexOperatorDescriptor externalFilesOp = new ExternalFilesIndexOperatorDescriptor(spec,
                 AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
                 secondaryFileSplitProvider, indexDataflowHelperFactory, localResourceFactoryProvider,
@@ -408,6 +408,7 @@ public class ExternalIndexingOperations {
                 : indexDropStmt.getDataverseName();
         String datasetName = indexDropStmt.getDatasetName();
         String indexName = indexDropStmt.getIndexName();
+        boolean temp = dataset.getDatasetDetails().isTemp();
         JobSpecification spec = JobSpecificationUtils.createJobSpecification();
         Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadataProvider
                 .splitProviderAndPartitionConstraintsForFilesIndex(dataverseName, datasetName, indexName, true);
@@ -420,7 +421,7 @@ public class ExternalIndexingOperations {
                         dataset.getDatasetId()), compactionInfo.first, compactionInfo.second,
                         new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
                         AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
-                        storageProperties.getBloomFilterFalsePositiveRate(), false, null, null, null, null));
+                        storageProperties.getBloomFilterFalsePositiveRate(), false, null, null, null, null, !temp));
         AlgebricksPartitionConstraintHelper
                 .setPartitionConstraintInJobSpec(spec, btreeDrop, splitsAndConstraint.second);
         spec.addRoot(btreeDrop);
@@ -488,11 +489,12 @@ public class ExternalIndexingOperations {
         AsterixStorageProperties storageProperties = asterixPropertiesProvider.getStorageProperties();
         Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(ds,
                 metadataProvider.getMetadataTxnContext());
+        boolean temp = ds.getDatasetDetails().isTemp();
         ILSMMergePolicyFactory mergePolicyFactory = compactionInfo.first;
         Map<String, String> mergePolicyFactoryProperties = compactionInfo.second;
         Pair<IFileSplitProvider, AlgebricksPartitionConstraint> filesIndexSplitsAndConstraint = metadataProvider
                 .splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
-                        getFilesIndexName(ds.getDatasetName()));
+                        getFilesIndexName(ds.getDatasetName()), temp);
         IFileSplitProvider filesIndexSplitProvider = filesIndexSplitsAndConstraint.first;
         ExternalBTreeDataflowHelperFactory filesIndexDataflowHelperFactory = getFilesIndexDataflowHelperFactory(ds,
                 mergePolicyFactory, mergePolicyFactoryProperties, storageProperties, spec);
@@ -508,7 +510,7 @@ public class ExternalIndexingOperations {
             if (isValidIndexName(index.getDatasetName(), index.getIndexName())) {
                 Pair<IFileSplitProvider, AlgebricksPartitionConstraint> indexSplitsAndConstraint = metadataProvider
                         .splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
-                                index.getIndexName());
+                                index.getIndexName(), temp);
                 if (index.getIndexType() == IndexType.BTREE) {
                     btreeDataflowHelperFactories.add(getBTreeDataflowHelperFactory(ds, index, mergePolicyFactory,
                             mergePolicyFactoryProperties, storageProperties, spec));
@@ -543,7 +545,7 @@ public class ExternalIndexingOperations {
                 new SecondaryIndexOperationTrackerProvider(ds.getDatasetId()),
                 AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
                 storageProperties.getBloomFilterFalsePositiveRate(),
-                ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(ds));
+                ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(ds), true);
     }
 
     private static ExternalBTreeWithBuddyDataflowHelperFactory getBTreeDataflowHelperFactory(Dataset ds, Index index,
@@ -554,7 +556,7 @@ public class ExternalIndexingOperations {
                 AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
                 LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE,
                 storageProperties.getBloomFilterFalsePositiveRate(), new int[] { index.getKeyFieldNames().size() },
-                ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(ds));
+                ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(ds), true);
     }
 
     @SuppressWarnings("rawtypes")
@@ -609,7 +611,7 @@ public class ExternalIndexingOperations {
                 AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMRTreeIOOperationCallbackFactory.INSTANCE,
                 AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
                 storageProperties.getBloomFilterFalsePositiveRate(), new int[] { index.getKeyFieldNames().size() },
-                ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(ds));
+                ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(ds), true);
     }
 
     public static JobSpecification buildAbortOp(Dataset ds, List<Index> indexes, AqlMetadataProvider metadataProvider)
@@ -621,9 +623,11 @@ public class ExternalIndexingOperations {
                 metadataProvider.getMetadataTxnContext());
         ILSMMergePolicyFactory mergePolicyFactory = compactionInfo.first;
         Map<String, String> mergePolicyFactoryProperties = compactionInfo.second;
+
+        boolean temp = ds.getDatasetDetails().isTemp();
         Pair<IFileSplitProvider, AlgebricksPartitionConstraint> filesIndexSplitsAndConstraint = metadataProvider
                 .splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
-                        getFilesIndexName(ds.getDatasetName()));
+                        getFilesIndexName(ds.getDatasetName()), temp);
         IFileSplitProvider filesIndexSplitProvider = filesIndexSplitsAndConstraint.first;
         ExternalBTreeDataflowHelperFactory filesIndexDataflowHelperFactory = getFilesIndexDataflowHelperFactory(ds,
                 mergePolicyFactory, mergePolicyFactoryProperties, storageProperties, spec);
@@ -639,7 +643,7 @@ public class ExternalIndexingOperations {
             if (isValidIndexName(index.getDatasetName(), index.getIndexName())) {
                 Pair<IFileSplitProvider, AlgebricksPartitionConstraint> indexSplitsAndConstraint = metadataProvider
                         .splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
-                                index.getIndexName());
+                                index.getIndexName(), temp);
                 if (index.getIndexType() == IndexType.BTREE) {
                     btreeDataflowHelperFactories.add(getBTreeDataflowHelperFactory(ds, index, mergePolicyFactory,
                             mergePolicyFactoryProperties, storageProperties, spec));
@@ -677,9 +681,11 @@ public class ExternalIndexingOperations {
                 metadataProvider.getMetadataTxnContext());
         ILSMMergePolicyFactory mergePolicyFactory = compactionInfo.first;
         Map<String, String> mergePolicyFactoryProperties = compactionInfo.second;
+        boolean temp = ds.getDatasetDetails().isTemp();
+
         Pair<IFileSplitProvider, AlgebricksPartitionConstraint> filesIndexSplitsAndConstraint = metadataProvider
                 .splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
-                        getFilesIndexName(ds.getDatasetName()));
+                        getFilesIndexName(ds.getDatasetName()), temp);
         IFileSplitProvider filesIndexSplitProvider = filesIndexSplitsAndConstraint.first;
         ExternalBTreeDataflowHelperFactory filesIndexDataflowHelperFactory = getFilesIndexDataflowHelperFactory(ds,
                 mergePolicyFactory, mergePolicyFactoryProperties, storageProperties, spec);
@@ -695,7 +701,7 @@ public class ExternalIndexingOperations {
             if (isValidIndexName(index.getDatasetName(), index.getIndexName())) {
                 Pair<IFileSplitProvider, AlgebricksPartitionConstraint> indexSplitsAndConstraint = metadataProvider
                         .splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
-                                index.getIndexName());
+                                index.getIndexName(), temp);
                 if (index.getIndexType() == IndexType.BTREE) {
                     btreeDataflowHelperFactories.add(getBTreeDataflowHelperFactory(ds, index, mergePolicyFactory,
                             mergePolicyFactoryProperties, storageProperties, spec));
@@ -740,7 +746,7 @@ public class ExternalIndexingOperations {
                 mergePolicyFactory, mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(
                         dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
                 LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties.getBloomFilterFalsePositiveRate(),
-                ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+                ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true);
         LSMTreeIndexCompactOperatorDescriptor compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
                 AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
                 secondaryFileSplitProvider, FilesIndexDescription.EXTERNAL_FILE_INDEX_TYPE_TRAITS,

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/ccd67fe8/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java
index 53dfc6e..5159277 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java
@@ -94,19 +94,22 @@ public class IndexOperations {
         String datasetName = indexDropStmt.getDatasetName();
         String indexName = indexDropStmt.getIndexName();
         JobSpecification spec = JobSpecificationUtils.createJobSpecification();
+        boolean temp = dataset.getDatasetDetails().isTemp();
 
         Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadataProvider
-                .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, indexName);
+                .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, indexName, temp);
         AsterixStorageProperties storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
         Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(dataset,
                 metadataProvider.getMetadataTxnContext());
+
+        // The index drop operation should be persistent regardless of temp datasets or permanent dataset.
         IndexDropOperatorDescriptor btreeDrop = new IndexDropOperatorDescriptor(spec,
                 AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
                 splitsAndConstraint.first, new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(
                         dataset.getDatasetId()), compactionInfo.first, compactionInfo.second,
                         new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
                         AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
-                        storageProperties.getBloomFilterFalsePositiveRate(), false, null, null, null, null));
+                        storageProperties.getBloomFilterFalsePositiveRate(), false, null, null, null, null, true));
         AlgebricksPartitionConstraintHelper
                 .setPartitionConstraintInJobSpec(spec, btreeDrop, splitsAndConstraint.second);
         spec.addRoot(btreeDrop);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/ccd67fe8/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeOperationsHelper.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeOperationsHelper.java
index a061451..333d6e7 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeOperationsHelper.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeOperationsHelper.java
@@ -92,12 +92,13 @@ public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelp
                     secondaryBTreeFields, secondaryFilterFields);
             localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(localResourceMetadata,
                     LocalResource.LSMBTreeResource);
+            // The index create operation should be persistent regardless of temp datasets or permanent dataset.
             indexDataflowHelperFactory = new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(
                     dataset.getDatasetId()), mergePolicyFactory, mergePolicyFactoryProperties,
                     new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
                     AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
                     storageProperties.getBloomFilterFalsePositiveRate(), false, filterTypeTraits, filterCmpFactories,
-                    secondaryBTreeFields, secondaryFilterFields);
+                    secondaryBTreeFields, secondaryFilterFields, true);
         } else {
             // External dataset local resource and dataflow helper
             int[] buddyBreeFields = new int[] { numSecondaryKeys };
@@ -111,7 +112,7 @@ public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelp
                     AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
                     LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE,
                     storageProperties.getBloomFilterFalsePositiveRate(), buddyBreeFields,
-                    ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+                    ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true);
         }
         TreeIndexCreateOperatorDescriptor secondaryIndexCreateOp = new TreeIndexCreateOperatorDescriptor(spec,
                 AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
@@ -163,7 +164,7 @@ public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelp
                             dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
                     LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE,
                     storageProperties.getBloomFilterFalsePositiveRate(), new int[] { numSecondaryKeys },
-                    ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+                    ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true);
             IOperatorDescriptor root;
             if (externalFiles != null) {
                 // Transaction load
@@ -216,6 +217,7 @@ public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelp
             ExternalSortOperatorDescriptor sortOp = createSortOp(spec, secondaryComparatorFactories, secondaryRecDesc);
 
             AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
+            boolean temp = dataset.getDatasetDetails().isTemp();
             // Create secondary BTree bulk load op.
             TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(
                     spec,
@@ -226,7 +228,7 @@ public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelp
                             AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
                             LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties
                                     .getBloomFilterFalsePositiveRate(), false, filterTypeTraits, filterCmpFactories,
-                            secondaryBTreeFields, secondaryFilterFields), GlobalConfig.DEFAULT_TREE_FILL_FACTOR);
+                            secondaryBTreeFields, secondaryFilterFields, !temp), GlobalConfig.DEFAULT_TREE_FILL_FACTOR);
 
             AlgebricksMetaOperatorDescriptor metaOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0,
                     new IPushRuntimeFactory[] { new SinkRuntimeFactory() }, new RecordDescriptor[] { secondaryRecDesc });
@@ -247,6 +249,7 @@ public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelp
         }
     }
 
+    @Override
     protected int getNumSecondaryKeys() {
         return numSecondaryKeys;
     }
@@ -256,6 +259,7 @@ public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelp
         JobSpecification spec = JobSpecificationUtils.createJobSpecification();
 
         AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
+        boolean temp = dataset.getDatasetDetails().isTemp();
         LSMTreeIndexCompactOperatorDescriptor compactOp;
         if (dataset.getDatasetType() == DatasetType.INTERNAL) {
             compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
@@ -267,7 +271,7 @@ public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelp
                                     dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
                             LSMBTreeIOOperationCallbackFactory.INSTANCE,
                             storageProperties.getBloomFilterFalsePositiveRate(), false, filterTypeTraits,
-                            filterCmpFactories, secondaryBTreeFields, secondaryFilterFields),
+                            filterCmpFactories, secondaryBTreeFields, secondaryFilterFields, !temp),
                     NoOpOperationCallbackFactory.INSTANCE);
         } else {
             // External dataset
@@ -280,7 +284,7 @@ public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelp
                             AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
                             LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE, storageProperties
                                     .getBloomFilterFalsePositiveRate(), new int[] { numSecondaryKeys },
-                            ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset)),
+                            ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true),
                     NoOpOperationCallbackFactory.INSTANCE);
         }
         AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, compactOp,

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/ccd67fe8/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java
index cccb461..07c8bab 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java
@@ -148,8 +148,8 @@ public abstract class SecondaryIndexOperationsHelper {
     }
 
     public static SecondaryIndexOperationsHelper createIndexOperationsHelper(IndexType indexType, String dataverseName,
-            String datasetName, String indexName, List<List<String>> secondaryKeyFields, List<IAType> secondaryKeyTypes,
-            boolean isEnforced, int gramLength, AqlMetadataProvider metadataProvider,
+            String datasetName, String indexName, List<List<String>> secondaryKeyFields,
+            List<IAType> secondaryKeyTypes, boolean isEnforced, int gramLength, AqlMetadataProvider metadataProvider,
             PhysicalOptimizationConfig physOptConf, ARecordType recType, ARecordType enforcedType)
             throws AsterixException, AlgebricksException {
         IAsterixPropertiesProvider asterixPropertiesProvider = AsterixAppContextInfo.getInstance();
@@ -198,12 +198,13 @@ public abstract class SecondaryIndexOperationsHelper {
         if (dataset == null) {
             throw new AsterixException("Unknown dataset " + datasetName);
         }
+        boolean temp = dataset.getDatasetDetails().isTemp();
         itemType = aRecType;
         enforcedItemType = enforcedType;
         payloadSerde = AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(itemType);
         numSecondaryKeys = secondaryKeyFields.size();
         Pair<IFileSplitProvider, AlgebricksPartitionConstraint> secondarySplitsAndConstraint = metadataProvider
-                .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, secondaryIndexName);
+                .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, secondaryIndexName, temp);
         secondaryFileSplitProvider = secondarySplitsAndConstraint.first;
         secondaryPartitionConstraint = secondarySplitsAndConstraint.second;
 
@@ -219,7 +220,7 @@ public abstract class SecondaryIndexOperationsHelper {
 
             numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size();
             Pair<IFileSplitProvider, AlgebricksPartitionConstraint> primarySplitsAndConstraint = metadataProvider
-                    .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, datasetName);
+                    .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, datasetName, temp);
             primaryFileSplitProvider = primarySplitsAndConstraint.first;
             primaryPartitionConstraint = primarySplitsAndConstraint.second;
             setPrimaryRecDescAndComparators();
@@ -291,9 +292,9 @@ public abstract class SecondaryIndexOperationsHelper {
         primaryRecDesc = new RecordDescriptor(primaryRecFields, primaryTypeTraits);
     }
 
-    protected abstract void setSecondaryRecDescAndComparators(IndexType indexType, List<List<String>> secondaryKeyFields,
-            List<IAType> secondaryKeyTypes, int gramLength, AqlMetadataProvider metadataProvider)
-            throws AlgebricksException, AsterixException;
+    protected abstract void setSecondaryRecDescAndComparators(IndexType indexType,
+            List<List<String>> secondaryKeyFields, List<IAType> secondaryKeyTypes, int gramLength,
+            AqlMetadataProvider metadataProvider) throws AlgebricksException, AsterixException;
 
     protected AbstractOperatorDescriptor createDummyKeyProviderOp(JobSpecification spec) throws AsterixException,
             AlgebricksException {
@@ -330,9 +331,10 @@ public abstract class SecondaryIndexOperationsHelper {
         IJobletEventListenerFactory jobEventListenerFactory = new JobEventListenerFactory(jobId, isWriteTransaction);
         spec.setJobletEventListenerFactory(jobEventListenerFactory);
 
-        ISearchOperationCallbackFactory searchCallbackFactory = new PrimaryIndexInstantSearchOperationCallbackFactory(
-                jobId, dataset.getDatasetId(), primaryBloomFilterKeyFields, txnSubsystemProvider,
-                ResourceType.LSM_BTREE);
+        boolean temp = dataset.getDatasetDetails().isTemp();
+        ISearchOperationCallbackFactory searchCallbackFactory = temp ? NoOpOperationCallbackFactory.INSTANCE
+                : new PrimaryIndexInstantSearchOperationCallbackFactory(jobId, dataset.getDatasetId(),
+                        primaryBloomFilterKeyFields, txnSubsystemProvider, ResourceType.LSM_BTREE);
         AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
         BTreeSearchOperatorDescriptor primarySearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
                 AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
@@ -343,7 +345,7 @@ public abstract class SecondaryIndexOperationsHelper {
                                 dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
                         LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties
                                 .getBloomFilterFalsePositiveRate(), true, filterTypeTraits, filterCmpFactories,
-                        primaryBTreeFields, primaryFilterFields), false, false, null,
+                        primaryBTreeFields, primaryFilterFields, !temp), false, false, null,
                 searchCallbackFactory, null, null);
 
         AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, primarySearchOp,

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/ccd67fe8/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexOperationsHelper.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexOperationsHelper.java
index 579af2f..74c4256 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexOperationsHelper.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexOperationsHelper.java
@@ -203,6 +203,7 @@ public class SecondaryInvertedIndexOperationsHelper extends SecondaryIndexOperat
 
     }
 
+    @Override
     protected int getNumSecondaryKeys() {
         return numTokenKeyPairFields - numPrimaryKeys;
     }
@@ -330,6 +331,7 @@ public class SecondaryInvertedIndexOperationsHelper extends SecondaryIndexOperat
 
     private IIndexDataflowHelperFactory createDataflowHelperFactory() {
         AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
+        boolean temp = dataset.getDatasetDetails().isTemp();
         if (!isPartitioned) {
             return new LSMInvertedIndexDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(
                     dataset.getDatasetId()), mergePolicyFactory, mergePolicyFactoryProperties,
@@ -338,7 +340,7 @@ public class SecondaryInvertedIndexOperationsHelper extends SecondaryIndexOperat
                     LSMInvertedIndexIOOperationCallbackFactory.INSTANCE,
                     storageProperties.getBloomFilterFalsePositiveRate(), invertedIndexFields, filterTypeTraits,
                     filterCmpFactories, secondaryFilterFields, secondaryFilterFieldsForNonBulkLoadOps,
-                    invertedIndexFieldsForNonBulkLoadOps);
+                    invertedIndexFieldsForNonBulkLoadOps, !temp);
         } else {
             return new PartitionedLSMInvertedIndexDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(
                     dataset.getDatasetId()), mergePolicyFactory, mergePolicyFactoryProperties,
@@ -347,7 +349,7 @@ public class SecondaryInvertedIndexOperationsHelper extends SecondaryIndexOperat
                     LSMInvertedIndexIOOperationCallbackFactory.INSTANCE,
                     storageProperties.getBloomFilterFalsePositiveRate(), invertedIndexFields, filterTypeTraits,
                     filterCmpFactories, secondaryFilterFields, secondaryFilterFieldsForNonBulkLoadOps,
-                    invertedIndexFieldsForNonBulkLoadOps);
+                    invertedIndexFieldsForNonBulkLoadOps, !temp);
         }
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/ccd67fe8/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java
index 8be9f59..0e49fb9 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java
@@ -92,7 +92,7 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
         JobSpecification spec = JobSpecificationUtils.createJobSpecification();
 
         AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
-
+        boolean temp = dataset.getDatasetDetails().isTemp();
         IIndexDataflowHelperFactory indexDataflowHelperFactory;
         ILocalResourceFactoryProvider localResourceFactoryProvider;
         if (dataset.getDatasetType() == DatasetType.INTERNAL) {
@@ -112,7 +112,7 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
                     AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMRTreeIOOperationCallbackFactory.INSTANCE,
                     AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
                     storageProperties.getBloomFilterFalsePositiveRate(), rtreeFields, primaryKeyFields,
-                    filterTypeTraits, filterCmpFactories, secondaryFilterFields);
+                    filterTypeTraits, filterCmpFactories, secondaryFilterFields, !temp);
         } else {
             // External dataset
             // Prepare a LocalResourceMetadata which will be stored in NC's local resource repository
@@ -131,7 +131,7 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
                     LSMRTreeIOOperationCallbackFactory.INSTANCE, AqlMetadataProvider.proposeLinearizer(keyType,
                             secondaryComparatorFactories.length), storageProperties.getBloomFilterFalsePositiveRate(),
                     new int[] { numNestedSecondaryKeyFields },
-                    ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+                    ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true);
         }
 
         TreeIndexCreateOperatorDescriptor secondaryIndexCreateOp = new TreeIndexCreateOperatorDescriptor(spec,
@@ -146,6 +146,7 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
         return spec;
     }
 
+    @Override
     protected int getNumSecondaryKeys() {
         return numNestedSecondaryKeyFields;
     }
@@ -233,6 +234,7 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
     @Override
     public JobSpecification buildLoadingJobSpec() throws AsterixException, AlgebricksException {
         JobSpecification spec = JobSpecificationUtils.createJobSpecification();
+        boolean temp = dataset.getDatasetDetails().isTemp();
         if (dataset.getDatasetType() == DatasetType.INTERNAL) {
             // Create dummy key provider for feeding the primary index scan. 
             AbstractOperatorDescriptor keyProviderOp = createDummyKeyProviderOp(spec);
@@ -273,7 +275,7 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
                             LSMRTreeIOOperationCallbackFactory.INSTANCE, AqlMetadataProvider.proposeLinearizer(keyType,
                                     secondaryComparatorFactories.length), storageProperties
                                     .getBloomFilterFalsePositiveRate(), rtreeFields, primaryKeyFields,
-                            filterTypeTraits, filterCmpFactories, secondaryFilterFields),
+                            filterTypeTraits, filterCmpFactories, secondaryFilterFields, !temp),
                     GlobalConfig.DEFAULT_TREE_FILL_FACTOR);
             AlgebricksMetaOperatorDescriptor metaOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0,
                     new IPushRuntimeFactory[] { new SinkRuntimeFactory() }, new RecordDescriptor[] {});
@@ -325,7 +327,7 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
                     AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMRTreeIOOperationCallbackFactory.INSTANCE,
                     AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
                     storageProperties.getBloomFilterFalsePositiveRate(), new int[] { numNestedSecondaryKeyFields },
-                    ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+                    ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true);
             // Create secondary RTree bulk load op.
             IOperatorDescriptor root;
             AbstractTreeIndexOperatorDescriptor secondaryBulkLoadOp;
@@ -364,6 +366,7 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
         JobSpecification spec = JobSpecificationUtils.createJobSpecification();
 
         AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
+        boolean temp = dataset.getDatasetDetails().isTemp();
         LSMTreeIndexCompactOperatorDescriptor compactOp;
         if (dataset.getDatasetType() == DatasetType.INTERNAL) {
             compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
@@ -377,7 +380,7 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
                             LSMRTreeIOOperationCallbackFactory.INSTANCE, AqlMetadataProvider.proposeLinearizer(keyType,
                                     secondaryComparatorFactories.length),
                             storageProperties.getBloomFilterFalsePositiveRate(), rtreeFields, primaryKeyFields,
-                            filterTypeTraits, filterCmpFactories, secondaryFilterFields),
+                            filterTypeTraits, filterCmpFactories, secondaryFilterFields, !temp),
                     NoOpOperationCallbackFactory.INSTANCE);
         } else {
             // External dataset
@@ -392,7 +395,7 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
                             LSMRTreeIOOperationCallbackFactory.INSTANCE, AqlMetadataProvider.proposeLinearizer(keyType,
                                     secondaryComparatorFactories.length), storageProperties
                                     .getBloomFilterFalsePositiveRate(), new int[] { numNestedSecondaryKeyFields },
-                            ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset)),
+                            ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true),
                     NoOpOperationCallbackFactory.INSTANCE);
         }
 


Mime
View raw message