asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amo...@apache.org
Subject [1/3] asterixdb git commit: Fix transaction logs and optimize upserts
Date Tue, 14 Mar 2017 18:13:06 GMT
Repository: asterixdb
Updated Branches:
  refs/heads/master 89328a88a -> 04075533a


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/04075533/asterixdb/asterix-transactions/src/test/java/org/apache/asterix/transaction/management/service/locking/Request.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-transactions/src/test/java/org/apache/asterix/transaction/management/service/locking/Request.java
b/asterixdb/asterix-transactions/src/test/java/org/apache/asterix/transaction/management/service/locking/Request.java
index 70a803b..fd4dae5 100644
--- a/asterixdb/asterix-transactions/src/test/java/org/apache/asterix/transaction/management/service/locking/Request.java
+++ b/asterixdb/asterix-transactions/src/test/java/org/apache/asterix/transaction/management/service/locking/Request.java
@@ -70,53 +70,63 @@ abstract class Request {
         switch (kind) {
             case INSTANT_TRY_LOCK:
                 return new Request(kind, txnCtx) {
+                    @Override
                     boolean execute(ILockManager lockMgr) throws ACIDException {
                         return lockMgr.instantTryLock(dsId, hashValue, lockMode, txnCtx);
                     }
 
+                    @Override
                     public String toString() {
                         return asString(kind, txnCtx, dsId, hashValue, lockMode);
                     }
                 };
             case INSTANT_LOCK:
                 return new Request(kind, txnCtx) {
+                    @Override
                     boolean execute(ILockManager lockMgr) throws ACIDException {
                         lockMgr.instantLock(dsId, hashValue, lockMode, txnCtx);
                         return true;
                     }
 
+                    @Override
                     public String toString() {
                         return asString(kind, txnCtx, dsId, hashValue, lockMode);
                     }
                 };
             case LOCK:
                 return new Request(kind, txnCtx) {
+                    @Override
                     boolean execute(ILockManager lockMgr) throws ACIDException {
                         lockMgr.lock(dsId, hashValue, lockMode, txnCtx);
                         return true;
                     }
 
+                    @Override
                     public String toString() {
                         return asString(kind, txnCtx, dsId, hashValue, lockMode);
                     }
                 };
             case TRY_LOCK:
                 return new Request(kind, txnCtx) {
+                    @Override
                     boolean execute(ILockManager lockMgr) throws ACIDException {
                         return lockMgr.tryLock(dsId, hashValue, lockMode, txnCtx);
                     }
 
+                    @Override
                     public String toString() {
                         return asString(kind, txnCtx, dsId, hashValue, lockMode);
                     }
                 };
             case UNLOCK:
                 return new Request(kind, txnCtx) {
+                    @Override
                     boolean execute(ILockManager lockMgr) throws ACIDException {
                         lockMgr.unlock(dsId, hashValue, lockMode, txnCtx);
                         return true;
                     }
 
+                    @Override
                     public String toString() {
                         return asString(kind, txnCtx, dsId, hashValue, lockMode);
                     }
@@ -129,11 +139,13 @@ abstract class Request {
     static Request create(final Kind kind, final ITransactionContext txnCtx) {
         if (kind == Kind.RELEASE) {
             return new Request(kind, txnCtx) {
+                @Override
                 boolean execute(ILockManager lockMgr) throws ACIDException {
                     lockMgr.releaseLocks(txnCtx);
                     return true;
                 }
 
+                @Override
                 public String toString() {
                     return txnCtx.getJobId().toString() + ":" + kind.name();
                 }
@@ -145,6 +157,7 @@ abstract class Request {
     static Request create(final Kind kind, final PrintStream out) {
         if (kind == Kind.PRINT) {
             return new Request(kind, null) {
+                @Override
                 boolean execute(ILockManager lockMgr) throws ACIDException {
                     if (out == null) {
                         return false;
@@ -157,6 +170,7 @@ abstract class Request {
                     return true;
                 }
 
+                @Override
                 public String toString() {
                     return kind.name();
                 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/04075533/asterixdb/asterix-yarn/pom.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-yarn/pom.xml b/asterixdb/asterix-yarn/pom.xml
index 66e3a96..81ca3c3 100644
--- a/asterixdb/asterix-yarn/pom.xml
+++ b/asterixdb/asterix-yarn/pom.xml
@@ -136,12 +136,10 @@
                 <include>org.apache.hadoop:hadoop-common</include>
                 <include>org.apache.hadoop:hadoop-hdfs</include>
                 <include>org.apache.hadoop:hadoop-auth</include>
-                <include>org.apache.hadoop:hadoop-yarn-client</include>
                 <include>org.apache.hadoop:hadoop-yarn-common</include>
                 <include>org.apache.hadoop:hadoop-yarn-api</include>
                 <include>org.apache.httpcomponents:httpcore</include>
                 <include>org.apache.httpcomponents:httpclient</include>
-                <include>org.htrace:htrace-core</include>
                 <include>commons-httpclient:commons-httpclient</include>
                 <include>com.google.protobuf:protobuf-java</include>
                 <include>com.google.guava:guava</include>

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/04075533/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputPushRuntime.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputPushRuntime.java
b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputPushRuntime.java
index 89d043c..85db3af 100644
--- a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputPushRuntime.java
+++ b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputPushRuntime.java
@@ -28,5 +28,4 @@ public abstract class AbstractOneInputOneOutputPushRuntime extends AbstractOneIn
     public void setInputRecordDescriptor(int index, RecordDescriptor recordDescriptor) {
         this.inputRecordDesc = recordDescriptor;
     }
-
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/04075533/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IModificationOperationCallback.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IModificationOperationCallback.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IModificationOperationCallback.java
index f22c239..902af7e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IModificationOperationCallback.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IModificationOperationCallback.java
@@ -25,19 +25,14 @@ import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
  * This operation callback allows for arbitrary actions to be taken while traversing
  * an index structure. The {@link IModificationOperationCallback} will be called on
  * all modifying operations (e.g. insert, update, delete...) for all indexes.
- *
- * @author zheilbron
  */
 public interface IModificationOperationCallback {
-    public enum Operation {
-        INSERT,
-        DELETE
-    }
 
     /**
      * This method is called on a tuple that is about to traverse an index's structure
      * (i.e. before any pages are pinned or latched).
      * The format of the tuple is the format that would be stored in the index itself.
+     *
      * @param tuple
      *            the tuple that is about to be operated on
      */
@@ -49,16 +44,9 @@ public interface IModificationOperationCallback {
      * matching key was found.
      * When found is called, the leaf page where the tuple resides will be pinned and latched,
      * so blocking operations should be avoided.
+     *
      * @param tuple
      *            a tuple with a matching key, otherwise null if none exists
      */
     public void found(ITupleReference before, ITupleReference after) throws HyracksDataException;
-
-    /**
-     * This call specifies the next operation to be performed. It is used to allow
-     * a single operator to perform different operations per tuple
-     * @param op
-     * @throws HyracksDataException
-     */
-    public void setOp(Operation op) throws HyracksDataException;
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/04075533/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/NoOpOperationCallback.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/NoOpOperationCallback.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/NoOpOperationCallback.java
index e8ab8dc..511d166 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/NoOpOperationCallback.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/NoOpOperationCallback.java
@@ -59,9 +59,4 @@ public enum NoOpOperationCallback implements IModificationOperationCallback,ISea
     public void complete(ITupleReference tuple) throws HyracksDataException {
         // Do nothing.
     }
-
-    @Override
-    public void setOp(Operation op) throws HyracksDataException {
-        // Do nothing.
-    }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/04075533/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index b78de8b..3ecb5e0 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -286,10 +286,10 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex
{
             case PHYSICALDELETE:
             case FLUSH:
             case DELETE:
+            case UPSERT:
                 operationalComponents.add(memoryComponents.get(cmc));
                 break;
             case INSERT:
-            case UPSERT:
                 addOperationalMutableComponents(operationalComponents);
                 operationalComponents.addAll(immutableComponents);
                 break;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/04075533/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java
index 90c70aa..b0f366b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexAccessor.java
@@ -174,6 +174,15 @@ public interface ILSMIndexAccessor extends IIndexAccessor {
     void forceDelete(ITupleReference tuple) throws HyracksDataException, IndexException;
 
     /**
+     * Force upserting the tuple into the memory component even if it is full
+     *
+     * @param tuple
+     * @throws HyracksDataException
+     * @throws IndexException
+     */
+    void forceUpsert(ITupleReference tuple) throws HyracksDataException, IndexException;
+
+    /**
      * Schedule a replication for disk components
      *
      * @param diskComponents

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/04075533/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
index 0fa69ec..1f93fc5 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
@@ -161,6 +161,12 @@ public abstract class LSMTreeIndexAccessor implements ILSMIndexAccessor
{
     }
 
     @Override
+    public void forceUpsert(ITupleReference tuple) throws HyracksDataException, IndexException
{
+        ctx.setOperation(IndexOperation.UPSERT);
+        lsmHarness.forceModify(ctx, tuple);
+    }
+
+    @Override
     public void forceDelete(ITupleReference tuple) throws HyracksDataException, IndexException
{
         ctx.setOperation(IndexOperation.DELETE);
         lsmHarness.forceModify(ctx, tuple);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/04075533/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
index 0a6ffd7..604a57c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
@@ -200,4 +200,9 @@ public class LSMInvertedIndexAccessor implements ILSMIndexAccessor, IInvertedInd
         lsmHarness.forceUpdateMeta(ctx, key, value);
     }
 
+    @Override
+    public void forceUpsert(ITupleReference tuple) throws HyracksDataException, IndexException
{
+        throw new UnsupportedOperationException("Upsert not supported by lsm inverted index.");
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/04075533/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/AbstractModificationOperationCallbackTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/AbstractModificationOperationCallbackTest.java
b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/AbstractModificationOperationCallbackTest.java
index 6da9334..65a3067 100644
--- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/AbstractModificationOperationCallbackTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/AbstractModificationOperationCallbackTest.java
@@ -98,11 +98,6 @@ public abstract class AbstractModificationOperationCallbackTest extends
Abstract
             }
             Assert.assertEquals(0, cmp.compare(AbstractModificationOperationCallbackTest.this.tuple,
after));
         }
-
-        @Override
-        public void setOp(Operation op) throws HyracksDataException {
-        }
-
     }
 
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/04075533/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/TestOperationCallback.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/TestOperationCallback.java
b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/TestOperationCallback.java
index 15d8a60..664d9d6 100644
--- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/TestOperationCallback.java
+++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/TestOperationCallback.java
@@ -65,10 +65,4 @@ public enum TestOperationCallback implements ISearchOperationCallback,IModificat
     public void complete(ITupleReference tuple) throws HyracksDataException {
         // Do nothing.
     }
-
-    @Override
-    public void setOp(Operation op) throws HyracksDataException {
-        // Do nothing.
-    }
-
 }


Mime
View raw message