asterixdb-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Luo Chen (Code Review)" <do-not-re...@asterixdb.incubator.apache.org>
Subject Change in asterixdb[master]: ASTERIXDB-1917: FLUSH_LSN for disk components is not correct...
Date Tue, 23 May 2017 02:54:34 GMT
Luo Chen has uploaded a new change for review.

  https://asterix-gerrit.ics.uci.edu/1771

Change subject: ASTERIXDB-1917: FLUSH_LSN for disk components is not correctly set
......................................................................

ASTERIXDB-1917: FLUSH_LSN for disk components is not correctly set

-Fixed a bug that FLUSH_LSN for flushed disk components is not
correctly set (not increasing) when an NC has multiple partitions.
-Added LSMIOOperationCallback unit tests to cover this bug

Change-Id: If438e34f8f612458d81f618eea04c0c72c49a9fe
---
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
A asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMBTreeIOOperationCallbackTest.java
A asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallbackTest.java
A asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMInvertedIndexIOOperationCallbackTest.java
A asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMRTreeIOOperationCallbackTest.java
5 files changed, 344 insertions(+), 2 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/71/1771/1

diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
index f903b65..d04443a 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
@@ -107,8 +107,14 @@
         return pointable.getLength() == 0 ? INVALID : pointable.longValue();
     }
 
-    public void updateLastLSN(long lastLSN) {
-        mutableLastLSNs[writeIndex] = lastLSN;
+    public synchronized void updateLastLSN(long lastLSN) {
+        if (!flushRequested[writeIndex]) {
+            //if the memory component pointed by writeIndex is being flushed, we should ignore
this update call
+            //since otherwise the original LSN is overwritten.
+            //Moreover, since the memory component is already being flushed, the next scheduleFlush
request must fail.
+            //See https://issues.apache.org/jira/browse/ASTERIXDB-1917
+            mutableLastLSNs[writeIndex] = lastLSN;
+        }
     }
 
     public void setFirstLSN(long firstLSN) {
diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMBTreeIOOperationCallbackTest.java
b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMBTreeIOOperationCallbackTest.java
new file mode 100644
index 0000000..bc206e2
--- /dev/null
+++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMBTreeIOOperationCallbackTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.test.ioopcallbacks;
+
+import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallback;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
+import org.junit.Assert;
+import org.mockito.Mockito;
+
+import junit.framework.TestCase;
+
+public class LSMBTreeIOOperationCallbackTest extends TestCase {
+
+    public void testNormalSequence() {
+        try {
+            LSMBTreeIOOperationCallback callback = new LSMBTreeIOOperationCallback();
+            callback.setNumOfMutableComponents(2);
+
+            //request to flush first component
+            callback.updateLastLSN(1);
+            callback.beforeOperation(LSMOperationType.FLUSH);
+
+            //request to flush second component
+            callback.updateLastLSN(2);
+            callback.beforeOperation(LSMOperationType.FLUSH);
+
+            Assert.assertEquals(1, callback.getComponentLSN(null));
+            callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+
+            Assert.assertEquals(2, callback.getComponentLSN(null));
+            callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+        } catch (Exception e) {
+            Assert.fail();
+        }
+    }
+
+    public void testOverWrittenLSN() {
+        try {
+            LSMBTreeIOOperationCallback callback = new LSMBTreeIOOperationCallback();
+            callback.setNumOfMutableComponents(2);
+
+            //request to flush first component
+            callback.updateLastLSN(1);
+            callback.beforeOperation(LSMOperationType.FLUSH);
+
+            //request to flush second component
+            callback.updateLastLSN(2);
+            callback.beforeOperation(LSMOperationType.FLUSH);
+
+            //request to flush first component again
+            //this call should fail
+            callback.updateLastLSN(3);
+            //there is no corresponding beforeOperation, since the first component is being
flush
+            //the scheduleFlush request would fail this time
+
+            Assert.assertEquals(1, callback.getComponentLSN(null));
+            callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+
+            Assert.assertEquals(2, callback.getComponentLSN(null));
+            callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+        } catch (Exception e) {
+            Assert.fail();
+        }
+    }
+
+}
diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallbackTest.java
b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallbackTest.java
new file mode 100644
index 0000000..fe84d4a
--- /dev/null
+++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallbackTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.test.ioopcallbacks;
+
+import org.apache.asterix.common.ioopcallbacks.LSMBTreeWithBuddyIOOperationCallback;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
+import org.junit.Assert;
+import org.mockito.Mockito;
+
+import junit.framework.TestCase;
+
+public class LSMBTreeWithBuddyIOOperationCallbackTest extends TestCase {
+
+    public void testNormalSequence() {
+        try {
+            LSMBTreeWithBuddyIOOperationCallback callback = new LSMBTreeWithBuddyIOOperationCallback();
+            callback.setNumOfMutableComponents(2);
+
+            //request to flush first component
+            callback.updateLastLSN(1);
+            callback.beforeOperation(LSMOperationType.FLUSH);
+
+            //request to flush second component
+            callback.updateLastLSN(2);
+            callback.beforeOperation(LSMOperationType.FLUSH);
+
+            Assert.assertEquals(1, callback.getComponentLSN(null));
+            callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+
+            Assert.assertEquals(2, callback.getComponentLSN(null));
+            callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+        } catch (Exception e) {
+            Assert.fail();
+        }
+    }
+
+    public void testOverWrittenLSN() {
+        try {
+            LSMBTreeWithBuddyIOOperationCallback callback = new LSMBTreeWithBuddyIOOperationCallback();
+            callback.setNumOfMutableComponents(2);
+
+            //request to flush first component
+            callback.updateLastLSN(1);
+            callback.beforeOperation(LSMOperationType.FLUSH);
+
+            //request to flush second component
+            callback.updateLastLSN(2);
+            callback.beforeOperation(LSMOperationType.FLUSH);
+
+            //request to flush first component again
+            //this call should fail
+            callback.updateLastLSN(3);
+            //there is no corresponding beforeOperation, since the first component is being
flush
+            //the scheduleFlush request would fail this time
+
+            Assert.assertEquals(1, callback.getComponentLSN(null));
+            callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+
+            Assert.assertEquals(2, callback.getComponentLSN(null));
+            callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+        } catch (Exception e) {
+            Assert.fail();
+        }
+    }
+
+}
diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMInvertedIndexIOOperationCallbackTest.java
b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMInvertedIndexIOOperationCallbackTest.java
new file mode 100644
index 0000000..1f99db3
--- /dev/null
+++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMInvertedIndexIOOperationCallbackTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.test.ioopcallbacks;
+
+import org.apache.asterix.common.ioopcallbacks.LSMInvertedIndexIOOperationCallback;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
+import org.junit.Assert;
+import org.mockito.Mockito;
+
+import junit.framework.TestCase;
+
+public class LSMInvertedIndexIOOperationCallbackTest extends TestCase {
+
+    public void testNormalSequence() {
+        try {
+            LSMInvertedIndexIOOperationCallback callback = new LSMInvertedIndexIOOperationCallback();
+            callback.setNumOfMutableComponents(2);
+
+            //request to flush first component
+            callback.updateLastLSN(1);
+            callback.beforeOperation(LSMOperationType.FLUSH);
+
+            //request to flush second component
+            callback.updateLastLSN(2);
+            callback.beforeOperation(LSMOperationType.FLUSH);
+
+            Assert.assertEquals(1, callback.getComponentLSN(null));
+            callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+
+            Assert.assertEquals(2, callback.getComponentLSN(null));
+            callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+        } catch (Exception e) {
+            Assert.fail();
+        }
+    }
+
+    public void testOverWrittenLSN() {
+        try {
+            LSMInvertedIndexIOOperationCallback callback = new LSMInvertedIndexIOOperationCallback();
+            callback.setNumOfMutableComponents(2);
+
+            //request to flush first component
+            callback.updateLastLSN(1);
+            callback.beforeOperation(LSMOperationType.FLUSH);
+
+            //request to flush second component
+            callback.updateLastLSN(2);
+            callback.beforeOperation(LSMOperationType.FLUSH);
+
+            //request to flush first component again
+            //this call should fail
+            callback.updateLastLSN(3);
+            //there is no corresponding beforeOperation, since the first component is being
flush
+            //the scheduleFlush request would fail this time
+
+            Assert.assertEquals(1, callback.getComponentLSN(null));
+            callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+
+            Assert.assertEquals(2, callback.getComponentLSN(null));
+            callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+        } catch (Exception e) {
+            Assert.fail();
+        }
+    }
+
+}
diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMRTreeIOOperationCallbackTest.java
b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMRTreeIOOperationCallbackTest.java
new file mode 100644
index 0000000..38f7ee9
--- /dev/null
+++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMRTreeIOOperationCallbackTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.test.ioopcallbacks;
+
+import org.apache.asterix.common.ioopcallbacks.LSMRTreeIOOperationCallback;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
+import org.junit.Assert;
+import org.mockito.Mockito;
+
+import junit.framework.TestCase;
+
+public class LSMRTreeIOOperationCallbackTest extends TestCase {
+
+    public void testNormalSequence() {
+        try {
+            LSMRTreeIOOperationCallback callback = new LSMRTreeIOOperationCallback();
+            callback.setNumOfMutableComponents(2);
+
+            //request to flush first component
+            callback.updateLastLSN(1);
+            callback.beforeOperation(LSMOperationType.FLUSH);
+
+            //request to flush second component
+            callback.updateLastLSN(2);
+            callback.beforeOperation(LSMOperationType.FLUSH);
+
+            Assert.assertEquals(1, callback.getComponentLSN(null));
+            callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+
+            Assert.assertEquals(2, callback.getComponentLSN(null));
+            callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+        } catch (Exception e) {
+            Assert.fail();
+        }
+    }
+
+    public void testOverWrittenLSN() {
+        try {
+            LSMRTreeIOOperationCallback callback = new LSMRTreeIOOperationCallback();
+            callback.setNumOfMutableComponents(2);
+
+            //request to flush first component
+            callback.updateLastLSN(1);
+            callback.beforeOperation(LSMOperationType.FLUSH);
+
+            //request to flush second component
+            callback.updateLastLSN(2);
+            callback.beforeOperation(LSMOperationType.FLUSH);
+
+            //request to flush first component again
+            //this call should fail
+            callback.updateLastLSN(3);
+            //there is no corresponding beforeOperation, since the first component is being
flush
+            //the scheduleFlush request would fail this time
+
+            Assert.assertEquals(1, callback.getComponentLSN(null));
+            callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+
+            Assert.assertEquals(2, callback.getComponentLSN(null));
+            callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class));
+        } catch (Exception e) {
+            Assert.fail();
+        }
+    }
+
+}

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1771
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: If438e34f8f612458d81f618eea04c0c72c49a9fe
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Luo Chen <cluo8@uci.edu>

Mime
View raw message