asterixdb-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Murtadha Hubail (Code Review)" <do-not-re...@asterixdb.incubator.apache.org>
Subject Change in asterixdb[master]: Fix Triggering Flush on Temporary Datasets
Date Tue, 05 Apr 2016 01:59:40 GMT
Murtadha Hubail has submitted this change and it was merged.

Change subject: Fix Triggering Flush on Temporary Datasets
......................................................................


Fix Triggering Flush on Temporary Datasets

Change-Id: I964566e7ba03bc674209b4d45d9abb82bb581240
Reviewed-on: https://asterix-gerrit.ics.uci.edu/785
Reviewed-by: Ian Maxon <imaxon@apache.org>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
---
A asterix-app/src/test/resources/runtimets/queries/temp-dataset/temp_primary_plus_ngram_flush/temp_primary_plus_ngram_flush.1.ddl.aql
A asterix-app/src/test/resources/runtimets/queries/temp-dataset/temp_primary_plus_ngram_flush/temp_primary_plus_ngram_flush.2.update.aql
A asterix-app/src/test/resources/runtimets/queries/temp-dataset/temp_primary_plus_ngram_flush/temp_primary_plus_ngram_flush.3.query.aql
A asterix-app/src/test/resources/runtimets/results/temp-dataset/temp_primary_plus_ngram_flush/temp_primary_plus_ngram_flush.3.adm
M asterix-app/src/test/resources/runtimets/testsuite.xml
M asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
6 files changed, 158 insertions(+), 4 deletions(-)

Approvals:
  Ian Maxon: Looks good to me, approved
  Jenkins: Verified



diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/temp_primary_plus_ngram_flush/temp_primary_plus_ngram_flush.1.ddl.aql
b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/temp_primary_plus_ngram_flush/temp_primary_plus_ngram_flush.1.ddl.aql
new file mode 100644
index 0000000..671e5a2
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/temp_primary_plus_ngram_flush/temp_primary_plus_ngram_flush.1.ddl.aql
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : temp_primary_plus_ngram_flush.aql
+ * Description     : Check that flush for temporary datasets with ngram indexes succeeds.
+ * Expected Result : Success
+ * Date            : Apr 4 2016
+ */
+
+drop dataverse recovery if exists;
+create dataverse recovery;
+use dataverse recovery;
+
+/* For raw Fragile data */
+create type FragileTypeRaw as closed {
+row_id: int32,
+sid: int32,
+date: string,
+day: int32,
+time: string,
+bpm: int32,
+RR: float,
+text: string,
+location: point,
+text2: string
+};
+
+/* For cleaned Fragile data */
+create type FragileType as closed {
+row_id: int32,
+sid: int32,
+date: date,
+day: int32,
+time: time,
+bpm: int32,
+RR: float,
+text: string,
+location: point,
+text2: string
+};
+
+/* Create dataset for loading raw Fragile data */
+create temporary dataset Fragile_raw (FragileTypeRaw)
+primary key row_id;
+
+/* Create dataset for cleaned Fragile data */
+create temporary dataset Fragile (FragileType)
+primary key row_id;
+
+/* Create default secondary index on dataset clean Fragile */
+create index cfText2Ix on Fragile(text2) type ngram(3);
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/temp_primary_plus_ngram_flush/temp_primary_plus_ngram_flush.2.update.aql
b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/temp_primary_plus_ngram_flush/temp_primary_plus_ngram_flush.2.update.aql
new file mode 100644
index 0000000..1b3cbd3
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/temp_primary_plus_ngram_flush/temp_primary_plus_ngram_flush.2.update.aql
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : temp_primary_plus_ngram_flush.aql
+ * Description     : Check that flush for temporary datasets with ngram indexes succeeds.
+ * Expected Result : Success
+ * Date            : Apr 4 2016
+ */
+
+use dataverse recovery;
+
+load dataset Fragile_raw using localfs
+(("path"="asterix_nc1://data/csv/fragile_02.adm"),("format"="adm")) pre-sorted;
+
+/* Load Fragile data from raw dataset into cleaned dataset */
+insert into dataset Fragile (
+for $t in dataset Fragile_raw
+where $t.row_id <= 1000
+return {
+"row_id": $t.row_id,
+"sid": $t.sid,
+"date": date($t.date),
+"day": $t.day,
+"time": parse-time($t.time, "h:m:s"),
+"bpm": $t.bpm,
+"RR": $t.RR,
+"text": $t.text,
+"location": $t.location,
+"text2": $t.text2}
+);
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/temp-dataset/temp_primary_plus_ngram_flush/temp_primary_plus_ngram_flush.3.query.aql
b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/temp_primary_plus_ngram_flush/temp_primary_plus_ngram_flush.3.query.aql
new file mode 100644
index 0000000..67c41d6
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/temp-dataset/temp_primary_plus_ngram_flush/temp_primary_plus_ngram_flush.3.query.aql
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : temp_primary_plus_ngram_flush.aql
+ * Description     : Check that flush for temporary datasets with ngram indexes succeeds.
+ * Expected Result : Success
+ * Date            : Apr 4 2016
+ */
+
+use dataverse recovery;
+
+count (for $x in dataset Fragile
+where contains($x.text2, "location") return $x);
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/temp-dataset/temp_primary_plus_ngram_flush/temp_primary_plus_ngram_flush.3.adm
b/asterix-app/src/test/resources/runtimets/results/temp-dataset/temp_primary_plus_ngram_flush/temp_primary_plus_ngram_flush.3.adm
new file mode 100644
index 0000000..e37d32a
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/temp-dataset/temp_primary_plus_ngram_flush/temp_primary_plus_ngram_flush.3.adm
@@ -0,0 +1 @@
+1000
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterix-app/src/test/resources/runtimets/testsuite.xml
index 40ba82d..e4e12f3 100644
--- a/asterix-app/src/test/resources/runtimets/testsuite.xml
+++ b/asterix-app/src/test/resources/runtimets/testsuite.xml
@@ -7038,5 +7038,10 @@
                 <output-dir compare="Text">insert-and-scan-dataset-with-index</output-dir>
             </compilation-unit>
         </test-case>
+        <test-case FilePath="temp-dataset">
+            <compilation-unit name="temp_primary_plus_ngram_flush">
+                <output-dir compare="Text">temp_primary_plus_ngram_flush</output-dir>
+            </compilation-unit>
+        </test-case>
     </test-group>
 </test-suite>
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
b/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
index dac05e7..b3eb281 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
@@ -121,8 +121,13 @@
                     }
                 }
             }
+            LogRecord logRecord = new LogRecord();
+            flushOnExit = false;
             if (dsInfo.isDurable()) {
-                LogRecord logRecord = new LogRecord();
+                /**
+                 * Generate a FLUSH log.
+                 * Flush will be triggered when the log is written to disk by LogFlusher.
+                 */
                 TransactionUtil.formFlushLogRecord(logRecord, datasetID, this, logManager.getNodeId(),
                         dsInfo.getDatasetIndexes().size());
                 try {
@@ -130,10 +135,11 @@
                 } catch (ACIDException e) {
                     throw new HyracksDataException("could not write flush log", e);
                 }
+                flushLogCreated = true;
+            } else {
+                //trigger flush for temporary indexes without generating a FLUSH log.
+                triggerScheduleFlush(logRecord);
             }
-
-            flushLogCreated = true;
-            flushOnExit = false;
         }
     }
 

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/785
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: merged
Gerrit-Change-Id: I964566e7ba03bc674209b4d45d9abb82bb581240
Gerrit-PatchSet: 4
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <hubailmor@gmail.com>
Gerrit-Reviewer: Ian Maxon <imaxon@apache.org>
Gerrit-Reviewer: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hubailmor@gmail.com>
Gerrit-Reviewer: abdullah alamoudi <bamousaa@gmail.com>

Mime
View raw message