hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From whe...@apache.org
Subject [31/50] [abbrv] hadoop git commit: [partial-ns] Add skip_wal option to HDFSDB.
Date Tue, 05 Jan 2016 19:52:31 GMT
[partial-ns] Add skip_wal option to HDFSDB.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c2cf9bc9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c2cf9bc9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c2cf9bc9

Branch: refs/heads/feature-HDFS-8286
Commit: c2cf9bc97f96699142826bff986243a719113322
Parents: af9ff74
Author: Haohui Mai <wheat9@apache.org>
Authored: Mon Nov 3 14:44:26 2014 -0800
Committer: Haohui Mai <wheat9@apache.org>
Committed: Fri Jun 12 13:57:00 2015 -0700

----------------------------------------------------------------------
 .../apache/hadoop/hdfs/hdfsdb/WriteOptions.java |  6 +++
 .../src/main/native/hdfsdb/db/db_impl.cc        | 48 +++++++++++---------
 .../native/hdfsdb/include/leveldb/options.h     |  5 +-
 .../src/main/native/jni/bindings.cc             |  5 ++
 4 files changed, 41 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2cf9bc9/hadoop-hdfs-project/hadoop-hdfsdb/src/main/java/org/apache/hadoop/hdfs/hdfsdb/WriteOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfsdb/src/main/java/org/apache/hadoop/hdfs/hdfsdb/WriteOptions.java
b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/java/org/apache/hadoop/hdfs/hdfsdb/WriteOptions.java
index 88ebcad..2dc04bc 100644
--- a/hadoop-hdfs-project/hadoop-hdfsdb/src/main/java/org/apache/hadoop/hdfs/hdfsdb/WriteOptions.java
+++ b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/java/org/apache/hadoop/hdfs/hdfsdb/WriteOptions.java
@@ -27,6 +27,11 @@ public class WriteOptions extends NativeObject {
     return this;
   }
 
+  public WriteOptions skipWal(boolean value) {
+    skipWal(nativeHandle, value);
+    return this;
+  }
+
   @Override
   public void close() {
     if (nativeHandle != 0) {
@@ -38,4 +43,5 @@ public class WriteOptions extends NativeObject {
   private static native long construct();
   private static native void destruct(long handle);
   private static native void sync(long handle, boolean value);
+  private static native void skipWal(long handle, boolean value);
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2cf9bc9/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/db_impl.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/db_impl.cc b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/db_impl.cc
index faf5e7d..1225412 100644
--- a/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/db_impl.cc
+++ b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/db_impl.cc
@@ -1183,29 +1183,33 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch*
my_batch) {
     WriteBatchInternal::SetSequence(updates, last_sequence + 1);
     last_sequence += WriteBatchInternal::Count(updates);
 
-    // Add to log and apply to memtable.  We can release the lock
-    // during this phase since &w is currently responsible for logging
-    // and protects against concurrent loggers and concurrent writes
-    // into mem_.
-    {
-      mutex_.Unlock();
-      status = log_->AddRecord(WriteBatchInternal::Contents(updates));
-      bool sync_error = false;
-      if (status.ok() && options.sync) {
-        status = logfile_->Sync();
-        if (!status.ok()) {
-          sync_error = true;
+    if (options.skip_wal) {
+      status = WriteBatchInternal::InsertInto(updates, mem_);
+    } else {
+      // Add to log and apply to memtable.  We can release the lock
+      // during this phase since &w is currently responsible for logging
+      // and protects against concurrent loggers and concurrent writes
+      // into mem_.
+      {
+        mutex_.Unlock();
+        status = log_->AddRecord(WriteBatchInternal::Contents(updates));
+        bool sync_error = false;
+        if (status.ok() && options.sync) {
+          status = logfile_->Sync();
+          if (!status.ok()) {
+            sync_error = true;
+          }
+        }
+        if (status.ok()) {
+          status = WriteBatchInternal::InsertInto(updates, mem_);
+        }
+        mutex_.Lock();
+        if (sync_error) {
+          // The state of the log file is indeterminate: the log record we
+          // just added may or may not show up when the DB is re-opened.
+          // So we force the DB into a mode where all future writes fail.
+          RecordBackgroundError(status);
         }
-      }
-      if (status.ok()) {
-        status = WriteBatchInternal::InsertInto(updates, mem_);
-      }
-      mutex_.Lock();
-      if (sync_error) {
-        // The state of the log file is indeterminate: the log record we
-        // just added may or may not show up when the DB is re-opened.
-        // So we force the DB into a mode where all future writes fail.
-        RecordBackgroundError(status);
       }
     }
     if (updates == tmp_batch_) tmp_batch_->Clear();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2cf9bc9/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/include/leveldb/options.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/include/leveldb/options.h
b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/include/leveldb/options.h
index fdda718..66eaf75 100644
--- a/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/include/leveldb/options.h
+++ b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/include/leveldb/options.h
@@ -184,9 +184,12 @@ struct WriteOptions {
   //
   // Default: false
   bool sync;
+  // If true, the write will not write the write-ahead log.
+  bool skip_wal;
 
   WriteOptions()
-      : sync(false) {
+      : sync(false)
+      , skip_wal(false) {
   }
 };
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2cf9bc9/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/jni/bindings.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/jni/bindings.cc b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/jni/bindings.cc
index 0b03766..33604b8 100644
--- a/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/jni/bindings.cc
+++ b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/jni/bindings.cc
@@ -229,6 +229,11 @@ void JNICALL Java_org_apache_hadoop_hdfs_hdfsdb_WriteOptions_sync(JNIEnv
*, jcla
   options->sync = value;
 }
 
+void JNICALL Java_org_apache_hadoop_hdfs_hdfsdb_WriteOptions_skipWal(JNIEnv *, jclass, jlong
handle, jboolean value) {
+  leveldb::WriteOptions *options = reinterpret_cast<leveldb::WriteOptions*>(handle);
+  options->skip_wal = value;
+}
+
 void JNICALL Java_org_apache_hadoop_hdfs_hdfsdb_WriteOptions_destruct(JNIEnv *, jclass, jlong
handle) {
   delete reinterpret_cast<leveldb::WriteOptions*>(handle);
 }


Mime
View raw message