qpid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kp...@apache.org
Subject [1/2] qpid-cpp git commit: QPID-7383: Fix for excessive time taken to overwrite files when overwrite-before-return option is in use. Also improves time taken to create new journal files for the pool.
Date Tue, 09 Aug 2016 20:58:01 GMT
Repository: qpid-cpp
Updated Branches:
  refs/heads/master 7c54578c6 -> 40a133d32


QPID-7383: Fix for excessive time taken to overwrite files when overwrite-before-return option
is in use. Also improves time taken to create new journal files for the pool.


Project: http://git-wip-us.apache.org/repos/asf/qpid-cpp/repo
Commit: http://git-wip-us.apache.org/repos/asf/qpid-cpp/commit/2dc01547
Tree: http://git-wip-us.apache.org/repos/asf/qpid-cpp/tree/2dc01547
Diff: http://git-wip-us.apache.org/repos/asf/qpid-cpp/diff/2dc01547

Branch: refs/heads/master
Commit: 2dc01547ba5481bfa7ad8e31c183f3ecf18704cd
Parents: 6136161
Author: Kim van der Riet <kpvdr@apache.org>
Authored: Tue Aug 9 16:56:20 2016 -0400
Committer: Kim van der Riet <kpvdr@apache.org>
Committed: Tue Aug 9 16:56:20 2016 -0400

----------------------------------------------------------------------
 src/qpid/linearstore/journal/EmptyFilePool.cpp | 63 +++++++++++++++------
 src/qpid/linearstore/journal/EmptyFilePool.h   |  8 +++
 2 files changed, 54 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/qpid-cpp/blob/2dc01547/src/qpid/linearstore/journal/EmptyFilePool.cpp
----------------------------------------------------------------------
diff --git a/src/qpid/linearstore/journal/EmptyFilePool.cpp b/src/qpid/linearstore/journal/EmptyFilePool.cpp
index 681ced3..3f3b61e 100644
--- a/src/qpid/linearstore/journal/EmptyFilePool.cpp
+++ b/src/qpid/linearstore/journal/EmptyFilePool.cpp
@@ -23,7 +23,6 @@
 
 #include <fstream>
 #include "qpid/linearstore/journal/EmptyFilePoolPartition.h"
-#include "qpid/linearstore/journal/jcfg.h"
 #include "qpid/linearstore/journal/jdir.h"
 #include "qpid/linearstore/journal/JournalLog.h"
 #include "qpid/linearstore/journal/slock.h"
@@ -37,11 +36,20 @@ namespace qpid {
 namespace linearstore {
 namespace journal {
 
-// static
-std::string EmptyFilePool::s_inuseFileDirectory_ = "in_use";
+#define FHDR_BUFF_SIZE (QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_KIB) * 1024
+#define ZERO_BUFF_SIZE QLS_SBLK_SIZE_KIB * 1024
 
-// static
+// tatic declarations
+
+std::string EmptyFilePool::s_inuseFileDirectory_ = "in_use";
 std::string EmptyFilePool::s_returnedFileDirectory_ = "returned";
+size_t EmptyFilePool::s_fhdr_buff_size_ = FHDR_BUFF_SIZE;
+unsigned char EmptyFilePool::s_fhdr_buff_[FHDR_BUFF_SIZE];
+smutex EmptyFilePool::s_fhdr_buff_mutex_;
+size_t EmptyFilePool::s_zero_buff_size_ = ZERO_BUFF_SIZE;
+unsigned char EmptyFilePool::s_zero_buff_[ZERO_BUFF_SIZE];
+bool EmptyFilePool::s_static_initializer_flag_ = false;
+
 
 EmptyFilePool::EmptyFilePool(const std::string& efpDirectory,
                              const EmptyFilePoolPartition* partitionPtr,
@@ -54,7 +62,12 @@ EmptyFilePool::EmptyFilePool(const std::string& efpDirectory,
                 overwriteBeforeReturnFlag_(overwriteBeforeReturnFlag),
                 truncateFlag_(truncateFlag),
                 journalLogRef_(journalLogRef)
-{}
+{
+    if (!s_static_initializer_flag_) {
+        initializeStaticBuffers();
+        s_static_initializer_flag_ = true;
+    }
+}
 
 EmptyFilePool::~EmptyFilePool() {}
 
@@ -250,19 +263,28 @@ void EmptyFilePool::initializeSubDirectory(const std::string& fqDirName)
{
 }
 
 void EmptyFilePool::overwriteFileContents(const std::string& fqFileName) {
-    ::file_hdr_t fh;
-    ::file_hdr_create(&fh, QLS_FILE_MAGIC, QLS_JRNL_VERSION, QLS_JRNL_FHDR_RES_SIZE_SBLKS,
partitionPtr_->getPartitionNumber(), efpDataSize_kib_);
-    std::ofstream ofs(fqFileName.c_str(), std::ofstream::out | std::ofstream::binary);
-    checkIosState(errno, ofs, jerrno::JERR_EFP_FOPEN, fqFileName, "constructor", "Failed
to create file", "EmptyFilePool", "overwriteFileContents");
-    ofs.write((char*)&fh, sizeof(::file_hdr_t));
-    checkIosState(errno, ofs, jerrno::JERR_EFP_FWRITE, fqFileName, "write()", "Failed to
write header", "EmptyFilePool", "overwriteFileContents");
-    uint64_t rem = ((efpDataSize_kib_ + (QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_KIB))
* 1024) - sizeof(::file_hdr_t);
-    while (rem--) {
-        ofs.put('\0');
-        checkIosState(errno, ofs, jerrno::JERR_EFP_FWRITE, fqFileName, "put()", "Failed to
put \0", "EmptyFilePool", "overwriteFileContents");
+    FILE* pFile;
+    pFile = ::fopen(fqFileName.c_str(), "wb");
+    {
+        slock l(s_fhdr_buff_mutex_);
+
+        // Initialize file header
+        ::file_hdr_create((::file_hdr_t*)s_fhdr_buff_,
+                          QLS_FILE_MAGIC,
+                          QLS_JRNL_VERSION,
+                          QLS_JRNL_FHDR_RES_SIZE_SBLKS,
+                          partitionPtr_->getPartitionNumber(),
+                          efpDataSize_kib_);
+
+        // Write file header
+        ::fwrite((void*)s_fhdr_buff_, 1, s_fhdr_buff_size_, pFile);
+    }
+
+    // Fill rest of file with zeros (buffer is 1 sblk in size)
+    for (efpDataSize_sblks_t i = 0; i < dataSize_sblks(); ++i) {
+        ::fwrite((void*)s_zero_buff_, 1, s_zero_buff_size_, pFile);
     }
-    ofs.close();
-//std::cout << "*** WARNING: EFP " << efpDirectory_ << " is empty - created
new journal file " << fqFileName.substr(fqFileName.rfind('/') + 1) << " on the
fly" << std::endl; // DEBUG
+    ::fclose(pFile);
 }
 
 std::string EmptyFilePool::popEmptyFile() {
@@ -476,4 +498,11 @@ bool EmptyFilePool::moveFile(const std::string& from,
     return true;
 }
 
+//static
+void EmptyFilePool::initializeStaticBuffers() {
+    // Overwrite buffers with zeros
+    ::memset(s_fhdr_buff_, 0, s_fhdr_buff_size_);
+    ::memset(s_zero_buff_, 0, s_zero_buff_size_);
+}
+
 }}}

http://git-wip-us.apache.org/repos/asf/qpid-cpp/blob/2dc01547/src/qpid/linearstore/journal/EmptyFilePool.h
----------------------------------------------------------------------
diff --git a/src/qpid/linearstore/journal/EmptyFilePool.h b/src/qpid/linearstore/journal/EmptyFilePool.h
index 935057f..aaf09d5 100644
--- a/src/qpid/linearstore/journal/EmptyFilePool.h
+++ b/src/qpid/linearstore/journal/EmptyFilePool.h
@@ -30,6 +30,7 @@ namespace journal {
 
 #include <deque>
 #include "qpid/linearstore/journal/EmptyFilePoolTypes.h"
+#include "qpid/linearstore/journal/jcfg.h"
 #include "qpid/linearstore/journal/smutex.h"
 
 namespace qpid {
@@ -48,6 +49,12 @@ protected:
 
     static std::string s_inuseFileDirectory_;
     static std::string s_returnedFileDirectory_;
+    static size_t s_fhdr_buff_size_;
+    static unsigned char s_fhdr_buff_[(QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_KIB)
* 1024];
+    static smutex s_fhdr_buff_mutex_;
+    static size_t s_zero_buff_size_;
+    static unsigned char s_zero_buff_[QLS_SBLK_SIZE_KIB * 1024];
+    static bool s_static_initializer_flag_;
 
     const std::string efpDirectory_;
     const efpDataSize_kib_t efpDataSize_kib_;
@@ -112,6 +119,7 @@ protected:
     static bool isSymlink(const std::string& fqName);
     static bool moveFile(const std::string& fromFqPath,
                          const std::string& toFqPath);
+    static void initializeStaticBuffers();
 };
 
 }}}


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@qpid.apache.org
For additional commands, e-mail: commits-help@qpid.apache.org


Mime
View raw message