htrace-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From iwasak...@apache.org
Subject incubator-htrace git commit: HTRACE-309. htraced: improve leveldb configuration (Colin Patrick McCabe via iwasakims)
Date Sun, 29 Nov 2015 15:49:56 GMT
Repository: incubator-htrace
Updated Branches:
  refs/heads/master 94bc87844 -> 3b10cfb53


HTRACE-309. htraced: improve leveldb configuration (Colin Patrick McCabe via iwasakims)


Project: http://git-wip-us.apache.org/repos/asf/incubator-htrace/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-htrace/commit/3b10cfb5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-htrace/tree/3b10cfb5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-htrace/diff/3b10cfb5

Branch: refs/heads/master
Commit: 3b10cfb534724eba81f6ef17657d32c42cf9a3cc
Parents: 94bc878
Author: Masatake Iwasaki <iwasakims@apache.org>
Authored: Sat Nov 28 09:54:06 2015 +0900
Committer: Masatake Iwasaki <iwasakims@apache.org>
Committed: Sat Nov 28 09:54:06 2015 +0900

----------------------------------------------------------------------
 .../src/org/apache/htrace/conf/config_keys.go   |  5 ++
 .../src/org/apache/htrace/htraced/datastore.go  | 60 +++++++++++++++++++-
 2 files changed, 63 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/3b10cfb5/htrace-htraced/go/src/org/apache/htrace/conf/config_keys.go
----------------------------------------------------------------------
diff --git a/htrace-htraced/go/src/org/apache/htrace/conf/config_keys.go b/htrace-htraced/go/src/org/apache/htrace/conf/config_keys.go
index 10204cd..6f6e8cf 100644
--- a/htrace-htraced/go/src/org/apache/htrace/conf/config_keys.go
+++ b/htrace-htraced/go/src/org/apache/htrace/conf/config_keys.go
@@ -96,6 +96,10 @@ const HTRACE_NUM_HRPC_HANDLERS = "num.hrpc.handlers"
 // this to read or write a message, we will abort the connection.
 const HTRACE_HRPC_IO_TIMEOUT_MS = "hrpc.io.timeout.ms"
 
+// The leveldb write buffer size, or 0 to use the library default, which is 4
+// MB in leveldb 1.16.  See leveldb's options.h for more details.
+const HTRACE_LEVELDB_WRITE_BUFFER_SIZE = "leveldb.write.buffer.size"
+
 // Default values for HTrace configuration keys.
 var DEFAULTS = map[string]string{
 	HTRACE_WEB_ADDRESS:  fmt.Sprintf("0.0.0.0:%d", HTRACE_WEB_ADDRESS_DEFAULT_PORT),
@@ -112,6 +116,7 @@ var DEFAULTS = map[string]string{
 	HTRACE_REAPER_HEARTBEAT_PERIOD_MS:    fmt.Sprintf("%d", 90*1000),
 	HTRACE_NUM_HRPC_HANDLERS:             "20",
 	HTRACE_HRPC_IO_TIMEOUT_MS:            "60000",
+	HTRACE_LEVELDB_WRITE_BUFFER_SIZE:     "0",
 }
 
 // Values to be used when creating test configurations

http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/3b10cfb5/htrace-htraced/go/src/org/apache/htrace/htraced/datastore.go
----------------------------------------------------------------------
diff --git a/htrace-htraced/go/src/org/apache/htrace/htraced/datastore.go b/htrace-htraced/go/src/org/apache/htrace/htraced/datastore.go
index 2a3d65c..828a6af 100644
--- a/htrace-htraced/go/src/org/apache/htrace/htraced/datastore.go
+++ b/htrace-htraced/go/src/org/apache/htrace/htraced/datastore.go
@@ -27,6 +27,7 @@ import (
 	"fmt"
 	"github.com/jmhodges/levigo"
 	"github.com/ugorji/go/codec"
+	"math"
 	"org/apache/htrace/common"
 	"org/apache/htrace/conf"
 	"os"
@@ -34,6 +35,7 @@ import (
 	"strings"
 	"sync"
 	"sync/atomic"
+	"syscall"
 	"time"
 )
 
@@ -460,6 +462,9 @@ type dataStore struct {
 
 	// When this datastore was started (in UTC milliseconds since the epoch)
 	startMs int64
+
+	// The maximum number of open files to allow per shard.
+	maxFdPerShard int
 }
 
 func CreateDataStore(cnf *conf.Config, writtenSpans *common.Semaphore) (*dataStore, error)
{
@@ -482,6 +487,7 @@ func CreateDataStore(cnf *conf.Config, writtenSpans *common.Semaphore)
(*dataSto
 
 	store.readOpts = levigo.NewReadOptions()
 	store.readOpts.SetFillCache(true)
+	store.readOpts.SetVerifyChecksums(false)
 	store.writeOpts = levigo.NewWriteOptions()
 	store.writeOpts.SetSync(false)
 
@@ -514,9 +520,49 @@ func CreateDataStore(cnf *conf.Config, writtenSpans *common.Semaphore)
(*dataSto
 		})
 	}
 	store.startMs = common.TimeToUnixMs(time.Now().UTC())
+	err = store.calculateMaxOpenFilesPerShard()
+	if err != nil {
+		lg.Warnf("Unable to calculate maximum open files per shard: %s\n",
+			err.Error())
+	}
 	return store, nil
 }
 
+// The maximum number of file descriptors we'll use on non-datastore things.
+const NON_DATASTORE_FD_MAX = 300
+
+// The minimum number of file descriptors per shard we will set.  Setting fewer
+// than this number could trigger a bug in some early versions of leveldb.
+const MIN_FDS_PER_SHARD = 80
+
+func (store *dataStore) calculateMaxOpenFilesPerShard() error {
+	var rlim syscall.Rlimit
+	err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlim)
+	if err != nil {
+		return err
+	}
+	// I think RLIMIT_NOFILE fits in 32 bits on all known operating systems,
+	// but there's no harm in being careful.  'int' in golang always holds at
+	// least 32 bits.
+	var maxFd int
+	if rlim.Cur > uint64(math.MaxInt32) {
+		maxFd = math.MaxInt32
+	} else {
+		maxFd = int(rlim.Cur)
+	}
+	fdsPerShard := (maxFd - NON_DATASTORE_FD_MAX) / len(store.shards)
+	if fdsPerShard < MIN_FDS_PER_SHARD {
+		return errors.New(fmt.Sprintf("Expected to be able to use at least %d " +
+			"fds per shard, but we have %d shards and %d total fds to allocate, " +
+			"giving us only %d FDs per shard.", MIN_FDS_PER_SHARD,
+			len(store.shards), maxFd - NON_DATASTORE_FD_MAX, fdsPerShard))
+	}
+	store.lg.Infof("maxFd = %d.  Setting maxFdPerShard = %d\n",
+		maxFd, fdsPerShard)
+	store.maxFdPerShard = fdsPerShard
+	return nil
+}
+
 func CreateShard(store *dataStore, cnf *conf.Config, path string,
 	clearStored bool) (*shard, error) {
 	lg := store.lg
@@ -543,6 +589,14 @@ func CreateShard(store *dataStore, cnf *conf.Config, path string,
 	}
 	var shd *shard
 	openOpts := levigo.NewOptions()
+	openOpts.SetParanoidChecks(false)
+	writeBufferSize := cnf.GetInt(conf.HTRACE_LEVELDB_WRITE_BUFFER_SIZE)
+	if writeBufferSize > 0 {
+		openOpts.SetWriteBufferSize(writeBufferSize)
+	}
+	if store.maxFdPerShard > 0 {
+		openOpts.SetMaxOpenFiles(store.maxFdPerShard)
+	}
 	defer openOpts.Close()
 	newlyCreated := false
 	ldb, err := levigo.Open(path, openOpts)
@@ -636,8 +690,10 @@ func (store *dataStore) Close() {
 		store.hb = nil
 	}
 	for idx := range store.shards {
-		store.shards[idx].Close()
-		store.shards[idx] = nil
+		if store.shards[idx] != nil {
+			store.shards[idx].Close()
+			store.shards[idx] = nil
+		}
 	}
 	if store.rpr != nil {
 		store.rpr.Shutdown()


Mime
View raw message