accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bil...@apache.org
Subject [1/3] accumulo git commit: ACCUMULO-3715 add configurable sampling to internal tracing
Date Sat, 11 Apr 2015 00:15:37 GMT
Repository: accumulo
Updated Branches:
  refs/heads/master 9c8dcaf0a -> 3a99300a2


ACCUMULO-3715 add configurable sampling to internal tracing


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/281c2f32
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/281c2f32
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/281c2f32

Branch: refs/heads/master
Commit: 281c2f3243bf3f90043ca8f17f7bed6a01b60297
Parents: 9c8dcaf
Author: Billie Rinaldi <billie.rinaldi@gmail.com>
Authored: Wed Apr 8 08:35:32 2015 -0700
Committer: Billie Rinaldi <billie.rinaldi@gmail.com>
Committed: Fri Apr 10 10:09:28 2015 -0700

----------------------------------------------------------------------
 .../org/apache/accumulo/core/conf/Property.java |  7 +++++
 .../accumulo/core/trace/ProbabilitySampler.java | 27 ++++++++++++++++++++
 .../accumulo/gc/SimpleGarbageCollector.java     |  4 +--
 .../master/replication/ReplicationDriver.java   |  4 +--
 .../replication/AccumuloReplicaSystem.java      |  5 +++-
 .../tserver/tablet/MinorCompactionTask.java     |  8 ++++--
 .../apache/accumulo/tserver/tablet/Tablet.java  | 10 +++++---
 7 files changed, 55 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/281c2f32/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 9dceb1e..f9d8662 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -275,6 +275,10 @@ public enum Property {
       "The maximum number of concurrent major compactions for a tablet server"),
   TSERV_MINC_MAXCONCURRENT("tserver.compaction.minor.concurrent.max", "4", PropertyType.COUNT,
       "The maximum number of concurrent minor compactions for a tablet server"),
+  TSERV_MAJC_TRACE_PERCENT("tserver.compaction.major.trace.percent", "0.1", PropertyType.FRACTION,
+      "The percent of major compactions to trace"),
+  TSERV_MINC_TRACE_PERCENT("tserver.compaction.minor.trace.percent", "0.1", PropertyType.FRACTION,
+      "The percent of minor compactions to trace"),
   TSERV_COMPACTION_WARN_TIME("tserver.compaction.warn.time", "10m", PropertyType.TIMEDURATION,
       "When a compaction has not made progress for this time period, a warning will be logged"),
   TSERV_BLOOM_LOAD_MAXCONCURRENT("tserver.bloom.load.concurrent.max", "4", PropertyType.COUNT,
@@ -344,6 +348,7 @@ public enum Property {
   GC_DELETE_THREADS("gc.threads.delete", "16", PropertyType.COUNT, "The number of threads
used to delete files"),
   GC_TRASH_IGNORE("gc.trash.ignore", "false", PropertyType.BOOLEAN, "Do not use the Trash,
even if it is configured"),
   GC_FILE_ARCHIVE("gc.file.archive", "false", PropertyType.BOOLEAN, "Archive any files/directories
instead of moving to the HDFS trash or deleting"),
+  GC_TRACE_PERCENT("gc.trace.percent", "0.01", PropertyType.FRACTION, "Percent of gc cycles
to trace"),
 
   // properties that are specific to the monitor server behavior
   MONITOR_PREFIX("monitor.", null, PropertyType.PREFIX, "Properties in this category affect
the behavior of the monitor web server."),
@@ -556,6 +561,8 @@ public enum Property {
       "Amount of time to wait before first checking for replication work, not useful outside
of tests"),
   REPLICATION_WORK_PROCESSOR_PERIOD("replication.work.processor.period", "0s", PropertyType.TIMEDURATION,
       "Amount of time to wait before re-checking for replication work, not useful outside
of tests"),
+  REPLICATION_TRACE_PERCENT("replication.trace.percent", "0.1", PropertyType.FRACTION,
+      "The sampling percentage to use for replication traces"),
 
   ;
 

http://git-wip-us.apache.org/repos/asf/accumulo/blob/281c2f32/core/src/main/java/org/apache/accumulo/core/trace/ProbabilitySampler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/trace/ProbabilitySampler.java b/core/src/main/java/org/apache/accumulo/core/trace/ProbabilitySampler.java
new file mode 100644
index 0000000..ff02fd7
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/trace/ProbabilitySampler.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.trace;
+
+import org.apache.htrace.HTraceConfiguration;
+
+import java.util.Collections;
+
+public class ProbabilitySampler extends org.apache.htrace.impl.ProbabilitySampler {
+  public ProbabilitySampler(double d) {
+    super(HTraceConfiguration.fromMap(Collections.singletonMap(ProbabilitySampler.SAMPLER_FRACTION_CONF_KEY,
Double.toString(d))));
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/281c2f32/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
----------------------------------------------------------------------
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index 04f21a1..ffa59cd 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@ -64,8 +64,8 @@ import org.apache.accumulo.core.replication.ReplicationTable;
 import org.apache.accumulo.core.replication.ReplicationTableOfflineException;
 import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.core.security.thrift.TCredentials;
-import org.apache.accumulo.core.trace.CountSampler;
 import org.apache.accumulo.core.trace.DistributedTrace;
+import org.apache.accumulo.core.trace.ProbabilitySampler;
 import org.apache.accumulo.core.trace.Span;
 import org.apache.accumulo.core.trace.Trace;
 import org.apache.accumulo.core.trace.thrift.TInfo;
@@ -526,7 +526,7 @@ public class SimpleGarbageCollector extends AccumuloServerContext implements
Ifa
       return;
     }
 
-    CountSampler sampler = new CountSampler(100);
+    ProbabilitySampler sampler = new ProbabilitySampler(getConfiguration().getFraction(Property.GC_TRACE_PERCENT));
 
     while (true) {
       Trace.on("gc", sampler);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/281c2f32/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationDriver.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationDriver.java
b/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationDriver.java
index 63c6b20..72b3bbd 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationDriver.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationDriver.java
@@ -21,7 +21,7 @@ import org.apache.accumulo.core.client.AccumuloSecurityException;
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.trace.CountSampler;
+import org.apache.accumulo.core.trace.ProbabilitySampler;
 import org.apache.accumulo.core.trace.Trace;
 import org.apache.accumulo.core.util.Daemon;
 import org.apache.accumulo.fate.util.UtilWaitThread;
@@ -53,7 +53,7 @@ public class ReplicationDriver extends Daemon {
 
   @Override
   public void run() {
-    CountSampler sampler = new CountSampler(10);
+    ProbabilitySampler sampler = new ProbabilitySampler(conf.getFraction(Property.REPLICATION_TRACE_PERCENT));
 
     long millisToWait = conf.getTimeInMillis(Property.REPLICATION_DRIVER_DELAY);
     log.debug("Waiting " + millisToWait + "ms before starting main replication loop");

http://git-wip-us.apache.org/repos/asf/accumulo/blob/281c2f32/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystem.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystem.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystem.java
index cb8ae13..d18e434 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystem.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystem.java
@@ -55,6 +55,7 @@ import org.apache.accumulo.core.replication.thrift.ReplicationServicer.Client;
 import org.apache.accumulo.core.replication.thrift.WalEdits;
 import org.apache.accumulo.core.security.Credentials;
 import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.trace.ProbabilitySampler;
 import org.apache.accumulo.core.trace.Span;
 import org.apache.accumulo.core.trace.Trace;
 import org.apache.accumulo.core.util.UtilWaitThread;
@@ -232,7 +233,9 @@ public class AccumuloReplicaSystem implements ReplicaSystem {
   private Status _replicate(final Path p, final Status status, final ReplicationTarget target,
final ReplicaSystemHelper helper,
       final AccumuloConfiguration localConf, final ClientContext peerContext, final UserGroupInformation
accumuloUgi) {
     try {
-      Trace.on("AccumuloReplicaSystem");
+      double tracePercent = localConf.getFraction(Property.REPLICATION_TRACE_PERCENT);
+      ProbabilitySampler sampler = new ProbabilitySampler(tracePercent);
+      Trace.on("AccumuloReplicaSystem", sampler);
 
       // Remote identifier is an integer (table id) in this case.
       final String remoteTableId = target.getRemoteIdentifier();

http://git-wip-us.apache.org/repos/asf/accumulo/blob/281c2f32/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactionTask.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactionTask.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactionTask.java
index 0f6a98d..53bf5c2 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactionTask.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactionTask.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.tserver.tablet;
 import java.io.IOException;
 
 import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import org.apache.accumulo.core.trace.ProbabilitySampler;
 import org.apache.accumulo.core.trace.Span;
 import org.apache.accumulo.core.trace.Trace;
 import org.apache.accumulo.server.fs.FileRef;
@@ -38,8 +39,9 @@ class MinorCompactionTask implements Runnable {
   private FileRef mergeFile;
   private long flushId;
   private MinorCompactionReason mincReason;
+  private double tracePercent;
 
-  MinorCompactionTask(Tablet tablet, FileRef mergeFile, CommitSession commitSession, long
flushId, MinorCompactionReason mincReason) {
+  MinorCompactionTask(Tablet tablet, FileRef mergeFile, CommitSession commitSession, long
flushId, MinorCompactionReason mincReason, double tracePercent) {
     this.tablet = tablet;
     queued = System.currentTimeMillis();
     tablet.minorCompactionWaitingToStart();
@@ -47,12 +49,14 @@ class MinorCompactionTask implements Runnable {
     this.mergeFile = mergeFile;
     this.flushId = flushId;
     this.mincReason = mincReason;
+    this.tracePercent = tracePercent;
   }
 
   @Override
   public void run() {
     tablet.minorCompactionStarted();
-    Span minorCompaction = Trace.on("minorCompaction");
+    ProbabilitySampler sampler = new ProbabilitySampler(tracePercent);
+    Span minorCompaction = Trace.on("minorCompaction", sampler);
     try {
       FileRef newMapfileLocation = tablet.getNextMapFilename(mergeFile == null ? "F" : "M");
       FileRef tmpFileRef = new FileRef(newMapfileLocation.path() + "_tmp");

http://git-wip-us.apache.org/repos/asf/accumulo/blob/281c2f32/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 43bd9d9..2342789 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -82,6 +82,7 @@ import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.core.security.ColumnVisibility;
 import org.apache.accumulo.core.tabletserver.log.LogEntry;
 import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
+import org.apache.accumulo.core.trace.ProbabilitySampler;
 import org.apache.accumulo.core.trace.Span;
 import org.apache.accumulo.core.trace.Trace;
 import org.apache.accumulo.core.util.LocalityGroupUtil;
@@ -996,7 +997,9 @@ public class Tablet implements TabletCommitter {
       mergeFile = getDatafileManager().reserveMergingMinorCompactionFile();
     }
 
-    return new MinorCompactionTask(this, mergeFile, oldCommitSession, flushId, mincReason);
+    double tracePercent = tabletServer.getConfiguration().getFraction(Property.TSERV_MINC_TRACE_PERCENT);
+
+    return new MinorCompactionTask(this, mergeFile, oldCommitSession, flushId, mincReason,
tracePercent);
 
   }
 
@@ -2075,8 +2078,9 @@ public class Tablet implements TabletCommitter {
     Span span = null;
 
     try {
-      // Always trace majC
-      span = Trace.on("majorCompaction");
+      double tracePercent = tabletServer.getConfiguration().getFraction(Property.TSERV_MAJC_TRACE_PERCENT);
+      ProbabilitySampler sampler = new ProbabilitySampler(tracePercent);
+      span = Trace.on("majorCompaction", sampler);
 
       majCStats = _majorCompact(reason);
       if (reason == MajorCompactionReason.CHOP) {


Mime
View raw message