Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id A69CC200CFC for ; Thu, 28 Sep 2017 21:50:25 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id A4DC41609CD; Thu, 28 Sep 2017 19:50:25 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id C8CE91609B4 for ; Thu, 28 Sep 2017 21:50:23 +0200 (CEST) Received: (qmail 2272 invoked by uid 500); 28 Sep 2017 19:50:16 -0000 Mailing-List: contact notifications-help@asterixdb.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@asterixdb.apache.org Delivered-To: mailing list notifications@asterixdb.apache.org Received: (qmail 2246 invoked by uid 99); 28 Sep 2017 19:50:16 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd1-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 28 Sep 2017 19:50:16 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd1-us-west.apache.org (ASF Mail Server at spamd1-us-west.apache.org) with ESMTP id A37FED319D for ; Thu, 28 Sep 2017 19:50:15 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd1-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: 2.127 X-Spam-Level: ** X-Spam-Status: No, score=2.127 tagged_above=-999 required=6.31 tests=[MISSING_HEADERS=1.207, SPF_FAIL=0.919, URIBL_BLOCKED=0.001] autolearn=disabled Received: from mx1-lw-us.apache.org ([10.40.0.8]) by localhost (spamd1-us-west.apache.org [10.40.0.7]) (amavisd-new, port 10024) with ESMTP id QsB2c3gHQq0v for ; Thu, 28 Sep 2017 19:50:11 +0000 (UTC) Received: from vitalstatistix.ics.uci.edu (vitalstatistix.ics.uci.edu [128.195.52.38]) by mx1-lw-us.apache.org (ASF Mail Server at mx1-lw-us.apache.org) with ESMTP id 35CDA5F5F7 for ; Thu, 28 Sep 2017 19:50:11 +0000 (UTC) Received: from localhost (localhost [127.0.0.1]) by vitalstatistix.ics.uci.edu (Postfix) with ESMTP id BABA1100728; Thu, 28 Sep 2017 12:50:10 -0700 (PDT) Date: Thu, 28 Sep 2017 12:50:10 -0700 From: "Till Westmann (Code Review)" CC: Jenkins , Michael Blow , abdullah alamoudi Reply-To: tillw@apache.org X-Gerrit-MessageType: merged Subject: Change in asterixdb[master]: [NO ISSUE] Add support for tracing X-Gerrit-Change-Id: Ifa83592870e7e38a84aa560e4edb7fa2ad2d5e74 X-Gerrit-ChangeURL: X-Gerrit-Commit: b67505d6a707537173833c701f6a86378c1cfc47 In-Reply-To: References: MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Content-Disposition: inline User-Agent: Gerrit/2.12.7 Message-Id: <20170928195010.BABA1100728@vitalstatistix.ics.uci.edu> archived-at: Thu, 28 Sep 2017 19:50:25 -0000 Till Westmann has submitted this change and it was merged. Change subject: [NO ISSUE] Add support for tracing ...................................................................... [NO ISSUE] Add support for tracing - using a format that can can be visualized using chrome://tracing in Chrome Change-Id: Ifa83592870e7e38a84aa560e4edb7fa2ad2d5e74 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2022 Tested-by: Jenkins Contrib: Jenkins Integration-Tests: Jenkins Reviewed-by: Michael Blow --- M hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/INCServiceContext.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetNodeDetailsJSONWork.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/application/NCServiceContext.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AsynchronousScheduler.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java A hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java M hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestNCServiceContext.java M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceTest.java M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java R hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java A hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java 26 files changed, 342 insertions(+), 38 deletions(-) Approvals: Jenkins: Verified; ; Verified Michael Blow: Looks good to me, approved Objections: Anon. E. Moose #1000171: Jenkins: Violations found diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/INCServiceContext.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/INCServiceContext.java index d72cc01..9a484ca 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/INCServiceContext.java +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/INCServiceContext.java @@ -22,6 +22,7 @@ import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.api.lifecycle.ILifeCycleComponentManager; import org.apache.hyracks.api.resources.memory.IMemoryManager; +import org.apache.hyracks.util.trace.Tracer; /** * Service Context at the Node Controller for an application. @@ -54,6 +55,13 @@ IMemoryManager getMemoryManager(); /** + * Get a Tracer to write trace events to. + * + * @return a Tracer + */ + Tracer getTracer(); + + /** * Set the handler for state dumps. * * @param handler diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetNodeDetailsJSONWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetNodeDetailsJSONWork.java index 3dec959..6433223 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetNodeDetailsJSONWork.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetNodeDetailsJSONWork.java @@ -39,7 +39,7 @@ import org.apache.hyracks.control.common.config.ConfigUtils; import org.apache.hyracks.control.common.controllers.CCConfig; import org.apache.hyracks.control.common.controllers.NCConfig; -import org.apache.hyracks.control.common.utils.PidHelper; +import org.apache.hyracks.util.PidHelper; import org.apache.hyracks.control.common.work.IPCResponder; import org.apache.hyracks.control.common.work.SynchronizableWork; diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java index 0fc3be7..02a469d 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java @@ -78,7 +78,8 @@ APP_CLASS(STRING, (String) null), NCSERVICE_PID(INTEGER, -1), COMMAND(STRING, "hyracksnc"), - JVM_ARGS(STRING, (String) null); + JVM_ARGS(STRING, (String) null), + TRACE_CATEGORIES(STRING_ARRAY, new String[0]); private final IOptionType parser; private final String defaultValueDescription; @@ -190,6 +191,8 @@ return "Command NCService should invoke to start the NCDriver"; case JVM_ARGS: return "JVM args to pass to the NCDriver"; + case TRACE_CATEGORIES: + return "Categories for tracing"; default: throw new IllegalStateException("NYI: " + this); } @@ -445,6 +448,14 @@ configManager.set(nodeId, Option.IODEVICES, iodevices); } + public String[] getTraceCategories() { + return appConfig.getStringArray(Option.TRACE_CATEGORIES); + } + + public void setTraceCategories(String[] traceCategories) { + configManager.set(nodeId, Option.TRACE_CATEGORIES, traceCategories); + } + public int getNetThreadCount() { return appConfig.getInt(Option.NET_THREAD_COUNT); } diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java index a426d47..b0a702d 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java @@ -69,7 +69,7 @@ import org.apache.hyracks.control.common.ipc.ClusterControllerRemoteProxy; import org.apache.hyracks.control.common.ipc.IControllerRemoteProxyIPCEventListener; import org.apache.hyracks.control.common.job.profiling.om.JobProfile; -import org.apache.hyracks.control.common.utils.PidHelper; +import org.apache.hyracks.util.PidHelper; import org.apache.hyracks.control.common.work.FutureValue; import org.apache.hyracks.control.common.work.WorkQueue; import org.apache.hyracks.control.nc.application.NCServiceContext; diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/application/NCServiceContext.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/application/NCServiceContext.java index d659fe6..d1d2dc9 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/application/NCServiceContext.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/application/NCServiceContext.java @@ -34,6 +34,7 @@ import org.apache.hyracks.control.nc.NodeControllerService; import org.apache.hyracks.control.nc.io.IOManager; import org.apache.hyracks.control.nc.resources.memory.MemoryManager; +import org.apache.hyracks.util.trace.Tracer; public class NCServiceContext extends ServiceContext implements INCServiceContext { private final ILifeCycleComponentManager lccm; @@ -43,9 +44,10 @@ private IStateDumpHandler sdh; private final NodeControllerService ncs; private IChannelInterfaceFactory messagingChannelInterfaceFactory; + private final Tracer tracer; - public NCServiceContext(NodeControllerService ncs, ServerContext serverCtx, IOManager ioManager, - String nodeId, MemoryManager memoryManager, ILifeCycleComponentManager lifeCyclecomponentManager, + public NCServiceContext(NodeControllerService ncs, ServerContext serverCtx, IOManager ioManager, String nodeId, + MemoryManager memoryManager, ILifeCycleComponentManager lifeCyclecomponentManager, IApplicationConfig appConfig) throws IOException { super(serverCtx, appConfig, new HyracksThreadFactory(nodeId)); this.lccm = lifeCyclecomponentManager; @@ -53,7 +55,8 @@ this.ioManager = ioManager; this.memoryManager = memoryManager; this.ncs = ncs; - sdh = lccm::dumpState; + this.sdh = lccm::dumpState; + this.tracer = new Tracer(nodeId, ncs.getConfiguration().getTraceCategories()); } @Override @@ -90,6 +93,11 @@ } @Override + public Tracer getTracer() { + return tracer; + } + + @Override public IControllerService getControllerService() { return ncs; } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml index b3798ab..48961db 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml @@ -86,5 +86,10 @@ hyracks-data-std ${project.version} + + org.apache.hyracks + hyracks-util + ${project.version} + diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java index c283502..dad56eb 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java @@ -77,6 +77,6 @@ mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx), opTrackerProvider.getOperationTracker(serviceCtx), ioSchedulerProvider.getIoScheduler(serviceCtx), ioOpCallbackFactory.createIoOpCallback(), isPrimary, filterTypeTraits, filterCmpFactories, btreeFields, - filterFields, durable, metadataPageManagerFactory, updateAware); + filterFields, durable, metadataPageManagerFactory, updateAware, serviceCtx.getTracer()); } } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java index 24b408c..1920592 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java @@ -76,6 +76,7 @@ import org.apache.hyracks.storage.common.ISearchPredicate; import org.apache.hyracks.storage.common.MultiComparator; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.util.trace.Tracer; public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { @@ -106,10 +107,10 @@ double bloomFilterFalsePositiveRate, int fieldCount, IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, int[] btreeFields, int[] filterFields, - boolean durable, boolean updateAware) throws HyracksDataException { + boolean durable, boolean updateAware, Tracer tracer) throws HyracksDataException { super(ioManager, virtualBufferCaches, diskBTreeFactory.getBufferCache(), fileManager, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory, - filterManager, filterFields, durable, filterHelper, btreeFields); + filterManager, filterFields, durable, filterHelper, btreeFields, tracer); this.insertLeafFrameFactory = insertLeafFrameFactory; this.deleteLeafFrameFactory = deleteLeafFrameFactory; this.cmpFactories = cmpFactories; diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java index 5e50eee..a9c7a80 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java @@ -53,6 +53,7 @@ import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager; import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.util.trace.Tracer; public class LSMBTreeUtil { @@ -65,7 +66,7 @@ ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields, int[] filterFields, boolean durable, - IMetadataPageManagerFactory freePageManagerFactory, boolean updateAware) + IMetadataPageManagerFactory freePageManagerFactory, boolean updateAware, Tracer tracer) throws HyracksDataException { LSMBTreeTupleWriterFactory insertTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, false, updateAware); @@ -109,7 +110,7 @@ deleteLeafFrameFactory, fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory, filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, typeTraits.length, cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, needKeyDupCheck, btreeFields, - filterFields, durable, updateAware); + filterFields, durable, updateAware, tracer); } public static ExternalBTree createExternalBTree(IIOManager ioManager, FileReference file, diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java index c5bf2ca..beed6b4 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java @@ -63,6 +63,7 @@ import org.apache.hyracks.storage.common.IModificationOperationCallback; import org.apache.hyracks.storage.common.ISearchOperationCallback; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.util.trace.Tracer; public abstract class AbstractLSMIndex implements ILSMIndex { protected final ILSMHarness lsmHarness; @@ -90,13 +91,14 @@ protected boolean isActive; protected final AtomicBoolean[] flushRequests; protected boolean memoryComponentsAllocated = false; + protected Tracer tracer; public AbstractLSMIndex(IIOManager ioManager, List virtualBufferCaches, IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager, int[] filterFields, boolean durable, - IComponentFilterHelper filterHelper, int[] treeFields) { + IComponentFilterHelper filterHelper, int[] treeFields, Tracer tracer) { this.ioManager = ioManager; this.virtualBufferCaches = virtualBufferCaches; this.diskBufferCache = diskBufferCache; @@ -112,6 +114,7 @@ this.filterFields = filterFields; this.inactiveDiskComponents = new LinkedList<>(); this.durable = durable; + this.tracer = tracer; lsmHarness = new LSMHarness(this, mergePolicy, opTracker, diskBufferCache.isReplicationEnabled()); isActive = false; diskComponents = new ArrayList<>(); @@ -333,7 +336,7 @@ opCtx.setOperation(IndexOperation.FLUSH); opCtx.getComponentHolder().add(flushingComponent); ILSMIOOperation flushOp = createFlushOperation(opCtx, flushingComponent, componentFileRefs, callback); - ioScheduler.scheduleOperation(flushOp); + ioScheduler.scheduleOperation(TracedIOOperation.wrap(flushOp, tracer)); } @Override @@ -348,7 +351,7 @@ ILSMDiskComponent lastComponent = (ILSMDiskComponent) mergingComponents.get(mergingComponents.size() - 1); LSMComponentFileReferences mergeFileRefs = getMergeFileReferences(firstComponent, lastComponent); ILSMIOOperation mergeOp = createMergeOperation(opCtx, mergingComponents, mergeFileRefs, callback); - ioScheduler.scheduleOperation(mergeOp); + ioScheduler.scheduleOperation(TracedIOOperation.wrap(mergeOp, tracer)); } private void addOperationalMutableComponents(List operationalComponents) { diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AsynchronousScheduler.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AsynchronousScheduler.java index 50e240e..5f6766f 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AsynchronousScheduler.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AsynchronousScheduler.java @@ -41,12 +41,13 @@ public final static AsynchronousScheduler INSTANCE = new AsynchronousScheduler(); private ExecutorService executor; private final Map runningFlushOperations = new HashMap(); - private final Map> waitingFlushOperations = new HashMap>(); + private final Map> waitingFlushOperations = + new HashMap>(); public void init(ThreadFactory threadFactory) { // Creating an executor with the same configuration of Executors.newCachedThreadPool. - executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, - new SynchronousQueue(), threadFactory) { + executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, new SynchronousQueue(), + threadFactory) { @Override protected RunnableFuture newTaskFor(Callable callable) { diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java index cafb008..3d65a6f 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java @@ -23,11 +23,12 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent; -public class FlushOperation extends AbstractIoOperation implements Comparable { +public class FlushOperation extends AbstractIoOperation implements Comparable { protected final ILSMMemoryComponent flushingComponent; @@ -73,16 +74,17 @@ } @Override - public int compareTo(FlushOperation o) { - return target.getFile().getName().compareTo(o.getTarget().getFile().getName()); + public int compareTo(ILSMIOOperation o) { + if (o instanceof FlushOperation) { + return target.getFile().getName().compareTo(((FlushOperation) o).getTarget().getFile().getName()); + } + return -1; } @Override public boolean equals(Object o) { - if (!(o instanceof FlushOperation)) { - return false; - } - return Objects.equals(target.getFile().getName(), ((FlushOperation) o).target.getFile().getName()); + return (o instanceof FlushOperation) + && Objects.equals(target.getFile().getName(), ((FlushOperation) o).target.getFile().getName()); } @Override diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java new file mode 100644 index 0000000..a58b0b1 --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hyracks.storage.am.lsm.common.impls; + +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.io.IODeviceHandle; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; +import org.apache.hyracks.util.trace.Tracer; + +class TracedIOOperation implements ILSMIOOperation { + protected final ILSMIOOperation ioOp; + private final LSMIOOpertionType ioOpType; + private final Tracer tracer; + private final String cat; + + protected TracedIOOperation(ILSMIOOperation ioOp, Tracer tracer) { + this.ioOp = ioOp; + this.tracer = tracer; + this.ioOpType = ioOp.getIOOpertionType(); + this.cat = ioOpType.name().toLowerCase(); + } + + public static ILSMIOOperation wrap(final ILSMIOOperation ioOp, final Tracer tracer) { + if (tracer != null && tracer.isEnabled()) { + return ioOp instanceof Comparable ? new ComparableTracedIOOperation(ioOp, tracer) + : new TracedIOOperation(ioOp, tracer); + } + return ioOp; + } + + protected ILSMIOOperation getIoOp() { + return ioOp; + } + + public IODeviceHandle getDevice() { + return ioOp.getDevice(); + } + + public ILSMIOOperationCallback getCallback() { + return ioOp.getCallback(); + } + + public String getIndexIdentifier() { + return ioOp.getIndexIdentifier(); + } + + public LSMIOOpertionType getIOOpertionType() { + return ioOpType; + } + + @Override + public Boolean call() throws HyracksDataException { + final long tid = tracer.durationB(getDevice().toString(), cat, null); + try { + return ioOp.call(); + } finally { + tracer.durationE(tid, "{\"optional\":\"value\"}"); + } + } +} + +class ComparableTracedIOOperation extends TracedIOOperation implements Comparable { + + protected ComparableTracedIOOperation(ILSMIOOperation ioOp, Tracer trace) { + super(ioOp, trace); + System.err.println("COMPARE ComparableTracedIOOperation"); + } + + public int compareTo(ILSMIOOperation other) { + System.err.println("COMPARE compareTo " + other.getClass().getSimpleName()); + if (other instanceof ComparableTracedIOOperation) { + other = ((ComparableTracedIOOperation) other).getIoOp(); + return ((Comparable) this.ioOp).compareTo(other); + } + throw new IllegalArgumentException("Comparing ioOps of type " + this.ioOp.getClass().getSimpleName() + " and " + + other.getClass().getSimpleName()); + } +} \ No newline at end of file diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java index 1f2a810..7994bf0 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java @@ -86,6 +86,7 @@ import org.apache.hyracks.storage.common.ISearchPredicate; import org.apache.hyracks.storage.common.MultiComparator; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.util.trace.Tracer; public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex { private static final Logger LOGGER = Logger.getLogger(LSMInvertedIndex.class.getName()); @@ -117,7 +118,7 @@ throws HyracksDataException { super(ioManager, virtualBufferCaches, diskInvIndexFactory.getBufferCache(), fileManager, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory, - filterManager, filterFields, durable, filterHelper, invertedIndexFields); + filterManager, filterFields, durable, filterHelper, invertedIndexFields, Tracer.none()); this.tokenizerFactory = tokenizerFactory; this.invListTypeTraits = invListTypeTraits; this.invListCmpFactories = invListCmpFactories; diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java index 6ca48b3..eb03696 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java @@ -57,6 +57,7 @@ import org.apache.hyracks.storage.common.ISearchOperationCallback; import org.apache.hyracks.storage.common.ISearchPredicate; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.util.trace.Tracer; public abstract class AbstractLSMRTree extends AbstractLSMIndex implements ITreeIndex { @@ -91,7 +92,7 @@ boolean isPointMBR, IBufferCache diskBufferCache) throws HyracksDataException { super(ioManager, virtualBufferCaches, diskBufferCache, fileManager, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory, filterManager, filterFields, durable, - filterHelper, rtreeFields); + filterHelper, rtreeFields, Tracer.none()); int i = 0; for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) { RTree memRTree = new RTree(virtualBufferCache, new VirtualFreePageManager(virtualBufferCache), diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestNCServiceContext.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestNCServiceContext.java index ee74b75..cbdb7b7 100644 --- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestNCServiceContext.java +++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestNCServiceContext.java @@ -32,6 +32,7 @@ import org.apache.hyracks.api.messages.IMessageBroker; import org.apache.hyracks.api.resources.memory.IMemoryManager; import org.apache.hyracks.api.service.IControllerService; +import org.apache.hyracks.util.trace.Tracer; public class TestNCServiceContext implements INCServiceContext { private final ILifeCycleComponentManager lccm; @@ -86,6 +87,11 @@ } @Override + public Tracer getTracer() { + return Tracer.none(); + } + + @Override public void setMessageBroker(IMessageBroker staticticsConnector) { } diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml index e3f49c7..55c5710 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml @@ -104,9 +104,13 @@ ${project.version} + org.apache.hyracks + hyracks-util + ${project.version} + + org.apache.commons commons-lang3 - 3.5 diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java index 39b16c6..56af40d 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java @@ -37,6 +37,7 @@ import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestHarness; import org.apache.hyracks.storage.am.lsm.btree.utils.LSMBTreeUtil; import org.apache.hyracks.storage.common.IIndexAccessor; +import org.apache.hyracks.util.trace.Tracer; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -53,7 +54,7 @@ bloomFilterKeyFields, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(), true, filterTypeTraits, filterCmpFactories, btreeFields, filterFields, true, - harness.getMetadataPageManagerFactory(), false); + harness.getMetadataPageManagerFactory(), false, Tracer.none()); } @Before diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java index 98672e7..87ccde7 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java @@ -29,6 +29,7 @@ import org.apache.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper; import org.apache.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallbackFactory; import org.apache.hyracks.storage.am.lsm.common.impls.NoOpOperationTrackerFactory; +import org.apache.hyracks.util.trace.Tracer; import org.junit.Test; public class LSMBTreeModificationOperationCallbackTest extends AbstractModificationOperationCallbackTest { @@ -52,7 +53,7 @@ harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), NoOpOperationTrackerFactory.INSTANCE.getOperationTracker(null), harness.getIOScheduler(), harness.getIOOperationCallback(), true, null, null, null, null, true, - harness.getMetadataPageManagerFactory(), false); + harness.getMetadataPageManagerFactory(), false, Tracer.none()); } @Override diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java index 8aa4428..73497a9 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java @@ -38,6 +38,7 @@ import org.apache.hyracks.storage.common.IIndexBulkLoader; import org.apache.hyracks.storage.common.IIndexCursor; import org.apache.hyracks.storage.common.ISearchOperationCallback; +import org.apache.hyracks.util.trace.Tracer; import org.junit.Assert; import org.junit.Test; @@ -58,7 +59,7 @@ harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), NoOpOperationTrackerFactory.INSTANCE.getOperationTracker(null), harness.getIOScheduler(), harness.getIOOperationCallback(), true, null, null, null, null, true, - harness.getMetadataPageManagerFactory(), false); + harness.getMetadataPageManagerFactory(), false, Tracer.none()); } @Override diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceTest.java index 4cfb102..2e68e44 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceTest.java @@ -39,6 +39,7 @@ import org.apache.hyracks.storage.am.lsm.common.impls.NoOpOperationTrackerFactory; import org.apache.hyracks.storage.common.IIndexAccessor; import org.apache.hyracks.storage.common.IModificationOperationCallback; +import org.apache.hyracks.util.trace.Tracer; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -74,7 +75,7 @@ harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), NoOpOperationTrackerFactory.INSTANCE.getOperationTracker(null), harness.getIOScheduler(), harness.getIOOperationCallback(), true, null, null, null, null, true, - harness.getMetadataPageManagerFactory(), true); + harness.getMetadataPageManagerFactory(), true, Tracer.all()); } @Override diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java index db626c8..4ebbe5f 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java @@ -32,6 +32,7 @@ import org.apache.hyracks.storage.am.common.datagen.ProbabilityHelper; import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestHarness; import org.apache.hyracks.storage.am.lsm.btree.utils.LSMBTreeUtil; +import org.apache.hyracks.util.trace.Tracer; public class LSMBTreeMultiThreadTest extends OrderedIndexMultiThreadTest { @@ -56,7 +57,7 @@ harness.getFileReference(), harness.getDiskBufferCache(), typeTraits, cmpFactories, bloomFilterKeyFields, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(), true, null, - null, null, null, true, harness.getMetadataPageManagerFactory(), false); + null, null, null, true, harness.getMetadataPageManagerFactory(), false, Tracer.none()); } @Override diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java index a7ed646..50078ef 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java @@ -48,6 +48,7 @@ import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.test.support.TestStorageManagerComponentHolder; import org.apache.hyracks.test.support.TestUtils; +import org.apache.hyracks.util.trace.Tracer; public class LSMTreeRunner implements IExperimentRunner { @@ -109,7 +110,7 @@ cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, new NoMergePolicy(), new ThreadCountingTracker(), ioScheduler, NoOpIOOperationCallbackFactory.INSTANCE.createIoOpCallback(), true, null, null, null, null, true, TestStorageManagerComponentHolder.getMetadataPageManagerFactory(), - false); + false, Tracer.none()); } @Override diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java index ec78f6a..6c95f4d 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java @@ -41,6 +41,7 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker; import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.util.trace.Tracer; @SuppressWarnings("rawtypes") public final class LSMBTreeTestContext extends OrderedIndexTestContext { @@ -95,14 +96,12 @@ lsmTree = LSMBTreeUtil.createLSMTree(ioManager, virtualBufferCaches, file, diskBufferCache, typeTraits, cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, needKeyDupCheck, filterTypeTraits, filterCmp, btreefields, filterfields, - true, - metadataPageManagerFactory, updateAware); + true, metadataPageManagerFactory, updateAware, Tracer.none()); } else { lsmTree = LSMBTreeUtil.createLSMTree(ioManager, virtualBufferCaches, file, diskBufferCache, typeTraits, cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, needKeyDupCheck, null, null, null, null, true, - metadataPageManagerFactory, - updateAware); + metadataPageManagerFactory, updateAware, Tracer.all()); } LSMBTreeTestContext testCtx = new LSMBTreeTestContext(fieldSerdes, lsmTree, filtered); return testCtx; diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/PidHelper.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java similarity index 97% rename from hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/PidHelper.java rename to hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java index 79642c0..410097e 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/PidHelper.java +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.hyracks.control.common.utils; +package org.apache.hyracks.util; import java.lang.management.ManagementFactory; import java.lang.management.RuntimeMXBean; diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java new file mode 100644 index 0000000..1c02ac8 --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hyracks.util.trace; + +import java.util.Arrays; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.hyracks.util.PidHelper; + +/** + * https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/edit + */ +public class Tracer { + + protected static final Level TRACE_LOG_LEVEL = Level.INFO; + + protected final Logger traceLog; + protected String[] categories; + + protected static final int pid = PidHelper.getPid(); + + public enum Phase { + // Duration Events + B, // begin + E, // end + // Complete Events + X, + // Instant Events + I, + // Counter Events + C, + // Async Events + b, // nestable start + n, // nestable instant + e, // nestable end + // Flow Events + s, // start + t, // step + f, // end + // Object Events + N, // created + O, // snapshot + D // destroyed + } + + public Tracer(String name, String[] categories) { + this.traceLog = Logger.getLogger(Tracer.class.getName() + "@" + name); + this.categories = categories; + } + + public static Tracer none() { + return new Tracer("None", new String[0]); + } + + public static Tracer all() { + return new Tracer("All", new String[] { "*" }); + } + + @Override + public String toString() { + return getName() + Arrays.toString(categories) + (isEnabled() ? "enabled" : "disabled"); + } + + public String getName() { + return traceLog.getName(); + } + + public boolean isEnabled() { + return categories.length > 0; + } + + public long durationB(String name, String cat, String args) { + Event e = Event.create(name, cat, Phase.B, pid, Thread.currentThread().getId(), args); + traceLog.log(TRACE_LOG_LEVEL, e.toJson()); + return e.tid; + } + + public void durationE(long tid, String args) { + Event e = Event.create(null, null, Phase.E, pid, tid, args); + traceLog.log(TRACE_LOG_LEVEL, e.toJson()); + } +} + +class Event { + public final String name; + public final String cat; + public final Tracer.Phase ph; + public final long ts; + public final int pid; + public final long tid; + public final String args; + + private Event(String name, String cat, Tracer.Phase ph, long ts, int pid, long tid, String args) { + this.name = name; + this.cat = cat; + this.ph = ph; + this.ts = ts; + this.pid = pid; + this.tid = tid; + this.args = args; + } + + private static long timestamp() { + return System.nanoTime() / 1000; + } + + public static Event create(String name, String cat, Tracer.Phase ph, int pid, long tid, String args) { + return new Event(name, cat, ph, timestamp(), pid, tid, args); + } + + public String toJson() { + return append(new StringBuilder()).toString(); + } + + public StringBuilder append(StringBuilder sb) { + sb.append("{"); + if (name != null) { + sb.append("\"name\":\"").append(name).append("\","); + } + if (cat != null) { + sb.append("\"cat\":\"").append(cat).append("\","); + } + sb.append("\"ph\":\"").append(ph).append("\","); + sb.append("\"pid\":\"").append(pid).append("\","); + sb.append("\"tid\":").append(tid).append(","); + sb.append("\"ts\":").append(ts); + if (args != null) { + sb.append(",\"args\":").append(args); + } + sb.append("}"); + return sb; + } +} -- To view, visit https://asterix-gerrit.ics.uci.edu/2022 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ifa83592870e7e38a84aa560e4edb7fa2ad2d5e74 Gerrit-PatchSet: 20 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Till Westmann Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Michael Blow Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: abdullah alamoudi