Return-Path: X-Original-To: apmail-drill-issues-archive@minotaur.apache.org Delivered-To: apmail-drill-issues-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id F075618122 for ; Wed, 12 Aug 2015 01:03:48 +0000 (UTC) Received: (qmail 5040 invoked by uid 500); 12 Aug 2015 01:03:45 -0000 Delivered-To: apmail-drill-issues-archive@drill.apache.org Received: (qmail 5009 invoked by uid 500); 12 Aug 2015 01:03:45 -0000 Mailing-List: contact issues-help@drill.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@drill.apache.org Delivered-To: mailing list issues@drill.apache.org Received: (qmail 4990 invoked by uid 99); 12 Aug 2015 01:03:45 -0000 Received: from arcas.apache.org (HELO arcas.apache.org) (140.211.11.28) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 12 Aug 2015 01:03:45 +0000 Date: Wed, 12 Aug 2015 01:03:45 +0000 (UTC) From: "ASF GitHub Bot (JIRA)" To: issues@drill.apache.org Message-ID: In-Reply-To: References: Subject: [jira] [Commented] (DRILL-1942) Improve off-heap memory usage tracking MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 7bit X-JIRA-FingerPrint: 30527f35849b9dde25b450d4833f0394 [ https://issues.apache.org/jira/browse/DRILL-1942?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14692699#comment-14692699 ] ASF GitHub Bot commented on DRILL-1942: --------------------------------------- Github user cwestin commented on a diff in the pull request: https://github.com/apache/drill/pull/105#discussion_r36817993 --- Diff: exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedConcurrent.java --- @@ -0,0 +1,177 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill; + +import java.io.IOException; +import java.util.IdentityHashMap; +import java.util.Random; +import java.util.concurrent.Semaphore; + +import org.apache.drill.QueryTestUtil; +import org.apache.drill.common.exceptions.UserException; +import org.apache.drill.common.util.TestTools; +import org.apache.drill.exec.proto.UserBitShared; +import org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState; +import org.apache.drill.exec.rpc.user.UserResultsListener; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +/* + * Note that the real interest here is that the drillbit doesn't become + * unstable from running a lot of queries concurrently -- it's not about + * any particular order of execution. We ignore the results. + */ +// TODO(cwestin) @Ignore +public class TestTpchDistributedConcurrent extends BaseTestQuery { + @Rule public final TestRule TIMEOUT = TestTools.getTimeoutRule(120000); // Longer timeout than usual. + + /* + * Valid test names taken from TestTpchDistributed. Fuller path prefixes are + * used so that tests may also be taken from other locations -- more variety + * is better as far as this test goes. + */ + private final static String queryFile[] = { + "queries/tpch/01.sql", + "queries/tpch/03.sql", + "queries/tpch/04.sql", + "queries/tpch/05.sql", + "queries/tpch/06.sql", + "queries/tpch/07.sql", + "queries/tpch/08.sql", + "queries/tpch/09.sql", + "queries/tpch/10.sql", + "queries/tpch/11.sql", + "queries/tpch/12.sql", + "queries/tpch/13.sql", + "queries/tpch/14.sql", + // "queries/tpch/15.sql", this creates a view + "queries/tpch/16.sql", + "queries/tpch/18.sql", + "queries/tpch/19_1.sql", + "queries/tpch/20.sql", + }; + + private final static int TOTAL_QUERIES = 115; + private final static int CONCURRENT_QUERIES = 15; + + private final static Random random = new Random(0xdeadbeef); + private final static String alterSession = "alter session set `planner.slice_target` = 10"; + + private int remainingQueries = TOTAL_QUERIES - CONCURRENT_QUERIES; + private final Semaphore completionSemaphore = new Semaphore(0); + private final Semaphore submissionSemaphore = new Semaphore(0); + private final IdentityHashMap listeners = new IdentityHashMap<>(); + + private void submitRandomQuery() { + final String filename = queryFile[random.nextInt(queryFile.length)]; + final String query; + try { + query = QueryTestUtil.normalizeQuery(getFile(filename)).replace(';', ' '); + } catch(IOException e) { + throw new RuntimeException("Caught exception", e); + } + final UserResultsListener listener = new ChainingSilentListener(query); + client.runQuery(UserBitShared.QueryType.SQL, query, listener); + synchronized(listeners) { + listeners.put(listener, listener); + } + } + + private class ChainingSilentListener extends SilentListener { + private final String query; + + public ChainingSilentListener(final String query) { + this.query = query; + } + + @Override + public void queryCompleted(QueryState state) { + super.queryCompleted(state); + + final Object object; + synchronized(listeners) { + object = listeners.remove(this); + } + assertNotNull("listener not found", object); --- End diff -- Done. > Improve off-heap memory usage tracking > -------------------------------------- > > Key: DRILL-1942 > URL: https://issues.apache.org/jira/browse/DRILL-1942 > Project: Apache Drill > Issue Type: Improvement > Components: Execution - Relational Operators > Reporter: Chris Westin > Assignee: Chris Westin > Fix For: 1.2.0 > > Attachments: DRILL-1942.1.patch.txt, DRILL-1942.2.patch.txt, DRILL-1942.3.patch.txt > > > We're using a lot more memory than we think we should. We may be leaking it, or not releasing it as soon as we could. > This is a call to come up with some improved tracking so that we can get statistics out about exactly where we're using it, and whether or not we can release it earlier. -- This message was sent by Atlassian JIRA (v6.3.4#6332)