Mailing-List: contact dev-help@tephra.incubator.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@tephra.incubator.apache.org
Date: Tue, 7 Feb 2017 07:50:42 +0000 (UTC)
From: "ASF GitHub Bot (JIRA)" <jira@apache.org>
To: dev@tephra.incubator.apache.org
Message-ID: <JIRA.13039308.1485900043000.13686.1486453842492@Atlassian.JIRA>
In-Reply-To: <JIRA.13039308.1485900043000@Atlassian.JIRA>
References: <JIRA.13039308.1485900043000@Atlassian.JIRA> <JIRA.13039308.1485900043928@jira-lw-us.apache.org>
Subject: [jira] [Commented] (TEPHRA-214) Provide a way to debug Transaction
 Pruning progress
MIME-Version: 1.0
Content-Type: text/plain; charset=utf-8
Content-Transfer-Encoding: 7bit
archived-at: Tue, 07 Feb 2017 07:50:51 -0000


    [ https://issues.apache.org/jira/browse/TEPHRA-214?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15855515#comment-15855515 ] 

ASF GitHub Bot commented on TEPHRA-214:
---------------------------------------

Github user poornachandra commented on a diff in the pull request:

    https://github.com/apache/incubator-tephra/pull/31#discussion_r99751506
  
    --- Diff: tephra-hbase-compat-1.1-base/src/main/java/org/apache/tephra/hbase/txprune/InvalidListPruningDebug.java ---
    @@ -0,0 +1,191 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements.  See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership.  The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License.  You may obtain a copy of the License at
    + *
    + *   http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing,
    + * software distributed under the License is distributed on an
    + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
    + * KIND, either express or implied.  See the License for the
    + * specific language governing permissions and limitations
    + * under the License.
    + */
    +
    +package org.apache.tephra.hbase.txprune;
    +
    +import com.google.common.collect.MinMaxPriorityQueue;
    +import com.google.gson.Gson;
    +import org.apache.hadoop.conf.Configuration;
    +import org.apache.hadoop.hbase.TableName;
    +import org.apache.hadoop.hbase.client.Connection;
    +import org.apache.hadoop.hbase.client.ConnectionFactory;
    +import org.apache.hadoop.hbase.client.Table;
    +import org.apache.hadoop.hbase.util.Bytes;
    +import org.apache.tephra.TxConstants;
    +import org.apache.tephra.txprune.RegionPruneInfo;
    +import org.slf4j.Logger;
    +import org.slf4j.LoggerFactory;
    +
    +import java.io.IOException;
    +import java.io.PrintWriter;
    +import java.util.Comparator;
    +import java.util.LinkedList;
    +import java.util.List;
    +import java.util.Queue;
    +import javax.annotation.Nullable;
    +
    +/**
    + * Invalid List Pruning Debug Tool.
    + */
    +public class InvalidListPruningDebug {
    +  private static final Logger LOG = LoggerFactory.getLogger(InvalidListPruningDebug.class);
    +  private static final Gson GSON = new Gson();
    +  private DataJanitorState dataJanitorState;
    +
    +  /**
    +   * Initialize the Invalid List Debug Tool.
    +   * @param conf {@link Configuration}
    +   * @throws IOException
    +   */
    +  public void initialize(final Configuration conf) throws IOException {
    +    LOG.info("InvalidListPruningDebugMain : initialize method called");
    +    final Connection connection = ConnectionFactory.createConnection(conf);
    +    dataJanitorState = new DataJanitorState(new DataJanitorState.TableSupplier() {
    +      @Override
    +      public Table get() throws IOException {
    +        return connection.getTable(TableName.valueOf(
    +          conf.get(TxConstants.TransactionPruning.PRUNE_STATE_TABLE,
    +                   TxConstants.TransactionPruning.DEFAULT_PRUNE_STATE_TABLE)));
    +      }
    +    });
    +  }
    +
    +  /**
    +   * Return a list of RegionPruneInfo. These regions are the ones that have the lowest prune upper bounds.
    +   * If -1 is passed in, all the regions and their prune upper bound will be returned.
    +   *
    +   * @param numRegions number of regions
    +   * @return Map of region name and its prune upper bound
    +   */
    +  public Queue<RegionPruneInfo> getIdleRegions(Integer numRegions) throws IOException {
    +    List<RegionPruneInfo> regionPruneInfos = dataJanitorState.getPruneInfoForRegions(null);
    +    if (regionPruneInfos.isEmpty()) {
    +      return new LinkedList<>();
    +    }
    +
    +    if (numRegions < 0) {
    +      numRegions = regionPruneInfos.size();
    +    }
    +    
    +    Queue<RegionPruneInfo> lowestPrunes = MinMaxPriorityQueue.orderedBy(new Comparator<RegionPruneInfo>() {
    +      @Override
    +      public int compare(RegionPruneInfo o1, RegionPruneInfo o2) {
    +        return (int) (o1.getPruneUpperBound() - o2.getPruneUpperBound());
    +      }
    +    }).maximumSize(numRegions).create();
    +
    +    for (RegionPruneInfo pruneInfo : regionPruneInfos) {
    +      lowestPrunes.add(pruneInfo);
    +    }
    +    return lowestPrunes;
    +  }
    +
    +  /**
    +   * Return the prune upper bound value of a given region. If no prune upper bound has been written for this region yet,
    +   * it will return a null.
    +   *
    +   * @param regionId region id
    +   * @return {@link RegionPruneInfo} of the region
    +   * @throws IOException if there are any errors while trying to fetch the {@link RegionPruneInfo}
    +   */
    +  @Nullable
    +  public RegionPruneInfo getRegionPruneInfo(String regionId) throws IOException {
    +    return dataJanitorState.getPruneInfoForRegion(Bytes.toBytesBinary(regionId));
    +  }
    +
    +  /**
    +   *
    +   * @param time Given a time, provide the {@link TimeRegions} at or before that time
    +   * @return time regions or null if no time regions are present at or before the given time
    +   * @throws IOException if there are any errors while trying to fetch the {@link TimeRegions}
    +   */
    +  @Nullable
    +  public String getRegionsOnOrBeforeTime(Long time) throws IOException {
    +    TimeRegions timeRegions = dataJanitorState.getRegionsOnOrBeforeTime(time);
    +    if (timeRegions == null) {
    +      return null;
    +    }
    +    return timeRegions.toString();
    +  }
    +
    +  private void printUsage(PrintWriter pw) {
    +    pw.println("Usage : org.apache.tephra.hbase.txprune.InvalidListPruning <command> <parameter>");
    +    pw.println("Available commands, corresponding parameters are:");
    +    pw.println("****************************************************");
    +    pw.println("timeregion ts");
    +    pw.println("Desc: Prints out a time region at time 'ts' (in milliseconds) or the latest before time 'ts'.");
    +    pw.println("idleregions numOfRegions");
    +    pw.println("Desc: Prints out 'numOfRegions' number of regions which has the lowest prune Upper bounds.");
    +    pw.println("pruneinfo regionNameAsString");
    +    pw.println("Desc: Prints out the Pruning information for the region 'regionNameAsString'");
    +  }
    +
    +  private boolean execute(String[] args) throws IOException {
    +    try (PrintWriter pw = new PrintWriter(System.out)) {
    +      if (args.length != 2) {
    +        printUsage(pw);
    +        return false;
    +      }
    +
    +      String command = args[0];
    +      String parameter = args[1];
    +      if ("timeregion".equals(command)) {
    +        Long time = Long.parseLong(parameter);
    +        String timeRegion = getRegionsOnOrBeforeTime(time);
    +        if (timeRegion != null) {
    +          pw.println(timeRegion);
    +        } else {
    +          pw.println(String.format("No TimeRegion found for time %d or before that.", time));
    +        }
    +        return true;
    +      } else if ("idleregions".equals(command)) {
    +        Integer numRegions = Integer.parseInt(parameter);
    +        Queue<RegionPruneInfo> regionPruneInfos = getIdleRegions(numRegions);
    +        pw.println(GSON.toJson(regionPruneInfos));
    +        return true;
    +      } else if ("pruneinfo".endsWith(command)) {
    --- End diff --
    
    This too can be `equals` instead of `endsWith`, right?


> Provide a way to debug Transaction Pruning progress
> ---------------------------------------------------
>
>                 Key: TEPHRA-214
>                 URL: https://issues.apache.org/jira/browse/TEPHRA-214
>             Project: Tephra
>          Issue Type: Improvement
>            Reporter: Gokul Gunasekaran
>            Assignee: Gokul Gunasekaran
>
> It would be good to get information about what regions are lagging that might be useful to debug why transaction pruning might not be happening. We do print debug information but this method could be invoked on a need-basis outside of the regular pruning operation.


--
This message was sent by Atlassian JIRA
(v6.3.15#6346)