hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ekoif...@apache.org
Subject [3/3] hive git commit: HIVE-16360 Improve 'No delta files or original files found to compact in' message (Eugene Koifman, reviewed by Alan Gates)
Date Tue, 26 Sep 2017 16:16:49 GMT
HIVE-16360 Improve 'No delta files or original files found to compact in' message (Eugene Koifman,
reviewed by Alan Gates)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a3b4bf89
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a3b4bf89
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a3b4bf89

Branch: refs/heads/master
Commit: a3b4bf89c4200e0a6eb6de47ecb12940a0f8b885
Parents: 1a7f853
Author: Eugene Koifman <ekoifman@hortonworks.com>
Authored: Tue Sep 26 08:48:05 2017 -0700
Committer: Eugene Koifman <ekoifman@hortonworks.com>
Committed: Tue Sep 26 09:16:22 2017 -0700

----------------------------------------------------------------------
 .../apache/hadoop/hive/common/ValidCompactorTxnList.java    | 8 ++++++--
 .../org/apache/hadoop/hive/common/ValidReadTxnList.java     | 7 ++++---
 .../java/org/apache/hadoop/hive/common/ValidTxnList.java    | 4 ++++
 .../apache/hadoop/hive/ql/txn/compactor/TestCompactor.java  | 4 +++-
 .../java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java | 8 +++++++-
 .../hive/metastore/txn/TestValidCompactorTxnList.java       | 6 +++---
 .../apache/hadoop/hive/ql/txn/compactor/CompactorMR.java    | 9 +++++++--
 7 files changed, 34 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/a3b4bf89/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java b/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java
index c022577..eaa0b34 100644
--- a/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java
+++ b/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java
@@ -39,14 +39,18 @@ public class ValidCompactorTxnList extends ValidReadTxnList {
   public ValidCompactorTxnList() {
     super();
   }
+  public ValidCompactorTxnList(long[] abortedTxnList, BitSet abortedBits, long highWatermark)
{
+    this(abortedTxnList, abortedBits, highWatermark, Long.MAX_VALUE);
+  }
   /**
    * @param abortedTxnList list of all aborted transactions
    * @param abortedBits bitset marking whether the corresponding transaction is aborted
    * @param highWatermark highest committed transaction to be considered for compaction,
    *                      equivalently (lowest_open_txn - 1).
    */
-  public ValidCompactorTxnList(long[] abortedTxnList, BitSet abortedBits, long highWatermark)
{
-    super(abortedTxnList, abortedBits, highWatermark); // abortedBits should be all true
as everything in exceptions are aborted txns
+  public ValidCompactorTxnList(long[] abortedTxnList, BitSet abortedBits, long highWatermark,
long minOpenTxnId) {
+    // abortedBits should be all true as everything in exceptions are aborted txns
+    super(abortedTxnList, abortedBits, highWatermark, minOpenTxnId);
     if(this.exceptions.length <= 0) {
       return;
     }

http://git-wip-us.apache.org/repos/asf/hive/blob/a3b4bf89/common/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java b/common/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java
index 4e57772..002afd6 100644
--- a/common/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java
+++ b/common/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java
@@ -193,9 +193,10 @@ public class ValidReadTxnList implements ValidTxnList {
   public long[] getInvalidTransactions() {
     return exceptions;
   }
-  @VisibleForTesting
-  public long getMinOpenTxn() {
-    return minOpenTxn;
+
+  @Override
+  public Long getMinOpenTxn() {
+    return minOpenTxn == Long.MAX_VALUE ? null : minOpenTxn;
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hive/blob/a3b4bf89/common/src/java/org/apache/hadoop/hive/common/ValidTxnList.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/ValidTxnList.java b/common/src/java/org/apache/hadoop/hive/common/ValidTxnList.java
index d4ac02c..108e5ca 100644
--- a/common/src/java/org/apache/hadoop/hive/common/ValidTxnList.java
+++ b/common/src/java/org/apache/hadoop/hive/common/ValidTxnList.java
@@ -105,4 +105,8 @@ public interface ValidTxnList {
    */
   public RangeResponse isTxnRangeAborted(long minTxnId, long maxTxnId);
 
+  /**
+   * Returns smallest Open transaction in this set, {@code null} if there is none.
+   */
+  Long getMinOpenTxn();
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/a3b4bf89/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
index 4bfc5be..aea1dfc 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
@@ -1150,7 +1150,6 @@ public class TestCompactor {
   /**
    * Users have the choice of specifying compaction related tblproperties either in CREATE
TABLE
    * statement or in ALTER TABLE .. COMPACT statement. This tests both cases.
-   * @throws Exception
    */
   @Test
   public void testTableProperties() throws Exception {
@@ -1320,6 +1319,9 @@ public class TestCompactor {
       }
 
       @Override
+      public Long getMinOpenTxn() { return null; }
+
+      @Override
       public long getHighWatermark() {
         return  Long.MAX_VALUE;
       }

http://git-wip-us.apache.org/repos/asf/hive/blob/a3b4bf89/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
index e6c62d3..30b155f 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
@@ -81,6 +81,7 @@ public class TxnUtils {
    * @return a valid txn list.
    */
   public static ValidTxnList createValidCompactTxnList(GetOpenTxnsInfoResponse txns) {
+    //highWater is the last txn id that has been allocated
     long highWater = txns.getTxn_high_water_mark();
     long minOpenTxn = Long.MAX_VALUE;
     long[] exceptions = new long[txns.getOpen_txnsSize()];
@@ -100,7 +101,12 @@ public class TxnUtils {
     highWater = minOpenTxn == Long.MAX_VALUE ? highWater : minOpenTxn - 1;
     BitSet bitSet = new BitSet(exceptions.length);
     bitSet.set(0, exceptions.length); // for ValidCompactorTxnList, everything in exceptions
are aborted
-    return new ValidCompactorTxnList(exceptions, bitSet, highWater);
+    if(minOpenTxn == Long.MAX_VALUE) {
+      return new ValidCompactorTxnList(exceptions, bitSet, highWater);
+    }
+    else {
+      return new ValidCompactorTxnList(exceptions, bitSet, highWater, minOpenTxn);
+    }
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hive/blob/a3b4bf89/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestValidCompactorTxnList.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestValidCompactorTxnList.java
b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestValidCompactorTxnList.java
index 91d6215..eb88e32 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestValidCompactorTxnList.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestValidCompactorTxnList.java
@@ -103,11 +103,11 @@ public class TestValidCompactorTxnList {
   public void readFromString() {
     ValidCompactorTxnList txns = new ValidCompactorTxnList("37:" + Long.MAX_VALUE + "::7,9,10");
     Assert.assertEquals(37L, txns.getHighWatermark());
-    Assert.assertEquals(Long.MAX_VALUE, txns.getMinOpenTxn());
+    Assert.assertNull(txns.getMinOpenTxn());
     Assert.assertArrayEquals(new long[]{7L, 9L, 10L}, txns.getInvalidTransactions());
     txns = new ValidCompactorTxnList("21:" + Long.MAX_VALUE + ":");
     Assert.assertEquals(21L, txns.getHighWatermark());
-    Assert.assertEquals(Long.MAX_VALUE, txns.getMinOpenTxn());
+    Assert.assertNull(txns.getMinOpenTxn());
     Assert.assertEquals(0, txns.getInvalidTransactions().length);
   }
 
@@ -115,7 +115,7 @@ public class TestValidCompactorTxnList {
   public void testAbortedTxn() throws Exception {
     ValidCompactorTxnList txnList = new ValidCompactorTxnList("5:4::1,2,3");
     Assert.assertEquals(5L, txnList.getHighWatermark());
-    Assert.assertEquals(4, txnList.getMinOpenTxn());
+    Assert.assertEquals(4, txnList.getMinOpenTxn().longValue());
     Assert.assertArrayEquals(new long[]{1L, 2L, 3L}, txnList.getInvalidTransactions());
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/a3b4bf89/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
index 04ef7fc..7f5c6d3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
@@ -265,10 +265,15 @@ public class CompactorMR {
         dirsToSearch.add(baseDir);
       }
     }
-
     if (parsedDeltas.size() == 0 && dir.getOriginalFiles().size() == 0) {
       // Skip compaction if there's no delta files AND there's no original files
-      LOG.error("No delta files or original files found to compact in " + sd.getLocation()
+ " for compactionId=" + ci.id);
+      String minOpenInfo = ".";
+      if(txns.getMinOpenTxn() != null) {
+        minOpenInfo = " with min Open " + JavaUtils.txnIdToString(txns.getMinOpenTxn()) +
+          ".  Compaction cannot compact above this txnid";
+      }
+      LOG.error("No delta files or original files found to compact in " + sd.getLocation()
+
+        " for compactionId=" + ci.id + minOpenInfo);
       return;
     }
 


Mime
View raw message