hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject hive git commit: HIVE-15143 : add logging for HIVE-15024 ADDENDUM (Sergey Shelukhin, reviewed by Siddharth Seth)
Date Wed, 11 Jan 2017 21:22:25 GMT
Repository: hive
Updated Branches:
  refs/heads/master 2d813f4d4 -> 58247c563


HIVE-15143 : add logging for HIVE-15024 ADDENDUM (Sergey Shelukhin, reviewed by Siddharth
Seth)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/58247c56
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/58247c56
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/58247c56

Branch: refs/heads/master
Commit: 58247c5635738888898a8aaa3feda5373abfb9db
Parents: 2d813f4
Author: Sergey Shelukhin <sershe@apache.org>
Authored: Wed Jan 11 13:18:20 2017 -0800
Committer: Sergey Shelukhin <sershe@apache.org>
Committed: Wed Jan 11 13:18:20 2017 -0800

----------------------------------------------------------------------
 .../hive/llap/cache/LowLevelCacheImpl.java      | 37 ++++++++++++++++++++
 .../hive/llap/cache/TestLowLevelCacheImpl.java  | 34 ++++++++++++++++++
 .../ql/io/orc/encoded/EncodedReaderImpl.java    | 13 +++----
 .../hadoop/hive/common/io/DiskRangeList.java    |  2 +-
 4 files changed, 79 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/58247c56/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java
index ea458ca..9ad0b26 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hive.llap.cache;
 
+import org.apache.orc.impl.RecordReaderUtils;
+
 import java.nio.ByteBuffer;
 import java.util.Iterator;
 import java.util.List;
@@ -108,6 +110,7 @@ public class LowLevelCacheImpl implements LowLevelCache, BufferUsageManager,
Lla
     } finally {
       subCache.decRef();
     }
+    boolean isInvalid = false;
     if (qfCounters != null) {
       DiskRangeList current = prev.next;
       long bytesHit = 0, bytesMissed = 0;
@@ -116,12 +119,46 @@ public class LowLevelCacheImpl implements LowLevelCache, BufferUsageManager,
Lla
         if (current.hasData()) {
           bytesHit += current.getLength();
         } else {
+          if (gotAllData.value) {
+            isInvalid = true;
+          }
           bytesMissed += current.getLength();
         }
         current = current.next;
       }
       qfCounters.recordCacheHit(bytesHit);
       qfCounters.recordCacheMiss(bytesMissed);
+    } else if (gotAllData.value) {
+      DiskRangeList current = prev.next;
+      while (current != null) {
+        if (!current.hasData()) {
+          isInvalid = true;
+          break;
+        }
+        current = current.next;
+      }
+    }
+    if (isInvalid) {
+      StringBuilder invalidMsg = new StringBuilder(
+          "Internal error - gotAllData=true but the resulting ranges are ").append(
+              RecordReaderUtils.stringifyDiskRanges(prev.next));
+      subCache = cache.get(fileKey);
+      if (subCache != null && subCache.incRef()) {
+        try {
+          invalidMsg.append("; cache ranges (not necessarily consistent) are ");
+          for (Map.Entry<Long, LlapDataBuffer> e : subCache.cache.entrySet()) {
+            long start = e.getKey(), end = start + e.getValue().declaredCachedLength;
+            invalidMsg.append("[").append(start).append(", ").append(end).append("), ");
+          }
+        } finally {
+          subCache.decRef();
+        }
+      } else {
+        invalidMsg.append("; cache ranges can no longer be determined");
+      }
+      String s = invalidMsg.toString();
+      LlapIoImpl.LOG.error(s);
+      throw new RuntimeException(s);
     }
     return prev.next;
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/58247c56/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelCacheImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelCacheImpl.java
b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelCacheImpl.java
index 0ac0a0d..6c3ec03 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelCacheImpl.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelCacheImpl.java
@@ -121,6 +121,23 @@ public class TestLowLevelCacheImpl {
     }
   }
 
+/*
+Example code to test specific scenarios:
+    LowLevelCacheImpl cache = new LowLevelCacheImpl(
+        LlapDaemonCacheMetrics.create("test", "1"), new DummyCachePolicy(),
+        new DummyAllocator(), true, -1); // no cleanup thread
+    final int FILE = 1;
+    cache.putFileData(FILE, gaps(3756206, 4261729, 7294767, 7547564), fbs(3), 0, Priority.NORMAL,
null);
+    cache.putFileData(FILE, gaps(7790545, 11051556), fbs(1), 0, Priority.NORMAL, null);
+    cache.putFileData(FILE, gaps(11864971, 11912961, 13350968, 13393630), fbs(3), 0, Priority.NORMAL,
null);
+    DiskRangeList dr = dr(3756206, 7313562);
+    MutateHelper mh = new MutateHelper(dr);
+    dr = dr.insertAfter(dr(7790545, 11051556));
+    dr = dr.insertAfter(dr(11864971, 13393630));
+    BooleanRef g = new BooleanRef();
+    dr = cache.getFileData(FILE, mh.next, 0, testFactory, null, g);
+*/
+
   @Test
   public void testGetPut() {
     LowLevelCacheImpl cache = new LowLevelCacheImpl(
@@ -488,6 +505,15 @@ public class TestLowLevelCacheImpl {
     return rv;
   }
 
+  private MemoryBuffer[] fbs(int count) {
+    MemoryBuffer[] rv = new MemoryBuffer[count];
+    for (int i = 0; i < count; ++i) {
+      rv[i] = fb();
+    }
+    return rv;
+  }
+
+
   private LlapDataBuffer fb() {
     LlapDataBuffer fake = LowLevelCacheImpl.allocateFake();
     fake.incRef();
@@ -506,6 +532,14 @@ public class TestLowLevelCacheImpl {
     return result;
   }
 
+  private DiskRange[] gaps(int... offsets) {
+    DiskRange[] result = new DiskRange[offsets.length - 1];
+    for (int i = 0; i < result .length; ++i) {
+      result[i] = new DiskRange(offsets[i], offsets[i + 1]);
+    }
+    return result;
+  }
+
   private int generateOffsets(int offsetsToUse, Random rdm, int[] offsets) {
     for (int j = 0; j < offsets.length; ++j) {
       offsets[j] = rdm.nextInt(offsetsToUse);

http://git-wip-us.apache.org/repos/asf/hive/blob/58247c56/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
index edf3218..0ac3ec5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
@@ -289,16 +289,17 @@ class EncodedReaderImpl implements EncodedReader {
 
     // 2. Now, read all of the ranges from cache or disk.
     DiskRangeList.MutateHelper toRead = new DiskRangeList.MutateHelper(listToRead.get());
-    if (isTracingEnabled && LOG.isInfoEnabled()) {
-      LOG.trace("Resulting disk ranges to read (file " + fileKey + "): "
+    if (/*isTracingEnabled && */LOG.isInfoEnabled()) {
+      LOG.info("Resulting disk ranges to read (file " + fileKey + "): "
           + RecordReaderUtils.stringifyDiskRanges(toRead.next));
     }
     BooleanRef isAllInCache = new BooleanRef();
     if (hasFileId) {
       cacheWrapper.getFileData(fileKey, toRead.next, stripeOffset, CC_FACTORY, isAllInCache);
-      if (isTracingEnabled && LOG.isInfoEnabled()) {
-        LOG.trace("Disk ranges after cache (file " + fileKey + ", base offset " + stripeOffset
-            + "): " + RecordReaderUtils.stringifyDiskRanges(toRead.next));
+      if (/*isTracingEnabled && */LOG.isInfoEnabled()) {
+        LOG.info("Disk ranges after cache (found everything " + isAllInCache.value + "; file
"
+            + fileKey + ", base offset " + stripeOffset  + "): "
+            + RecordReaderUtils.stringifyDiskRanges(toRead.next));
       }
     }
 
@@ -640,7 +641,7 @@ class EncodedReaderImpl implements EncodedReader {
         : prepareRangesForUncompressedRead(
             cOffset, endCOffset, streamOffset, unlockUntilCOffset, current, csd);
     } catch (Exception ex) {
-      LOG.error("Failed " + (isCompressed ? "" : "un") + " compressed read; cOffset " + cOffset
+      LOG.error("Failed " + (isCompressed ? "" : "un") + "compressed read; cOffset " + cOffset
           + ", endCOffset " + endCOffset + ", streamOffset " + streamOffset
           + ", unlockUntilCOffset " + unlockUntilCOffset + "; ranges passed in "
           + RecordReaderUtils.stringifyDiskRanges(start) + "; ranges passed to prepare "

http://git-wip-us.apache.org/repos/asf/hive/blob/58247c56/storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRangeList.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRangeList.java b/storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRangeList.java
index 62f9d8e..5de225e 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRangeList.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRangeList.java
@@ -169,7 +169,7 @@ public class DiskRangeList extends DiskRange {
     public void addOrMerge(long offset, long end, boolean doMerge, boolean doLogNew) {
       if (doMerge && tail != null && tail.merge(offset, end)) return;
       if (doLogNew) {
-        LOG.info("Creating new range; last range (which can include some previous adds) was
"
+        LOG.debug("Creating new range; last range (which can include some previous adds)
was "
             + tail);
       }
       DiskRangeList node = new DiskRangeList(offset, end);


Mime
View raw message