FAQ
Repository: hive
Updated Branches:
   refs/heads/branch-2.0 f6170a0e8 -> a49f5d898
   refs/heads/master e7abf72c7 -> 688a3e17f


HIVE-12599 : Add logging to debug rare unexpected refCount error from the LLAP IO layer (Sergey Shelukhin, reviewed by Siddharth Seth)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a49f5d89
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a49f5d89
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a49f5d89

Branch: refs/heads/branch-2.0
Commit: a49f5d898ae79cf5002dcf92535119a33c133dbb
Parents: f6170a0
Author: Sergey Shelukhin <sershe@apache.org>
Authored: Thu Dec 10 12:26:58 2015 -0800
Committer: Sergey Shelukhin <sershe@apache.org>
Committed: Thu Dec 10 12:27:21 2015 -0800

----------------------------------------------------------------------
  .../ql/io/orc/encoded/EncodedReaderImpl.java | 32 ++++++++++++++++++--
  1 file changed, 29 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/a49f5d89/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
index deeed52..ea7e0fa 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
@@ -386,12 +386,17 @@ class EncodedReaderImpl implements EncodedReader {
              cb = sctx.stripeLevelStream;
            } else {
              // This stream can be separated by RG using index. Let's do that.
+ // Offset to where this RG begins.
              long cOffset = sctx.offset + index.getPositions(sctx.streamIndexOffset);
+ // Offset relative to the beginning of the stream of where this RG ends.
              long nextCOffsetRel = isLastRg ? sctx.length
                  : nextIndex.getPositions(sctx.streamIndexOffset);
+ // Offset before which this RG is guaranteed to end. Can only be estimated.
              // We estimate the same way for compressed and uncompressed for now.
              long endCOffset = sctx.offset + RecordReaderUtils.estimateRgEndOffset(
                  isCompressed, isLastRg, nextCOffsetRel, sctx.length, bufferSize);
+ // As we read, we can unlock initial refcounts for the buffers that end before
+ // the data that we need for this RG.
              long unlockUntilCOffset = sctx.offset + nextCOffsetRel;
              cb = createRgColumnStreamData(
                  rgIx, isLastRg, ctx.colIx, sctx, cOffset, endCOffset, isCompressed);
@@ -543,6 +548,12 @@ class EncodedReaderImpl implements EncodedReader {
      }

      @Override
+ public String toString() {
+ return super.toString() + ", original is set " + (this.originalData != null)
+ + ", buffer was replaced " + (originalCbIndex == -1);
+ }
+
+ @Override
      public void handleCacheCollision(DataCache cache, MemoryBuffer replacementBuffer,
          List<MemoryBuffer> cacheBuffers) {
        assert originalCbIndex >= 0;
@@ -1020,17 +1031,32 @@ class EncodedReaderImpl implements EncodedReader {

    private void ponderReleaseInitialRefcount(
        long unlockUntilCOffset, long streamStartOffset, CacheChunk cc) {
+ // Don't release if the buffer contains any data beyond the acceptable boundary.
      if (cc.getEnd() > unlockUntilCOffset) return;
      assert cc.getBuffer() != null;
- releaseInitialRefcount(cc, false);
- // Release all the previous buffers that we may not have been able to release due to reuse.
+ try {
+ releaseInitialRefcount(cc, false);
+ } catch (AssertionError e) {
+ LOG.error("BUG: releasing initial refcount; stream start " + streamStartOffset + ", "
+ + "unlocking until " + unlockUntilCOffset + " from [" + cc + "]: " + e.getMessage());
+ throw e;
+ }
+ // Release all the previous buffers that we may not have been able to release due to reuse,
+ // as long as they are still in the same stream and are not already released.
      DiskRangeList prev = cc.prev;
      while (true) {
        if ((prev == null) || (prev.getEnd() <= streamStartOffset)
!(prev instanceof CacheChunk)) break;
        CacheChunk prevCc = (CacheChunk)prev;
        if (prevCc.buffer == null) break;
- releaseInitialRefcount(prevCc, true);
+ try {
+ releaseInitialRefcount(prevCc, true);
+ } catch (AssertionError e) {
+ LOG.error("BUG: releasing initial refcount; stream start " + streamStartOffset + ", "
+ + "unlocking until " + unlockUntilCOffset + " from [" + cc + "] and backtracked to ["
+ + prevCc + "]: " + e.getMessage());
+ throw e;
+ }
        prev = prev.prev;
      }
    }

Search Discussions

  • Sershe at Dec 10, 2015 at 8:28 pm
    HIVE-12599 : Add logging to debug rare unexpected refCount error from the LLAP IO layer (Sergey Shelukhin, reviewed by Siddharth Seth)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/688a3e17
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/688a3e17
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/688a3e17

    Branch: refs/heads/master
    Commit: 688a3e17f3489f6246661c9ab52800b1f035a60e
    Parents: e7abf72
    Author: Sergey Shelukhin <sershe@apache.org>
    Authored: Thu Dec 10 12:26:58 2015 -0800
    Committer: Sergey Shelukhin <sershe@apache.org>
    Committed: Thu Dec 10 12:27:51 2015 -0800

    ----------------------------------------------------------------------
      .../ql/io/orc/encoded/EncodedReaderImpl.java | 32 ++++++++++++++++++--
      1 file changed, 29 insertions(+), 3 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/688a3e17/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
    index deeed52..ea7e0fa 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
    @@ -386,12 +386,17 @@ class EncodedReaderImpl implements EncodedReader {
                  cb = sctx.stripeLevelStream;
                } else {
                  // This stream can be separated by RG using index. Let's do that.
    + // Offset to where this RG begins.
                  long cOffset = sctx.offset + index.getPositions(sctx.streamIndexOffset);
    + // Offset relative to the beginning of the stream of where this RG ends.
                  long nextCOffsetRel = isLastRg ? sctx.length
                      : nextIndex.getPositions(sctx.streamIndexOffset);
    + // Offset before which this RG is guaranteed to end. Can only be estimated.
                  // We estimate the same way for compressed and uncompressed for now.
                  long endCOffset = sctx.offset + RecordReaderUtils.estimateRgEndOffset(
                      isCompressed, isLastRg, nextCOffsetRel, sctx.length, bufferSize);
    + // As we read, we can unlock initial refcounts for the buffers that end before
    + // the data that we need for this RG.
                  long unlockUntilCOffset = sctx.offset + nextCOffsetRel;
                  cb = createRgColumnStreamData(
                      rgIx, isLastRg, ctx.colIx, sctx, cOffset, endCOffset, isCompressed);
    @@ -543,6 +548,12 @@ class EncodedReaderImpl implements EncodedReader {
          }

          @Override
    + public String toString() {
    + return super.toString() + ", original is set " + (this.originalData != null)
    + + ", buffer was replaced " + (originalCbIndex == -1);
    + }
    +
    + @Override
          public void handleCacheCollision(DataCache cache, MemoryBuffer replacementBuffer,
              List<MemoryBuffer> cacheBuffers) {
            assert originalCbIndex >= 0;
    @@ -1020,17 +1031,32 @@ class EncodedReaderImpl implements EncodedReader {

        private void ponderReleaseInitialRefcount(
            long unlockUntilCOffset, long streamStartOffset, CacheChunk cc) {
    + // Don't release if the buffer contains any data beyond the acceptable boundary.
          if (cc.getEnd() > unlockUntilCOffset) return;
          assert cc.getBuffer() != null;
    - releaseInitialRefcount(cc, false);
    - // Release all the previous buffers that we may not have been able to release due to reuse.
    + try {
    + releaseInitialRefcount(cc, false);
    + } catch (AssertionError e) {
    + LOG.error("BUG: releasing initial refcount; stream start " + streamStartOffset + ", "
    + + "unlocking until " + unlockUntilCOffset + " from [" + cc + "]: " + e.getMessage());
    + throw e;
    + }
    + // Release all the previous buffers that we may not have been able to release due to reuse,
    + // as long as they are still in the same stream and are not already released.
          DiskRangeList prev = cc.prev;
          while (true) {
            if ((prev == null) || (prev.getEnd() <= streamStartOffset)
    !(prev instanceof CacheChunk)) break;
            CacheChunk prevCc = (CacheChunk)prev;
            if (prevCc.buffer == null) break;
    - releaseInitialRefcount(prevCc, true);
    + try {
    + releaseInitialRefcount(prevCc, true);
    + } catch (AssertionError e) {
    + LOG.error("BUG: releasing initial refcount; stream start " + streamStartOffset + ", "
    + + "unlocking until " + unlockUntilCOffset + " from [" + cc + "] and backtracked to ["
    + + prevCc + "]: " + e.getMessage());
    + throw e;
    + }
            prev = prev.prev;
          }
        }

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedDec 10, '15 at 8:28p
activeDec 10, '15 at 8:28p
posts2
users1
websitehive.apache.org

1 user in discussion

Sershe: 2 posts

People

Translate

site design / logo © 2021 Grokbase