FAQ
Author: gangtimliu
Date: Fri Mar 29 00:26:46 2013
New Revision: 1462363

URL: http://svn.apache.org/r1462363
Log:
HIVE-4157: ORC runs out of heap when writing (Kevin Wilfong vi Gang Tim Liu)

Modified:
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java?rev=1462363&r1=1462362&r2=1462363&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java Fri Mar 29 00:26:46 2013
@@ -45,17 +45,14 @@ class OutStream extends PositionedOutput
      this.bufferSize = bufferSize;
      this.codec = codec;
      this.receiver = receiver;
- getNewInputBuffer();
    }

    public void clear() throws IOException {
- current.position(codec == null ? 0 : HEADER_SIZE);
- if (compressed != null) {
- compressed.clear();
- }
- if (overflow != null) {
- overflow.clear();
- }
+ uncompressedBytes = 0;
+ compressedBytes = 0;
+ compressed = null;
+ overflow = null;
+ current = null;
    }

    /**
@@ -100,6 +97,9 @@ class OutStream extends PositionedOutput

    @Override
    public void write(int i) throws IOException {
+ if (current == null) {
+ getNewInputBuffer();
+ }
      if (current.remaining() < 1) {
        spill();
      }
@@ -109,6 +109,9 @@ class OutStream extends PositionedOutput

    @Override
    public void write(byte[] bytes, int offset, int length) throws IOException {
+ if (current == null) {
+ getNewInputBuffer();
+ }
      int remaining = Math.min(current.remaining(), length);
      current.put(bytes, offset, remaining);
      uncompressedBytes += remaining;
@@ -125,7 +128,7 @@ class OutStream extends PositionedOutput

    private void spill() throws java.io.IOException {
      // if there isn't anything in the current buffer, don't spill
- if (current.position() == (codec == null ? 0 : HEADER_SIZE)) {
+ if (current == null || current.position() == (codec == null ? 0 : HEADER_SIZE)) {
        return;
      }
      flip();
@@ -211,8 +214,7 @@ class OutStream extends PositionedOutput
        receiver.output(compressed);
        compressed = null;
      }
- uncompressedBytes = 0;
- compressedBytes = 0;
+ clear();
    }

    @Override

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 1 of 1 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedMar 29, '13 at 12:27a
activeMar 29, '13 at 12:27a
posts1
users1
websitehive.apache.org

1 user in discussion

Gangtimliu: 1 post

People

Translate

site design / logo © 2021 Grokbase