FAQ
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RedBlackTree.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RedBlackTree.java?rev=1452992&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RedBlackTree.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RedBlackTree.java Tue Mar 5 20:44:50 2013
@@ -0,0 +1,326 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io.orc;
+
+/**
+ * A memory efficient red-black tree that does not allocate any objects per
+ * an element. This class is abstract and assumes that the child class
+ * handles the key and comparisons with the key.
+ */
+abstract class RedBlackTree {
+ public static final int NULL = -1;
+ private static final int DEFAULT_INITIAL_CAPACITY = 16 * 1024;
+
+ // Various values controlling the offset of the data within the array.
+ private static final int LEFT_OFFSET = 0;
+ private static final int RIGHT_OFFSET = 1;
+ private static final int COUNT_OFFSET = 2;
+ private static final int ELEMENT_SIZE = 3;
+
+ protected int size = 0;
+ private final DynamicIntArray data;
+ protected int root = NULL;
+ protected int lastAdd = 0;
+ private boolean wasAdd = false;
+
+ /**
+ * Create a set with a default initial capacity.
+ */
+ public RedBlackTree() {
+ data = new DynamicIntArray(DEFAULT_INITIAL_CAPACITY * ELEMENT_SIZE);
+ }
+
+ /**
+ * Create a set with the given initial capacity.
+ */
+ public RedBlackTree(int initialCapacity) {
+ data = new DynamicIntArray(initialCapacity * ELEMENT_SIZE);
+ }
+
+ /**
+ * Insert a new node into the data array, growing the array as necessary.
+ *
+ * @return Returns the position of the new node.
+ */
+ private int insert(int left, int right, boolean isRed) {
+ int position = size;
+ size += 1;
+ setLeft(position, left, isRed);
+ setRight(position, right);
+ setCount(position, 1);
+ return position;
+ }
+
+ /**
+ * Compare the value at the given position to the new value.
+ * @return 0 if the values are the same, -1 if the new value is smaller and
+ * 1 if the new value is larger.
+ */
+ protected abstract int compareValue(int position);
+
+ /**
+ * Is the given node red as opposed to black? To prevent having an extra word
+ * in the data array, we just the low bit on the left child index.
+ */
+ protected boolean isRed(int position) {
+ return position != NULL &&
+ (data.get(position * ELEMENT_SIZE + LEFT_OFFSET) & 1) == 1;
+ }
+
+ /**
+ * Set the red bit true or false.
+ */
+ private void setRed(int position, boolean isRed) {
+ int offset = position * ELEMENT_SIZE + LEFT_OFFSET;
+ if (isRed) {
+ data.set(offset, data.get(offset) | 1);
+ } else {
+ data.set(offset, data.get(offset) & ~1);
+ }
+ }
+
+ /**
+ * Get the left field of the given position.
+ */
+ protected int getLeft(int position) {
+ return data.get(position * ELEMENT_SIZE + LEFT_OFFSET) >> 1;
+ }
+
+ /**
+ * Get the right field of the given position.
+ */
+ protected int getRight(int position) {
+ return data.get(position * ELEMENT_SIZE + RIGHT_OFFSET);
+ }
+
+ protected int getCount(int position) {
+ return data.get(position * ELEMENT_SIZE + COUNT_OFFSET);
+ }
+
+ private void setCount(int position, int value) {
+ data.set(position * ELEMENT_SIZE + COUNT_OFFSET, value);
+ }
+
+ private void incrementCount(int position, int value) {
+ data.increment(position * ELEMENT_SIZE + COUNT_OFFSET, value);
+ }
+
+ /**
+ * Set the left field of the given position.
+ * Note that we are storing the node color in the low bit of the left pointer.
+ */
+ private void setLeft(int position, int left) {
+ int offset = position * ELEMENT_SIZE + LEFT_OFFSET;
+ data.set(offset, (left << 1) | (data.get(offset) & 1));
+ }
+
+ /**
+ * Set the left field of the given position.
+ * Note that we are storing the node color in the low bit of the left pointer.
+ */
+ private void setLeft(int position, int left, boolean isRed) {
+ int offset = position * ELEMENT_SIZE + LEFT_OFFSET;
+ data.set(offset, (left << 1) | (isRed ? 1 : 0));
+ }
+
+ /**
+ * Set the right field of the given position.
+ */
+ private void setRight(int position, int right) {
+ data.set(position * ELEMENT_SIZE + RIGHT_OFFSET, right);
+ }
+
+ /**
+ * Insert or find a given key in the tree and rebalance the tree correctly.
+ * Rebalancing restores the red-black aspect of the tree to maintain the
+ * invariants:
+ * 1. If a node is red, both of its children are black.
+ * 2. Each child of a node has the same black height (the number of black
+ * nodes between it and the leaves of the tree).
+ *
+ * Inserted nodes are at the leaves and are red, therefore there is at most a
+ * violation of rule 1 at the node we just put in. Instead of always keeping
+ * the parents, this routine passing down the context.
+ *
+ * The fix is broken down into 6 cases (1.{1,2,3} and 2.{1,2,3} that are
+ * left-right mirror images of each other). See Algorighms by Cormen,
+ * Leiserson, and Rivest for the explaination of the subcases.
+ *
+ * @param node The node that we are fixing right now.
+ * @param fromLeft Did we come down from the left?
+ * @param parent Nodes' parent
+ * @param grandparent Parent's parent
+ * @param greatGrandparent Grandparent's parent
+ * @return Does parent also need to be checked and/or fixed?
+ */
+ private boolean add(int node, boolean fromLeft, int parent,
+ int grandparent, int greatGrandparent) {
+ if (node == NULL) {
+ if (root == NULL) {
+ lastAdd = insert(NULL, NULL, false);
+ root = lastAdd;
+ wasAdd = true;
+ return false;
+ } else {
+ lastAdd = insert(NULL, NULL, true);
+ node = lastAdd;
+ wasAdd = true;
+ // connect the new node into the tree
+ if (fromLeft) {
+ setLeft(parent, node);
+ } else {
+ setRight(parent, node);
+ }
+ }
+ } else {
+ int compare = compareValue(node);
+ boolean keepGoing;
+
+ // Recurse down to find where the node needs to be added
+ if (compare < 0) {
+ keepGoing = add(getLeft(node), true, node, parent, grandparent);
+ } else if (compare > 0) {
+ keepGoing = add(getRight(node), false, node, parent, grandparent);
+ } else {
+ lastAdd = node;
+ wasAdd = false;
+ incrementCount(node, 1);
+ return false;
+ }
+
+ // we don't need to fix the root (because it is always set to black)
+ if (node == root || !keepGoing) {
+ return false;
+ }
+ }
+
+
+ // Do we need to fix this node? Only if there are two reds right under each
+ // other.
+ if (isRed(node) && isRed(parent)) {
+ if (parent == getLeft(grandparent)) {
+ int uncle = getRight(grandparent);
+ if (isRed(uncle)) {
+ // case 1.1
+ setRed(parent, false);
+ setRed(uncle, false);
+ setRed(grandparent, true);
+ return true;
+ } else {
+ if (node == getRight(parent)) {
+ // case 1.2
+ // swap node and parent
+ int tmp = node;
+ node = parent;
+ parent = tmp;
+ // left-rotate on node
+ setLeft(grandparent, parent);
+ setRight(node, getLeft(parent));
+ setLeft(parent, node);
+ }
+
+ // case 1.2 and 1.3
+ setRed(parent, false);
+ setRed(grandparent, true);
+
+ // right-rotate on grandparent
+ if (greatGrandparent == NULL) {
+ root = parent;
+ } else if (getLeft(greatGrandparent) == grandparent) {
+ setLeft(greatGrandparent, parent);
+ } else {
+ setRight(greatGrandparent, parent);
+ }
+ setLeft(grandparent, getRight(parent));
+ setRight(parent, grandparent);
+ return false;
+ }
+ } else {
+ int uncle = getLeft(grandparent);
+ if (isRed(uncle)) {
+ // case 2.1
+ setRed(parent, false);
+ setRed(uncle, false);
+ setRed(grandparent, true);
+ return true;
+ } else {
+ if (node == getLeft(parent)) {
+ // case 2.2
+ // swap node and parent
+ int tmp = node;
+ node = parent;
+ parent = tmp;
+ // right-rotate on node
+ setRight(grandparent, parent);
+ setLeft(node, getRight(parent));
+ setRight(parent, node);
+ }
+ // case 2.2 and 2.3
+ setRed(parent, false);
+ setRed(grandparent, true);
+ // left-rotate on grandparent
+ if (greatGrandparent == NULL) {
+ root = parent;
+ } else if (getRight(greatGrandparent) == grandparent) {
+ setRight(greatGrandparent, parent);
+ } else {
+ setLeft(greatGrandparent, parent);
+ }
+ setRight(grandparent, getLeft(parent));
+ setLeft(parent, grandparent);
+ return false;
+ }
+ }
+ } else {
+ return true;
+ }
+ }
+
+ /**
+ * Add the new key to the tree.
+ * @return true if the element is a new one.
+ */
+ protected boolean add() {
+ add(root, false, NULL, NULL, NULL);
+ if (wasAdd) {
+ setRed(root, false);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * Get the number of elements in the set.
+ */
+ public int size() {
+ return size;
+ }
+
+ /**
+ * Reset the table to empty.
+ */
+ public void clear() {
+ root = NULL;
+ size = 0;
+ data.clear();
+ }
+
+}
+

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteReader.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteReader.java?rev=1452992&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteReader.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteReader.java Tue Mar 5 20:44:50 2013
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.io.orc;
+
+import java.io.EOFException;
+import java.io.IOException;
+
+/**
+ * A reader that reads a sequence of bytes. A control byte is read before
+ * each run with positive values 0 to 127 meaning 3 to 130 repetitions. If the
+ * byte is -1 to -128, 1 to 128 literal byte values follow.
+ */
+class RunLengthByteReader {
+ private final InStream input;
+ private final byte[] literals =
+ new byte[RunLengthByteWriter.MAX_LITERAL_SIZE];
+ private int numLiterals = 0;
+ private int used = 0;
+ private boolean repeat = false;
+
+ RunLengthByteReader(InStream input) throws IOException {
+ this.input = input;
+ }
+
+ private void readValues() throws IOException {
+ int control = input.read();
+ used = 0;
+ if (control == -1) {
+ throw new EOFException("Read past end of buffer RLE byte from " + input);
+ } else if (control < 0x80) {
+ repeat = true;
+ numLiterals = control + RunLengthByteWriter.MIN_REPEAT_SIZE;
+ int val = input.read();
+ if (val == -1) {
+ throw new EOFException("Reading RLE byte got EOF");
+ }
+ literals[0] = (byte) val;
+ } else {
+ repeat = false;
+ numLiterals = 0x100 - control;
+ int bytes = 0;
+ while (bytes < numLiterals) {
+ int result = input.read(literals, bytes, numLiterals - bytes);
+ if (result == -1) {
+ throw new EOFException("Reading RLE byte literal got EOF");
+ }
+ bytes += result;
+ }
+ }
+ }
+
+ boolean hasNext() throws IOException {
+ return used != numLiterals || input.available() > 0;
+ }
+
+ byte next() throws IOException {
+ byte result;
+ if (used == numLiterals) {
+ readValues();
+ }
+ if (repeat) {
+ used += 1;
+ result = literals[0];
+ } else {
+ result = literals[used++];
+ }
+ return result;
+ }
+
+ void seek(PositionProvider index) throws IOException {
+ input.seek(index);
+ int consumed = (int) index.getNext();
+ if (consumed != 0) {
+ // a loop is required for cases where we break the run into two parts
+ while (consumed > 0) {
+ readValues();
+ used = consumed;
+ consumed -= numLiterals;
+ }
+ } else {
+ used = 0;
+ numLiterals = 0;
+ }
+ }
+
+ void skip(long items) throws IOException {
+ while (items > 0) {
+ if (used == numLiterals) {
+ readValues();
+ }
+ long consume = Math.min(items, numLiterals - used);
+ used += consume;
+ items -= consume;
+ }
+ }
+}

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteWriter.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteWriter.java?rev=1452992&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteWriter.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteWriter.java Tue Mar 5 20:44:50 2013
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.io.orc;
+
+import java.io.IOException;
+
+/**
+ * A streamFactory that writes a sequence of bytes. A control byte is written before
+ * each run with positive values 0 to 127 meaning 2 to 129 repetitions. If the
+ * bytes is -1 to -128, 1 to 128 literal byte values follow.
+ */
+class RunLengthByteWriter {
+ static final int MIN_REPEAT_SIZE = 3;
+ static final int MAX_LITERAL_SIZE = 128;
+ static final int MAX_REPEAT_SIZE= 127 + MIN_REPEAT_SIZE;
+ private final PositionedOutputStream output;
+ private final byte[] literals = new byte[MAX_LITERAL_SIZE];
+ private int numLiterals = 0;
+ private boolean repeat = false;
+ private int tailRunLength = 0;
+
+ RunLengthByteWriter(PositionedOutputStream output) {
+ this.output = output;
+ }
+
+ private void writeValues() throws IOException {
+ if (numLiterals != 0) {
+ if (repeat) {
+ output.write(numLiterals - MIN_REPEAT_SIZE);
+ output.write(literals, 0, 1);
+ } else {
+ output.write(-numLiterals);
+ output.write(literals, 0, numLiterals);
+ }
+ repeat = false;
+ tailRunLength = 0;
+ numLiterals = 0;
+ }
+ }
+
+ void flush() throws IOException {
+ writeValues();
+ output.flush();
+ }
+
+ void write(byte value) throws IOException {
+ if (numLiterals == 0) {
+ literals[numLiterals++] = value;
+ tailRunLength = 1;
+ } else if (repeat) {
+ if (value == literals[0]) {
+ numLiterals += 1;
+ if (numLiterals == MAX_REPEAT_SIZE) {
+ writeValues();
+ }
+ } else {
+ writeValues();
+ literals[numLiterals++] = value;
+ tailRunLength = 1;
+ }
+ } else {
+ if (value == literals[numLiterals - 1]) {
+ tailRunLength += 1;
+ } else {
+ tailRunLength = 1;
+ }
+ if (tailRunLength == MIN_REPEAT_SIZE) {
+ if (numLiterals + 1 == MIN_REPEAT_SIZE) {
+ repeat = true;
+ numLiterals += 1;
+ } else {
+ numLiterals -= MIN_REPEAT_SIZE - 1;
+ writeValues();
+ literals[0] = value;
+ repeat = true;
+ numLiterals = MIN_REPEAT_SIZE;
+ }
+ } else {
+ literals[numLiterals++] = value;
+ if (numLiterals == MAX_LITERAL_SIZE) {
+ writeValues();
+ }
+ }
+ }
+ }
+
+ void getPosition(PositionRecorder recorder) throws IOException {
+ output.getPosition(recorder);
+ recorder.addPosition(numLiterals);
+ }
+}

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerReader.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerReader.java?rev=1452992&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerReader.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerReader.java Tue Mar 5 20:44:50 2013
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.io.orc;
+
+import java.io.EOFException;
+import java.io.IOException;
+
+/**
+ * A reader that reads a sequence of integers.
+ * */
+class RunLengthIntegerReader {
+ private final InStream input;
+ private final boolean signed;
+ private final long[] literals =
+ new long[RunLengthIntegerWriter.MAX_LITERAL_SIZE];
+ private int numLiterals = 0;
+ private int delta = 0;
+ private int used = 0;
+ private boolean repeat = false;
+
+ RunLengthIntegerReader(InStream input, boolean signed) throws IOException {
+ this.input = input;
+ this.signed = signed;
+ }
+
+ private void readValues() throws IOException {
+ int control = input.read();
+ if (control == -1) {
+ throw new EOFException("Read past end of RLE integer from " + input);
+ } else if (control < 0x80) {
+ numLiterals = control + RunLengthIntegerWriter.MIN_REPEAT_SIZE;
+ used = 0;
+ repeat = true;
+ delta = input.read();
+ if (delta == -1) {
+ throw new EOFException("End of stream in RLE Integer from " + input);
+ }
+ // convert from 0 to 255 to -128 to 127 by converting to a signed byte
+ delta = (byte) (0 + delta);
+ if (signed) {
+ literals[0] = SerializationUtils.readVslong(input);
+ } else {
+ literals[0] = SerializationUtils.readVulong(input);
+ }
+ } else {
+ repeat = false;
+ numLiterals = 0x100 - control;
+ used = 0;
+ for(int i=0; i < numLiterals; ++i) {
+ if (signed) {
+ literals[i] = SerializationUtils.readVslong(input);
+ } else {
+ literals[i] = SerializationUtils.readVulong(input);
+ }
+ }
+ }
+ }
+
+ boolean hasNext() throws IOException {
+ return used != numLiterals || input.available() > 0;
+ }
+
+ long next() throws IOException {
+ long result;
+ if (used == numLiterals) {
+ readValues();
+ }
+ if (repeat) {
+ result = literals[0] + (used++) * delta;
+ } else {
+ result = literals[used++];
+ }
+ return result;
+ }
+
+ void seek(PositionProvider index) throws IOException {
+ input.seek(index);
+ int consumed = (int) index.getNext();
+ if (consumed != 0) {
+ // a loop is required for cases where we break the run into two parts
+ while (consumed > 0) {
+ readValues();
+ used = consumed;
+ consumed -= numLiterals;
+ }
+ } else {
+ used = 0;
+ numLiterals = 0;
+ }
+ }
+
+ void skip(long numValues) throws IOException {
+ while (numValues > 0) {
+ if (used == numLiterals) {
+ readValues();
+ }
+ long consume = Math.min(numValues, numLiterals - used);
+ used += consume;
+ numValues -= consume;
+ }
+ }
+}

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriter.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriter.java?rev=1452992&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriter.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriter.java Tue Mar 5 20:44:50 2013
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.io.orc;
+
+import java.io.IOException;
+
+/**
+ * A streamFactory that writes a sequence of integers. A control byte is written before
+ * each run with positive values 0 to 127 meaning 3 to 130 repetitions, each
+ * repetition is offset by a delta. If the control byte is -1 to -128, 1 to 128
+ * literal vint values follow.
+ */
+class RunLengthIntegerWriter {
+ static final int MIN_REPEAT_SIZE = 3;
+ static final int MAX_DELTA = 127;
+ static final int MIN_DELTA = -128;
+ static final int MAX_LITERAL_SIZE = 128;
+ private static final int MAX_REPEAT_SIZE = 127 + MIN_REPEAT_SIZE;
+ private final PositionedOutputStream output;
+ private final boolean signed;
+ private final long[] literals = new long[MAX_LITERAL_SIZE];
+ private int numLiterals = 0;
+ private long delta = 0;
+ private boolean repeat = false;
+ private int tailRunLength = 0;
+
+ RunLengthIntegerWriter(PositionedOutputStream output,
+ boolean signed) {
+ this.output = output;
+ this.signed = signed;
+ }
+
+ private void writeValues() throws IOException {
+ if (numLiterals != 0) {
+ if (repeat) {
+ output.write(numLiterals - MIN_REPEAT_SIZE);
+ output.write((byte) delta);
+ if (signed) {
+ SerializationUtils.writeVslong(output, literals[0]);
+ } else {
+ SerializationUtils.writeVulong(output, literals[0]);
+ }
+ } else {
+ output.write(-numLiterals);
+ for(int i=0; i < numLiterals; ++i) {
+ if (signed) {
+ SerializationUtils.writeVslong(output, literals[i]);
+ } else {
+ SerializationUtils.writeVulong(output, literals[i]);
+ }
+ }
+ }
+ repeat = false;
+ numLiterals = 0;
+ tailRunLength = 0;
+ }
+ }
+
+ void flush() throws IOException {
+ writeValues();
+ output.flush();
+ }
+
+ void write(long value) throws IOException {
+ if (numLiterals == 0) {
+ literals[numLiterals++] = value;
+ tailRunLength = 1;
+ } else if (repeat) {
+ if (value == literals[0] + delta * numLiterals) {
+ numLiterals += 1;
+ if (numLiterals == MAX_REPEAT_SIZE) {
+ writeValues();
+ }
+ } else {
+ writeValues();
+ literals[numLiterals++] = value;
+ tailRunLength = 1;
+ }
+ } else {
+ if (tailRunLength == 1) {
+ delta = value - literals[numLiterals - 1];
+ if (delta < MIN_DELTA || delta > MAX_DELTA) {
+ tailRunLength = 1;
+ } else {
+ tailRunLength = 2;
+ }
+ } else if (value == literals[numLiterals - 1] + delta) {
+ tailRunLength += 1;
+ } else {
+ delta = value - literals[numLiterals - 1];
+ if (delta < MIN_DELTA || delta > MAX_DELTA) {
+ tailRunLength = 1;
+ } else {
+ tailRunLength = 2;
+ }
+ }
+ if (tailRunLength == MIN_REPEAT_SIZE) {
+ if (numLiterals + 1 == MIN_REPEAT_SIZE) {
+ repeat = true;
+ numLiterals += 1;
+ } else {
+ numLiterals -= MIN_REPEAT_SIZE - 1;
+ long base = literals[numLiterals];
+ writeValues();
+ literals[0] = base;
+ repeat = true;
+ numLiterals = MIN_REPEAT_SIZE;
+ }
+ } else {
+ literals[numLiterals++] = value;
+ if (numLiterals == MAX_LITERAL_SIZE) {
+ writeValues();
+ }
+ }
+ }
+ }
+
+ void getPosition(PositionRecorder recorder) throws IOException {
+ output.getPosition(recorder);
+ recorder.addPosition(numLiterals);
+ }
+}

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SerializationUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SerializationUtils.java?rev=1452992&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SerializationUtils.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SerializationUtils.java Tue Mar 5 20:44:50 2013
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io.orc;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+final class SerializationUtils {
+
+ // unused
+ private SerializationUtils() {}
+
+ static void writeVulong(OutputStream output, long value) throws IOException {
+ while (true) {
+ if ((value & ~0x7f) == 0) {
+ output.write((byte) value);
+ return;
+ } else {
+ output.write((byte) (0x80 | (value & 0x7f)));
+ value >>>= 7;
+ }
+ }
+ }
+
+ static void writeVslong(OutputStream output, long value) throws IOException {
+ writeVulong(output, (value << 1) ^ (value >> 63));
+ }
+
+
+ static long readVulong(InputStream in) throws IOException {
+ long result = 0;
+ long b;
+ int offset = 0;
+ do {
+ b = in.read();
+ if (b == -1) {
+ throw new EOFException("Reading Vulong past EOF");
+ }
+ result |= (0x7f & b) << offset;
+ offset += 7;
+ } while (b >= 0x80);
+ return result;
+ }
+
+ static long readVslong(InputStream in) throws IOException {
+ long result = readVulong(in);
+ return (result >>> 1) ^ -(result & 1);
+ }
+
+ static float readFloat(InputStream in) throws IOException {
+ int ser = in.read() | (in.read() << 8) | (in.read() << 16) |
+ (in.read() << 24);
+ return Float.intBitsToFloat(ser);
+ }
+
+ static void writeFloat(OutputStream output, float value) throws IOException {
+ int ser = Float.floatToIntBits(value);
+ output.write(ser & 0xff);
+ output.write((ser >> 8) & 0xff);
+ output.write((ser >> 16) & 0xff);
+ output.write((ser >> 24) & 0xff);
+ }
+
+ static double readDouble(InputStream in) throws IOException {
+ long ser = (long) in.read() |
+ ((long) in.read() << 8) |
+ ((long) in.read() << 16) |
+ ((long) in.read() << 24) |
+ ((long) in.read() << 32) |
+ ((long) in.read() << 40) |
+ ((long) in.read() << 48) |
+ ((long) in.read() << 56);
+ return Double.longBitsToDouble(ser);
+ }
+
+ static void writeDouble(OutputStream output,
+ double value) throws IOException {
+ long ser = Double.doubleToLongBits(value);
+ output.write(((int) ser) & 0xff);
+ output.write(((int) (ser >> 8)) & 0xff);
+ output.write(((int) (ser >> 16)) & 0xff);
+ output.write(((int) (ser >> 24)) & 0xff);
+ output.write(((int) (ser >> 32)) & 0xff);
+ output.write(((int) (ser >> 40)) & 0xff);
+ output.write(((int) (ser >> 48)) & 0xff);
+ output.write(((int) (ser >> 56)) & 0xff);
+ }
+}

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SnappyCodec.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SnappyCodec.java?rev=1452992&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SnappyCodec.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SnappyCodec.java Tue Mar 5 20:44:50 2013
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io.orc;
+
+import org.iq80.snappy.Snappy;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+class SnappyCodec implements CompressionCodec {
+
+ @Override
+ public boolean compress(ByteBuffer in, ByteBuffer out,
+ ByteBuffer overflow) throws IOException {
+ int inBytes = in.remaining();
+ // I should work on a patch for Snappy to support an overflow buffer
+ // to prevent the extra buffer copy.
+ byte[] compressed = new byte[Snappy.maxCompressedLength(inBytes)];
+ int outBytes =
+ Snappy.compress(in.array(), in.arrayOffset() + in.position(), inBytes,
+ compressed, 0);
+ if (outBytes < inBytes) {
+ int remaining = out.remaining();
+ if (remaining >= outBytes) {
+ System.arraycopy(compressed, 0, out.array(), out.arrayOffset() +
+ out.position(), outBytes);
+ out.position(out.position() + outBytes);
+ } else {
+ System.arraycopy(compressed, 0, out.array(), out.arrayOffset() +
+ out.position(), remaining);
+ out.position(out.limit());
+ System.arraycopy(compressed, remaining, overflow.array(),
+ overflow.arrayOffset(), outBytes - remaining);
+ overflow.position(outBytes - remaining);
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public void decompress(ByteBuffer in, ByteBuffer out) throws IOException {
+ int inOffset = in.position();
+ int uncompressLen =
+ Snappy.uncompress(in.array(), in.arrayOffset() + inOffset,
+ in.limit() - inOffset, out.array(), out.arrayOffset() + out.position());
+ out.position(uncompressLen + out.position());
+ out.flip();
+ }
+}

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StreamName.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StreamName.java?rev=1452992&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StreamName.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StreamName.java Tue Mar 5 20:44:50 2013
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io.orc;
+
+/**
+ * The name of a stream within a stripe.
+ */
+class StreamName implements Comparable<StreamName> {
+ private final int column;
+ private final OrcProto.Stream.Kind kind;
+
+ public static enum Area {
+ DATA, INDEX
+ }
+
+ public StreamName(int column, OrcProto.Stream.Kind kind) {
+ this.column = column;
+ this.kind = kind;
+ }
+
+ public boolean equals(Object obj) {
+ if (obj != null && obj instanceof StreamName) {
+ StreamName other = (StreamName) obj;
+ return other.column == column && other.kind == kind;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public int compareTo(StreamName streamName) {
+ if (streamName == null) {
+ return -1;
+ }
+ Area area = getArea(kind);
+ Area otherArea = streamName.getArea(streamName.kind);
+ if (area != otherArea) {
+ return -area.compareTo(otherArea);
+ }
+ if (column != streamName.column) {
+ return column < streamName.column ? -1 : 1;
+ }
+ return kind.compareTo(streamName.kind);
+ }
+
+ public int getColumn() {
+ return column;
+ }
+
+ public OrcProto.Stream.Kind getKind() {
+ return kind;
+ }
+
+ public Area getArea() {
+ return getArea(kind);
+ }
+
+ public static Area getArea(OrcProto.Stream.Kind kind) {
+ switch (kind) {
+ case ROW_INDEX:
+ case DICTIONARY_COUNT:
+ return Area.INDEX;
+ default:
+ return Area.DATA;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "Stream for column " + column + " kind " + kind;
+ }
+
+ @Override
+ public int hashCode() {
+ return column * 101 + kind.getNumber();
+ }
+}
+

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringColumnStatistics.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringColumnStatistics.java?rev=1452992&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringColumnStatistics.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringColumnStatistics.java Tue Mar 5 20:44:50 2013
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.io.orc;
+
+/**
+ * Statistics for string columns.
+ */
+public interface StringColumnStatistics extends ColumnStatistics {
+ /**
+ * Get the minimum string.
+ * @return the minimum
+ */
+ String getMinimum();
+
+ /**
+ * Get the maximum string.
+ * @return the maximum
+ */
+ String getMaximum();
+}

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringRedBlackTree.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringRedBlackTree.java?rev=1452992&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringRedBlackTree.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringRedBlackTree.java Tue Mar 5 20:44:50 2013
@@ -0,0 +1,176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.io.orc;
+
+import org.apache.hadoop.io.Text;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * A red-black tree that stores strings. The strings are stored as UTF-8 bytes
+ * and an offset/length for each entry.
+ */
+class StringRedBlackTree extends RedBlackTree {
+ private final DynamicByteArray byteArray = new DynamicByteArray();
+ private final DynamicIntArray keySizes = new DynamicIntArray();
+ private final Text newKey = new Text();
+
+ public StringRedBlackTree() {
+ // PASS
+ }
+
+ public StringRedBlackTree(int initialCapacity) {
+ super(initialCapacity);
+ }
+
+ public int add(String value) {
+ newKey.set(value);
+ // if the key is new, add it to our byteArray and store the offset & length
+ if (add()) {
+ int len = newKey.getLength();
+ keySizes.add(byteArray.add(newKey.getBytes(), 0, len));
+ keySizes.add(len);
+ }
+ return lastAdd;
+ }
+
+ @Override
+ protected int compareValue(int position) {
+ return byteArray.compare(newKey.getBytes(), 0, newKey.getLength(),
+ keySizes.get(2 * position), keySizes.get(2 * position + 1));
+ }
+
+ /**
+ * The information about each node.
+ */
+ public interface VisitorContext {
+ /**
+ * Get the position where the key was originally added.
+ * @return the number returned by add.
+ */
+ int getOriginalPosition();
+
+ /**
+ * Write the bytes for the string to the given output stream.
+ * @param out the stream to write to.
+ * @throws IOException
+ */
+ void writeBytes(OutputStream out) throws IOException;
+
+ /**
+ * Get the original string.
+ * @return the string
+ */
+ Text getText();
+
+ /**
+ * Get the number of bytes.
+ * @return the string's length in bytes
+ */
+ int getLength();
+
+ /**
+ * Get the count for this key.
+ * @return the number of times this key was added
+ */
+ int getCount();
+ }
+
+ /**
+ * The interface for visitors.
+ */
+ public interface Visitor {
+ /**
+ * Called once for each node of the tree in sort order.
+ * @param context the information about each node
+ * @throws IOException
+ */
+ void visit(VisitorContext context) throws IOException;
+ }
+
+ private class VisitorContextImpl implements VisitorContext {
+ private int originalPosition;
+ private final Text text = new Text();
+
+ public int getOriginalPosition() {
+ return originalPosition;
+ }
+
+ public Text getText() {
+ byteArray.setText(text, keySizes.get(originalPosition * 2), getLength());
+ return text;
+ }
+
+ public void writeBytes(OutputStream out) throws IOException {
+ byteArray.write(out, keySizes.get(originalPosition * 2), getLength());
+ }
+
+ public int getLength() {
+ return keySizes.get(originalPosition * 2 + 1);
+ }
+
+ public int getCount() {
+ return StringRedBlackTree.this.getCount(originalPosition);
+ }
+ }
+
+ private void recurse(int node, Visitor visitor, VisitorContextImpl context
+ ) throws IOException {
+ if (node != NULL) {
+ recurse(getLeft(node), visitor, context);
+ context.originalPosition = node;
+ visitor.visit(context);
+ recurse(getRight(node), visitor, context);
+ }
+ }
+
+ /**
+ * Visit all of the nodes in the tree in sorted order.
+ * @param visitor the action to be applied to each ndoe
+ * @throws IOException
+ */
+ public void visit(Visitor visitor) throws IOException {
+ recurse(root, visitor, new VisitorContextImpl());
+ }
+
+ /**
+ * Reset the table to empty.
+ */
+ public void clear() {
+ super.clear();
+ byteArray.clear();
+ keySizes.clear();
+ }
+
+ /**
+ * Get the size of the character data in the table.
+ * @return the bytes used by the table
+ */
+ public int getCharacterSize() {
+ return byteArray.size();
+ }
+
+ /**
+ * Calculate the approximate size in memory.
+ * @return the number of bytes used in storing the tree.
+ */
+ public long getByteSize() {
+ return byteArray.size() + 5 * 4 * size();
+ }
+}

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StripeInformation.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StripeInformation.java?rev=1452992&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StripeInformation.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StripeInformation.java Tue Mar 5 20:44:50 2013
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.io.orc;
+
+/**
+ * Information about the stripes in an ORC file that is provided by the Reader.
+ */
+public interface StripeInformation {
+ /**
+ * Get the byte offset of the start of the stripe.
+ * @return the bytes from the start of the file
+ */
+ long getOffset();
+
+ /**
+ * Get the length of the stripe's indexes.
+ * @return the number of bytes in the index
+ */
+ long getIndexLength();
+
+ /**
+ * Get the length of the stripe's data.
+ * @return the number of bytes in the stripe
+ */
+ long getDataLength();
+
+ /**
+ * Get the length of the stripe's tail section, which contains its index.
+ * @return the number of bytes in the tail
+ */
+ long getFooterLength();
+
+ /**
+ * Get the number of rows in the stripe.
+ * @return a count of the number of rows
+ */
+ long getNumberOfRows();
+}

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Writer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Writer.java?rev=1452992&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Writer.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Writer.java Tue Mar 5 20:44:50 2013
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io.orc;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+/**
+ * The interface for writing ORC files.
+ */
+public interface Writer {
+ /**
+ * Add arbitrary meta-data to the ORC file. This may be called at any point
+ * until the Writer is closed. If the same key is passed a second time, the
+ * second value will replace the first.
+ * @param key a key to label the data with.
+ * @param value the contents of the metadata.
+ */
+ void addUserMetadata(String key, ByteBuffer value);
+
+ /**
+ * Add a row to the ORC file.
+ * @param row the row to add
+ * @throws IOException
+ */
+ void addRow(Object row) throws IOException;
+
+ /**
+ * Flush all of the buffers and close the file. No methods on this writer
+ * should be called afterwards.
+ * @throws IOException
+ */
+ void close() throws IOException;
+}

Search Discussions

Discussion Posts

Previous

Follow ups

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 2 of 8 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedMar 5, '13 at 8:45p
activeMar 5, '13 at 8:45p
posts8
users1
websitehive.apache.org

1 user in discussion

Kevinwilfong: 8 posts

People

Translate

site design / logo © 2021 Grokbase