FAQ
Author: kevinwilfong
Date: Tue Mar 5 20:44:50 2013
New Revision: 1452992

URL: http://svn.apache.org/r1452992
Log:
HIVE-3874. Create a new Optimized Row Columnar file format for Hive. (Owen O'Malley via kevinwilfong)

Added:
     hive/trunk/ql/src/gen/protobuf/
     hive/trunk/ql/src/gen/protobuf/gen-java/
     hive/trunk/ql/src/gen/protobuf/gen-java/org/
     hive/trunk/ql/src/gen/protobuf/gen-java/org/apache/
     hive/trunk/ql/src/gen/protobuf/gen-java/org/apache/hadoop/
     hive/trunk/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/
     hive/trunk/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/
     hive/trunk/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/
     hive/trunk/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/
     hive/trunk/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/BitFieldReader.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/BitFieldWriter.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/BooleanColumnStatistics.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatistics.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/CompressionCodec.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/CompressionKind.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DoubleColumnStatistics.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DynamicByteArray.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DynamicIntArray.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/InStream.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/IntegerColumnStatistics.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUnion.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/PositionProvider.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/PositionRecorder.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/PositionedOutputStream.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReader.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RedBlackTree.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteReader.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteWriter.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerReader.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriter.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SerializationUtils.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SnappyCodec.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StreamName.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringColumnStatistics.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringRedBlackTree.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StripeInformation.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Writer.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ZlibCodec.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/package-info.java
     hive/trunk/ql/src/protobuf/
     hive/trunk/ql/src/protobuf/org/
     hive/trunk/ql/src/protobuf/org/apache/
     hive/trunk/ql/src/protobuf/org/apache/hadoop/
     hive/trunk/ql/src/protobuf/org/apache/hadoop/hive/
     hive/trunk/ql/src/protobuf/org/apache/hadoop/hive/ql/
     hive/trunk/ql/src/protobuf/org/apache/hadoop/hive/ql/io/
     hive/trunk/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/
     hive/trunk/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto
     hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/
     hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestBitFieldReader.java
     hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestDynamicArray.java
     hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java
     hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInStream.java
     hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
     hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
     hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcStruct.java
     hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRunLengthByteReader.java
     hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRunLengthIntegerReader.java
     hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestSerializationUtils.java
     hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStreamName.java
     hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringRedBlackTree.java
     hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestZlib.java
     hive/trunk/ql/src/test/resources/orc-file-dump.out
Modified:
     hive/trunk/build.properties
     hive/trunk/build.xml
     hive/trunk/ivy/libraries.properties
     hive/trunk/ql/build.xml
     hive/trunk/ql/ivy.xml

Modified: hive/trunk/build.properties
URL: http://svn.apache.org/viewvc/hive/trunk/build.properties?rev=1452992&r1=1452991&r2=1452992&view=diff
==============================================================================
--- hive/trunk/build.properties (original)
+++ hive/trunk/build.properties Tue Mar 5 20:44:50 2013
@@ -76,6 +76,7 @@ iterate.hive.all=ant,shims,common,serde,
  iterate.hive.modules=shims,common,serde,metastore,ql,contrib,service,cli,jdbc,hwi,hbase-handler,pdk,builtins
  iterate.hive.tests=ql,contrib,hbase-handler,hwi,jdbc,metastore,odbc,serde,service
  iterate.hive.thrift=ql,service,metastore,serde
+iterate.hive.protobuf=ql
  iterate.hive.cpp=odbc

  #

Modified: hive/trunk/build.xml
URL: http://svn.apache.org/viewvc/hive/trunk/build.xml?rev=1452992&r1=1452991&r2=1452992&view=diff
==============================================================================
--- hive/trunk/build.xml (original)
+++ hive/trunk/build.xml Tue Mar 5 20:44:50 2013
@@ -256,6 +256,11 @@
      <iterate target="thriftif" iterate="${iterate.hive.thrift}"/>
    </target>

+ <target name="protobuf">
+ <echo message="Project: ${ant.project.name}"/>
+ <iterate target="protobuf" iterate="${iterate.hive.protobuf}"/>
+ </target>
+
    <target name="jar"
            depends="init"
            description="Build JAR artifacts">

Modified: hive/trunk/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/hive/trunk/ivy/libraries.properties?rev=1452992&r1=1452991&r2=1452992&view=diff
==============================================================================
--- hive/trunk/ivy/libraries.properties (original)
+++ hive/trunk/ivy/libraries.properties Tue Mar 5 20:44:50 2013
@@ -60,9 +60,11 @@ libthrift.version=0.9.0
  log4j.version=1.2.16
  maven-ant-tasks.version=2.1.0
  mockito-all.version=1.8.2
+protobuf.version=2.4.1
  rat.version=0.8
  slf4j-api.version=1.6.1
  slf4j-log4j12.version=1.6.1
+snappy.version=0.2
  velocity.version=1.5
  zookeeper.version=3.4.3
  javolution.version=5.5.1

Modified: hive/trunk/ql/build.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/build.xml?rev=1452992&r1=1452991&r2=1452992&view=diff
==============================================================================
--- hive/trunk/ql/build.xml (original)
+++ hive/trunk/ql/build.xml Tue Mar 5 20:44:50 2013
@@ -22,6 +22,9 @@

    <property name="ql.lib.dir" value="${basedir}/lib"/>
    <property name="src.dir" location="${basedir}/src/java"/>
+ <property name="protobuf.src.dir" location="${basedir}/src/protobuf"/>
+ <property name="protobuf.build.dir"
+ location="${basedir}/src/gen/protobuf/gen-java"/>
    <property name="ql.test.query.dir" location="${basedir}/src/test/queries"/>
    <property name="ql.test.template.dir" location="${basedir}/src/test/templates"/>
    <property name="ql.test.results.dir" location="${basedir}/src/test/results"/>
@@ -147,16 +150,28 @@
      </java>
    </target>

+ <target name="protobuf">
+ <echo message="Project: ${ant.project.name}"/>
+ <echo>Building ORC Protobuf</echo>
+ <mkdir dir="${protobuf.build.dir}"/>
+ <exec executable="protoc" failonerror="true">
+ <arg value="--java_out=${protobuf.build.dir}"/>
+ <arg value="-I=${protobuf.src.dir}/org/apache/hadoop/hive/ql/io/orc"/>
+ <arg value="${protobuf.src.dir}/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto"/>
+ </exec>
+ </target>
+
    <target name="ql-init">
      <echo message="Project: ${ant.project.name}"/>
      <mkdir dir="${build.dir}/gen/antlr/gen-java/org/apache/hadoop/hive/ql/parse"/>
    </target>

- <target name="compile" depends="init, ql-init, ivy-retrieve, build-grammar">
+ <target name="compile"
+ depends="init, ql-init, ivy-retrieve, build-grammar">
      <echo message="Project: ${ant.project.name}"/>
      <javac
       encoding="${build.encoding}"
- srcdir="${src.dir}:${basedir}/src/gen/thrift/gen-javabean:${build.dir}/gen/antlr/gen-java"
+ srcdir="${src.dir}:${basedir}/src/gen/thrift/gen-javabean:${build.dir}/gen/antlr/gen-java:${protobuf.build.dir}"
       includes="**/*.java"
       destdir="${build.classes}"
       debug="${javac.debug}"

Modified: hive/trunk/ql/ivy.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/ivy.xml?rev=1452992&r1=1452991&r2=1452992&view=diff
==============================================================================
--- hive/trunk/ql/ivy.xml (original)
+++ hive/trunk/ql/ivy.xml Tue Mar 5 20:44:50 2013
@@ -38,6 +38,10 @@
                  conf="test->default" transitive="false"/>
      <dependency org="org.antlr" name="antlr" rev="${antlr.version}"
                  transitive="false"/>
+ <dependency org="com.google.protobuf" name="protobuf-java"
+ rev="${protobuf.version}" transitive="false"/>
+ <dependency org="org.iq80.snappy" name="snappy"
+ rev="${snappy.version}" transitive="false"/>

      <!-- hadoop specific guava -->
      <dependency org="com.google.guava" name="guava" rev="${guava-hadoop20.version}"

Search Discussions

Discussion Posts

Follow ups

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 1 of 8 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedMar 5, '13 at 8:45p
activeMar 5, '13 at 8:45p
posts8
users1
websitehive.apache.org

1 user in discussion

Kevinwilfong: 8 posts

People

Translate

site design / logo © 2021 Grokbase