FAQ
Author: omalley
Date: Wed Nov 6 21:47:24 2013
New Revision: 1539472

URL: http://svn.apache.org/r1539472
Log:
HIVE-5425 Provide a configuration option to control the default stripe
size for ORC. (omalley reviewed by gunther)

Modified:
     hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java

Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1539472&r1=1539471&r2=1539472&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Wed Nov 6 21:47:24 2013
@@ -507,6 +507,9 @@ public class HiveConf extends Configurat
      HIVE_ORC_FILE_MEMORY_POOL("hive.exec.orc.memory.pool", 0.5f), // 50%
      // Define the version of the file to write
      HIVE_ORC_WRITE_FORMAT("hive.exec.orc.write.format", null),
+ // Define the default ORC stripe size
+ HIVE_ORC_DEFAULT_STRIPE_SIZE("hive.exec.orc.default.stripe.size",
+ 256L * 1024 * 1024),

      HIVE_ORC_DICTIONARY_KEY_SIZE_THRESHOLD("hive.exec.orc.dictionary.key.size.threshold", 0.8f),


Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java?rev=1539472&r1=1539471&r2=1539472&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java Wed Nov 6 21:47:24 2013
@@ -103,7 +103,8 @@ public final class OrcFile {
    public static final String ENABLE_INDEXES = "orc.create.index";
    public static final String BLOCK_PADDING = "orc.block.padding";

- static final long DEFAULT_STRIPE_SIZE = 256 * 1024 * 1024;
+ static final long DEFAULT_STRIPE_SIZE =
+ HiveConf.ConfVars.HIVE_ORC_DEFAULT_STRIPE_SIZE.defaultLongVal;
    static final CompressionKind DEFAULT_COMPRESSION_KIND =
      CompressionKind.ZLIB;
    static final int DEFAULT_BUFFER_SIZE = 256 * 1024;
@@ -132,7 +133,7 @@ public final class OrcFile {
      private final Configuration configuration;
      private FileSystem fileSystemValue = null;
      private ObjectInspector inspectorValue = null;
- private long stripeSizeValue = DEFAULT_STRIPE_SIZE;
+ private long stripeSizeValue;
      private int rowIndexStrideValue = DEFAULT_ROW_INDEX_STRIDE;
      private int bufferSizeValue = DEFAULT_BUFFER_SIZE;
      private boolean blockPaddingValue = DEFAULT_BLOCK_PADDING;
@@ -143,6 +144,9 @@ public final class OrcFile {
      WriterOptions(Configuration conf) {
        configuration = conf;
        memoryManagerValue = getMemoryManager(conf);
+ stripeSizeValue =
+ conf.getLong(HiveConf.ConfVars.HIVE_ORC_DEFAULT_STRIPE_SIZE.varname,
+ DEFAULT_STRIPE_SIZE);
        String versionName =
          conf.get(HiveConf.ConfVars.HIVE_ORC_WRITE_FORMAT.varname);
        if (versionName == null) {

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedNov 6, '13 at 9:47p
activeNov 6, '13 at 9:47p
posts1
users1
websitehive.apache.org

1 user in discussion

Omalley: 1 post

People

Translate

site design / logo © 2021 Grokbase