FAQ
Author: brock
Date: Sun Mar 1 19:08:50 2015
New Revision: 1663147

URL: http://svn.apache.org/r1663147
Log:
HIVE-9823 - Load spark-defaults.conf from classpath [Spark Branch] (Brock via Xuefu)

Modified:
     hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java

Modified: hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java?rev=1663147&r1=1663146&r2=1663147&view=diff
==============================================================================
--- hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java (original)
+++ hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java Sun Mar 1 19:08:50 2015
@@ -24,12 +24,14 @@ import com.google.common.base.Preconditi
  import com.google.common.base.Throwables;
  import com.google.common.collect.Lists;
  import com.google.common.collect.Maps;
+import com.google.common.io.Resources;

  import io.netty.channel.ChannelHandlerContext;
  import io.netty.util.concurrent.GenericFutureListener;
  import io.netty.util.concurrent.Promise;

  import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
  import java.io.File;
  import java.io.FileOutputStream;
  import java.io.IOException;
@@ -39,6 +41,7 @@ import java.io.OutputStreamWriter;
  import java.io.Serializable;
  import java.io.Writer;
  import java.net.URI;
+import java.net.URL;
  import java.util.List;
  import java.util.Map;
  import java.util.Properties;
@@ -249,6 +252,18 @@ class SparkClientImpl implements SparkCl
        }

        Properties allProps = new Properties();
+ // first load the defaults from spark-defaults.conf if available
+ try {
+ URL sparkDefaultsUrl = Thread.currentThread().getContextClassLoader().getResource("spark-defaults.conf");
+ if (sparkDefaultsUrl != null) {
+ LOG.info("Loading spark defaults: " + sparkDefaultsUrl);
+ allProps.load(new ByteArrayInputStream(Resources.toByteArray(sparkDefaultsUrl)));
+ }
+ } catch (Exception e) {
+ String msg = "Exception trying to load spark-defaults.conf: " + e;
+ throw new IOException(msg, e);
+ }
+ // then load the SparkClientImpl config
        for (Map.Entry<String, String> e : conf.entrySet()) {
          allProps.put(e.getKey(), conf.get(e.getKey()));
        }

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedMar 1, '15 at 7:08p
activeMar 1, '15 at 7:08p
posts1
users1
websitehive.apache.org

1 user in discussion

Brock: 1 post

People

Translate

site design / logo © 2021 Grokbase