FAQ
Author: szehon
Date: Thu Jul 24 01:28:14 2014
New Revision: 1612978

URL: http://svn.apache.org/r1612978
Log:
HIVE-5160 : HS2 should support .hiverc (Dong Chen via Szehon)

Added:
     hive/trunk/common/src/java/org/apache/hadoop/hive/common/cli/HiveFileProcessor.java
     hive/trunk/common/src/java/org/apache/hadoop/hive/common/cli/IHiveFileProcessor.java
     hive/trunk/service/src/test/org/apache/hive/service/cli/session/TestSessionGlobalInitFile.java
Modified:
     hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
     hive/trunk/conf/hive-default.xml.template
     hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java
     hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
     hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java
     hive/trunk/service/src/java/org/apache/hive/service/cli/session/SessionManager.java

Added: hive/trunk/common/src/java/org/apache/hadoop/hive/common/cli/HiveFileProcessor.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/common/cli/HiveFileProcessor.java?rev=1612978&view=auto
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/common/cli/HiveFileProcessor.java (added)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/common/cli/HiveFileProcessor.java Thu Jul 24 01:28:14 2014
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common.cli;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.io.IOUtils;
+
+/**
+ * HiveFileProcessor is used for processing a file consist of Hive executable
+ * statements
+ */
+public abstract class HiveFileProcessor implements IHiveFileProcessor {
+
+ public int processFile(String fileName) throws IOException {
+ BufferedReader bufferedReader = null;
+ try {
+ bufferedReader = loadFile(fileName);
+ return (processReader(bufferedReader));
+ } finally {
+ IOUtils.closeStream(bufferedReader);
+ }
+ }
+
+ /**
+ * load commands into buffered reader from the file
+ * @param fileName
+ * @return
+ * @throws IOException
+ */
+ protected abstract BufferedReader loadFile(String fileName)
+ throws IOException;
+
+ /**
+ * execute the buffered reader which stores the commands
+ * @param reader the buffered reader
+ * @return the return code of the execution result
+ * @throws IOException
+ */
+ protected int processReader(BufferedReader reader) throws IOException {
+ String line;
+ StringBuilder qsb = new StringBuilder();
+ while ((line = reader.readLine()) != null) {
+ if (!line.startsWith("--")) {
+ qsb.append(line);
+ }
+ }
+ return processLine(qsb.toString());
+ }
+
+ /**
+ * process the Hive command by lines
+ * @param line contains a legal Hive command
+ * @return the return code of the execution result
+ */
+ protected int processLine(String line) {
+ int lastRet = 0, ret = 0;
+ String command = "";
+ for (String oneCmd : line.split(";")) {
+ if (StringUtils.indexOf(oneCmd, "\\") != -1) {
+ command += StringUtils.join(oneCmd.split("\\\\"));
+ } else {
+ command += oneCmd;
+ }
+ if (StringUtils.isBlank(command)) {
+ continue;
+ }
+
+ ret = processCmd(command);
+ command = "";
+ lastRet = ret;
+ if (ret != 0) {
+ return ret;
+ }
+ }
+ return lastRet;
+ }
+
+ /**
+ * define the processor for each Hive command supported by Hive
+ * @param cmd
+ * @return the return code of the execution result
+ */
+ protected abstract int processCmd(String cmd);
+}

Added: hive/trunk/common/src/java/org/apache/hadoop/hive/common/cli/IHiveFileProcessor.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/common/cli/IHiveFileProcessor.java?rev=1612978&view=auto
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/common/cli/IHiveFileProcessor.java (added)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/common/cli/IHiveFileProcessor.java Thu Jul 24 01:28:14 2014
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common.cli;
+
+import java.io.IOException;
+
+/**
+ * Hive file processor is used for processing an executable Hive file
+ */
+public interface IHiveFileProcessor {
+ /**
+ * Parse the file and execute the Hive commands in it.
+ * @param fileName the name of the file
+ * @exception IOException if an I/O error occurs.
+ */
+ public int processFile(String fileName) throws IOException;
+}

Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1612978&r1=1612977&r2=1612978&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Thu Jul 24 01:28:14 2014
@@ -1385,6 +1385,12 @@ public class HiveConf extends Configurat
          "If the property is set, the value must be a valid URI (java.net.URI, e.g. \"file:///tmp/my-logging.properties\"), \n" +
          "which you can then extract a URL from and pass to PropertyConfigurator.configure(URL)."),

+ // Hive global init file location
+ HIVE_GLOBAL_INIT_FILE_LOCATION("hive.global.init.file.location", System.getenv("HIVE_CONF_DIR"),
+ "The location of HS2 global init file (.hiverc).\n" +
+ "If the property is not set, then HS2 will search for the file in $HIVE_CONF_DIR/.\n" +
+ "If the property is set, the value must be a valid path where the init file is located."),
+
      // prefix used to auto generated column aliases (this should be started with '_')
      HIVE_AUTOGEN_COLUMNALIAS_PREFIX_LABEL("hive.autogen.columnalias.prefix.label", "_c",
          "String used as a prefix when auto generating column alias.\n" +

Modified: hive/trunk/conf/hive-default.xml.template
URL: http://svn.apache.org/viewvc/hive/trunk/conf/hive-default.xml.template?rev=1612978&r1=1612977&r2=1612978&view=diff
==============================================================================
--- hive/trunk/conf/hive-default.xml.template (original)
+++ hive/trunk/conf/hive-default.xml.template Thu Jul 24 01:28:14 2014
@@ -2452,6 +2452,15 @@
      </description>
    </property>
    <property>
+ <key>hive.global.init.file.location</key>
+ <value/>
+ <description>
+ The location of HS2 global init file (.hiverc).
+ If the property is not set, then HS2 will search for the file in $HIVE_CONF_DIR/.
+ If the property is set, the value must be a valid path where the init file is located.
+ </description>
+ </property>
+ <property>
      <key>hive.autogen.columnalias.prefix.label</key>
      <value>_c</value>
      <description>

Modified: hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java
URL: http://svn.apache.org/viewvc/hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java?rev=1612978&r1=1612977&r2=1612978&view=diff
==============================================================================
--- hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java (original)
+++ hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java Thu Jul 24 01:28:14 2014
@@ -24,6 +24,8 @@ import org.apache.hive.service.cli.Sessi
  import org.apache.hive.service.cli.operation.OperationManager;
  import org.apache.hive.service.cli.thrift.TProtocolVersion;

+import java.util.Map;
+
  /**
   * Methods that don't need to be executed under a doAs
   * context are here. Rest of them in HiveSession interface
@@ -49,6 +51,12 @@ public interface HiveSessionBase {
     */
    public void setOperationManager(OperationManager operationManager);

+ /**
+ * Initialize the session
+ * @param sessionConfMap
+ */
+ public void initialize(Map<String, String> sessionConfMap);
+
    public SessionHandle getSessionHandle();

    public String getUsername();

Modified: hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
URL: http://svn.apache.org/viewvc/hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java?rev=1612978&r1=1612977&r2=1612978&view=diff
==============================================================================
--- hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java (original)
+++ hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java Thu Jul 24 01:28:14 2014
@@ -18,7 +18,11 @@

  package org.apache.hive.service.cli.session;

+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
  import java.io.IOException;
+import java.io.InputStreamReader;
  import java.util.HashSet;
  import java.util.List;
  import java.util.Map;
@@ -26,6 +30,8 @@ import java.util.Set;

  import org.apache.commons.logging.Log;
  import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.common.cli.HiveFileProcessor;
+import org.apache.hadoop.hive.common.cli.IHiveFileProcessor;
  import org.apache.hadoop.hive.conf.HiveConf;
  import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
  import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
@@ -82,7 +88,7 @@ public class HiveSessionImpl implements
    private final Set<OperationHandle> opHandleSet = new HashSet<OperationHandle>();

    public HiveSessionImpl(TProtocolVersion protocol, String username, String password,
- HiveConf serverhiveConf, Map<String, String> sessionConfMap, String ipAddress) {
+ HiveConf serverhiveConf, String ipAddress) {
      this.username = username;
      this.password = password;
      this.sessionHandle = new SessionHandle(protocol);
@@ -101,6 +107,13 @@ public class HiveSessionImpl implements
      sessionState.setUserIpAddress(ipAddress);
      sessionState.setIsHiveServerQuery(true);
      SessionState.start(sessionState);
+ }
+
+ @Override
+ public void initialize(Map<String, String> sessionConfMap) {
+ //process global init file: .hiverc
+ processGlobalInitFile();
+ SessionState.setCurrentSessionState(sessionState);

      //set conf properties specified by user from client side
      if (sessionConfMap != null) {
@@ -108,6 +121,53 @@ public class HiveSessionImpl implements
      }
    }

+ /**
+ * It is used for processing hiverc file from HiveServer2 side.
+ */
+ private class GlobalHivercFileProcessor extends HiveFileProcessor {
+ @Override
+ protected BufferedReader loadFile(String fileName) throws IOException {
+ FileInputStream initStream = null;
+ BufferedReader bufferedReader = null;
+ initStream = new FileInputStream(fileName);
+ bufferedReader = new BufferedReader(new InputStreamReader(initStream));
+ return bufferedReader;
+ }
+
+ @Override
+ protected int processCmd(String cmd) {
+ int rc = 0;
+ String cmd_trimed = cmd.trim();
+ try {
+ executeStatementInternal(cmd_trimed, null, false);
+ } catch (HiveSQLException e) {
+ rc = -1;
+ LOG.warn("Failed to execute HQL command in global .hiverc file.", e);
+ }
+ return rc;
+ }
+ }
+
+ private void processGlobalInitFile() {
+ IHiveFileProcessor processor = new GlobalHivercFileProcessor();
+
+ try {
+ if (hiveConf.getVar(ConfVars.HIVE_GLOBAL_INIT_FILE_LOCATION) != null) {
+ String hiverc = hiveConf.getVar(ConfVars.HIVE_GLOBAL_INIT_FILE_LOCATION)
+ + File.separator + SessionManager.HIVERCFILE;
+ if (new File(hiverc).exists()) {
+ LOG.info("Running global init file: " + hiverc);
+ int rc = processor.processFile(hiverc);
+ if (rc != 0) {
+ LOG.warn("Failed on initializing global .hiverc file");
+ }
+ }
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed on initializing global .hiverc file", e);
+ }
+ }
+
    private void configureSession(Map<String, String> sessionConfMap) {
      for (Map.Entry<String, String> entry : sessionConfMap.entrySet()) {
        String key = entry.getKey();

Modified: hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java
URL: http://svn.apache.org/viewvc/hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java?rev=1612978&r1=1612977&r2=1612978&view=diff
==============================================================================
--- hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java (original)
+++ hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java Thu Jul 24 01:28:14 2014
@@ -44,9 +44,8 @@ public class HiveSessionImplwithUGI exte
    private HiveSession proxySession = null;

    public HiveSessionImplwithUGI(TProtocolVersion protocol, String username, String password,
- HiveConf hiveConf, Map<String, String> sessionConf, String ipAddress,
- String delegationToken) throws HiveSQLException {
- super(protocol, username, password, hiveConf, sessionConf, ipAddress);
+ HiveConf hiveConf, String ipAddress, String delegationToken) throws HiveSQLException {
+ super(protocol, username, password, hiveConf, ipAddress);
      setSessionUGI(username);
      setDelegationToken(delegationToken);
    }

Modified: hive/trunk/service/src/java/org/apache/hive/service/cli/session/SessionManager.java
URL: http://svn.apache.org/viewvc/hive/trunk/service/src/java/org/apache/hive/service/cli/session/SessionManager.java?rev=1612978&r1=1612977&r2=1612978&view=diff
==============================================================================
--- hive/trunk/service/src/java/org/apache/hive/service/cli/session/SessionManager.java (original)
+++ hive/trunk/service/src/java/org/apache/hive/service/cli/session/SessionManager.java Thu Jul 24 01:28:14 2014
@@ -47,6 +47,7 @@ import org.apache.hive.service.cli.thrif
  public class SessionManager extends CompositeService {

    private static final Log LOG = LogFactory.getLog(CompositeService.class);
+ public static final String HIVERCFILE = ".hiverc";
    private HiveConf hiveConf;
    private final Map<SessionHandle, HiveSession> handleToSession =
        new ConcurrentHashMap<SessionHandle, HiveSession>();
@@ -122,15 +123,16 @@ public class SessionManager extends Comp
      HiveSession session;
      if (withImpersonation) {
        HiveSessionImplwithUGI hiveSessionUgi = new HiveSessionImplwithUGI(protocol, username, password,
- hiveConf, sessionConf, TSetIpAddressProcessor.getUserIpAddress(), delegationToken);
+ hiveConf, TSetIpAddressProcessor.getUserIpAddress(), delegationToken);
        session = HiveSessionProxy.getProxy(hiveSessionUgi, hiveSessionUgi.getSessionUgi());
        hiveSessionUgi.setProxySession(session);
      } else {
- session = new HiveSessionImpl(protocol, username, password, hiveConf, sessionConf,
+ session = new HiveSessionImpl(protocol, username, password, hiveConf,
            TSetIpAddressProcessor.getUserIpAddress());
      }
      session.setSessionManager(this);
      session.setOperationManager(operationManager);
+ session.initialize(sessionConf);
      session.open();
      handleToSession.put(session.getSessionHandle(), session);


Added: hive/trunk/service/src/test/org/apache/hive/service/cli/session/TestSessionGlobalInitFile.java
URL: http://svn.apache.org/viewvc/hive/trunk/service/src/test/org/apache/hive/service/cli/session/TestSessionGlobalInitFile.java?rev=1612978&view=auto
==============================================================================
--- hive/trunk/service/src/test/org/apache/hive/service/cli/session/TestSessionGlobalInitFile.java (added)
+++ hive/trunk/service/src/test/org/apache/hive/service/cli/session/TestSessionGlobalInitFile.java Thu Jul 24 01:28:14 2014
@@ -0,0 +1,159 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hive.service.cli.session;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import junit.framework.Assert;
+import junit.framework.TestCase;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hive.service.cli.*;
+import org.apache.hive.service.cli.thrift.ThriftBinaryCLIService;
+import org.apache.hive.service.cli.thrift.ThriftCLIServiceClient;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestSessionGlobalInitFile extends TestCase {
+
+ private FakeEmbeddedThriftBinaryCLIService service;
+ private ThriftCLIServiceClient client;
+ private File initFile;
+ private String tmpDir;
+
+ /**
+ * This class is almost the same as EmbeddedThriftBinaryCLIService,
+ * except its constructor having a HiveConf param for test usage.
+ */
+ private class FakeEmbeddedThriftBinaryCLIService extends ThriftBinaryCLIService {
+ public FakeEmbeddedThriftBinaryCLIService(HiveConf hiveConf) {
+ super(new CLIService());
+ isEmbedded = true;
+ cliService.init(hiveConf);
+ cliService.start();
+ }
+
+ public ICLIService getService() {
+ return cliService;
+ }
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+
+ // create and put .hiverc sample file to default directory
+ initFile = File.createTempFile("test", "hive");
+ tmpDir =
+ initFile.getParentFile().getAbsoluteFile() + File.separator
+ + "TestSessionGlobalInitFile";
+ initFile.delete();
+ FileUtils.deleteDirectory(new File(tmpDir));
+
+ initFile =
+ new File(tmpDir + File.separator + SessionManager.HIVERCFILE);
+ initFile.getParentFile().mkdirs();
+ initFile.createNewFile();
+
+ String[] fileContent =
+ new String[] { "-- global init hive file for test", "set a=1;",
+ "set hiveconf:b=1;", "set hivevar:c=1;", "set d\\", " =1;",
+ "add jar " + initFile.getAbsolutePath() };
+ FileUtils.writeLines(initFile, Arrays.asList(fileContent));
+
+ // set up service and client
+ HiveConf hiveConf = new HiveConf();
+ hiveConf.setVar(HiveConf.ConfVars.HIVE_GLOBAL_INIT_FILE_LOCATION,
+ initFile.getParentFile().getAbsolutePath());
+ service = new FakeEmbeddedThriftBinaryCLIService(hiveConf);
+ service.init(new HiveConf());
+ client = new ThriftCLIServiceClient(service);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ // restore
+ FileUtils.deleteDirectory(new File(tmpDir));
+ }
+
+ @Test
+ public void testSessionGlobalInitFile() throws Exception {
+ /**
+ * create session, and fetch the property set in global init file. Test if
+ * the global init file .hiverc is loaded correctly by checking the expected
+ * setting property.
+ */
+ SessionHandle sessionHandle = client.openSession(null, null, null);
+
+ verifyInitProperty("a", "1", sessionHandle);
+ verifyInitProperty("b", "1", sessionHandle);
+ verifyInitProperty("c", "1", sessionHandle);
+ verifyInitProperty("hivevar:c", "1", sessionHandle);
+ verifyInitProperty("d", "1", sessionHandle);
+
+ /**
+ * TODO: client.executeStatement do not support listing resources command
+ * (beeline> list jar)
+ */
+ // Assert.assertEquals("expected uri", api.getAddedResource("jar"));
+
+ client.closeSession(sessionHandle);
+ }
+
+ @Test
+ public void testSessionGlobalInitFileWithUser() throws Exception {
+ //Test when the session is opened by a user. (HiveSessionImplwithUGI)
+ SessionHandle sessionHandle = client.openSession("hive", "password", null);
+ verifyInitProperty("a", "1", sessionHandle);
+ client.closeSession(sessionHandle);
+ }
+
+ @Test
+ public void testSessionGlobalInitFileAndConfOverlay() throws Exception {
+ // Test if the user session specific conf overlaying global init conf.
+ Map<String, String> confOverlay = new HashMap<String, String>();
+ confOverlay.put("a", "2");
+ confOverlay.put("set:hiveconf:b", "2");
+ confOverlay.put("set:hivevar:c", "2");
+
+ SessionHandle sessionHandle = client.openSession(null, null, confOverlay);
+ verifyInitProperty("a", "2", sessionHandle);
+ verifyInitProperty("b", "2", sessionHandle);
+ verifyInitProperty("c", "2", sessionHandle);
+ client.closeSession(sessionHandle);
+
+ sessionHandle = client.openSession("hive", "password", confOverlay);
+ verifyInitProperty("a", "2", sessionHandle);
+ client.closeSession(sessionHandle);
+ }
+
+ private void verifyInitProperty(String key, String value,
+ SessionHandle sessionHandle) throws Exception {
+ OperationHandle operationHandle =
+ client.executeStatement(sessionHandle, "set " + key, null);
+ RowSet rowSet = client.fetchResults(operationHandle);
+ Assert.assertEquals(1, rowSet.numRows());
+ // we know rowSet has only one element
+ Assert.assertEquals(key + "=" + value, rowSet.iterator().next()[0]);
+ }
+}

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedJul 24, '14 at 1:28a
activeJul 24, '14 at 1:28a
posts1
users1
websitehive.apache.org

1 user in discussion

Szehon: 1 post

People

Translate

site design / logo © 2021 Grokbase