FAQ
Author: hashutosh
Date: Thu Apr 12 18:57:52 2012
New Revision: 1325446

URL: http://svn.apache.org/viewvc?rev=1325446&view=rev
Log:
HIVE-2767 [jira] Optionally use framed transport with metastore
(Travis Crawford via Ashutosh Chauhan)

Summary:
Add support for optionally using the thrift framed transport, enabling
integration with environments where that is necessary.

Users may want/need to use thrift's framed transport when communicating with the
Hive MetaStore. This patch adds a new property
hive.metastore.thrift.framed.transport.enabled that enables the framed transport
(defaults to off, aka no change from before the patch). This property must be
set for both clients and the HMS server.

It wasn't immediately clear how to use the framed transport with SASL, so as
written an exception is thrown if you try starting the server with both options.
If SASL and the framed transport will indeed work together I can update the
patch (although I don't have a secured environment to test in).

Test Plan:
Tested locally that client and server can connect, both with and
without the flag. Tests pass.

Reviewers: JIRA, ashutoshc

Reviewed By: ashutoshc

Differential Revision: https://reviews.facebook.net/D2661

Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/conf/hive-default.xml.template
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java

Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1325446&r1=1325445&r2=1325446&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Thu Apr 12 18:57:52 2012
@@ -268,6 +268,7 @@ public class HiveConf extends Configurat
METASTORE_KERBEROS_PRINCIPAL("hive.metastore.kerberos.principal",
"hive-metastore/_HOST@EXAMPLE.COM"),
METASTORE_USE_THRIFT_SASL("hive.metastore.sasl.enabled", false),
+ METASTORE_USE_THRIFT_FRAMED_TRANSPORT("hive.metastore.thrift.framed.transport.enabled", false),
METASTORE_CLUSTER_DELEGATION_TOKEN_STORE_CLS(
"hive.cluster.delegation.token.store.class",
"org.apache.hadoop.hive.thrift.MemoryTokenStore"),

Modified: hive/trunk/conf/hive-default.xml.template
URL: http://svn.apache.org/viewvc/hive/trunk/conf/hive-default.xml.template?rev=1325446&r1=1325445&r2=1325446&view=diff
==============================================================================
--- hive/trunk/conf/hive-default.xml.template (original)
+++ hive/trunk/conf/hive-default.xml.template Thu Apr 12 18:57:52 2012
@@ -748,6 +748,12 @@
</property>

<property>
+ <name>hive.metastore.thrift.framed.transport.enabled</name>
+ <value>false</value>
+ <description>If true, the metastore thrift interface will use TFramedTransport. When false (default) a standard TTransport is used.</description>
+</property>
+
+<property>
<name>hive.metastore.kerberos.keytab.file</name>
<value></value>
<description>The path to the Kerberos Keytab file containing the metastore thrift server's service principal.</description>

Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1325446&r1=1325445&r2=1325446&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Thu Apr 12 18:57:52 2012
@@ -122,8 +122,10 @@ import org.apache.thrift.TProcessor;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.server.TServer;
import org.apache.thrift.server.TThreadPoolServer;
+import org.apache.thrift.transport.TFramedTransport;
import org.apache.thrift.transport.TServerSocket;
import org.apache.thrift.transport.TServerTransport;
+import org.apache.thrift.transport.TTransport;
import org.apache.thrift.transport.TTransportFactory;

import com.facebook.fb303.FacebookBase;
@@ -144,6 +146,23 @@ public class HiveMetaStore extends Thrif
private static HadoopThriftAuthBridge.Server saslServer;
private static boolean useSasl;

+ private static final class ChainedTTransportFactory extends TTransportFactory {
+ private final TTransportFactory parentTransFactory;
+ private final TTransportFactory childTransFactory;
+
+ private ChainedTTransportFactory(
+ TTransportFactory parentTransFactory,
+ TTransportFactory childTransFactory) {
+ this.parentTransFactory = parentTransFactory;
+ this.childTransFactory = childTransFactory;
+ }
+
+ @Override
+ public TTransport getTransport(TTransport trans) {
+ return childTransFactory.getTransport(parentTransFactory.getTransport(trans));
+ }
+ }
+
public static class HMSHandler extends FacebookBase implements
ThriftHiveMetastore.Iface {
public static final Log LOG = HiveMetaStore.LOG;
@@ -2989,6 +3008,7 @@ public class HiveMetaStore extends Thrif
int minWorkerThreads = conf.getIntVar(HiveConf.ConfVars.METASTORESERVERMINTHREADS);
int maxWorkerThreads = conf.getIntVar(HiveConf.ConfVars.METASTORESERVERMAXTHREADS);
boolean tcpKeepAlive = conf.getBoolVar(HiveConf.ConfVars.METASTORE_TCP_KEEP_ALIVE);
+ boolean useFramedTransport = conf.getBoolVar(ConfVars.METASTORE_USE_THRIFT_FRAMED_TRANSPORT);
useSasl = conf.getBoolVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL);

TServerTransport serverTransport = tcpKeepAlive ?
@@ -2998,6 +3018,9 @@ public class HiveMetaStore extends Thrif
TTransportFactory transFactory;
if (useSasl) {
// we are in secure mode.
+ if (useFramedTransport) {
+ throw new HiveMetaException("Framed transport is not supported with SASL enabled.");
+ }
saslServer = bridge.createServer(
conf.getVar(HiveConf.ConfVars.METASTORE_KERBEROS_KEYTAB_FILE),
conf.getVar(HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL));
@@ -3010,12 +3033,18 @@ public class HiveMetaStore extends Thrif
} else {
// we are in unsecure mode.
HMSHandler handler = new HMSHandler("new db based metaserver", conf);
+
if (conf.getBoolVar(ConfVars.METASTORE_EXECUTE_SET_UGI)) {
- transFactory = new TUGIContainingTransport.Factory();
+ transFactory = useFramedTransport ?
+ new ChainedTTransportFactory(new TFramedTransport.Factory(),
+ new TUGIContainingTransport.Factory())
+ : new TUGIContainingTransport.Factory();
+
processor = new TUGIBasedProcessor<HMSHandler>(handler);
LOG.info("Starting DB backed MetaStore Server with SetUGI enabled");
} else {
- transFactory = new TTransportFactory();
+ transFactory = useFramedTransport ?
+ new TFramedTransport.Factory() : new TTransportFactory();
processor = new TSetIpAddressProcessor<HMSHandler>(handler);
LOG.info("Starting DB backed MetaStore Server");
}

Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java?rev=1325446&r1=1325445&r2=1325446&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java Thu Apr 12 18:57:52 2012
@@ -70,6 +70,7 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.util.StringUtils;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TBinaryProtocol;
+import org.apache.thrift.transport.TFramedTransport;
import org.apache.thrift.transport.TSocket;
import org.apache.thrift.transport.TTransport;
import org.apache.thrift.transport.TTransportException;
@@ -215,9 +216,11 @@ public class HiveMetaStoreClient impleme
TTransportException tte = null;
HadoopShims shim = ShimLoader.getHadoopShims();
boolean useSasl = conf.getBoolVar(ConfVars.METASTORE_USE_THRIFT_SASL);
+ boolean useFramedTransport = conf.getBoolVar(ConfVars.METASTORE_USE_THRIFT_FRAMED_TRANSPORT);
transport = new TSocket(store.getHost(), store.getPort());
- ((TSocket)transport).setTimeout(1000 * conf.getIntVar(ConfVars.
- METASTORE_CLIENT_SOCKET_TIMEOUT));
+ int clientSocketTimeout = conf.getIntVar(ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT);
+
+ transport = new TSocket(store.getHost(), store.getPort(), 1000 * clientSocketTimeout);

if (useSasl) {
// Wrap thrift connection with SASL for secure connection.
@@ -248,6 +251,8 @@ public class HiveMetaStoreClient impleme
LOG.error("Couldn't create client transport", ioe);
throw new MetaException(ioe.toString());
}
+ } else if (useFramedTransport) {
+ transport = new TFramedTransport(transport);
}

client = new ThriftHiveMetastore.Client(new TBinaryProtocol(transport));

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedApr 12, '12 at 6:58p
activeApr 12, '12 at 6:58p
posts1
users1
websitehive.apache.org

1 user in discussion

Hashutosh: 1 post

People

Translate

site design / logo © 2021 Grokbase