Grokbase Groups Hive commits May 2015
FAQ
Repository: hive
Updated Branches:
   refs/heads/master 61763335b -> 42f88ca9a


HIVE-10453 : HS2 leaking open file descriptors when using UDFs (Yongzhi Chen via Szehon)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/42f88ca9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/42f88ca9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/42f88ca9

Branch: refs/heads/master
Commit: 42f88ca9a105956a2085b75dae681d62dd784cef
Parents: 6176333
Author: Szehon Ho <szehon@cloudera.com>
Authored: Thu May 7 11:20:53 2015 -0700
Committer: Szehon Ho <szehon@cloudera.com>
Committed: Thu May 7 11:20:53 2015 -0700

----------------------------------------------------------------------
  .../apache/hadoop/hive/ql/exec/Registry.java | 29 ++++++++++++++++++--
  .../apache/hadoop/hive/ql/exec/Utilities.java | 5 ++++
  .../hadoop/hive/ql/session/SessionState.java | 4 +--
  3 files changed, 34 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/42f88ca9/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
index 2ba91d0..a5d59ae 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
@@ -20,8 +20,10 @@ package org.apache.hadoop.hive.ql.exec;

  import com.google.common.base.Splitter;
  import com.google.common.collect.Sets;
+
  import org.apache.commons.logging.Log;
  import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.common.JavaUtils;
  import org.apache.hadoop.hive.ql.exec.FunctionInfo.FunctionResource;
  import org.apache.hadoop.hive.ql.ErrorMsg;
  import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -43,9 +45,12 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
  import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
  import org.apache.hadoop.util.ReflectionUtils;

+import java.io.IOException;
+import java.net.URLClassLoader;
  import java.util.Collections;
  import java.util.HashSet;
  import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
  import java.util.List;
  import java.util.Map;
  import java.util.Set;
@@ -66,6 +71,7 @@ public class Registry {
     */
    private final Map<String, FunctionInfo> mFunctions = new LinkedHashMap<String, FunctionInfo>();
    private final Set<Class<?>> builtIns = Collections.synchronizedSet(new HashSet<Class<?>>());
+ private final Set<ClassLoader> mSessionUDFLoaders = new LinkedHashSet<ClassLoader>();

    private final boolean isNative;

@@ -443,7 +449,6 @@ public class Registry {
    // should be called after session registry is checked
    private FunctionInfo registerToSessionRegistry(String qualifiedName, FunctionInfo function) {
      FunctionInfo ret = null;
-
      ClassLoader prev = Utilities.getSessionSpecifiedClassLoader();
      try {
        // Found UDF in metastore - now add it to the function registry
@@ -455,7 +460,6 @@ public class Registry {
          LOG.error("Unable to load resources for " + qualifiedName + ":" + e, e);
          return null;
        }
-
        ClassLoader loader = Utilities.getSessionSpecifiedClassLoader();
        Class<?> udfClass = Class.forName(function.getClassName(), true, loader);

@@ -463,6 +467,9 @@ public class Registry {
        if (ret == null) {
          LOG.error(function.getClassName() + " is not a valid UDF class and was not registered.");
        }
+ if (SessionState.get().isHiveServerQuery()) {
+ SessionState.getRegistryForWrite().addToUDFLoaders(loader);
+ }
      } catch (ClassNotFoundException e) {
        // Lookup of UDf class failed
        LOG.error("Unable to load UDF class: " + e);
@@ -489,6 +496,24 @@ public class Registry {
      builtIns.clear();
    }

+ public synchronized void closeCUDFLoaders() {
+ try {
+ for(ClassLoader loader: mSessionUDFLoaders) {
+ JavaUtils.closeClassLoader(loader);
+ }
+ } catch (IOException ie) {
+ LOG.error("Error in close loader: " + ie);
+ }
+ mSessionUDFLoaders.clear();
+ }
+
+ public synchronized void addToUDFLoaders(ClassLoader loader) {
+ mSessionUDFLoaders.add(loader);
+ }
+ public synchronized void removeFromUDFLoaders(ClassLoader loader) {
+ mSessionUDFLoaders.remove(loader);
+ }
+
    /**
     * Setup blocked flag for all builtin UDFs as per udf whitelist and blacklist
     * @param whiteListStr

http://git-wip-us.apache.org/repos/asf/hive/blob/42f88ca9/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index ad5c8f8..7b48b8b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -2250,6 +2250,11 @@ public final class Utilities {
        }
      }
      JavaUtils.closeClassLoader(loader);
+//this loader is closed, remove it from cached registry loaders to avoid remove it again.
+ Registry reg = SessionState.getRegistry();
+ if(reg != null) {
+ reg.removeFromUDFLoaders(loader);
+ }

      loader = new URLClassLoader(newPath.toArray(new URL[0]));
      curThread.setContextClassLoader(loader);

http://git-wip-us.apache.org/repos/asf/hive/blob/42f88ca9/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 8db78e5..37b6d6f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -1458,7 +1458,7 @@ public class SessionState {
    }

    public void close() throws IOException {
- registry.clear();;
+ registry.clear();
      if (txnMgr != null) txnMgr.closeTxnManager();
      JavaUtils.closeClassLoadersTo(conf.getClassLoader(), parentLoader);
      File resourceDir =
@@ -1493,7 +1493,7 @@ public class SessionState {
          sparkSession = null;
        }
      }
-
+ registry.closeCUDFLoaders();
      dropSessionPaths(conf);
    }

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 1 of 1 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedMay 7, '15 at 6:21p
activeMay 7, '15 at 6:21p
posts1
users1
websitehive.apache.org

1 user in discussion

Szehon: 1 post

People

Translate

site design / logo © 2021 Grokbase