Grokbase Groups Hive commits May 2016
FAQ
Repository: hive
Updated Branches:
   refs/heads/branch-1 cb3636f3f -> 5d00e7c5c


HIVE-13551: Make cleardanglingscratchdir work on Windows (Daniel Dai, reviewed by Thejas Nair)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5d00e7c5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5d00e7c5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5d00e7c5

Branch: refs/heads/branch-1
Commit: 5d00e7c5c40b5d470aa7c2c9b5c350709f387c1b
Parents: cb3636f
Author: Daniel Dai <daijy@hortonworks.com>
Authored: Wed May 25 15:00:05 2016 -0700
Committer: Daniel Dai <daijy@hortonworks.com>
Committed: Wed May 25 15:03:18 2016 -0700

----------------------------------------------------------------------
  bin/ext/cleardanglingscratchdir.cmd | 1 -
  bin/hive.cmd | 3 ++
  .../ql/session/TestClearDanglingScratchDir.java | 5 +++
  .../ql/session/ClearDanglingScratchDir.java | 46 +++++++++++++++-----
  4 files changed, 44 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/5d00e7c5/bin/ext/cleardanglingscratchdir.cmd
----------------------------------------------------------------------
diff --git a/bin/ext/cleardanglingscratchdir.cmd b/bin/ext/cleardanglingscratchdir.cmd
index 31104af..1bb0453 100644
--- a/bin/ext/cleardanglingscratchdir.cmd
+++ b/bin/ext/cleardanglingscratchdir.cmd
@@ -16,7 +16,6 @@

  set CLASS=org.apache.hadoop.hive.ql.session.ClearDanglingScratchDir
  set HIVE_OPTS=
-set HADOOP_CLASSPATH=

  pushd %HIVE_LIB%
  for /f %%a IN ('dir /b hive-exec-*.jar') do (

http://git-wip-us.apache.org/repos/asf/hive/blob/5d00e7c5/bin/hive.cmd
----------------------------------------------------------------------
diff --git a/bin/hive.cmd b/bin/hive.cmd
index 9080796..79d6d1b 100644
--- a/bin/hive.cmd
+++ b/bin/hive.cmd
@@ -361,6 +361,9 @@ goto :EOF

   set /a SERVICE_COUNT = %SERVICE_COUNT% + 1
   set VAR%SERVICE_COUNT%=schematool
+
+ set /a SERVICE_COUNT = %SERVICE_COUNT% + 1
+ set VAR%SERVICE_COUNT%=cleardanglingscratchdir
  goto :EOF

  :AddToAuxParam

http://git-wip-us.apache.org/repos/asf/hive/blob/5d00e7c5/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestClearDanglingScratchDir.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestClearDanglingScratchDir.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestClearDanglingScratchDir.java
index 599e6a4..f02c6bc 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestClearDanglingScratchDir.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestClearDanglingScratchDir.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.fs.Path;
  import org.apache.hadoop.fs.permission.FsPermission;
  import org.apache.hadoop.hdfs.MiniDFSCluster;
  import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.WindowsPathUtil;
+import org.apache.hadoop.util.Shell;
  import org.junit.AfterClass;
  import org.junit.Assert;
  import org.junit.BeforeClass;
@@ -47,6 +49,9 @@ public class TestClearDanglingScratchDir {
    static public void oneTimeSetup() throws Exception {
      m_dfs = new MiniDFSCluster.Builder(new Configuration()).numDataNodes(1).format(true).build();
      conf = new HiveConf();
+ if (Shell.WINDOWS) {
+ WindowsPathUtil.convertPathsFromWindowsToHdfs(conf);
+ }
      conf.set(HiveConf.ConfVars.HIVE_SCRATCH_DIR_LOCK.toString(), "true");
      conf.set(HiveConf.ConfVars.METASTORE_AUTO_CREATE_SCHEMA.toString(), "true");
      LoggerFactory.getLogger("SessionState");

http://git-wip-us.apache.org/repos/asf/hive/blob/5d00e7c5/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java b/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java
index 8543768..ee012c2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java
@@ -25,6 +25,7 @@ import org.apache.commons.cli.GnuParser;
  import org.apache.commons.cli.HelpFormatter;
  import org.apache.commons.cli.OptionBuilder;
  import org.apache.commons.cli.Options;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
  import org.apache.hadoop.fs.FileStatus;
  import org.apache.hadoop.fs.FileSystem;
  import org.apache.hadoop.fs.Path;
@@ -66,6 +67,7 @@ public class ClearDanglingScratchDir {

      if (cli.hasOption("r")) {
        dryRun = true;
+ SessionState.getConsole().printInfo("dry-run mode on");
      }

      if (cli.hasOption("v")) {
@@ -99,24 +101,48 @@ public class ClearDanglingScratchDir {
            }
            continue;
          }
+ boolean removable = false;
+ boolean inuse = false;
          try {
            IOUtils.closeStream(fs.append(lockFilePath));
- scratchDirToRemove.add(scratchDir.getPath());
- } catch (RemoteException e) {
+ removable = true;
+ } catch (RemoteException eAppend) {
            // RemoteException with AlreadyBeingCreatedException will be thrown
            // if the file is currently held by a writer
- if(AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){
- // Cannot open the lock file for writing, must be held by a live process
- String message = scratchDir.getPath() + " is being used by live process";
- if (verbose) {
- SessionState.getConsole().printInfo(message);
- } else {
- SessionState.getConsole().logInfo(message);
+ if(AlreadyBeingCreatedException.class.getName().equals(eAppend.getClassName())){
+ inuse = true;
+ } else if (UnsupportedOperationException.class.getName().equals(eAppend.getClassName())) {
+ // Append is not supported in the cluster, try to use create
+ try {
+ IOUtils.closeStream(fs.create(lockFilePath, false));
+ } catch (RemoteException eCreate) {
+ if (AlreadyBeingCreatedException.class.getName().equals(eCreate.getClassName())){
+ // If the file is held by a writer, will throw AlreadyBeingCreatedException
+ inuse = true;
+ } else {
+ SessionState.getConsole().printInfo("Unexpected error:" + eCreate.getMessage());
+ }
+ } catch (FileAlreadyExistsException eCreateNormal) {
+ // Otherwise, throw FileAlreadyExistsException, which means the file owner is
+ // dead
+ removable = true;
              }
            } else {
- throw e;
+ SessionState.getConsole().printInfo("Unexpected error:" + eAppend.getMessage());
            }
          }
+ if (inuse) {
+ // Cannot open the lock file for writing, must be held by a live process
+ String message = scratchDir.getPath() + " is being used by live process";
+ if (verbose) {
+ SessionState.getConsole().printInfo(message);
+ } else {
+ SessionState.getConsole().logInfo(message);
+ }
+ }
+ if (removable) {
+ scratchDirToRemove.add(scratchDir.getPath());
+ }
        }
      }

Search Discussions

  • Daijy at May 25, 2016 at 11:09 pm
    Repository: hive
    Updated Branches:
       refs/heads/branch-1 5d00e7c5c -> fcc83ac25


    HIVE-13513: cleardanglingscratchdir does not work in some version of HDFS (Daniel Dai, reviewed by Thejas Nair)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fcc83ac2
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fcc83ac2
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fcc83ac2

    Branch: refs/heads/branch-1
    Commit: fcc83ac2515187e860d2591f9ef1b72f64b22dff
    Parents: 5d00e7c
    Author: Daniel Dai <daijy@hortonworks.com>
    Authored: Wed May 25 15:24:56 2016 -0700
    Committer: Daniel Dai <daijy@hortonworks.com>
    Committed: Wed May 25 15:24:56 2016 -0700

    ----------------------------------------------------------------------
      .../hadoop/hive/ql/session/ClearDanglingScratchDir.java | 6 ++++++
      .../org/apache/hadoop/hive/ql/session/SessionState.java | 9 ++++++---
      2 files changed, 12 insertions(+), 3 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/fcc83ac2/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java b/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java
    index ee012c2..725f954 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java
    @@ -30,6 +30,8 @@ import org.apache.hadoop.fs.FileStatus;
      import org.apache.hadoop.fs.FileSystem;
      import org.apache.hadoop.fs.Path;
      import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
    +import org.apache.hadoop.hive.common.LogUtils;
    +import org.apache.hadoop.hive.common.LogUtils.LogInitializationException;
      import org.apache.hadoop.hive.conf.HiveConf;
      import org.apache.hadoop.io.IOUtils;
      import org.apache.hadoop.ipc.RemoteException;
    @@ -52,6 +54,10 @@ import org.apache.hadoop.ipc.RemoteException;
      public class ClearDanglingScratchDir {

        public static void main(String[] args) throws Exception {
    + try {
    + LogUtils.initHiveLog4j();
    + } catch (LogInitializationException e) {
    + }
          Options opts = createOptions();
          CommandLine cli = new GnuParser().parse(opts, args);


    http://git-wip-us.apache.org/repos/asf/hive/blob/fcc83ac2/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
    index 07ce8cc..0cf77e9 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
    @@ -111,6 +111,7 @@ public class SessionState {
        private static final String HDFS_SESSION_PATH_KEY = "_hive.hdfs.session.path";
        private static final String TMP_TABLE_SPACE_KEY = "_hive.tmp_table_space";
        static final String LOCK_FILE_NAME = "inuse.lck";
    + static final String INFO_FILE_NAME = "inuse.info";

        private final Map<String, Map<String, Table>> tempTables = new HashMap<String, Map<String, Table>>();
        private final Map<String, Map<String, ColumnStatisticsObj>> tempTableColStats =
    @@ -615,10 +616,12 @@ public class SessionState {
          // 5. hold a lock file in HDFS session dir to indicate the it is in use
          if (conf.getBoolVar(HiveConf.ConfVars.HIVE_SCRATCH_DIR_LOCK)) {
            FileSystem fs = hdfsSessionPath.getFileSystem(conf);
    + FSDataOutputStream hdfsSessionPathInfoFile = fs.create(new Path(hdfsSessionPath, INFO_FILE_NAME),
    + true);
    + hdfsSessionPathInfoFile.writeUTF("process: " + ManagementFactory.getRuntimeMXBean().getName()
    + +"\n");
    + hdfsSessionPathInfoFile.close();
            hdfsSessionPathLockFile = fs.create(new Path(hdfsSessionPath, LOCK_FILE_NAME), true);
    - hdfsSessionPathLockFile.writeUTF("hostname: " + InetAddress.getLocalHost().getHostName() + "\n");
    - hdfsSessionPathLockFile.writeUTF("process: " + ManagementFactory.getRuntimeMXBean().getName() + "\n");
    - hdfsSessionPathLockFile.hsync();
          }
          // 6. Local session path
          localSessionPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.LOCALSCRATCHDIR), sessionId);
  • Daijy at May 26, 2016 at 4:52 pm
    Repository: hive
    Updated Branches:
       refs/heads/branch-2.1 6edd5056d -> ffbd633fa


    HIVE-13551: Make cleardanglingscratchdir work on Windows (Daniel Dai, reviewed by Thejas Nair)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3648d35f
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3648d35f
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3648d35f

    Branch: refs/heads/branch-2.1
    Commit: 3648d35f51c92c44f4ba96495241daa9114aee42
    Parents: 6edd505
    Author: Daniel Dai <daijy@hortonworks.com>
    Authored: Wed May 25 15:00:05 2016 -0700
    Committer: Daniel Dai <daijy@hortonworks.com>
    Committed: Thu May 26 09:50:38 2016 -0700

    ----------------------------------------------------------------------
      bin/ext/cleardanglingscratchdir.cmd | 1 -
      bin/hive.cmd | 3 ++
      .../ql/session/TestClearDanglingScratchDir.java | 5 +++
      .../ql/session/ClearDanglingScratchDir.java | 46 +++++++++++++++-----
      4 files changed, 44 insertions(+), 11 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/3648d35f/bin/ext/cleardanglingscratchdir.cmd
    ----------------------------------------------------------------------
    diff --git a/bin/ext/cleardanglingscratchdir.cmd b/bin/ext/cleardanglingscratchdir.cmd
    index 31104af..1bb0453 100644
    --- a/bin/ext/cleardanglingscratchdir.cmd
    +++ b/bin/ext/cleardanglingscratchdir.cmd
    @@ -16,7 +16,6 @@

      set CLASS=org.apache.hadoop.hive.ql.session.ClearDanglingScratchDir
      set HIVE_OPTS=
    -set HADOOP_CLASSPATH=

      pushd %HIVE_LIB%
      for /f %%a IN ('dir /b hive-exec-*.jar') do (

    http://git-wip-us.apache.org/repos/asf/hive/blob/3648d35f/bin/hive.cmd
    ----------------------------------------------------------------------
    diff --git a/bin/hive.cmd b/bin/hive.cmd
    index 9080796..79d6d1b 100644
    --- a/bin/hive.cmd
    +++ b/bin/hive.cmd
    @@ -361,6 +361,9 @@ goto :EOF

       set /a SERVICE_COUNT = %SERVICE_COUNT% + 1
       set VAR%SERVICE_COUNT%=schematool
    +
    + set /a SERVICE_COUNT = %SERVICE_COUNT% + 1
    + set VAR%SERVICE_COUNT%=cleardanglingscratchdir
      goto :EOF

      :AddToAuxParam

    http://git-wip-us.apache.org/repos/asf/hive/blob/3648d35f/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestClearDanglingScratchDir.java
    ----------------------------------------------------------------------
    diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestClearDanglingScratchDir.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestClearDanglingScratchDir.java
    index 3cb80a7..185dbd5 100644
    --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestClearDanglingScratchDir.java
    +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestClearDanglingScratchDir.java
    @@ -28,6 +28,8 @@ import org.apache.hadoop.fs.Path;
      import org.apache.hadoop.fs.permission.FsPermission;
      import org.apache.hadoop.hdfs.MiniDFSCluster;
      import org.apache.hadoop.hive.conf.HiveConf;
    +import org.apache.hadoop.hive.ql.WindowsPathUtil;
    +import org.apache.hadoop.util.Shell;
      import org.junit.AfterClass;
      import org.junit.Assert;
      import org.junit.BeforeClass;
    @@ -47,6 +49,9 @@ public class TestClearDanglingScratchDir {
        static public void oneTimeSetup() throws Exception {
          m_dfs = new MiniDFSCluster.Builder(new Configuration()).numDataNodes(1).format(true).build();
          conf = new HiveConf();
    + if (Shell.WINDOWS) {
    + WindowsPathUtil.convertPathsFromWindowsToHdfs(conf);
    + }
          conf.set(HiveConf.ConfVars.HIVE_SCRATCH_DIR_LOCK.toString(), "true");
          conf.set(HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL.toString(), "true");
          LoggerFactory.getLogger("SessionState");

    http://git-wip-us.apache.org/repos/asf/hive/blob/3648d35f/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java b/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java
    index 8543768..ee012c2 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java
    @@ -25,6 +25,7 @@ import org.apache.commons.cli.GnuParser;
      import org.apache.commons.cli.HelpFormatter;
      import org.apache.commons.cli.OptionBuilder;
      import org.apache.commons.cli.Options;
    +import org.apache.hadoop.fs.FileAlreadyExistsException;
      import org.apache.hadoop.fs.FileStatus;
      import org.apache.hadoop.fs.FileSystem;
      import org.apache.hadoop.fs.Path;
    @@ -66,6 +67,7 @@ public class ClearDanglingScratchDir {

          if (cli.hasOption("r")) {
            dryRun = true;
    + SessionState.getConsole().printInfo("dry-run mode on");
          }

          if (cli.hasOption("v")) {
    @@ -99,24 +101,48 @@ public class ClearDanglingScratchDir {
                }
                continue;
              }
    + boolean removable = false;
    + boolean inuse = false;
              try {
                IOUtils.closeStream(fs.append(lockFilePath));
    - scratchDirToRemove.add(scratchDir.getPath());
    - } catch (RemoteException e) {
    + removable = true;
    + } catch (RemoteException eAppend) {
                // RemoteException with AlreadyBeingCreatedException will be thrown
                // if the file is currently held by a writer
    - if(AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){
    - // Cannot open the lock file for writing, must be held by a live process
    - String message = scratchDir.getPath() + " is being used by live process";
    - if (verbose) {
    - SessionState.getConsole().printInfo(message);
    - } else {
    - SessionState.getConsole().logInfo(message);
    + if(AlreadyBeingCreatedException.class.getName().equals(eAppend.getClassName())){
    + inuse = true;
    + } else if (UnsupportedOperationException.class.getName().equals(eAppend.getClassName())) {
    + // Append is not supported in the cluster, try to use create
    + try {
    + IOUtils.closeStream(fs.create(lockFilePath, false));
    + } catch (RemoteException eCreate) {
    + if (AlreadyBeingCreatedException.class.getName().equals(eCreate.getClassName())){
    + // If the file is held by a writer, will throw AlreadyBeingCreatedException
    + inuse = true;
    + } else {
    + SessionState.getConsole().printInfo("Unexpected error:" + eCreate.getMessage());
    + }
    + } catch (FileAlreadyExistsException eCreateNormal) {
    + // Otherwise, throw FileAlreadyExistsException, which means the file owner is
    + // dead
    + removable = true;
                  }
                } else {
    - throw e;
    + SessionState.getConsole().printInfo("Unexpected error:" + eAppend.getMessage());
                }
              }
    + if (inuse) {
    + // Cannot open the lock file for writing, must be held by a live process
    + String message = scratchDir.getPath() + " is being used by live process";
    + if (verbose) {
    + SessionState.getConsole().printInfo(message);
    + } else {
    + SessionState.getConsole().logInfo(message);
    + }
    + }
    + if (removable) {
    + scratchDirToRemove.add(scratchDir.getPath());
    + }
            }
          }
  • Daijy at May 26, 2016 at 4:52 pm
    HIVE-13729: FileSystem leaks in FileUtils.checkFileAccessWithImpersonation (Daniel Dai, reviewed by Thejas Nair)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ffbd633f
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ffbd633f
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ffbd633f

    Branch: refs/heads/branch-2.1
    Commit: ffbd633fa448651e5c15c61826427537b31c160c
    Parents: 2209cab
    Author: Daniel Dai <daijy@hortonworks.com>
    Authored: Wed May 25 21:44:54 2016 -0700
    Committer: Daniel Dai <daijy@hortonworks.com>
    Committed: Thu May 26 09:52:00 2016 -0700

    ----------------------------------------------------------------------
      .../apache/hadoop/hive/common/FileUtils.java | 20 ++++++++++++--------
      1 file changed, 12 insertions(+), 8 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/ffbd633f/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
    ----------------------------------------------------------------------
    diff --git a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
    index 5cf4d39..d755798 100644
    --- a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
    +++ b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
    @@ -387,14 +387,18 @@ public final class FileUtils {
          // Otherwise, try user impersonation. Current user must be configured to do user impersonation.
          UserGroupInformation proxyUser = UserGroupInformation.createProxyUser(
              user, UserGroupInformation.getLoginUser());
    - proxyUser.doAs(new PrivilegedExceptionAction<Object>() {
    - @Override
    - public Object run() throws Exception {
    - FileSystem fsAsUser = FileSystem.get(fs.getUri(), fs.getConf());
    - ShimLoader.getHadoopShims().checkFileAccess(fsAsUser, stat, action);
    - return null;
    - }
    - });
    + try {
    + proxyUser.doAs(new PrivilegedExceptionAction<Object>() {
    + @Override
    + public Object run() throws Exception {
    + FileSystem fsAsUser = FileSystem.get(fs.getUri(), fs.getConf());
    + ShimLoader.getHadoopShims().checkFileAccess(fsAsUser, stat, action);
    + return null;
    + }
    + });
    + } finally {
    + FileSystem.closeAllForUGI(proxyUser);
    + }
        }

        /**
  • Daijy at May 26, 2016 at 4:52 pm
    HIVE-12279: Testcase to verify session temporary files are removed after HIVE-11768


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2209cab5
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2209cab5
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2209cab5

    Branch: refs/heads/branch-2.1
    Commit: 2209cab57486f70aebc25312c1f5f04cfce6f009
    Parents: b448993
    Author: Daniel Dai <daijy@hortonworks.com>
    Authored: Wed May 25 15:44:32 2016 -0700
    Committer: Daniel Dai <daijy@hortonworks.com>
    Committed: Thu May 26 09:51:01 2016 -0700

    ----------------------------------------------------------------------
      .../service/cli/session/TestSessionCleanup.java | 77 ++++++++++++++++++++
      1 file changed, 77 insertions(+)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/2209cab5/service/src/test/org/apache/hive/service/cli/session/TestSessionCleanup.java
    ----------------------------------------------------------------------
    diff --git a/service/src/test/org/apache/hive/service/cli/session/TestSessionCleanup.java b/service/src/test/org/apache/hive/service/cli/session/TestSessionCleanup.java
    new file mode 100644
    index 0000000..e38a52a
    --- /dev/null
    +++ b/service/src/test/org/apache/hive/service/cli/session/TestSessionCleanup.java
    @@ -0,0 +1,77 @@
    +/**
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +
    +package org.apache.hive.service.cli.session;
    +
    +import java.io.File;
    +import java.io.FilenameFilter;
    +import java.util.Arrays;
    +import java.util.Collections;
    +import java.util.HashSet;
    +import java.util.Set;
    +
    +import junit.framework.TestCase;
    +
    +import org.apache.hadoop.hive.conf.HiveConf;
    +import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
    +import org.apache.hive.service.cli.SessionHandle;
    +import org.apache.hive.service.cli.thrift.EmbeddedThriftBinaryCLIService;
    +import org.apache.hive.service.cli.thrift.ThriftCLIServiceClient;
    +import org.junit.Assert;
    +import org.junit.Test;
    +
    +public class TestSessionCleanup extends TestCase {
    +
    + @Test
    + // This is to test session temporary files are cleaned up after HIVE-11768
    + public void testTempSessionFileCleanup() throws Exception {
    + EmbeddedThriftBinaryCLIService service = new EmbeddedThriftBinaryCLIService();
    + service.init(null);
    + ThriftCLIServiceClient client = new ThriftCLIServiceClient(service);
    +
    + Set<String> existingPipeoutFiles = new HashSet<String>(Arrays.asList(getPipeoutFiles()));
    + SessionHandle sessionHandle = client.openSession("user1", "foobar",
    + Collections.<String, String>emptyMap());
    + client.executeStatement(sessionHandle, "set a=b", null);
    + File operationLogRootDir = new File(
    + new HiveConf().getVar(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LOG_LOCATION));
    + Assert.assertNotEquals(operationLogRootDir.list().length, 0);
    + client.closeSession(sessionHandle);
    +
    + // Check if session files are removed
    + Assert.assertEquals(operationLogRootDir.list().length, 0);
    +
    + // Check if the pipeout files are removed
    + Set<String> finalPipeoutFiles = new HashSet<String>(Arrays.asList(getPipeoutFiles()));
    + finalPipeoutFiles.removeAll(existingPipeoutFiles);
    + Assert.assertTrue(finalPipeoutFiles.isEmpty());
    + }
    +
    + private String[] getPipeoutFiles() {
    + File localScratchDir = new File(
    + new HiveConf().getVar(HiveConf.ConfVars.LOCALSCRATCHDIR));
    + String[] pipeoutFiles = localScratchDir.list(new FilenameFilter() {
    + @Override
    + public boolean accept(File dir, String name) {
    + if (name.endsWith("pipeout")) return true;
    + return false;
    + }
    + });
    + return pipeoutFiles;
    + }
    +}
  • Daijy at May 26, 2016 at 4:52 pm
    HIVE-13513: cleardanglingscratchdir does not work in some version of HDFS (Daniel Dai, reviewed by Thejas Nair)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b4489936
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b4489936
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b4489936

    Branch: refs/heads/branch-2.1
    Commit: b4489936e84b5f9fdc13bc9d12f152c111e779f9
    Parents: 3648d35
    Author: Daniel Dai <daijy@hortonworks.com>
    Authored: Wed May 25 15:23:57 2016 -0700
    Committer: Daniel Dai <daijy@hortonworks.com>
    Committed: Thu May 26 09:50:44 2016 -0700

    ----------------------------------------------------------------------
      .../hadoop/hive/ql/session/ClearDanglingScratchDir.java | 6 ++++++
      .../org/apache/hadoop/hive/ql/session/SessionState.java | 9 ++++++---
      2 files changed, 12 insertions(+), 3 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/b4489936/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java b/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java
    index ee012c2..725f954 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java
    @@ -30,6 +30,8 @@ import org.apache.hadoop.fs.FileStatus;
      import org.apache.hadoop.fs.FileSystem;
      import org.apache.hadoop.fs.Path;
      import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
    +import org.apache.hadoop.hive.common.LogUtils;
    +import org.apache.hadoop.hive.common.LogUtils.LogInitializationException;
      import org.apache.hadoop.hive.conf.HiveConf;
      import org.apache.hadoop.io.IOUtils;
      import org.apache.hadoop.ipc.RemoteException;
    @@ -52,6 +54,10 @@ import org.apache.hadoop.ipc.RemoteException;
      public class ClearDanglingScratchDir {

        public static void main(String[] args) throws Exception {
    + try {
    + LogUtils.initHiveLog4j();
    + } catch (LogInitializationException e) {
    + }
          Options opts = createOptions();
          CommandLine cli = new GnuParser().parse(opts, args);


    http://git-wip-us.apache.org/repos/asf/hive/blob/b4489936/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
    index 37ef165..ce43f7d 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
    @@ -109,6 +109,7 @@ public class SessionState {
        private static final String HDFS_SESSION_PATH_KEY = "_hive.hdfs.session.path";
        private static final String TMP_TABLE_SPACE_KEY = "_hive.tmp_table_space";
        static final String LOCK_FILE_NAME = "inuse.lck";
    + static final String INFO_FILE_NAME = "inuse.info";

        private final Map<String, Map<String, Table>> tempTables = new HashMap<String, Map<String, Table>>();
        private final Map<String, Map<String, ColumnStatisticsObj>> tempTableColStats =
    @@ -643,10 +644,12 @@ public class SessionState {
          // 5. hold a lock file in HDFS session dir to indicate the it is in use
          if (conf.getBoolVar(HiveConf.ConfVars.HIVE_SCRATCH_DIR_LOCK)) {
            FileSystem fs = hdfsSessionPath.getFileSystem(conf);
    + FSDataOutputStream hdfsSessionPathInfoFile = fs.create(new Path(hdfsSessionPath, INFO_FILE_NAME),
    + true);
    + hdfsSessionPathInfoFile.writeUTF("process: " + ManagementFactory.getRuntimeMXBean().getName()
    + +"\n");
    + hdfsSessionPathInfoFile.close();
            hdfsSessionPathLockFile = fs.create(new Path(hdfsSessionPath, LOCK_FILE_NAME), true);
    - hdfsSessionPathLockFile.writeUTF("hostname: " + InetAddress.getLocalHost().getHostName() + "\n");
    - hdfsSessionPathLockFile.writeUTF("process: " + ManagementFactory.getRuntimeMXBean().getName() + "\n");
    - hdfsSessionPathLockFile.hsync();
          }
          // 6. Local session path
          localSessionPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.LOCALSCRATCHDIR), sessionId);

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedMay 25, '16 at 10:54p
activeMay 26, '16 at 4:52p
posts6
users1
websitehive.apache.org

1 user in discussion

Daijy: 6 posts

People

Translate

site design / logo © 2021 Grokbase