FAQ
Repository: hive
Updated Branches:
   refs/heads/branch-1 983af1ec6 -> da56164de


HIVE-12965: Insert overwrite local directory should perserve the overwritten directory permission (Chaoyu Tang, reviewed by Xuefu Zhang)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/da56164d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/da56164d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/da56164d

Branch: refs/heads/branch-1
Commit: da56164de81ae373199e8275da02fcd402451dbf
Parents: 983af1e
Author: ctang <ctang.ma@gmail.com>
Authored: Fri Feb 12 11:44:01 2016 -0500
Committer: ctang <ctang@cloudera.com>
Committed: Fri Feb 12 12:18:50 2016 -0500

----------------------------------------------------------------------
  .../apache/hadoop/hive/ql/exec/MoveTask.java | 124 ++++++++++---------
  1 file changed, 66 insertions(+), 58 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/da56164d/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index c428812..8ee9f4c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@ -18,6 +18,7 @@

  package org.apache.hadoop.hive.ql.exec;

+import java.io.FileNotFoundException;
  import java.io.IOException;
  import java.io.Serializable;
  import java.util.ArrayList;
@@ -31,7 +32,6 @@ import org.apache.commons.logging.Log;
  import org.apache.commons.logging.LogFactory;
  import org.apache.hadoop.fs.FileStatus;
  import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
  import org.apache.hadoop.fs.Path;
  import org.apache.hadoop.hive.common.FileUtils;
  import org.apache.hadoop.hive.common.HiveStatsUtils;
@@ -86,73 +86,81 @@ public class MoveTask extends Task<MoveWork> implements Serializable {
    }

    private void moveFile(Path sourcePath, Path targetPath, boolean isDfsDir)
- throws Exception {
- FileSystem fs = sourcePath.getFileSystem(conf);
- if (isDfsDir) {
- // Just do a rename on the URIs, they belong to the same FS
- String mesg = "Moving data to: " + targetPath.toString();
+ throws HiveException {
+ try {
+ String mesg = "Moving data to " + (isDfsDir ? "" : "local ") + "directory "
+ + targetPath.toString();
        String mesg_detail = " from " + sourcePath.toString();
        console.printInfo(mesg, mesg_detail);

- // if source exists, rename. Otherwise, create a empty directory
- if (fs.exists(sourcePath)) {
- Path deletePath = null;
- // If it multiple level of folder are there fs.rename is failing so first
- // create the targetpath.getParent() if it not exist
- if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_INSERT_INTO_MULTILEVEL_DIRS)) {
- deletePath = createTargetPath(targetPath, fs);
- }
- if (!Hive.moveFile(conf, sourcePath, targetPath, true, false)) {
- try {
- if (deletePath != null) {
- fs.delete(deletePath, true);
- }
- } catch (IOException e) {
- LOG.info("Unable to delete the path created for facilitating rename"
- + deletePath);
- }
- throw new HiveException("Unable to rename: " + sourcePath
- + " to: " + targetPath);
- }
- } else if (!fs.mkdirs(targetPath)) {
- throw new HiveException("Unable to make directory: " + targetPath);
+ FileSystem fs = sourcePath.getFileSystem(conf);
+ if (isDfsDir) {
+ moveFileInDfs (sourcePath, targetPath, fs);
+ } else {
+ // This is a local file
+ FileSystem dstFs = FileSystem.getLocal(conf);
+ moveFileFromDfsToLocal(sourcePath, targetPath, fs, dstFs);
        }
- } else {
- // This is a local file
- String mesg = "Copying data to local directory " + targetPath.toString();
- String mesg_detail = " from " + sourcePath.toString();
- console.printInfo(mesg, mesg_detail);
-
- // delete the existing dest directory
- LocalFileSystem dstFs = FileSystem.getLocal(conf);
+ } catch (Exception e) {
+ throw new HiveException("Unable to move source " + sourcePath + " to destination "
+ + targetPath, e);
+ }
+ }

- if (dstFs.delete(targetPath, true) || !dstFs.exists(targetPath)) {
- // if source exists, rename. Otherwise, create a empty directory
- if (fs.exists(sourcePath)) {
- try {
- // create the destination if it does not exist
- if (!dstFs.exists(targetPath)) {
- FileUtils.mkdir(dstFs, targetPath, false, conf);
- }
- } catch (IOException e) {
- throw new HiveException("Unable to create target directory for copy" + targetPath, e);
+ private void moveFileInDfs (Path sourcePath, Path targetPath, FileSystem fs)
+ throws HiveException, IOException {
+ // if source exists, rename. Otherwise, create a empty directory
+ if (fs.exists(sourcePath)) {
+ Path deletePath = null;
+ // If it multiple level of folder are there fs.rename is failing so first
+ // create the targetpath.getParent() if it not exist
+ if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_INSERT_INTO_MULTILEVEL_DIRS)) {
+ deletePath = createTargetPath(targetPath, fs);
+ }
+ if (!Hive.moveFile(conf, sourcePath, targetPath, true, false)) {
+ try {
+ if (deletePath != null) {
+ fs.delete(deletePath, true);
            }
+ } catch (IOException e) {
+ LOG.info("Unable to delete the path created for facilitating rename"
+ + deletePath);
+ }
+ throw new HiveException("Unable to rename: " + sourcePath
+ + " to: " + targetPath);
+ }
+ } else if (!fs.mkdirs(targetPath)) {
+ throw new HiveException("Unable to make directory: " + targetPath);
+ }
+ }

- FileSystem srcFs = sourcePath.getFileSystem(conf);
- FileStatus[] srcs = srcFs.listStatus(sourcePath, FileUtils.HIDDEN_FILES_PATH_FILTER);
- for (FileStatus status : srcs) {
- fs.copyToLocalFile(status.getPath(), targetPath);
- }
- } else {
- if (!dstFs.mkdirs(targetPath)) {
- throw new HiveException("Unable to make local directory: "
- + targetPath);
+ private void moveFileFromDfsToLocal(Path sourcePath, Path targetPath, FileSystem fs,
+ FileSystem dstFs) throws HiveException, IOException {
+ // RawLocalFileSystem seems not able to get the right permissions for a local file, it
+ // always returns hdfs default permission (00666). So we can not overwrite a directory
+ // by deleting and recreating the directory and restoring its permissions. We should
+ // delete all its files and subdirectories instead.
+ if (dstFs.exists(targetPath)) {
+ if (dstFs.isDirectory(targetPath)) {
+ FileStatus[] destFiles = dstFs.listStatus(targetPath);
+ for (FileStatus destFile : destFiles) {
+ if (!dstFs.delete(destFile.getPath(), true)) {
+ throw new IOException("Unable to clean the destination directory: " + targetPath);
            }
          }
        } else {
- throw new AccessControlException(
- "Unable to delete the existing destination directory: "
- + targetPath);
+ throw new HiveException("Target " + targetPath + " is not a local directory.");
+ }
+ } else {
+ if (!FileUtils.mkdir(dstFs, targetPath, false, conf)) {
+ throw new HiveException("Failed to create local target directory " + targetPath);
+ }
+ }
+
+ if (fs.exists(sourcePath)) {
+ FileStatus[] srcs = fs.listStatus(sourcePath, FileUtils.HIDDEN_FILES_PATH_FILTER);
+ for (FileStatus status : srcs) {
+ fs.copyToLocalFile(status.getPath(), targetPath);
        }
      }
    }

Search Discussions

Discussion Posts

Previous

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 2 of 2 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedFeb 12, '16 at 4:44p
activeFeb 12, '16 at 5:19p
posts2
users1
websitehive.apache.org

1 user in discussion

Ctang: 2 posts

People

Translate

site design / logo © 2021 Grokbase