FAQ
Author: gunther
Date: Tue Apr 7 18:04:40 2015
New Revision: 1671903

URL: http://svn.apache.org/r1671903
Log:
HIVE-10223: Consolidate several redundant FileSystem API calls. (Chris Nauroth via Gunther Hagleitner)

Modified:
     hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
     hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java

Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java?rev=1671903&r1=1671902&r2=1671903&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java Tue Apr 7 18:04:40 2015
@@ -335,16 +335,16 @@ public final class FileUtils {
     * @param fs
     * file system
     * @param path
- * @return the argument path if it exists or a parent path exists. Returns
- * NULL root is only parent that exists
+ * @return FileStatus for argument path if it exists or the first ancestor in the path that exists
     * @throws IOException
     */
- public static Path getPathOrParentThatExists(FileSystem fs, Path path) throws IOException {
- if (!fs.exists(path)) {
- Path parentPath = path.getParent();
- return getPathOrParentThatExists(fs, parentPath);
+ public static FileStatus getPathOrParentThatExists(FileSystem fs, Path path) throws IOException {
+ FileStatus stat = FileUtils.getFileStatusOrNull(fs, path);
+ if (stat != null) {
+ return stat;
      }
- return path;
+ Path parentPath = path.getParent();
+ return getPathOrParentThatExists(fs, parentPath);
    }

    /**
@@ -743,4 +743,20 @@ public final class FileUtils {

    }

+ /**
+ * Attempts to get file status. This method differs from the FileSystem API in that it returns
+ * null instead of throwing FileNotFoundException if the path does not exist.
+ *
+ * @param fs file system to check
+ * @param path file system path to check
+ * @return FileStatus for path or null if path does not exist
+ * @throws IOException if there is an I/O error
+ */
+ public static FileStatus getFileStatusOrNull(FileSystem fs, Path path) throws IOException {
+ try {
+ return fs.getFileStatus(path);
+ } catch (FileNotFoundException e) {
+ return null;
+ }
+ }
  }

Modified: hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java?rev=1671903&r1=1671902&r2=1671903&view=diff
==============================================================================
--- hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java (original)
+++ hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java Tue Apr 7 18:04:40 2015
@@ -510,8 +510,9 @@ class FileOutputCommitterContainer exten
      }

      final Path finalOutputPath = getFinalPath(fs, file, srcDir, destDir, immutable);
+ FileStatus fileStatus = fs.getFileStatus(file);

- if (fs.isFile(file)) {
+ if (fileStatus.isFile()) {
        if (dryRun){
          if (immutable){
            // Dryrun checks are meaningless for mutable table - we should always succeed
@@ -541,7 +542,7 @@ class FileOutputCommitterContainer exten
            }
          }
        }
- } else if(fs.getFileStatus(file).isDir()) {
+ } else if (fileStatus.isDirectory()) {

        FileStatus[] children = fs.listStatus(file);
        FileStatus firstChild = null;

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java?rev=1671903&r1=1671902&r2=1671903&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java Tue Apr 7 18:04:40 2015
@@ -45,6 +45,7 @@ import org.apache.hadoop.conf.Configurat
  import org.apache.hadoop.fs.FileStatus;
  import org.apache.hadoop.fs.FileSystem;
  import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.FileUtils;
  import org.apache.hadoop.hive.conf.HiveConf;
  import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
  import org.apache.hadoop.hive.ql.Context;
@@ -926,8 +927,9 @@ public class DagUtils {
      throws IOException {
      FileSystem destFS = dest.getFileSystem(conf);
      FileSystem sourceFS = src.getFileSystem(conf);
- if (destFS.exists(dest)) {
- return (sourceFS.getFileStatus(src).getLen() == destFS.getFileStatus(dest).getLen());
+ FileStatus destStatus = FileUtils.getFileStatusOrNull(destFS, dest);
+ if (destStatus != null) {
+ return (sourceFS.getFileStatus(src).getLen() == destStatus.getLen());
      }
      return false;
    }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java?rev=1671903&r1=1671902&r2=1671903&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java Tue Apr 7 18:04:40 2015
@@ -460,7 +460,7 @@ public class OrcRawRecordMerger implemen
          Path deltaFile = AcidUtils.createBucketFile(delta, bucket);
          FileSystem fs = deltaFile.getFileSystem(conf);
          long length = getLastFlushLength(fs, deltaFile);
- if (fs.exists(deltaFile) && length != -1) {
+ if (length != -1 && fs.exists(deltaFile)) {
            Reader deltaReader = OrcFile.createReader(deltaFile,
                OrcFile.readerOptions(conf).maxLength(length));
            ReaderPair deltaPair = new ReaderPair(key, deltaReader, bucket, minKey,

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1671903&r1=1671902&r2=1671903&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Tue Apr 7 18:04:40 2015
@@ -2416,7 +2416,7 @@ private void constructOneLBLocationMap(F

      List<List<Path[]>> result = new ArrayList<List<Path[]>>();
      try {
- FileStatus destStatus = !replace && fs.exists(destf) ? fs.getFileStatus(destf) : null;
+ FileStatus destStatus = !replace ? FileUtils.getFileStatusOrNull(fs, destf) : null;
        if (destStatus != null && !destStatus.isDir()) {
          throw new HiveException("checkPaths: destination " + destf
              + " should be a directory");

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java?rev=1671903&r1=1671902&r2=1671903&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java Tue Apr 7 18:04:40 2015
@@ -361,19 +361,22 @@ public class StorageBasedAuthorizationPr

      final FileSystem fs = path.getFileSystem(conf);

- if (fs.exists(path)) {
- checkPermissions(fs, path, actions, authenticator.getUserName());
+ FileStatus pathStatus = FileUtils.getFileStatusOrNull(fs, path);
+ if (pathStatus != null) {
+ checkPermissions(fs, pathStatus, actions, authenticator.getUserName());
      } else if (path.getParent() != null) {
        // find the ancestor which exists to check its permissions
        Path par = path.getParent();
+ FileStatus parStatus = null;
        while (par != null) {
- if (fs.exists(par)) {
+ parStatus = FileUtils.getFileStatusOrNull(fs, par);
+ if (parStatus != null) {
            break;
          }
          par = par.getParent();
        }

- checkPermissions(fs, par, actions, authenticator.getUserName());
+ checkPermissions(fs, parStatus, actions, authenticator.getUserName());
      }
    }

@@ -382,18 +385,20 @@ public class StorageBasedAuthorizationPr
     * does not exists, it returns.
     */
    @SuppressWarnings("deprecation")
- protected static void checkPermissions(final FileSystem fs, final Path path,
+ protected static void checkPermissions(final FileSystem fs, final FileStatus stat,
        final EnumSet<FsAction> actions, String user) throws IOException,
        AccessControlException, HiveException {

- try {
- FileStatus stat = fs.getFileStatus(path);
- for (FsAction action : actions) {
- FileUtils.checkFileAccessWithImpersonation(fs, stat, action, user);
- }
- } catch (FileNotFoundException fnfe) {
+ if (stat == null) {
        // File named by path doesn't exist; nothing to validate.
        return;
+ }
+ FsAction checkActions = FsAction.NONE;
+ for (FsAction action : actions) {
+ checkActions = checkActions.or(action);
+ }
+ try {
+ FileUtils.checkFileAccessWithImpersonation(fs, stat, checkActions, user);
      } catch (org.apache.hadoop.fs.permission.AccessControlException ace) {
        // Older hadoop version will throw this @deprecated Exception.
        throw accessControlException(ace);

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java?rev=1671903&r1=1671902&r2=1671903&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java Tue Apr 7 18:04:40 2015
@@ -385,8 +385,7 @@ public class SQLAuthorizationUtils {
      FileSystem fs;
      try {
        fs = FileSystem.get(filePath.toUri(), conf);
- Path path = FileUtils.getPathOrParentThatExists(fs, filePath);
- FileStatus fileStatus = fs.getFileStatus(path);
+ FileStatus fileStatus = FileUtils.getPathOrParentThatExists(fs, filePath);
        if (FileUtils.isOwnerOfFileHierarchy(fs, fileStatus, userName)) {
          availPrivs.addPrivilege(SQLPrivTypeGrant.OWNER_PRIV);
        }

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedApr 7, '15 at 6:04p
activeApr 7, '15 at 6:04p
posts1
users1
websitehive.apache.org

1 user in discussion

Gunther: 1 post

People

Translate

site design / logo © 2022 Grokbase