Author: gunther
Date: Tue Apr 7 18:04:40 2015
New Revision: 1671903
URL: http://svn.apache.org/r1671903
Log:
HIVE-10223: Consolidate several redundant FileSystem API calls. (Chris Nauroth via Gunther Hagleitner)
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java?rev=1671903&r1=1671902&r2=1671903&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java Tue Apr 7 18:04:40 2015
@@ -335,16 +335,16 @@ public final class FileUtils {
* @param fs
* file system
* @param path
- * @return the argument path if it exists or a parent path exists. Returns
- * NULL root is only parent that exists
+ * @return FileStatus for argument path if it exists or the first ancestor in the path that exists
* @throws IOException
*/
- public static Path getPathOrParentThatExists(FileSystem fs, Path path) throws IOException {
- if (!fs.exists(path)) {
- Path parentPath = path.getParent();
- return getPathOrParentThatExists(fs, parentPath);
+ public static FileStatus getPathOrParentThatExists(FileSystem fs, Path path) throws IOException {
+ FileStatus stat = FileUtils.getFileStatusOrNull(fs, path);
+ if (stat != null) {
+ return stat;
}
- return path;
+ Path parentPath = path.getParent();
+ return getPathOrParentThatExists(fs, parentPath);
}
/**
@@ -743,4 +743,20 @@ public final class FileUtils {
}
+ /**
+ * Attempts to get file status. This method differs from the FileSystem API in that it returns
+ * null instead of throwing FileNotFoundException if the path does not exist.
+ *
+ * @param fs file system to check
+ * @param path file system path to check
+ * @return FileStatus for path or null if path does not exist
+ * @throws IOException if there is an I/O error
+ */
+ public static FileStatus getFileStatusOrNull(FileSystem fs, Path path) throws IOException {
+ try {
+ return fs.getFileStatus(path);
+ } catch (FileNotFoundException e) {
+ return null;
+ }
+ }
}
Modified: hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java?rev=1671903&r1=1671902&r2=1671903&view=diff
==============================================================================
--- hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java (original)
+++ hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java Tue Apr 7 18:04:40 2015
@@ -510,8 +510,9 @@ class FileOutputCommitterContainer exten
}
final Path finalOutputPath = getFinalPath(fs, file, srcDir, destDir, immutable);
+ FileStatus fileStatus = fs.getFileStatus(file);
- if (fs.isFile(file)) {
+ if (fileStatus.isFile()) {
if (dryRun){
if (immutable){
// Dryrun checks are meaningless for mutable table - we should always succeed
@@ -541,7 +542,7 @@ class FileOutputCommitterContainer exten
}
}
}
- } else if(fs.getFileStatus(file).isDir()) {
+ } else if (fileStatus.isDirectory()) {
FileStatus[] children = fs.listStatus(file);
FileStatus firstChild = null;
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java?rev=1671903&r1=1671902&r2=1671903&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java Tue Apr 7 18:04:40 2015
@@ -45,6 +45,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.Context;
@@ -926,8 +927,9 @@ public class DagUtils {
throws IOException {
FileSystem destFS = dest.getFileSystem(conf);
FileSystem sourceFS = src.getFileSystem(conf);
- if (destFS.exists(dest)) {
- return (sourceFS.getFileStatus(src).getLen() == destFS.getFileStatus(dest).getLen());
+ FileStatus destStatus = FileUtils.getFileStatusOrNull(destFS, dest);
+ if (destStatus != null) {
+ return (sourceFS.getFileStatus(src).getLen() == destStatus.getLen());
}
return false;
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java?rev=1671903&r1=1671902&r2=1671903&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java Tue Apr 7 18:04:40 2015
@@ -460,7 +460,7 @@ public class OrcRawRecordMerger implemen
Path deltaFile = AcidUtils.createBucketFile(delta, bucket);
FileSystem fs = deltaFile.getFileSystem(conf);
long length = getLastFlushLength(fs, deltaFile);
- if (fs.exists(deltaFile) && length != -1) {
+ if (length != -1 && fs.exists(deltaFile)) {
Reader deltaReader = OrcFile.createReader(deltaFile,
OrcFile.readerOptions(conf).maxLength(length));
ReaderPair deltaPair = new ReaderPair(key, deltaReader, bucket, minKey,
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1671903&r1=1671902&r2=1671903&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Tue Apr 7 18:04:40 2015
@@ -2416,7 +2416,7 @@ private void constructOneLBLocationMap(F
List<List<Path[]>> result = new ArrayList<List<Path[]>>();
try {
- FileStatus destStatus = !replace && fs.exists(destf) ? fs.getFileStatus(destf) : null;
+ FileStatus destStatus = !replace ? FileUtils.getFileStatusOrNull(fs, destf) : null;
if (destStatus != null && !destStatus.isDir()) {
throw new HiveException("checkPaths: destination " + destf
+ " should be a directory");
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java?rev=1671903&r1=1671902&r2=1671903&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java Tue Apr 7 18:04:40 2015
@@ -361,19 +361,22 @@ public class StorageBasedAuthorizationPr
final FileSystem fs = path.getFileSystem(conf);
- if (fs.exists(path)) {
- checkPermissions(fs, path, actions, authenticator.getUserName());
+ FileStatus pathStatus = FileUtils.getFileStatusOrNull(fs, path);
+ if (pathStatus != null) {
+ checkPermissions(fs, pathStatus, actions, authenticator.getUserName());
} else if (path.getParent() != null) {
// find the ancestor which exists to check its permissions
Path par = path.getParent();
+ FileStatus parStatus = null;
while (par != null) {
- if (fs.exists(par)) {
+ parStatus = FileUtils.getFileStatusOrNull(fs, par);
+ if (parStatus != null) {
break;
}
par = par.getParent();
}
- checkPermissions(fs, par, actions, authenticator.getUserName());
+ checkPermissions(fs, parStatus, actions, authenticator.getUserName());
}
}
@@ -382,18 +385,20 @@ public class StorageBasedAuthorizationPr
* does not exists, it returns.
*/
@SuppressWarnings("deprecation")
- protected static void checkPermissions(final FileSystem fs, final Path path,
+ protected static void checkPermissions(final FileSystem fs, final FileStatus stat,
final EnumSet<FsAction> actions, String user) throws IOException,
AccessControlException, HiveException {
- try {
- FileStatus stat = fs.getFileStatus(path);
- for (FsAction action : actions) {
- FileUtils.checkFileAccessWithImpersonation(fs, stat, action, user);
- }
- } catch (FileNotFoundException fnfe) {
+ if (stat == null) {
// File named by path doesn't exist; nothing to validate.
return;
+ }
+ FsAction checkActions = FsAction.NONE;
+ for (FsAction action : actions) {
+ checkActions = checkActions.or(action);
+ }
+ try {
+ FileUtils.checkFileAccessWithImpersonation(fs, stat, checkActions, user);
} catch (org.apache.hadoop.fs.permission.AccessControlException ace) {
// Older hadoop version will throw this @deprecated Exception.
throw accessControlException(ace);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java?rev=1671903&r1=1671902&r2=1671903&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java Tue Apr 7 18:04:40 2015
@@ -385,8 +385,7 @@ public class SQLAuthorizationUtils {
FileSystem fs;
try {
fs = FileSystem.get(filePath.toUri(), conf);
- Path path = FileUtils.getPathOrParentThatExists(fs, filePath);
- FileStatus fileStatus = fs.getFileStatus(path);
+ FileStatus fileStatus = FileUtils.getPathOrParentThatExists(fs, filePath);
if (FileUtils.isOwnerOfFileHierarchy(fs, fileStatus, userName)) {
availPrivs.addPrivilege(SQLPrivTypeGrant.OWNER_PRIV);
}
