FAQ
Author: ecapriolo
Date: Fri Jun 22 22:11:23 2012
New Revision: 1353044

URL: http://svn.apache.org/viewvc?rev=1353044&view=rev
Log:
HIVE-3128 Use commons-compress instead of forking tar process (Kanna Karanam via egc)

Modified:
hive/trunk/common/ivy.xml
hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
hive/trunk/ivy/libraries.properties

Modified: hive/trunk/common/ivy.xml
URL: http://svn.apache.org/viewvc/hive/trunk/common/ivy.xml?rev=1353044&r1=1353043&r2=1353044&view=diff
==============================================================================
--- hive/trunk/common/ivy.xml (original)
+++ hive/trunk/common/ivy.xml Fri Jun 22 22:11:23 2012
@@ -84,6 +84,7 @@
<dependency org="org.apache.hive" name="hive-shims" rev="${version}"
conf="compile->default" transitive="false" />
<dependency org="commons-cli" name="commons-cli" rev="${commons-cli.version}"/>
+ <dependency org="org.apache.commons" name="commons-compress" rev="${commons-compress.version}"/>
<dependency org="commons-lang" name="commons-lang" rev="${commons-lang.version}"/>
<dependency org="commons-logging" name="commons-logging" rev="${commons-logging.version}"
transitive="false"/>

Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java?rev=1353044&r1=1353043&r2=1353044&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java Fri Jun 22 22:11:23 2012
@@ -18,16 +18,23 @@

package org.apache.hadoop.hive.common;

+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.util.BitSet;
import java.util.List;

+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
+import org.apache.commons.compress.utils.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.util.Shell.ShellCommandExecutor;

/**
* Collection of file manipulation utilities common across Hive.
@@ -248,20 +255,27 @@ public final class FileUtils {
*/
public static void tar(String parentDir, String[] inputFiles, String outputFile)
throws IOException {
- StringBuffer tarCommand = new StringBuffer();
- tarCommand.append("cd " + parentDir + " ; ");
- tarCommand.append(" tar -zcvf ");
- tarCommand.append(" " + outputFile);
- for (int i = 0; i < inputFiles.length; i++) {
- tarCommand.append(" " + inputFiles[i]);
- }
- String[] shellCmd = {"bash", "-c", tarCommand.toString()};
- ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);
- shexec.execute();
- int exitcode = shexec.getExitCode();
- if (exitcode != 0) {
- throw new IOException("Error tarring file " + outputFile
- + ". Tar process exited with exit code " + exitcode);
+ TarArchiveOutputStream tOut = null;
+
+ try {
+ tOut = new TarArchiveOutputStream(
+ new GzipCompressorOutputStream(
+ new BufferedOutputStream(
+ new FileOutputStream(new File(parentDir, outputFile)))));
+
+ for (int i = 0; i < inputFiles.length; i++) {
+ File f = new File(inputFiles[i]);
+ TarArchiveEntry tarEntry = new TarArchiveEntry(f, f.getName());
+ tOut.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
+ tOut.putArchiveEntry(tarEntry);
+ IOUtils.copy(new FileInputStream(f), tOut);
+ tOut.closeArchiveEntry();
+ }
+ } finally {
+ if(tOut != null ) {
+ tOut.finish();
+ tOut.close();
+ }
}
}
}

Modified: hive/trunk/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/hive/trunk/ivy/libraries.properties?rev=1353044&r1=1353043&r2=1353044&view=diff
==============================================================================
--- hive/trunk/ivy/libraries.properties (original)
+++ hive/trunk/ivy/libraries.properties Fri Jun 22 22:11:23 2012
@@ -33,6 +33,7 @@ findbugs.version=1.3.9
commons-cli.version=1.2
commons-codec.version=1.4
commons-collections.version=3.2.1
+commons-compress.version=1.4.1
commons-configuration.version=1.6
commons-dbcp.version=1.4
commons-httpclient.version=3.0.1

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedJun 22, '12 at 10:11p
activeJun 22, '12 at 10:11p
posts1
users1
websitehive.apache.org

1 user in discussion

Ecapriolo: 1 post

People

Translate

site design / logo © 2021 Grokbase