FAQ
Author: jdere
Date: Wed Oct 29 00:34:46 2014
New Revision: 1635013

URL: http://svn.apache.org/r1635013
Log:
HIVE-8148: HDFS Path named with file:// instead of file:/// results in Unit test failures in Windows (Hari Subramaniyan via Jason Dere)

Modified:
     hive/trunk/cli/src/test/org/apache/hadoop/hive/cli/TestRCFileCat.java
     hive/trunk/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
     hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
     hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationProviderWithACL.java
     hive/trunk/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
     hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java
     hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java
     hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java

Modified: hive/trunk/cli/src/test/org/apache/hadoop/hive/cli/TestRCFileCat.java
URL: http://svn.apache.org/viewvc/hive/trunk/cli/src/test/org/apache/hadoop/hive/cli/TestRCFileCat.java?rev=1635013&r1=1635012&r2=1635013&view=diff
==============================================================================
--- hive/trunk/cli/src/test/org/apache/hadoop/hive/cli/TestRCFileCat.java (original)
+++ hive/trunk/cli/src/test/org/apache/hadoop/hive/cli/TestRCFileCat.java Wed Oct 29 00:34:46 2014
@@ -25,6 +25,7 @@ import java.io.ByteArrayOutputStream;
  import java.io.File;
  import java.io.IOException;
  import java.io.PrintStream;
+import java.net.URI;

  import org.apache.hadoop.conf.Configuration;
  import org.apache.hadoop.fs.FileSystem;
@@ -89,19 +90,19 @@ public class TestRCFileCat {


      try {
- String[] params = {"--verbose","file://" + template.getAbsolutePath() };
+ String[] params = {"--verbose","file://" + template.toURI().getPath() };

        assertEquals(0, fileCat.run(params));
        assertTrue(dataOut.toString().contains("123\t456\t789\t1000\t5.3\thive and hadoop\t\tNULL"));
        assertTrue(dataOut.toString().contains("100\t200\t123\t1000\t5.3\thive and hadoop\t\tNULL"));
        assertTrue(dataOut.toString().contains("200\t400\t678\t1000\t4.8\thive and hadoop\t\tTEST"));
        dataOut.reset();
- params = new String[] { "--start=-10","--file-sizes","file://" + template.getAbsolutePath() };
+ params = new String[] { "--start=-10","--file-sizes", "file://" + template.toURI().getPath() };
        assertEquals(0, fileCat.run(params));
        assertTrue(dataOut.toString().contains("File size (uncompressed): 105. File size (compressed): 134. Number of rows: 3."));
        dataOut.reset();

- params = new String[] {"--start=0", "--column-sizes","file://" + template.getAbsolutePath() };
+ params = new String[] {"--start=0", "--column-sizes","file://" + template.toURI().getPath() };
        assertEquals(0, fileCat.run(params));
        assertTrue(dataOut.toString().contains("0\t9\t17"));
        assertTrue(dataOut.toString().contains("1\t9\t17"));
@@ -112,7 +113,8 @@ public class TestRCFileCat {


        dataOut.reset();
- params = new String[] {"--start=0", "--column-sizes-pretty","file://" + template.getAbsolutePath() };
+ params = new String[] {"--start=0", "--column-sizes-pretty",
+ "file://" + template.toURI().getPath() };
        assertEquals(0, fileCat.run(params));
        assertTrue(dataOut.toString().contains("Column 0: Uncompressed size: 9 Compressed size: 17"));
        assertTrue(dataOut.toString().contains("Column 1: Uncompressed size: 9 Compressed size: 17"));
@@ -127,7 +129,8 @@ public class TestRCFileCat {
            "[--column-sizes | --column-sizes-pretty] [--file-sizes] fileName"));

        dataErr.reset();
- params = new String[] { "--fakeParameter","file://" + template.getAbsolutePath()};
+ params = new String[] { "--fakeParameter",
+ "file://" + template.toURI().getPath()};
        assertEquals(-1, fileCat.run(params));
        assertTrue(dataErr.toString().contains("RCFileCat [--start=start_offet] [--length=len] [--verbose] " +
            "[--column-sizes | --column-sizes-pretty] [--file-sizes] fileName"));

Modified: hive/trunk/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java?rev=1635013&r1=1635012&r2=1635013&view=diff
==============================================================================
--- hive/trunk/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java (original)
+++ hive/trunk/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java Wed Oct 29 00:34:46 2014
@@ -731,7 +731,7 @@ public class TestStreaming {
            throws Exception {
      Database db = new Database();
      db.setName(databaseName);
- String dbLocation = "raw://" + dbFolder.newFolder(databaseName + ".db").getCanonicalPath();
+ String dbLocation = "raw://" + dbFolder.newFolder(databaseName + ".db").toURI().getPath();
      db.setLocationUri(dbLocation);
      client.createDatabase(db);


Modified: hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java?rev=1635013&r1=1635012&r2=1635013&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java (original)
+++ hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java Wed Oct 29 00:34:46 2014
@@ -157,7 +157,7 @@ public class TestHCatClient {
      assertTrue(testDb.getProperties().size() == 0);
      String warehouseDir = System
        .getProperty("test.warehouse.dir", "/user/hive/warehouse");
- String expectedDir = fixPath(warehouseDir).replaceFirst("pfile:///", "pfile:/");
+ String expectedDir = warehouseDir.replaceFirst("pfile:///", "pfile:/");
      assertEquals(expectedDir + "/" + db + ".db", testDb.getLocation());
      ArrayList<HCatFieldSchema> cols = new ArrayList<HCatFieldSchema>();
      cols.add(new HCatFieldSchema("id", Type.INT, "id comment"));

Modified: hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationProviderWithACL.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationProviderWithACL.java?rev=1635013&r1=1635012&r2=1635013&view=diff
==============================================================================
--- hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationProviderWithACL.java (original)
+++ hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationProviderWithACL.java Wed Oct 29 00:34:46 2014
@@ -68,6 +68,10 @@ public class TestStorageBasedMetastoreAu
      conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, warehouseDir.toString());
      conf.setBoolVar(HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS, true);

+ // Set up scratch directory
+ Path scratchDir = new Path(new Path(fs.getUri()), "/scratchdir");
+ conf.setVar(HiveConf.ConfVars.SCRATCHDIR, scratchDir.toString());
+
      return conf;
    }


Modified: hive/trunk/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java?rev=1635013&r1=1635012&r2=1635013&view=diff
==============================================================================
--- hive/trunk/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java (original)
+++ hive/trunk/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java Wed Oct 29 00:34:46 2014
@@ -176,7 +176,7 @@ public class MiniHS2 extends AbstractHiv
        baseDfsDir = new Path(new Path(fs.getUri()), "/base");
      } else {
        fs = FileSystem.getLocal(hiveConf);
- baseDfsDir = new Path("file://"+ baseDir.getPath());
+ baseDfsDir = new Path("file://"+ baseDir.toURI().getPath());
      }
      if (useMiniKdc) {
        hiveConf.setVar(ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL, serverPrincipal);

Modified: hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java?rev=1635013&r1=1635012&r2=1635013&view=diff
==============================================================================
--- hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java (original)
+++ hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java Wed Oct 29 00:34:46 2014
@@ -83,10 +83,15 @@ public abstract class FolderPermissionBa
      fs.mkdirs(warehouseDir);
      conf.setVar(ConfVars.METASTOREWAREHOUSE, warehouseDir.toString());

+ // Assuming the tests are run either in C or D drive in Windows OS!
      dataFileDir = conf.get("test.data.files").replace('\\', '/')
- .replace("c:", "");
+ .replace("c:", "").replace("C:", "").replace("D:", "").replace("d:", "");
      dataFilePath = new Path(dataFileDir, "kv1.txt");

+ // Set up scratch directory
+ Path scratchDir = new Path(baseDfsDir, "scratchdir");
+ conf.setVar(HiveConf.ConfVars.SCRATCHDIR, scratchDir.toString());
+
      //set hive conf vars
      conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
      conf.setBoolVar(HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS, true);

Modified: hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java?rev=1635013&r1=1635012&r2=1635013&view=diff
==============================================================================
--- hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java (original)
+++ hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java Wed Oct 29 00:34:46 2014
@@ -21,6 +21,7 @@ import static org.junit.Assert.assertEqu
  import static org.junit.Assert.fail;

  import java.io.File;
+import java.net.URLEncoder;
  import java.sql.Connection;
  import java.sql.DriverManager;
  import java.sql.ResultSet;
@@ -55,8 +56,8 @@ public class TestSSL {
    private Connection hs2Conn = null;
    private String dataFileDir = conf.get("test.data.files");
    private Map<String, String> confOverlay;
- private final String SSL_CONN_PARAMS = ";ssl=true;sslTrustStore=" + dataFileDir + File.separator +
- TRUST_STORE_NAME + ";trustStorePassword=" + KEY_STORE_PASSWORD;
+ private final String SSL_CONN_PARAMS = ";ssl=true;sslTrustStore=" + URLEncoder.encode(dataFileDir + File.separator +
+ TRUST_STORE_NAME) + ";trustStorePassword=" + KEY_STORE_PASSWORD;

    @BeforeClass
    public static void beforeTest() throws Exception {

Modified: hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1635013&r1=1635012&r2=1635013&view=diff
==============================================================================
--- hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java Wed Oct 29 00:34:46 2014
@@ -305,7 +305,7 @@ public class QTestUtil {
      this.outDir = outDir;
      this.logDir = logDir;
      if (confDir != null && !confDir.isEmpty()) {
- HiveConf.setHiveSiteLocation(new URL("file://"+confDir+"/hive-site.xml"));
+ HiveConf.setHiveSiteLocation(new URL("file://"+ new File(confDir).toURI().getPath() + "/hive-site.xml"));
        System.out.println("Setting hive-site: "+HiveConf.getHiveSiteLocation());
      }
      conf = new HiveConf(Driver.class);

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedOct 29, '14 at 12:35a
activeOct 29, '14 at 12:35a
posts1
users1
websitehive.apache.org

1 user in discussion

Jdere: 1 post

People

Translate

site design / logo © 2022 Grokbase