FAQ
Repository: hive
Updated Branches:
   refs/heads/master e09e14a48 -> 84d38ef0a


HIVE-11066 Ensure tests don't share directories on FS


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/84d38ef0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/84d38ef0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/84d38ef0

Branch: refs/heads/master
Commit: 84d38ef0a5aac97d44c99dd252dfc8c0c58bfd12
Parents: e09e14a
Author: Eugene Koifman <ekoifman@hortonworks.com>
Authored: Sat Jun 20 17:10:03 2015 -0700
Committer: Eugene Koifman <ekoifman@hortonworks.com>
Committed: Sat Jun 20 17:11:21 2015 -0700

----------------------------------------------------------------------
  .../hcatalog/pig/TestHCatLoaderEncryption.java | 18 ++++++++++++++----
  .../hive/ql/txn/compactor/TestCompactor.java | 11 +++++++----
  2 files changed, 21 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/84d38ef0/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java
----------------------------------------------------------------------
diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java
index da8deb5..3b8076b 100644
--- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java
+++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java
@@ -28,7 +28,9 @@ import java.util.HashSet;
  import java.util.Iterator;
  import java.util.List;
  import java.util.Map;
+import java.util.Random;
  import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;

  import org.apache.commons.io.FileUtils;

@@ -84,12 +86,13 @@ import static org.junit.Assume.assumeTrue;

  @RunWith(Parameterized.class)
  public class TestHCatLoaderEncryption {
+ private static final AtomicInteger salt = new AtomicInteger(new Random().nextInt());
    private static final Logger LOG = LoggerFactory.getLogger(TestHCatLoader.class);
- private static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(System.getProperty
+ private final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(System.getProperty
        ("java.io.tmpdir") + File.separator + TestHCatLoader.class.getCanonicalName() + "-" +
- System.currentTimeMillis());
- private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
- private static final String BASIC_FILE_NAME = TEST_DATA_DIR + "/basic.input.data";
+ System.currentTimeMillis() + "_" + salt.getAndIncrement());
+ private final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
+ private final String BASIC_FILE_NAME = TEST_DATA_DIR + "/basic.input.data";
    private static final String BASIC_TABLE = "junit_unparted_basic";
    private static final String ENCRYPTED_TABLE = "encrypted_table";
    private static final String SECURITY_KEY_PROVIDER_URI_NAME = "dfs.encryption.key.provider.uri";
@@ -180,6 +183,13 @@ public class TestHCatLoaderEncryption {
      hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
      hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, TEST_WAREHOUSE_DIR);

+ String s = hiveConf.get("hdfs.minidfs.basedir");
+ if(s == null || s.length() <= 0) {
+ //return System.getProperty("test.build.data", "build/test/data") + "/dfs/";
+ hiveConf.set("hdfs.minidfs.basedir",
+ System.getProperty("test.build.data", "build/test/data") + "_" + System.currentTimeMillis() +
+ "_" + salt.getAndIncrement() + "/dfs/");
+ }
      if (Shell.WINDOWS) {
        WindowsPathUtil.convertPathsFromWindowsToHdfs(hiveConf);
      }

http://git-wip-us.apache.org/repos/asf/hive/blob/84d38ef0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
index c682c3e..abca1ce 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
@@ -54,18 +54,21 @@ import java.util.ArrayList;
  import java.util.Arrays;
  import java.util.List;
  import java.util.Map;
+import java.util.Random;
  import java.util.SortedSet;
  import java.util.TreeSet;
  import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;

  /**
   */
  public class TestCompactor {
+ private static final AtomicInteger salt = new AtomicInteger(new Random().nextInt());
    private static final Logger LOG = LoggerFactory.getLogger(TestCompactor.class);
- private static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(System.getProperty("java.io.tmpdir") +
- File.separator + TestCompactor.class.getCanonicalName() + "-" + System.currentTimeMillis());
- private static final String BASIC_FILE_NAME = TEST_DATA_DIR + "/basic.input.data";
- private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
+ private final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(System.getProperty("java.io.tmpdir") +
+ File.separator + TestCompactor.class.getCanonicalName() + "-" + System.currentTimeMillis() + "_" + salt.getAndIncrement());
+ private final String BASIC_FILE_NAME = TEST_DATA_DIR + "/basic.input.data";
+ private final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";

    @Rule
    public TemporaryFolder stagingFolder = new TemporaryFolder();

Search Discussions

  • Ekoifman at Jun 21, 2015 at 12:49 am
    Repository: hive
    Updated Branches:
       refs/heads/branch-1.2 ad308db50 -> 85676d5dd


    HIVE-11066 Ensure tests don't share directories on FS


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/85676d5d
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/85676d5d
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/85676d5d

    Branch: refs/heads/branch-1.2
    Commit: 85676d5dd05d42fdadf36bcc47f8e27fc79d3aec
    Parents: ad308db
    Author: Eugene Koifman <ekoifman@hortonworks.com>
    Authored: Sat Jun 20 17:49:24 2015 -0700
    Committer: Eugene Koifman <ekoifman@hortonworks.com>
    Committed: Sat Jun 20 17:49:24 2015 -0700

    ----------------------------------------------------------------------
      .../hcatalog/pig/TestHCatLoaderEncryption.java | 18 ++++++++++++++----
      .../hive/ql/txn/compactor/TestCompactor.java | 11 +++++++----
      2 files changed, 21 insertions(+), 8 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/85676d5d/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java
    ----------------------------------------------------------------------
    diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java
    index da8deb5..3b8076b 100644
    --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java
    +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java
    @@ -28,7 +28,9 @@ import java.util.HashSet;
      import java.util.Iterator;
      import java.util.List;
      import java.util.Map;
    +import java.util.Random;
      import java.util.Set;
    +import java.util.concurrent.atomic.AtomicInteger;

      import org.apache.commons.io.FileUtils;

    @@ -84,12 +86,13 @@ import static org.junit.Assume.assumeTrue;

      @RunWith(Parameterized.class)
      public class TestHCatLoaderEncryption {
    + private static final AtomicInteger salt = new AtomicInteger(new Random().nextInt());
        private static final Logger LOG = LoggerFactory.getLogger(TestHCatLoader.class);
    - private static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(System.getProperty
    + private final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(System.getProperty
            ("java.io.tmpdir") + File.separator + TestHCatLoader.class.getCanonicalName() + "-" +
    - System.currentTimeMillis());
    - private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
    - private static final String BASIC_FILE_NAME = TEST_DATA_DIR + "/basic.input.data";
    + System.currentTimeMillis() + "_" + salt.getAndIncrement());
    + private final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
    + private final String BASIC_FILE_NAME = TEST_DATA_DIR + "/basic.input.data";
        private static final String BASIC_TABLE = "junit_unparted_basic";
        private static final String ENCRYPTED_TABLE = "encrypted_table";
        private static final String SECURITY_KEY_PROVIDER_URI_NAME = "dfs.encryption.key.provider.uri";
    @@ -180,6 +183,13 @@ public class TestHCatLoaderEncryption {
          hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
          hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, TEST_WAREHOUSE_DIR);

    + String s = hiveConf.get("hdfs.minidfs.basedir");
    + if(s == null || s.length() <= 0) {
    + //return System.getProperty("test.build.data", "build/test/data") + "/dfs/";
    + hiveConf.set("hdfs.minidfs.basedir",
    + System.getProperty("test.build.data", "build/test/data") + "_" + System.currentTimeMillis() +
    + "_" + salt.getAndIncrement() + "/dfs/");
    + }
          if (Shell.WINDOWS) {
            WindowsPathUtil.convertPathsFromWindowsToHdfs(hiveConf);
          }

    http://git-wip-us.apache.org/repos/asf/hive/blob/85676d5d/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
    ----------------------------------------------------------------------
    diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
    index c682c3e..abca1ce 100644
    --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
    +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
    @@ -54,18 +54,21 @@ import java.util.ArrayList;
      import java.util.Arrays;
      import java.util.List;
      import java.util.Map;
    +import java.util.Random;
      import java.util.SortedSet;
      import java.util.TreeSet;
      import java.util.concurrent.atomic.AtomicBoolean;
    +import java.util.concurrent.atomic.AtomicInteger;

      /**
       */
      public class TestCompactor {
    + private static final AtomicInteger salt = new AtomicInteger(new Random().nextInt());
        private static final Logger LOG = LoggerFactory.getLogger(TestCompactor.class);
    - private static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(System.getProperty("java.io.tmpdir") +
    - File.separator + TestCompactor.class.getCanonicalName() + "-" + System.currentTimeMillis());
    - private static final String BASIC_FILE_NAME = TEST_DATA_DIR + "/basic.input.data";
    - private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
    + private final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(System.getProperty("java.io.tmpdir") +
    + File.separator + TestCompactor.class.getCanonicalName() + "-" + System.currentTimeMillis() + "_" + salt.getAndIncrement());
    + private final String BASIC_FILE_NAME = TEST_DATA_DIR + "/basic.input.data";
    + private final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";

        @Rule
        public TemporaryFolder stagingFolder = new TemporaryFolder();
  • Ekoifman at Jun 21, 2015 at 1:01 am
    Repository: hive
    Updated Branches:
       refs/heads/branch-1 072241209 -> cce0e8e6f


    HIVE-11066 Ensure tests don't share directories on FS


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cce0e8e6
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cce0e8e6
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cce0e8e6

    Branch: refs/heads/branch-1
    Commit: cce0e8e6f128418228fcfff0eb0018b3ea7d2bdd
    Parents: 0722412
    Author: Eugene Koifman <ekoifman@hortonworks.com>
    Authored: Sat Jun 20 18:00:51 2015 -0700
    Committer: Eugene Koifman <ekoifman@hortonworks.com>
    Committed: Sat Jun 20 18:00:51 2015 -0700

    ----------------------------------------------------------------------
      .../hcatalog/pig/TestHCatLoaderEncryption.java | 18 ++++++++++++++----
      .../hive/ql/txn/compactor/TestCompactor.java | 11 +++++++----
      2 files changed, 21 insertions(+), 8 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/cce0e8e6/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java
    ----------------------------------------------------------------------
    diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java
    index da8deb5..3b8076b 100644
    --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java
    +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java
    @@ -28,7 +28,9 @@ import java.util.HashSet;
      import java.util.Iterator;
      import java.util.List;
      import java.util.Map;
    +import java.util.Random;
      import java.util.Set;
    +import java.util.concurrent.atomic.AtomicInteger;

      import org.apache.commons.io.FileUtils;

    @@ -84,12 +86,13 @@ import static org.junit.Assume.assumeTrue;

      @RunWith(Parameterized.class)
      public class TestHCatLoaderEncryption {
    + private static final AtomicInteger salt = new AtomicInteger(new Random().nextInt());
        private static final Logger LOG = LoggerFactory.getLogger(TestHCatLoader.class);
    - private static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(System.getProperty
    + private final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(System.getProperty
            ("java.io.tmpdir") + File.separator + TestHCatLoader.class.getCanonicalName() + "-" +
    - System.currentTimeMillis());
    - private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
    - private static final String BASIC_FILE_NAME = TEST_DATA_DIR + "/basic.input.data";
    + System.currentTimeMillis() + "_" + salt.getAndIncrement());
    + private final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
    + private final String BASIC_FILE_NAME = TEST_DATA_DIR + "/basic.input.data";
        private static final String BASIC_TABLE = "junit_unparted_basic";
        private static final String ENCRYPTED_TABLE = "encrypted_table";
        private static final String SECURITY_KEY_PROVIDER_URI_NAME = "dfs.encryption.key.provider.uri";
    @@ -180,6 +183,13 @@ public class TestHCatLoaderEncryption {
          hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
          hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, TEST_WAREHOUSE_DIR);

    + String s = hiveConf.get("hdfs.minidfs.basedir");
    + if(s == null || s.length() <= 0) {
    + //return System.getProperty("test.build.data", "build/test/data") + "/dfs/";
    + hiveConf.set("hdfs.minidfs.basedir",
    + System.getProperty("test.build.data", "build/test/data") + "_" + System.currentTimeMillis() +
    + "_" + salt.getAndIncrement() + "/dfs/");
    + }
          if (Shell.WINDOWS) {
            WindowsPathUtil.convertPathsFromWindowsToHdfs(hiveConf);
          }

    http://git-wip-us.apache.org/repos/asf/hive/blob/cce0e8e6/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
    ----------------------------------------------------------------------
    diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
    index c682c3e..abca1ce 100644
    --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
    +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
    @@ -54,18 +54,21 @@ import java.util.ArrayList;
      import java.util.Arrays;
      import java.util.List;
      import java.util.Map;
    +import java.util.Random;
      import java.util.SortedSet;
      import java.util.TreeSet;
      import java.util.concurrent.atomic.AtomicBoolean;
    +import java.util.concurrent.atomic.AtomicInteger;

      /**
       */
      public class TestCompactor {
    + private static final AtomicInteger salt = new AtomicInteger(new Random().nextInt());
        private static final Logger LOG = LoggerFactory.getLogger(TestCompactor.class);
    - private static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(System.getProperty("java.io.tmpdir") +
    - File.separator + TestCompactor.class.getCanonicalName() + "-" + System.currentTimeMillis());
    - private static final String BASIC_FILE_NAME = TEST_DATA_DIR + "/basic.input.data";
    - private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
    + private final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(System.getProperty("java.io.tmpdir") +
    + File.separator + TestCompactor.class.getCanonicalName() + "-" + System.currentTimeMillis() + "_" + salt.getAndIncrement());
    + private final String BASIC_FILE_NAME = TEST_DATA_DIR + "/basic.input.data";
    + private final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";

        @Rule
        public TemporaryFolder stagingFolder = new TemporaryFolder();

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedJun 21, '15 at 12:11a
activeJun 21, '15 at 1:01a
posts3
users1
websitehive.apache.org

1 user in discussion

Ekoifman: 3 posts

People

Translate

site design / logo © 2021 Grokbase