crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject git commit: CRUNCH-477: Fix HFileTargetIT failures on hadoop1 with java7/8.
Date Wed, 22 Oct 2014 03:34:05 GMT
Repository: crunch
Updated Branches:
  refs/heads/master 8f525cc62 -> 24c1c176c


CRUNCH-477: Fix HFileTargetIT failures on hadoop1 with java7/8.


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/24c1c176
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/24c1c176
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/24c1c176

Branch: refs/heads/master
Commit: 24c1c176c9cad115b8cb517f33d2cdaf83ddaecc
Parents: 8f525cc
Author: Josh Wills <jwills@apache.org>
Authored: Sun Oct 19 15:15:26 2014 -0700
Committer: Josh Wills <jwills@apache.org>
Committed: Sun Oct 19 15:15:26 2014 -0700

----------------------------------------------------------------------
 .../apache/crunch/io/hbase/HFileTargetIT.java   | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/24c1c176/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java
----------------------------------------------------------------------
diff --git a/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java b/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java
index 25cec98..7d8ae83 100644
--- a/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java
+++ b/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java
@@ -66,8 +66,10 @@ import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
 
+import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.io.Serializable;
 import java.util.List;
@@ -104,6 +106,24 @@ public class HFileTargetIT implements Serializable {
     // We have to use mini mapreduce cluster, because LocalJobRunner allows only a single
reducer
     // (we will need it to test bulk load against multiple regions).
     Configuration conf = HBaseConfiguration.create();
+
+    // Workaround for HBASE-5711, we need to set config value dfs.datanode.data.dir.perm
+    // equal to the permissions of the temp dirs on the filesystem. These temp dirs were
+    // probably created using this process' umask. So we guess the temp dir permissions as
+    // 0777 & ~umask, and use that to set the config value.
+    Process process = Runtime.getRuntime().exec("/bin/sh -c umask");
+    BufferedReader br = new BufferedReader(new InputStreamReader(process.getInputStream()));
+    int rc = process.waitFor();
+    if(rc == 0) {
+      String umask = br.readLine();
+
+      int umaskBits = Integer.parseInt(umask, 8);
+      int permBits = 0777 & ~umaskBits;
+      String perms = Integer.toString(permBits, 8);
+
+      conf.set("dfs.datanode.data.dir.perm", perms);
+    }
+
     HBASE_TEST_UTILITY = new HBaseTestingUtility(conf);
     HBASE_TEST_UTILITY.startMiniCluster(1);
   }


Mime
View raw message