accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject [1/3] accumulo git commit: ACCUMULO-3664 Ensure bulk import tests set permissions on import directory
Date Thu, 19 Mar 2015 20:08:06 GMT
Repository: accumulo
Updated Branches:
  refs/heads/master c2fa74cbc -> 261801559


ACCUMULO-3664 Ensure bulk import tests set permissions on import directory

Accumulo will attempt to move the files out of the import directory
as a part of the process. This fails when HDFS permissions are actually
enabled. Make sure the tests perform a chmod.


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/838a5fcc
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/838a5fcc
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/838a5fcc

Branch: refs/heads/master
Commit: 838a5fcca6887c2ba14ed4eb55cdf820f75a9a08
Parents: c2fa74c
Author: Josh Elser <elserj@apache.org>
Authored: Thu Mar 19 12:06:39 2015 -0400
Committer: Josh Elser <elserj@apache.org>
Committed: Thu Mar 19 16:04:31 2015 -0400

----------------------------------------------------------------------
 .../mapreduce/bulk/BulkIngestExample.java       |  4 ++++
 .../accumulo/test/BulkImportVolumeIT.java       |  3 +++
 .../apache/accumulo/test/functional/BulkIT.java | 22 +++++++++-----------
 .../functional/BulkSplitOptimizationIT.java     |  1 +
 .../accumulo/test/functional/CompactionIT.java  |  1 +
 .../test/functional/FunctionalTestUtils.java    |  7 +++++++
 6 files changed, 26 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/838a5fcc/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/bulk/BulkIngestExample.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/bulk/BulkIngestExample.java
b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/bulk/BulkIngestExample.java
index e4000e4..fbe92b5 100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/bulk/BulkIngestExample.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/bulk/BulkIngestExample.java
@@ -32,6 +32,7 @@ import org.apache.accumulo.core.util.TextUtil;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsShell;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
@@ -144,6 +145,9 @@ public class BulkIngestExample extends Configured implements Tool {
       Path failures = new Path(opts.workDir, "failures");
       fs.delete(failures, true);
       fs.mkdirs(new Path(opts.workDir, "failures"));
+      // With HDFS permissions on, we need to make sure the Accumulo user can read/move the
rfiles
+      FsShell fsShell = new FsShell(conf);
+      fsShell.run(new String[] {"-chmod", "-R", "777", opts.workDir});
       connector.tableOperations().importDirectory(opts.getTableName(), opts.workDir + "/files",
opts.workDir + "/failures", false);
 
     } catch (Exception e) {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/838a5fcc/test/src/test/java/org/apache/accumulo/test/BulkImportVolumeIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/BulkImportVolumeIT.java b/test/src/test/java/org/apache/accumulo/test/BulkImportVolumeIT.java
index 493e07f..cc8fcc9 100644
--- a/test/src/test/java/org/apache/accumulo/test/BulkImportVolumeIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/BulkImportVolumeIT.java
@@ -27,6 +27,7 @@ import org.apache.accumulo.harness.AccumuloClusterIT;
 import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsShell;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RawLocalFileSystem;
 import org.junit.Test;
@@ -84,6 +85,8 @@ public class BulkImportVolumeIT extends AccumuloClusterIT {
     fs.create(bogus).close();
     log.info("bogus: {}", bogus);
     assertTrue(fs.exists(bogus));
+    FsShell fsShell = new FsShell(fs.getConf());
+    assertEquals("Failed to chmod " + rootPath, 0, fsShell.run(new String[] {"-chmod", "-R",
"777", rootPath.toString()}));
     log.info("Importing {} into {} with failures directory {}", bulk, tableName, err);
     to.importDirectory(tableName, bulk.toString(), err.toString(), false);
     assertEquals(1, fs.listStatus(err).length);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/838a5fcc/test/src/test/java/org/apache/accumulo/test/functional/BulkIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/BulkIT.java b/test/src/test/java/org/apache/accumulo/test/functional/BulkIT.java
index a2a05f6..8ad3d26 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BulkIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BulkIT.java
@@ -16,16 +16,9 @@
  */
 package org.apache.accumulo.test.functional;
 
-import java.io.IOException;
-
 import org.apache.accumulo.core.cli.BatchWriterOpts;
 import org.apache.accumulo.core.cli.ScannerOpts;
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
 import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.MutationsRejectedException;
-import org.apache.accumulo.core.client.TableExistsException;
-import org.apache.accumulo.core.client.TableNotFoundException;
 import org.apache.accumulo.core.util.CachedConfiguration;
 import org.apache.accumulo.harness.AccumuloClusterIT;
 import org.apache.accumulo.test.TestIngest;
@@ -33,12 +26,13 @@ import org.apache.accumulo.test.TestIngest.Opts;
 import org.apache.accumulo.test.VerifyIngest;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsShell;
 import org.apache.hadoop.fs.Path;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
-// TODO Change test to support bulk ingest on any filesystem, not just the local FS.
 public class BulkIT extends AccumuloClusterIT {
 
   private static final int N = 100000;
@@ -67,12 +61,11 @@ public class BulkIT extends AccumuloClusterIT {
 
   @Test
   public void test() throws Exception {
-    runTest(getConnector(), getCluster().getFileSystem(), getCluster().getTemporaryPath(),
getAdminPrincipal(), getUniqueNames(1)[0], this.getClass().getName(),
-        testName.getMethodName());
+    runTest(getConnector(), getCluster().getFileSystem(), getCluster().getTemporaryPath(),
getAdminPrincipal(), getUniqueNames(1)[0],
+        this.getClass().getName(), testName.getMethodName());
   }
 
-  static void runTest(Connector c, FileSystem fs, Path basePath, String principal, String
tableName, String filePrefix, String dirSuffix)
-      throws AccumuloException, AccumuloSecurityException, TableExistsException, IOException,
TableNotFoundException, MutationsRejectedException {
+  static void runTest(Connector c, FileSystem fs, Path basePath, String principal, String
tableName, String filePrefix, String dirSuffix) throws Exception {
     c.tableOperations().create(tableName);
     CachedConfiguration.setInstance(fs.getConf());
 
@@ -104,6 +97,11 @@ public class BulkIT extends AccumuloClusterIT {
     opts.rows = 1;
     // create an rfile with one entry, there was a bug with this:
     TestIngest.ingest(c, opts, BWOPTS);
+
+    // Make sure the server can modify the files
+    FsShell fsShell = new FsShell(fs.getConf());
+    Assert.assertEquals("Failed to chmod " + base.toString(), 0, fsShell.run(new String[]
{"-chmod", "-R", "777", base.toString()}));
+
     c.tableOperations().importDirectory(tableName, files.toString(), bulkFailures.toString(),
false);
     VerifyIngest.Opts vopts = new VerifyIngest.Opts();
     vopts.setTableName(tableName);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/838a5fcc/test/src/test/java/org/apache/accumulo/test/functional/BulkSplitOptimizationIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/BulkSplitOptimizationIT.java
b/test/src/test/java/org/apache/accumulo/test/functional/BulkSplitOptimizationIT.java
index 5beed91..fe26870 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BulkSplitOptimizationIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BulkSplitOptimizationIT.java
@@ -95,6 +95,7 @@ public class BulkSplitOptimizationIT extends AccumuloClusterIT {
     Path testDir = new Path(getUsableDir(), "testmf");
     FunctionalTestUtils.createRFiles(c, fs, testDir.toString(), ROWS, SPLITS, 8);
     FileStatus[] stats = fs.listStatus(testDir);
+
     System.out.println("Number of generated files: " + stats.length);
     FunctionalTestUtils.bulkImport(c, fs, tableName, testDir.toString());
     FunctionalTestUtils.checkSplits(c, tableName, 0, 0);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/838a5fcc/test/src/test/java/org/apache/accumulo/test/functional/CompactionIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/CompactionIT.java b/test/src/test/java/org/apache/accumulo/test/functional/CompactionIT.java
index 6acabc6..03646c1 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/CompactionIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/CompactionIT.java
@@ -114,6 +114,7 @@ public class CompactionIT extends AccumuloClusterIT {
     Path root = new Path(cluster.getTemporaryPath(), getClass().getName());
     Path testrf = new Path(root, "testrf");
     FunctionalTestUtils.createRFiles(c, fs, testrf.toString(), 500000, 59, 4);
+
     FunctionalTestUtils.bulkImport(c, fs, tableName, testrf.toString());
     int beforeCount = countFiles(c);
 

http://git-wip-us.apache.org/repos/asf/accumulo/blob/838a5fcc/test/src/test/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java
b/test/src/test/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java
index 5f59a19..876eb60 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java
@@ -46,8 +46,10 @@ import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
 import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl.LogWriter;
 import org.apache.accumulo.test.TestIngest;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsShell;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
+import org.junit.Assert;
 
 public class FunctionalTestUtils {
 
@@ -101,6 +103,11 @@ public class FunctionalTestUtils {
     fs.delete(failPath, true);
     fs.mkdirs(failPath);
 
+    // Ensure server can read/modify files
+    FsShell fsShell = new FsShell(fs.getConf());
+    Assert.assertEquals("Failed to chmod " + dir, 0, fsShell.run(new String[] {"-chmod",
"-R", "777", dir}));
+    Assert.assertEquals("Failed to chmod " + failDir, 0, fsShell.run(new String[] {"-chmod",
"-R", "777", failDir}));
+
     c.tableOperations().importDirectory(table, dir, failDir, false);
 
     if (fs.listStatus(failPath).length > 0) {


Mime
View raw message