hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From inigo...@apache.org
Subject [26/50] [abbrv] hadoop git commit: HDFS-11598. Improve -setrep for Erasure Coded files. Contributed by Yiqun Lin.
Date Fri, 07 Apr 2017 01:59:22 GMT
HDFS-11598. Improve -setrep for Erasure Coded files. Contributed by Yiqun Lin.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b4d65779
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b4d65779
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b4d65779

Branch: refs/heads/HDFS-10467
Commit: b4d65779841d824d0bff5f106b2888e8d4dbe8a8
Parents: 285478a
Author: Wei-Chiu Chuang <weichiu@apache.org>
Authored: Mon Apr 3 07:57:28 2017 -0700
Committer: Inigo <inigoiri@apache.org>
Committed: Thu Apr 6 18:58:21 2017 -0700

----------------------------------------------------------------------
 .../apache/hadoop/fs/shell/SetReplication.java  | 17 ++++++--
 .../hadoop/hdfs/TestSetrepIncreasing.java       | 44 ++++++++++++++++++++
 2 files changed, 57 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4d65779/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java
index 2231c58..16e6e92 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java
@@ -85,11 +85,20 @@ class SetReplication extends FsCommand {
     }
     
     if (item.stat.isFile()) {
-      if (!item.fs.setReplication(item.path, newRep)) {
-        throw new IOException("Could not set replication for: " + item);
+      // Do the checking if the file is erasure coded since
+      // replication factor for an EC file is meaningless.
+      if (!item.stat.isErasureCoded()) {
+        if (!item.fs.setReplication(item.path, newRep)) {
+          throw new IOException("Could not set replication for: " + item);
+        }
+        out.println("Replication " + newRep + " set: " + item);
+        if (waitOpt) {
+          waitList.add(item);
+        }
+      } else {
+        out.println("Did not set replication for: " + item
+            + ", because it's an erasure coded file.");
       }
-      out.println("Replication " + newRep + " set: " + item);
-      if (waitOpt) waitList.add(item);
     } 
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4d65779/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
index fee30b5..50d7b27 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
@@ -20,7 +20,9 @@ package org.apache.hadoop.hdfs;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.PrintStream;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.BlockLocation;
@@ -28,6 +30,7 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FsShell;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
 import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
 import org.junit.Test;
 
@@ -102,4 +105,45 @@ public class TestSetrepIncreasing {
       cluster.shutdown();
     }
  }
+
+  @Test
+  public void testSetRepOnECFile() throws Exception {
+    ClientProtocol client;
+    Configuration conf = new HdfsConfiguration();
+    conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
+        StripedFileTestUtil.getDefaultECPolicy().getName());
+    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
+        .build();
+    cluster.waitActive();
+    client = NameNodeProxies.createProxy(conf,
+        cluster.getFileSystem(0).getUri(), ClientProtocol.class).getProxy();
+    client.setErasureCodingPolicy("/",
+        StripedFileTestUtil.getDefaultECPolicy().getName());
+
+    FileSystem dfs = cluster.getFileSystem();
+    try {
+      Path d = new Path("/tmp");
+      dfs.mkdirs(d);
+      Path f = new Path(d, "foo");
+      dfs.createNewFile(f);
+      FileStatus file = dfs.getFileStatus(f);
+      assertTrue(file.isErasureCoded());
+
+      ByteArrayOutputStream out = new ByteArrayOutputStream();
+      System.setOut(new PrintStream(out));
+      String[] args = {"-setrep", "2", "" + f};
+      FsShell shell = new FsShell();
+      shell.setConf(conf);
+      assertEquals(0, shell.run(args));
+      assertTrue(
+          out.toString().contains("Did not set replication for: /tmp/foo"));
+
+      // verify the replication factor of the EC file
+      file = dfs.getFileStatus(f);
+      assertEquals(1, file.getReplication());
+    } finally {
+      dfs.close();
+      cluster.shutdown();
+    }
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message