hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s..@apache.org
Subject svn commit: r898131 - in /hadoop/hdfs/trunk: ./ src/java/ src/java/org/apache/hadoop/hdfs/server/common/ src/java/org/apache/hadoop/hdfs/server/datanode/ src/test/hdfs/org/apache/hadoop/hdfs/ src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/
Date Tue, 12 Jan 2010 00:21:54 GMT
Author: shv
Date: Tue Jan 12 00:21:53 2010
New Revision: 898131

URL: http://svn.apache.org/viewvc?rev=898131&view=rev
Log:
HDFS-873. Configuration specifies data-node storage directories as URIs. Contributed by Konstantin
Shvachko.

Added:
    hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDatanodeConfig.java   (with
props)
Modified:
    hadoop/hdfs/trunk/CHANGES.txt
    hadoop/hdfs/trunk/src/java/hdfs-default.xml
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Util.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
    hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java
    hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java

Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=898131&r1=898130&r2=898131&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Tue Jan 12 00:21:53 2010
@@ -122,6 +122,12 @@
     HDFS-512. Block.equals() and compareTo() compare blocks based
     only on block Ids, ignoring generation stamps. (shv)
 
+    HDFS-512. Block.equals() and compareTo() compare blocks based
+    only on block Ids, ignoring generation stamps. (shv)
+
+    HDFS-873. Configuration specifies data-node storage directories as URIs.
+    (shv)
+
   NEW FEATURES
 
     HDFS-436. Introduce AspectJ framework for HDFS code and tests.

Modified: hadoop/hdfs/trunk/src/java/hdfs-default.xml
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/hdfs-default.xml?rev=898131&r1=898130&r2=898131&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/hdfs-default.xml (original)
+++ hadoop/hdfs/trunk/src/java/hdfs-default.xml Tue Jan 12 00:21:53 2010
@@ -236,7 +236,7 @@
 
 <property>
   <name>dfs.datanode.data.dir</name>
-  <value>${hadoop.tmp.dir}/dfs/data</value>
+  <value>file://${hadoop.tmp.dir}/dfs/data</value>
   <description>Determines where on the local filesystem an DFS data node
   should store its blocks.  If this is a comma-delimited
   list of directories, then data will be stored in all named

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Util.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Util.java?rev=898131&r1=898130&r2=898131&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Util.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Util.java Tue Jan 12 00:21:53
2010
@@ -52,12 +52,14 @@
     try {
       u = new URI(s);
     } catch (URISyntaxException e){
-      LOG.warn("Path " + s + " should be specified as a URI " +
-      "in configuration files. Please update hdfs configuration.", e);
+      LOG.error("Syntax error in URI " + s
+          + ". Please check hdfs configuration.", e);
     }
 
     // if URI is null or scheme is undefined, then assume it's file://
     if(u == null || u.getScheme() == null){
+      LOG.warn("Path " + s + " should be specified as a URI "
+          + "in configuration files. Please update hdfs configuration.");
       u = fileAsURI(new File(s));
     }
     return u;

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=898131&r1=898130&r2=898131&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java Tue Jan
12 00:21:53 2010
@@ -26,6 +26,7 @@
 import java.net.ServerSocket;
 import java.net.Socket;
 import java.net.SocketTimeoutException;
+import java.net.URI;
 import java.net.UnknownHostException;
 import java.nio.channels.ServerSocketChannel;
 import java.nio.channels.SocketChannel;
@@ -63,6 +64,7 @@
 import org.apache.hadoop.hdfs.server.common.HdfsConstants;
 import org.apache.hadoop.hdfs.server.common.IncorrectVersionException;
 import org.apache.hadoop.hdfs.server.common.Storage;
+import org.apache.hadoop.hdfs.server.common.Util;
 import org.apache.hadoop.hdfs.server.common.HdfsConstants.ReplicaState;
 import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
 import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics;
@@ -401,7 +403,8 @@
     //init ipc server
     InetSocketAddress ipcAddr = NetUtils.createSocketAddr(
         conf.get("dfs.datanode.ipc.address"));
-    ipcServer = RPC.getServer(this, ipcAddr.getHostName(), ipcAddr.getPort(), 
+    ipcServer = RPC.getServer(DataNode.class, this,
+        ipcAddr.getHostName(), ipcAddr.getPort(), 
         conf.getInt("dfs.datanode.handler.count", 3), false, conf);
     ipcServer.start();
     dnRegistration.setIpcPort(ipcServer.getListenerAddress().getPort());
@@ -1365,12 +1368,18 @@
           " anymore. RackID resolution is handled by the NameNode.");
       System.exit(-1);
     }
-    String[] dataDirs = conf.getStrings(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY);
+    Collection<URI> dataDirs = getStorageDirs(conf);
     dnThreadName = "DataNode: [" +
-                        StringUtils.arrayToString(dataDirs) + "]";
+                    StringUtils.uriToString(dataDirs.toArray(new URI[0])) + "]";
     return makeInstance(dataDirs, conf);
   }
 
+  static Collection<URI> getStorageDirs(Configuration conf) {
+    Collection<String> dirNames =
+      conf.getStringCollection(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY);
+    return Util.stringCollectionAsURIs(dirNames);
+  }
+
   /** Instantiate & Start a single datanode daemon and wait for it to finish.
    *  If this thread is specifically interrupted, it will stop waiting.
    */
@@ -1400,21 +1409,27 @@
    * no directory from this directory list can be created.
    * @throws IOException
    */
-  public static DataNode makeInstance(String[] dataDirs, Configuration conf)
+  static DataNode makeInstance(Collection<URI> dataDirs, Configuration conf)
     throws IOException {
     ArrayList<File> dirs = new ArrayList<File>();
-    for (int i = 0; i < dataDirs.length; i++) {
-      File data = new File(dataDirs[i]);
+    for(URI dirURI : dataDirs) {
+      if(! "file".equalsIgnoreCase(dirURI.getScheme())) {
+        LOG.warn("Unsupported URI schema in " + dirURI  + ". Ignoring ...");
+        continue;
+      }
+      File data = new File(dirURI.getPath());
       try {
         DiskChecker.checkDir(data);
         dirs.add(data);
       } catch(DiskErrorException e) {
-        LOG.warn("Invalid directory in dfs.data.dir: " + e.getMessage());
+        LOG.warn("Invalid directory in "
+            + DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY + ": " + e.getMessage());
       }
     }
     if (dirs.size() > 0) 
       return new DataNode(conf, dirs);
-    LOG.error("All directories in dfs.data.dir are invalid.");
+    LOG.error("All directories in "
+        + DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY + " are invalid.");
     return null;
   }
 

Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=898131&r1=898130&r2=898131&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java Tue Jan 12
00:21:53 2010
@@ -310,6 +310,14 @@
   }
 
   /**
+   * Get configuration.
+   * @return Configuration of this MiniDFSCluster
+   */
+  public Configuration getConfiguration() {
+    return conf;
+  }
+
+  /**
    * wait for the cluster to get out of 
    * safemode.
    */
@@ -415,14 +423,16 @@
           throw new IOException("Mkdirs failed to create directory for DataNode "
                                 + i + ": " + dir1 + " or " + dir2);
         }
-        dnConf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dir1.getPath() + "," + dir2.getPath());

+        dnConf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY,
+                    fileAsURI(dir1) + "," + fileAsURI(dir2));
       }
       if (simulatedCapacities != null) {
         dnConf.setBoolean("dfs.datanode.simulateddatastorage", true);
         dnConf.setLong(SimulatedFSDataset.CONFIG_PROPERTY_CAPACITY,
             simulatedCapacities[i-curDatanodesNum]);
       }
-      System.out.println("Starting DataNode " + i + " with dfs.data.dir: " 
+      System.out.println("Starting DataNode " + i + " with "
+                         + DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY + ": "
                          + dnConf.get(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY));
       if (hosts != null) {
         dnConf.set(DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY, hosts[i - curDatanodesNum]);
@@ -441,6 +451,9 @@
         NetUtils.addStaticResolution(hosts[i - curDatanodesNum], "localhost");
       }
       DataNode dn = DataNode.instantiateDataNode(dnArgs, dnConf);
+      if(dn == null)
+        throw new IOException("Cannot start DataNode in "
+            + dnConf.get(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY));
       //since the HDFS does things based on IP:port, we need to add the mapping
       //for IP:port to rackId
       String ipAddr = dn.getSelfAddr().getAddress().getHostAddress();

Added: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDatanodeConfig.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDatanodeConfig.java?rev=898131&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDatanodeConfig.java (added)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDatanodeConfig.java Tue Jan
12 00:21:53 2010
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import static org.apache.hadoop.hdfs.server.common.Util.fileAsURI;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Tests if a data-node can startup depending on configuration parameters.
+ */
+public class TestDatanodeConfig {
+  private static final File BASE_DIR =
+                                new File(MiniDFSCluster.getBaseDirectory());
+
+  private static MiniDFSCluster cluster;
+
+  @BeforeClass
+  public static void setUp() throws Exception {
+    clearBaseDir();
+    Configuration conf = new HdfsConfiguration();
+    cluster = new MiniDFSCluster(conf, 0, true, null);
+    cluster.waitActive();
+  }
+
+  @AfterClass
+  public static void tearDown() throws Exception {
+    if(cluster != null)
+      cluster.shutdown();
+    clearBaseDir();
+  }
+
+  private static void clearBaseDir() throws IOException {
+    if(BASE_DIR.exists() && ! FileUtil.fullyDelete(BASE_DIR))
+      throw new IOException("Cannot clear BASE_DIR " + BASE_DIR);
+  }
+
+  /**
+   * Test that a data-node does not start if configuration specifies
+   * incorrect URI scheme in data directory.
+   * Test that a data-node starts if data directory is specified as
+   * URI = "file:///path" or as a non URI path.
+   */
+  @Test
+  public void testDataDirectories() throws IOException {
+    File dataDir = new File(BASE_DIR, "data").getCanonicalFile();
+    Configuration conf = cluster.getConfiguration();
+    // 1. Test unsupported schema. Only "file:" is supported.
+    String dnDir = makeURI("shv", null, fileAsURI(dataDir).getPath());
+    conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dnDir);
+    DataNode dn = DataNode.createDataNode(new String[]{}, conf);
+    assertNull("Data-node startup should have failed.", dn);
+
+    // 2. Test "file:" schema and no schema (path-only). Both should work.
+    String dnDir1 = fileAsURI(dataDir).toString() + "1";
+    String dnDir2 = makeURI("file", "localhost",
+                    fileAsURI(dataDir).getPath() + "2");
+    String dnDir3 = dataDir.getAbsolutePath() + "3";
+    conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY,
+                dnDir1 + "," + dnDir2 + "," + dnDir3);
+    cluster.startDataNodes(conf, 1, false, StartupOption.REGULAR, null);
+    assertTrue("Data-node should startup.", cluster.isDataNodeUp());
+  }
+
+  private static String makeURI(String scheme, String host, String path)
+  throws IOException {
+    try {
+      URI uDir = new URI(scheme, host, path, null);
+      return uDir.toString();
+    } catch(URISyntaxException e) {
+      throw new IOException("Bad URI", e);
+    }
+  }
+}

Propchange: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDatanodeConfig.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java?rev=898131&r1=898130&r2=898131&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java
(original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java
Tue Jan 12 00:21:53 2010
@@ -82,9 +82,7 @@
  
   protected void setUp() throws Exception {
     config = new HdfsConfiguration();
-    String baseDir = System.getProperty("test.build.data",  "build/test/data");
-    
-    hdfsDir = new File(baseDir, "dfs");
+    hdfsDir = new File(MiniDFSCluster.getBaseDirectory()).getCanonicalFile();
     if ( hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir) ) {
       throw new IOException("Could not delete hdfs directory '" + hdfsDir + "'");
     }
@@ -133,8 +131,7 @@
     Iterator<StorageDirectory> it = fi.dirIterator();
     while(it.hasNext()) {
       StorageDirectory sd = it.next();
-      if(sd.getRoot().getAbsolutePath().equals(path2.getAbsolutePath()) ||
-          sd.getRoot().getAbsolutePath().equals(path3.getAbsolutePath())) {
+      if(sd.getRoot().equals(path2) || sd.getRoot().equals(path3)) {
         al.add(sd);
       }
     }
@@ -260,8 +257,8 @@
       // should be different
       //assertTrue(fsImg1.length() != fsImg2.length());
       //assertTrue(fsImg1.length() != fsImg3.length());
-      assertTrue(fsEdits1.length() != fsEdits2.length());
-      assertTrue(fsEdits1.length() != fsEdits3.length());
+      assertTrue("edits1 = edits2", fsEdits1.length() != fsEdits2.length());
+      assertTrue("edits1 = edits3", fsEdits1.length() != fsEdits3.length());
       
       assertTrue(!md5_1.equals(md5_2));
       assertTrue(!md5_1.equals(md5_3));



Mime
View raw message