hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ste...@apache.org
Subject svn commit: r897029 [1/4] - in /hadoop/common/branches/HADOOP-6194: ./ bin/ src/contrib/cloud/ src/contrib/cloud/lib/ src/contrib/cloud/src/ src/contrib/cloud/src/integration-test/ src/contrib/cloud/src/py/ src/contrib/cloud/src/py/hadoop/ src/contrib/...
Date Thu, 07 Jan 2010 22:04:43 GMT
Author: stevel
Date: Thu Jan  7 22:04:37 2010
New Revision: 897029

URL: http://svn.apache.org/viewvc?rev=897029&view=rev
Log:
Merge with SVN_HEAD of 2010-01-07

Added:
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/
      - copied from r888072, hadoop/common/trunk/src/contrib/cloud/
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/README.txt
      - copied, changed from r888072, hadoop/common/trunk/src/contrib/cloud/README.txt
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/build.xml
      - copied unchanged from r897004, hadoop/common/trunk/src/contrib/cloud/build.xml
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/lib/
      - copied from r897004, hadoop/common/trunk/src/contrib/cloud/lib/
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/lib/pyAntTasks-1.3-LICENSE.txt
      - copied unchanged from r897004, hadoop/common/trunk/src/contrib/cloud/lib/pyAntTasks-1.3-LICENSE.txt
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/lib/pyAntTasks-1.3.jar
      - copied unchanged from r897004, hadoop/common/trunk/src/contrib/cloud/lib/pyAntTasks-1.3.jar
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/
      - copied from r888072, hadoop/common/trunk/src/contrib/cloud/src/
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/integration-test/
      - copied from r888072, hadoop/common/trunk/src/contrib/cloud/src/integration-test/
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/integration-test/create-ebs-snapshot.sh
      - copied unchanged from r888072, hadoop/common/trunk/src/contrib/cloud/src/integration-test/create-ebs-snapshot.sh
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/integration-test/ebs-storage-spec.json
      - copied, changed from r888072, hadoop/common/trunk/src/contrib/cloud/src/integration-test/ebs-storage-spec.json
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/integration-test/persistent-cluster.sh
      - copied, changed from r888072, hadoop/common/trunk/src/contrib/cloud/src/integration-test/persistent-cluster.sh
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/integration-test/transient-cluster.sh
      - copied, changed from r888072, hadoop/common/trunk/src/contrib/cloud/src/integration-test/transient-cluster.sh
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/
      - copied from r888072, hadoop/common/trunk/src/contrib/cloud/src/py/
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/
      - copied from r888072, hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop-ec2   (props changed)
      - copied unchanged from r888072, hadoop/common/trunk/src/contrib/cloud/src/py/hadoop-ec2
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/__init__.py
      - copied unchanged from r888072, hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/__init__.py
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/
      - copied from r888072, hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/__init__.py
      - copied, changed from r888072, hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/__init__.py
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/cli.py
      - copied, changed from r888072, hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/cli.py
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/cluster.py
      - copied, changed from r888072, hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/cluster.py
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/data/
      - copied from r897004, hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/data/
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/data/hadoop-ec2-init-remote.sh
      - copied unchanged from r897004, hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/data/hadoop-ec2-init-remote.sh
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/data/zookeeper-ec2-init-remote.sh
      - copied unchanged from r897004, hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/data/zookeeper-ec2-init-remote.sh
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/providers/
      - copied from r888072, hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/providers/
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/providers/__init__.py
      - copied unchanged from r888072, hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/providers/__init__.py
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/providers/dummy.py
      - copied unchanged from r888072, hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/providers/dummy.py
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/providers/ec2.py
      - copied, changed from r888072, hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/providers/ec2.py
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/service.py
      - copied unchanged from r897004, hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/service.py
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/storage.py
      - copied, changed from r888072, hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/storage.py
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/util.py
      - copied unchanged from r888072, hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/util.py
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/setup.py
      - copied unchanged from r897004, hadoop/common/trunk/src/contrib/cloud/src/py/setup.py
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/test/
      - copied from r888072, hadoop/common/trunk/src/contrib/cloud/src/test/
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/test/hadoop/
      - copied from r888072, hadoop/common/trunk/src/contrib/cloud/src/test/hadoop/
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/test/hadoop/cloud/
      - copied from r888072, hadoop/common/trunk/src/contrib/cloud/src/test/hadoop/cloud/
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/test/py/
      - copied from r897004, hadoop/common/trunk/src/contrib/cloud/src/test/py/
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/test/py/testcluster.py
      - copied unchanged from r897004, hadoop/common/trunk/src/contrib/cloud/src/test/py/testcluster.py
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/test/py/teststorage.py
      - copied unchanged from r897004, hadoop/common/trunk/src/contrib/cloud/src/test/py/teststorage.py
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/test/py/testuserdata.py
      - copied unchanged from r897004, hadoop/common/trunk/src/contrib/cloud/src/test/py/testuserdata.py
    hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/test/py/testutil.py
      - copied unchanged from r897004, hadoop/common/trunk/src/contrib/cloud/src/test/py/testutil.py
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/conf/ConfServlet.java
      - copied unchanged from r897004, hadoop/common/trunk/src/java/org/apache/hadoop/conf/ConfServlet.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/io/serializer/avro/AvroComparator.java
      - copied unchanged from r897004, hadoop/common/trunk/src/java/org/apache/hadoop/io/serializer/avro/AvroComparator.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/ipc/AvroRpcEngine.java
      - copied unchanged from r897004, hadoop/common/trunk/src/java/org/apache/hadoop/ipc/AvroRpcEngine.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/ipc/RpcEngine.java
      - copied unchanged from r897004, hadoop/common/trunk/src/java/org/apache/hadoop/ipc/RpcEngine.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/ipc/WritableRpcEngine.java
      - copied unchanged from r897004, hadoop/common/trunk/src/java/org/apache/hadoop/ipc/WritableRpcEngine.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/security/GroupMappingServiceProvider.java
      - copied unchanged from r897004, hadoop/common/trunk/src/java/org/apache/hadoop/security/GroupMappingServiceProvider.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/security/Groups.java
      - copied unchanged from r897004, hadoop/common/trunk/src/java/org/apache/hadoop/security/Groups.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/security/RefreshUserToGroupMappingsProtocol.java
      - copied unchanged from r897004, hadoop/common/trunk/src/java/org/apache/hadoop/security/RefreshUserToGroupMappingsProtocol.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
      - copied unchanged from r897004, hadoop/common/trunk/src/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/security/token/
      - copied from r897004, hadoop/common/trunk/src/java/org/apache/hadoop/security/token/
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/security/token/SecretManager.java
      - copied unchanged from r897004, hadoop/common/trunk/src/java/org/apache/hadoop/security/token/SecretManager.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/security/token/Token.java
      - copied unchanged from r897004, hadoop/common/trunk/src/java/org/apache/hadoop/security/token/Token.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/security/token/TokenIdentifier.java
      - copied unchanged from r897004, hadoop/common/trunk/src/java/org/apache/hadoop/security/token/TokenIdentifier.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/util/AsyncDiskService.java
      - copied unchanged from r897004, hadoop/common/trunk/src/java/org/apache/hadoop/util/AsyncDiskService.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/cli/CLITestHelper.java
      - copied unchanged from r897004, hadoop/common/trunk/src/test/core/org/apache/hadoop/cli/CLITestHelper.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/cli/util/CLICommands.java
      - copied unchanged from r897004, hadoop/common/trunk/src/test/core/org/apache/hadoop/cli/util/CLICommands.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/cli/util/CmdFactory.java
      - copied unchanged from r897004, hadoop/common/trunk/src/test/core/org/apache/hadoop/cli/util/CmdFactory.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/conf/TestConfServlet.java
      - copied unchanged from r897004, hadoop/common/trunk/src/test/core/org/apache/hadoop/conf/TestConfServlet.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/fs/FileContextTestHelper.java
      - copied unchanged from r897004, hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/FileContextTestHelper.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/http/TestHttpServerLifecycle.java
      - copied unchanged from r897004, hadoop/common/trunk/src/test/core/org/apache/hadoop/http/TestHttpServerLifecycle.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/io/serializer/TestRawComparators.java
      - copied unchanged from r897004, hadoop/common/trunk/src/test/core/org/apache/hadoop/io/serializer/TestRawComparators.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/security/token/
      - copied from r897004, hadoop/common/trunk/src/test/core/org/apache/hadoop/security/token/
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/security/token/TestToken.java
      - copied unchanged from r897004, hadoop/common/trunk/src/test/core/org/apache/hadoop/security/token/TestToken.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/util/TestAsyncDiskService.java
      - copied unchanged from r897004, hadoop/common/trunk/src/test/core/org/apache/hadoop/util/TestAsyncDiskService.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/util/TestReflectionUtils.java
      - copied unchanged from r897004, hadoop/common/trunk/src/test/core/org/apache/hadoop/util/TestReflectionUtils.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/util/TestRunJar.java
      - copied unchanged from r897004, hadoop/common/trunk/src/test/core/org/apache/hadoop/util/TestRunJar.java
Removed:
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/ipc/AvroRpc.java
Modified:
    hadoop/common/branches/HADOOP-6194/   (props changed)
    hadoop/common/branches/HADOOP-6194/CHANGES.txt   (contents, props changed)
    hadoop/common/branches/HADOOP-6194/bin/hadoop
    hadoop/common/branches/HADOOP-6194/bin/hdfs
    hadoop/common/branches/HADOOP-6194/ivybuild.xml
    hadoop/common/branches/HADOOP-6194/src/contrib/ec2/   (props changed)
    hadoop/common/branches/HADOOP-6194/src/docs/   (props changed)
    hadoop/common/branches/HADOOP-6194/src/java/   (props changed)
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/conf/Configuration.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/ChecksumFs.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/DF.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FSInputChecker.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FsShell.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/http/HttpServer.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/io/serializer/JavaSerialization.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/io/serializer/LegacySerialization.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/io/serializer/SerializationBase.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/io/serializer/WritableSerialization.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/io/serializer/avro/AvroGenericSerialization.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/io/serializer/avro/AvroReflectSerialization.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/io/serializer/avro/AvroSerialization.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/io/serializer/avro/AvroSpecificSerialization.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/ipc/RPC.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/ipc/Server.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/security/SecurityUtil.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/security/UnixUserGroupInformation.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/util/GenericOptionsParser.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/util/RunJar.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/util/Service.java
    hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/util/Shell.java
    hadoop/common/branches/HADOOP-6194/src/test/core/   (props changed)
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/cli/TestCLI.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/cli/testConf.xml
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/cli/testConf.xsl
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/cli/util/CommandExecutor.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/conf/TestConfiguration.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/fs/FileContextCreateMkdirBaseTest.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/fs/FileContextPermissionBase.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/fs/FileContextURIBase.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/fs/TestChecksumFileSystem.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/fs/TestDFVariations.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/fs/TestFileContextDeleteOnExit.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/io/TestUTF8.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/io/serializer/TestWritableSerialization.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/io/serializer/avro/TestAvroSerialization.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/ipc/TestAvroRpc.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/ipc/TestIPCServerResponder.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/ipc/TestRPC.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/security/TestUnixUserGroupInformation.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/security/authorize/TestConfiguredPolicy.java
    hadoop/common/branches/HADOOP-6194/src/test/core/org/apache/hadoop/util/TestGenericOptionsParser.java

Propchange: hadoop/common/branches/HADOOP-6194/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jan  7 22:04:37 2010
@@ -1,2 +1,2 @@
-/hadoop/common/trunk:804966-888120
+/hadoop/common/trunk:804966-897004
 /hadoop/core/branches/branch-0.19/core:713112

Modified: hadoop/common/branches/HADOOP-6194/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/CHANGES.txt?rev=897029&r1=897028&r2=897029&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/CHANGES.txt (original)
+++ hadoop/common/branches/HADOOP-6194/CHANGES.txt Thu Jan  7 22:04:37 2010
@@ -16,6 +16,28 @@
 
     HADOOP-6108. Add support for EBS storage on EC2. (tomwhite)
 
+    Hadoop-6223. Add new file system interface AbstractFileSystem with
+    implementation of some file systems that delegate to old FileSystem.
+    (Sanjay Radia via suresh)
+
+    HADOOP-6392. Run namenode and jobtracker on separate EC2 instances.
+    (tomwhite)
+
+    HADOOP-6323. Add comparators to the serialization API.
+    (Aaron Kimball via cutting)
+
+    HADOOP-6433. Introduce asychronous deletion of files via a pool of
+    threads. This can be used to delete files in the Distributed
+    Cache. (Zheng Shao via dhruba)
+
+    HADOOP-6415. Adds a common token interface for both job token and 
+    delegation token. (Kan Zhang via ddas)
+
+    HADOOP-6466. Add a ZooKeeper service to the cloud scripts. (tomwhite)
+
+    HADOOP-6408. Add a /conf servlet to dump running configuration.
+    (Todd Lipcon via tomwhite)
+
   IMPROVEMENTS
 
     HADOOP-6283. Improve the exception messages thrown by
@@ -27,19 +49,54 @@
     HADOOP-6305. Unify build property names to facilitate cross-projects
     modifications (cos)
 
-    HADOOP-6204. Implementing aspects development and fault injeciton
-    framework for Hadoop (cos)
-
     HADOOP-6312. Remove unnecessary debug logging in Configuration constructor.
     (Aaron Kimball via cdouglas)
 
-    HADOOP-6326. Hundson runs should check for AspectJ warnings and report
-    failure if any is present (cos)
-
     HADOOP-6366. Reduce ivy console output to ovservable level (cos)
 
     HADOOP-6400. Log errors getting Unix UGI. (Todd Lipcon via tomwhite)
 
+    HADOOP-6346. Add support for specifying unpack pattern regex to
+    RunJar.unJar. (Todd Lipcon via tomwhite)
+
+    HADOOP-6422. Make RPC backend plugable, protocol-by-protocol, to
+    ease evolution towards Avro.  (cutting)
+
+    HADOOP-5958. Use JDK 1.6 File APIs in DF.java wherever possible.
+    (Aaron Kimball via tomwhite)
+
+    HADOOP-6222. Core doesn't have TestCommonCLI facility. (cos)
+
+    HADOOP-6394. Add a helper class to simplify FileContext related tests and
+    improve code reusability. (Jitendra Nath Pandey via suresh)
+
+    HADOOP-6426. Create ant build for running EC2 unit tests. (tomwhite)
+
+    HADOOP-4656. Add a user to groups mapping service. (boryas, acmurthy)
+
+    HADOOP-6444. Support additional security group option in hadoop-ec2 script.
+    (Paul Egan via tomwhite)
+
+    HADOOP-6454. Create setup.py for EC2 cloud scripts. (tomwhite)
+
+    HADOOP-6435. Make RPC.waitForProxy with timeout public. (Steve Loughran
+    via tomwhite)
+  
+    HADOOP-6472. add tokenCache option to GenericOptionsParser for passing
+     file with secret keys to a map reduce job. (boryas)
+
+    HADOOP-6443. Serialization classes accept invalid metadata.
+    (Aaron Kimball via tomwhite)
+
+    HADOOP-3205. Read multiple chunks directly from FSInputChecker subclass
+    into user buffers. (Todd Lipcon via tomwhite)
+
+    HADOOP-6479. TestUTF8 assertions could fail with better text.
+    (Steve Loughran via tomwhite)
+
+    HADOOP-6420. Add functionality permitting subsets of Configuration to be
+    interpreted as Map<String,String>. (Aaron Kimball via cdouglas)
+
   OPTIMIZATIONS
 
   BUG FIXES
@@ -52,13 +109,30 @@
     HADOOP-6314. Fix "fs -help" for the "-count" commond.  (Ravi Phulari via
     szetszwo)
 
-    HADOOP-6398. Build is broken after HADOOP-6395 patch has been applied (cos)
-
     HADOOP-6405. Update Eclipse configuration to match changes to Ivy
     configuration (Edwin Chan via cos)
 
     HADOOP-6411. Remove deprecated file src/test/hadoop-site.xml. (cos)
 
+    HADOOP-6386. NameNode's HttpServer can't instantiate InetSocketAddress:
+    IllegalArgumentException is thrown (cos)
+
+    HADOOP-6254. Slow reads cause s3n to fail with SocketTimeoutException.
+    (Andrew Hitchcock via tomwhite)
+
+    HADOOP-6428. HttpServer sleeps with negative values. (cos)
+
+    HADOOP-6414. Add command line help for -expunge command.
+    (Ravi Phulari via tomwhite)
+
+    HADOOP-6391. Classpath should not be part of command line arguments.
+    (Cristian Ivascu via tomwhite)
+
+    HADOOP-6462. Target "compile" does not exist in contrib/cloud. (tomwhite)
+
+    HADOOP-6402. testConf.xsl is not well-formed XML. (Steve Loughran
+    via tomwhite)
+
 Release 0.21.0 - Unreleased
 
   INCOMPATIBLE CHANGES
@@ -271,13 +345,12 @@
     HADOOP-6240. Add new FileContext rename operation that posix compliant
     that allows overwriting existing destination. (suresh)
 
+    HADOOP-6204. Implementing aspects development and fault injeciton
+    framework for Hadoop (cos)
+
     HADOOP-6313. Implement Syncable interface in FSDataOutputStream to expose
     flush APIs to application users. (Hairong Kuang via suresh)
 
-    Hadoop-6223. Add new file system interface AbstractFileSystem with
-    implementation of some file systems that delegate to old FileSystem.
-    (Sanjay Radia via suresh)
-
   IMPROVEMENTS
 
     HADOOP-4565. Added CombineFileInputFormat to use data locality information
@@ -659,6 +732,9 @@
     HADOOP-6309. Change build.xml to run tests with java asserts.  (Eli
     Collins via szetszwo)
 
+    HADOOP-6326. Hundson runs should check for AspectJ warnings and report
+    failure if any is present (cos)
+
     HADOOP-6329. Add build-fi directory to the ignore lists.  (szetszwo)
 
     HADOOP-5107. Use Maven ant tasks to publish the subproject jars.
@@ -673,6 +749,10 @@
     HADOOP-6395. Upgrade some libraries to be consistent across common, hdfs,
     and mapreduce. (omalley)
 
+    HADOOP-6398. Build is broken after HADOOP-6395 patch has been applied (cos)
+
+    HADOOP-6413. Move TestReflectionUtils to Common. (Todd Lipcon via tomwhite)
+
   OPTIMIZATIONS
 
     HADOOP-5595. NameNode does not need to run a replicator to choose a
@@ -1185,6 +1265,9 @@
     HADOOP-6375. Sync documentation for FsShell du with its implementation.
     (Todd Lipcon via cdouglas)
 
+    HADOOP-6441. Protect web ui from cross site scripting attacks (XSS) on
+    the host http header and using encoded utf-7. (omalley)
+
 Release 0.20.2 - Unreleased
 
   NEW FEATURES
@@ -1207,6 +1290,10 @@
     HADOOP-6269. Fix threading issue with defaultResource in Configuration.
     (Sreekanth Ramakrishnan via cdouglas)
 
+    HADOOP-6460. Reinitializes buffers used for serializing responses in ipc
+    server on exceeding maximum response size to free up Java heap. (suresh)
+    
+
 Release 0.20.1 - 2009-09-01
 
   INCOMPATIBLE CHANGES

Propchange: hadoop/common/branches/HADOOP-6194/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jan  7 22:04:37 2010
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/CHANGES.txt:804966-888120
+/hadoop/common/trunk/CHANGES.txt:804966-897004
 /hadoop/core/branches/branch-0.18/CHANGES.txt:727226
 /hadoop/core/branches/branch-0.19/CHANGES.txt:713112
 /hadoop/core/trunk/CHANGES.txt:776175-785643,785929-786278

Modified: hadoop/common/branches/HADOOP-6194/bin/hadoop
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/bin/hadoop?rev=897029&r1=897028&r2=897029&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/bin/hadoop (original)
+++ hadoop/common/branches/HADOOP-6194/bin/hadoop Thu Jan  7 22:04:37 2010
@@ -112,7 +112,8 @@
     if $cygwin; then
       CLASSPATH=`cygpath -p -w "$CLASSPATH"`
     fi
-    exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS -classpath "$CLASSPATH" $CLASS "$@"
+    export CLASSPATH=$CLASSPATH
+    exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS $CLASS "$@"
     ;;
 
 esac

Modified: hadoop/common/branches/HADOOP-6194/bin/hdfs
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/bin/hdfs?rev=897029&r1=897028&r2=897029&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/bin/hdfs (original)
+++ hadoop/common/branches/HADOOP-6194/bin/hdfs Thu Jan  7 22:04:37 2010
@@ -106,5 +106,5 @@
 if $cygwin; then
   CLASSPATH=`cygpath -p -w "$CLASSPATH"`
 fi
-
-exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS -classpath "$CLASSPATH" $CLASS "$@"
+export CLASSPATH=$CLASSPATH
+exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS $CLASS "$@"

Modified: hadoop/common/branches/HADOOP-6194/ivybuild.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/ivybuild.xml?rev=897029&r1=897028&r2=897029&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/ivybuild.xml (original)
+++ hadoop/common/branches/HADOOP-6194/ivybuild.xml Thu Jan  7 22:04:37 2010
@@ -379,10 +379,6 @@
       <arg value="changelist"/>
       <arg value="${ivy.issue}"/>
       <!-- core -->
-      <!-- question this one -->
-      <!--
-      <arg value="${core/hadoop}/security/authorize/ConfiguredPolicy.java" />
-      -->
       <arg value="build.xml" />
       <arg value="ivy/hadoop-core.pom" />
       <arg value="ivy/libraries.properties" />

Copied: hadoop/common/branches/HADOOP-6194/src/contrib/cloud/README.txt (from r888072, hadoop/common/trunk/src/contrib/cloud/README.txt)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/contrib/cloud/README.txt?p2=hadoop/common/branches/HADOOP-6194/src/contrib/cloud/README.txt&p1=hadoop/common/trunk/src/contrib/cloud/README.txt&r1=888072&r2=897029&rev=897029&view=diff
==============================================================================
--- hadoop/common/trunk/src/contrib/cloud/README.txt (original)
+++ hadoop/common/branches/HADOOP-6194/src/contrib/cloud/README.txt Thu Jan  7 22:04:37 2010
@@ -61,8 +61,18 @@
 
 % hadoop-ec2 launch-cluster my-hadoop-cluster 10
 
-This will boot the master node and 10 worker nodes. When the nodes have started
-and the Hadoop cluster has come up, the console will display a message like
+This will boot the master node and 10 worker nodes. The master node runs the
+namenode, secondary namenode, and jobtracker, and each worker node runs a
+datanode and a tasktracker. Equivalently the cluster could be launched as:
+
+% hadoop-ec2 launch-cluster my-hadoop-cluster 1 nn,snn,jt 10 dn,tt
+
+Note that using this notation you can launch a split namenode/jobtracker cluster
+
+% hadoop-ec2 launch-cluster my-hadoop-cluster 1 nn,snn 1 jt 10 dn,tt
+
+When the nodes have started and the Hadoop cluster has come up, the console will
+display a message like
 
   Browse the cluster at http://ec2-xxx-xxx-xxx-xxx.compute-1.amazonaws.com/
 
@@ -70,7 +80,8 @@
 opened for access from your client machine. You may change the firewall settings
 (to allow access from a network, rather than just a single machine, for example)
 by using the Amazon EC2 command line tools, or by using a tool like Elastic Fox.
-The security group to change is the one named <cluster-name>-master.
+There is a security group for each node's role. The one for the namenode
+is <cluster-name>-nn, for example.
 
 For security reasons, traffic from the network your client is running on is
 proxied through the master node of the cluster using an SSH tunnel (a SOCKS
@@ -109,13 +120,13 @@
 
 % hadoop-ec2 create-formatted-snapshot my-ebs-cluster 100
 
-We create storage for a single master and for two slaves. The volumes to create
-are described in a JSON spec file, which references the snapshot we just
+We create storage for a single namenode and for two datanodes. The volumes to
+create are described in a JSON spec file, which references the snapshot we just
 created. Here is the contents of a JSON file, called
 my-ebs-cluster-storage-spec.json:
 
 {
-  "master": [
+  "nn": [
     {
       "device": "/dev/sdj",
       "mount_point": "/ebs1",
@@ -129,7 +140,7 @@
       "snapshot_id": "snap-268e704f"
     }
   ],
-  "slave": [
+  "dn": [
     {
       "device": "/dev/sdj",
       "mount_point": "/ebs1",
@@ -146,7 +157,7 @@
 }
 
 
-Each role (here "master" and "slave") is the key to an array of volume
+Each role (here "nn" and "dn") is the key to an array of volume
 specifications. In this example, the "slave" role has two devices ("/dev/sdj"
 and "/dev/sdk") with different mount points, sizes, and generated from an EBS
 snapshot. The snapshot is the formatted snapshot created earlier, so that the
@@ -155,9 +166,9 @@
 
 Let's create actual volumes using this file.
 
-% hadoop-ec2 create-storage my-ebs-cluster master 1 \
+% hadoop-ec2 create-storage my-ebs-cluster nn 1 \
     my-ebs-cluster-storage-spec.json
-% hadoop-ec2 create-storage my-ebs-cluster slave 2 \
+% hadoop-ec2 create-storage my-ebs-cluster dn 2 \
     my-ebs-cluster-storage-spec.json
 
 Now let's start the cluster with 2 slave nodes:
@@ -214,7 +225,7 @@
 
 Of course, these examples assume that you have installed Hadoop on your local
 machine. It is also possible to launch jobs from within the cluster. First log
-into the master node:
+into the namenode:
 
 % hadoop-ec2 login my-hadoop-cluster
 
@@ -305,3 +316,24 @@
 It's possible to use any image, as long as it i) runs (gzip compressed) user
 data on boot, and ii) has Java installed.
 
+OTHER SERVICES
+==============
+
+ZooKeeper
+=========
+
+You can run ZooKeeper by setting the "service" parameter to "zookeeper". For
+example:
+
+[my-zookeeper-cluster]
+service=zookeeper
+ami=ami-ed59bf84
+instance_type=m1.small
+key_name=tom
+availability_zone=us-east-1c
+public_key=PATH_TO_PUBLIC_KEY
+private_key=PATH_TO_PRIVATE_KEY
+
+Then to launch a three-node ZooKeeper ensemble, run:
+
+% ./hadoop-ec2 launch-cluster my-zookeeper-cluster 3 zk

Copied: hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/integration-test/ebs-storage-spec.json (from r888072, hadoop/common/trunk/src/contrib/cloud/src/integration-test/ebs-storage-spec.json)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/integration-test/ebs-storage-spec.json?p2=hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/integration-test/ebs-storage-spec.json&p1=hadoop/common/trunk/src/contrib/cloud/src/integration-test/ebs-storage-spec.json&r1=888072&r2=897029&rev=897029&view=diff
==============================================================================
--- hadoop/common/trunk/src/contrib/cloud/src/integration-test/ebs-storage-spec.json (original)
+++ hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/integration-test/ebs-storage-spec.json Thu Jan  7 22:04:37 2010
@@ -1,5 +1,5 @@
 {
-  "master": [
+  "nn": [
     {
       "device": "/dev/sdj",
       "mount_point": "/ebs1",
@@ -13,7 +13,7 @@
       "snapshot_id": "snap-fe44bb97"
     }
   ],
-  "slave": [
+  "dn": [
     {
       "device": "/dev/sdj",
       "mount_point": "/ebs1",

Copied: hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/integration-test/persistent-cluster.sh (from r888072, hadoop/common/trunk/src/contrib/cloud/src/integration-test/persistent-cluster.sh)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/integration-test/persistent-cluster.sh?p2=hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/integration-test/persistent-cluster.sh&p1=hadoop/common/trunk/src/contrib/cloud/src/integration-test/persistent-cluster.sh&r1=888072&r2=897029&rev=897029&view=diff
==============================================================================
--- hadoop/common/trunk/src/contrib/cloud/src/integration-test/persistent-cluster.sh (original)
+++ hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/integration-test/persistent-cluster.sh Thu Jan  7 22:04:37 2010
@@ -71,10 +71,10 @@
 
 # Create storage
 $HADOOP_CLOUD_SCRIPT create-storage --config-dir=$CONFIG_DIR \
-  --availability-zone=$AVAILABILITY_ZONE $CLUSTER master 1 \
+  --availability-zone=$AVAILABILITY_ZONE $CLUSTER nn 1 \
   $bin/ebs-storage-spec.json
 $HADOOP_CLOUD_SCRIPT create-storage --config-dir=$CONFIG_DIR \
-  --availability-zone=$AVAILABILITY_ZONE $CLUSTER slave 1 \
+  --availability-zone=$AVAILABILITY_ZONE $CLUSTER dn 1 \
   $bin/ebs-storage-spec.json
 
 # Launch a cluster

Copied: hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/integration-test/transient-cluster.sh (from r888072, hadoop/common/trunk/src/contrib/cloud/src/integration-test/transient-cluster.sh)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/integration-test/transient-cluster.sh?p2=hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/integration-test/transient-cluster.sh&p1=hadoop/common/trunk/src/contrib/cloud/src/integration-test/transient-cluster.sh&r1=888072&r2=897029&rev=897029&view=diff
==============================================================================
--- hadoop/common/trunk/src/contrib/cloud/src/integration-test/transient-cluster.sh (original)
+++ hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/integration-test/transient-cluster.sh Thu Jan  7 22:04:37 2010
@@ -41,6 +41,7 @@
 HADOOP_CLOUD_PROVIDER=${HADOOP_CLOUD_PROVIDER:-ec2}
 SSH_OPTIONS=${SSH_OPTIONS:-"-i ~/.$HADOOP_CLOUD_PROVIDER/id_rsa-$KEY_NAME \
   -o StrictHostKeyChecking=no"}
+LAUNCH_ARGS=${LAUNCH_ARGS:-1} # Try LAUNCH_ARGS="1 nn,snn 1 jt 1 dn,tt"
 
 HADOOP_CLOUD_SCRIPT=$HADOOP_CLOUD_HOME/hadoop-$HADOOP_CLOUD_PROVIDER
 export HADOOP_CONF_DIR=$CONFIG_DIR/$CLUSTER
@@ -56,7 +57,12 @@
 # Launch a cluster
 $HADOOP_CLOUD_SCRIPT launch-cluster --config-dir=$CONFIG_DIR \
   --image-id=$IMAGE_ID --key-name=$KEY_NAME --auto-shutdown=$AUTO_SHUTDOWN \
-  --availability-zone=$AVAILABILITY_ZONE $CLIENT_CIDRS $ENVS $CLUSTER 1
+  --availability-zone=$AVAILABILITY_ZONE $CLIENT_CIDRS $ENVS $CLUSTER \
+  $LAUNCH_ARGS
+  
+# List clusters
+$HADOOP_CLOUD_SCRIPT list --config-dir=$CONFIG_DIR
+$HADOOP_CLOUD_SCRIPT list --config-dir=$CONFIG_DIR $CLUSTER
 
 # Run a proxy and save its pid in HADOOP_CLOUD_PROXY_PID
 eval `$HADOOP_CLOUD_SCRIPT proxy --config-dir=$CONFIG_DIR \

Propchange: hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop-ec2
------------------------------------------------------------------------------
    svn:executable = *

Copied: hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/__init__.py (from r888072, hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/__init__.py)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/__init__.py?p2=hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/__init__.py&p1=hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/__init__.py&r1=888072&r2=897029&rev=897029&view=diff
==============================================================================
--- hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/__init__.py (original)
+++ hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/__init__.py Thu Jan  7 22:04:37 2010
@@ -11,4 +11,5 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations under the License.
\ No newline at end of file
+# limitations under the License.
+VERSION="0.22.0"
\ No newline at end of file

Copied: hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/cli.py (from r888072, hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/cli.py)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/cli.py?p2=hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/cli.py&p1=hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/cli.py&r1=888072&r2=897029&rev=897029&view=diff
==============================================================================
--- hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/cli.py (original)
+++ hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/cli.py Thu Jan  7 22:04:37 2010
@@ -16,22 +16,24 @@
 from __future__ import with_statement
 
 import ConfigParser
-import hadoop.cloud.commands as commands
+from hadoop.cloud import VERSION
 from hadoop.cloud.cluster import get_cluster
-from hadoop.cloud.cluster import TimeoutException
-from hadoop.cloud.providers.ec2 import Ec2Storage
+from hadoop.cloud.service import get_service
+from hadoop.cloud.service import InstanceTemplate
+from hadoop.cloud.service import NAMENODE
+from hadoop.cloud.service import SECONDARY_NAMENODE
+from hadoop.cloud.service import JOBTRACKER
+from hadoop.cloud.service import DATANODE
+from hadoop.cloud.service import TASKTRACKER
 from hadoop.cloud.util import merge_config_with_options
 from hadoop.cloud.util import xstr
 import logging
 from optparse import OptionParser
 from optparse import make_option
 import os
-import subprocess
 import sys
 
-version_file = os.path.join(sys.path[0], "VERSION")
-VERSION = open(version_file, "r").read().strip()
-
+DEFAULT_SERVICE_NAME = 'hadoop'
 DEFAULT_CLOUD_PROVIDER = 'ec2'
 
 DEFAULT_CONFIG_DIR_NAME = '.hadoop-cloud'
@@ -81,6 +83,9 @@
   make_option("--client-cidr", metavar="CIDR", action="append",
     help="The CIDR of the client, which is used to allow access through the \
 firewall to the master node. (May be specified multiple times.)"),
+  make_option("--security-group", metavar="SECURITY_GROUP", action="append",
+    default=[], help="Additional security groups within which the instances \
+should be run. (Amazon EC2 only.) (May be specified multiple times.)"),
   make_option("--public-key", metavar="FILE",
     help="The public key to authorize on launching instances. (Non-EC2 \
 providers only.)"),
@@ -125,8 +130,8 @@
                                         or instances in CLUSTER
   launch-master CLUSTER               launch or find a master in CLUSTER
   launch-slaves CLUSTER NUM_SLAVES    launch NUM_SLAVES slaves in CLUSTER
-  launch-cluster CLUSTER NUM_SLAVES   launch a master and NUM_SLAVES slaves
-                                        in CLUSTER
+  launch-cluster CLUSTER (NUM_SLAVES| launch a master and NUM_SLAVES slaves or
+    N ROLE [N ROLE ...])                N instances in ROLE in CLUSTER
   create-formatted-snapshot CLUSTER   create an empty, formatted snapshot of
     SIZE                                size SIZE GiB
   list-storage CLUSTER                list storage volumes for CLUSTER
@@ -147,6 +152,9 @@
 Use %(script)s COMMAND --help to see additional options for specific commands.
 """ % locals()
 
+def print_deprecation(script, replacement):
+  print "Deprecated. Use '%(script)s %(replacement)s'." % locals()
+
 def parse_options_and_config(command, option_list=[], extra_arguments=(),
                              unbounded_args=False):
   """
@@ -176,8 +184,11 @@
   cluster_name = args[0]
   opt = merge_config_with_options(cluster_name, config, options_dict)
   logging.debug("Options: %s", str(opt))
-  return (opt, args, get_cluster(get_cloud_provider(opt))(cluster_name,
-                                                          config_dir))
+  service_name = get_service_name(opt)
+  cloud_provider = get_cloud_provider(opt)
+  cluster = get_cluster(cloud_provider)(cluster_name, config_dir)
+  service = get_service(service_name, cloud_provider)(cluster)
+  return (opt, args, service)
 
 def parse_options(command, option_list=[], expected_arguments=(),
                   unbounded_args=False):
@@ -215,6 +226,12 @@
     config_dir = DEFAULT_CONFIG_DIR
   return config_dir
 
+def get_service_name(options_dict):
+  service_name = options_dict.get("service", None)
+  if service_name is None:
+    service_name = DEFAULT_SERVICE_NAME
+  return service_name
+
 def get_cloud_provider(options_dict):
   provider = options_dict.get("cloud_provider", None)
   if provider is None:
@@ -242,10 +259,6 @@
   else:
     return options.get('image_id')
 
-def _prompt(prompt):
-  """ Returns true if user responds "yes" to prompt. """
-  return raw_input("%s [yes or no]: " % prompt).lower() == "yes"
-
 def main():
   # Use HADOOP_CLOUD_LOGGING_LEVEL=DEBUG to enable debugging output.
   logging.basicConfig(level=getattr(logging,
@@ -261,194 +274,155 @@
   if command == 'list':
     (opt, args) = parse_options(command, BASIC_OPTIONS, unbounded_args=True)
     if len(args) == 0:
-      commands.list_all(get_cloud_provider(opt))
+      service_name = get_service_name(opt)
+      cloud_provider = get_cloud_provider(opt)
+      service = get_service(service_name, cloud_provider)(None)
+      service.list_all(cloud_provider)
     else:
-      (opt, args, cluster) = parse_options_and_config(command, BASIC_OPTIONS)
-      commands.list_cluster(cluster)
+      (opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS)
+      service.list()
 
   elif command == 'launch-master':
-    (opt, args, cluster) = parse_options_and_config(command, LAUNCH_OPTIONS)
-    check_launch_options_set(cluster, opt)
+    (opt, args, service) = parse_options_and_config(command, LAUNCH_OPTIONS)
+    check_launch_options_set(service.cluster, opt)
     config_dir = get_config_dir(opt)
-    commands.launch_master(cluster, config_dir, get_image_id(cluster, opt),
-      opt.get('instance_type'),
-      opt.get('key_name'), opt.get('public_key'), opt.get('user_data_file'),
-      opt.get('availability_zone'), opt.get('user_packages'),
-      opt.get('auto_shutdown'), opt.get('env'), opt.get('client_cidr'))
-    commands.attach_storage(cluster, (commands.MASTER,))
-    try:
-      commands.wait_for_hadoop(cluster, 0)
-    except TimeoutException:
-      print "Timeout while waiting for Hadoop to start. Please check logs on" +\
-        " master."
-    commands.print_master_url(cluster)
+    template = InstanceTemplate((NAMENODE, SECONDARY_NAMENODE, JOBTRACKER), 1,
+                         get_image_id(service.cluster, opt),
+                         opt.get('instance_type'), opt.get('key_name'),
+                         opt.get('public_key'), opt.get('user_data_file'),
+                         opt.get('availability_zone'), opt.get('user_packages'),
+                         opt.get('auto_shutdown'), opt.get('env'),
+                         opt.get('security_group'))
+    service.launch_master(template, config_dir, opt.get('client_cidr'))
 
   elif command == 'launch-slaves':
-    (opt, args, cluster) = parse_options_and_config(command, LAUNCH_OPTIONS,
+    (opt, args, service) = parse_options_and_config(command, LAUNCH_OPTIONS,
                                                     ("NUM_SLAVES",))
     number_of_slaves = int(args[1])
-    check_launch_options_set(cluster, opt)
-    config_dir = get_config_dir(opt)
-    commands.launch_slaves(cluster, number_of_slaves, get_image_id(cluster, opt),
-      opt.get('instance_type'),
-      opt.get('key_name'), opt.get('public_key'), opt.get('user_data_file'),
-      opt.get('availability_zone'), opt.get('user_packages'),
-      opt.get('auto_shutdown'), opt.get('env'))
-    commands.attach_storage(cluster, (commands.SLAVE,))
-    commands.print_master_url(cluster)
+    check_launch_options_set(service.cluster, opt)
+    template = InstanceTemplate((DATANODE, TASKTRACKER), number_of_slaves,
+                         get_image_id(service.cluster, opt),
+                         opt.get('instance_type'), opt.get('key_name'),
+                         opt.get('public_key'), opt.get('user_data_file'),
+                         opt.get('availability_zone'), opt.get('user_packages'),
+                         opt.get('auto_shutdown'), opt.get('env'),
+                         opt.get('security_group'))
+    service.launch_slaves(template)
 
   elif command == 'launch-cluster':
-    (opt, args, cluster) = parse_options_and_config(command, LAUNCH_OPTIONS,
-                                                    ("NUM_SLAVES",))
-    number_of_slaves = int(args[1])
-    check_launch_options_set(cluster, opt)
+    (opt, args, service) = parse_options_and_config(command, LAUNCH_OPTIONS,
+                                                    ("NUM_SLAVES",),
+                                                    unbounded_args=True)
+    check_launch_options_set(service.cluster, opt)
     config_dir = get_config_dir(opt)
-    commands.launch_master(cluster, config_dir, get_image_id(cluster, opt),
-      opt.get('instance_type'),
-      opt.get('key_name'), opt.get('public_key'), opt.get('user_data_file'),
-      opt.get('availability_zone'), opt.get('user_packages'),
-      opt.get('auto_shutdown'), opt.get('env'), opt.get('client_cidr'))
-    commands.launch_slaves(cluster, number_of_slaves, get_image_id(cluster, opt),
-      opt.get('instance_type'),
-      opt.get('key_name'), opt.get('public_key'), opt.get('user_data_file'),
-      opt.get('availability_zone'), opt.get('user_packages'),
-      opt.get('auto_shutdown'), opt.get('env'))
-    commands.attach_storage(cluster, commands.ROLES)
-    try:
-      commands.wait_for_hadoop(cluster, number_of_slaves)
-    except TimeoutException:
-      print "Timeout while waiting for Hadoop to start. Please check logs on" +\
-        " cluster."
-    commands.print_master_url(cluster)
+    instance_templates = []
+    if len(args) == 2:
+      number_of_slaves = int(args[1])
+      print_deprecation(sys.argv[0], 'launch-cluster %s 1 nn,snn,jt %s dn,tt' %
+                        (service.cluster.name, number_of_slaves))
+      instance_templates = [
+        InstanceTemplate((NAMENODE, SECONDARY_NAMENODE, JOBTRACKER), 1,
+                         get_image_id(service.cluster, opt),
+                         opt.get('instance_type'), opt.get('key_name'),
+                         opt.get('public_key'), opt.get('user_data_file'),
+                         opt.get('availability_zone'), opt.get('user_packages'),
+                         opt.get('auto_shutdown'), opt.get('env'),
+                         opt.get('security_group')),
+        InstanceTemplate((DATANODE, TASKTRACKER), number_of_slaves,
+                         get_image_id(service.cluster, opt),
+                         opt.get('instance_type'), opt.get('key_name'),
+                         opt.get('public_key'), opt.get('user_data_file'),
+                         opt.get('availability_zone'), opt.get('user_packages'),
+                         opt.get('auto_shutdown'), opt.get('env'),
+                         opt.get('security_group')),
+                         ]
+    elif len(args) > 2 and len(args) % 2 == 0:
+      print_usage(sys.argv[0])
+      sys.exit(1)
+    else:
+      for i in range(len(args) / 2):
+        number = int(args[2 * i + 1])
+        roles = args[2 * i + 2].split(",")
+        instance_templates.append(
+          InstanceTemplate(roles, number, get_image_id(service.cluster, opt),
+                           opt.get('instance_type'), opt.get('key_name'),
+                           opt.get('public_key'), opt.get('user_data_file'),
+                           opt.get('availability_zone'),
+                           opt.get('user_packages'),
+                           opt.get('auto_shutdown'), opt.get('env'),
+                           opt.get('security_group')))
+
+    service.launch_cluster(instance_templates, config_dir,
+                           opt.get('client_cidr'))
 
   elif command == 'login':
-    (opt, args, cluster) = parse_options_and_config(command, SSH_OPTIONS)
-    instances = cluster.check_running(commands.MASTER, 1)
-    if not instances:
-      sys.exit(1)
-    subprocess.call('ssh %s root@%s' % \
-                    (xstr(opt.get('ssh_options')), instances[0].public_ip),
-                    shell=True)
+    (opt, args, service) = parse_options_and_config(command, SSH_OPTIONS)
+    service.login(opt.get('ssh_options'))
 
   elif command == 'proxy':
-    (opt, args, cluster) = parse_options_and_config(command, SSH_OPTIONS)
-    instances = cluster.check_running(commands.MASTER, 1)
-    if not instances:
-      sys.exit(1)
-    options = '-o "ConnectTimeout 10" -o "ServerAliveInterval 60" ' \
-              '-N -D 6666'
-    process = subprocess.Popen('ssh %s %s root@%s' %
-      (xstr(opt.get('ssh_options')), options, instances[0].public_ip),
-      stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-      shell=True)
-    print """export HADOOP_CLOUD_PROXY_PID=%s;
-echo Proxy pid %s;""" % (process.pid, process.pid)
+    (opt, args, service) = parse_options_and_config(command, SSH_OPTIONS)
+    service.proxy(opt.get('ssh_options'))
 
   elif command == 'push':
-    (opt, args, cluster) = parse_options_and_config(command, SSH_OPTIONS,
+    (opt, args, service) = parse_options_and_config(command, SSH_OPTIONS,
                                                     ("FILE",))
-    instances = cluster.check_running(commands.MASTER, 1)
-    if not instances:
-      sys.exit(1)
-    subprocess.call('scp %s -r %s root@%s:' % (xstr(opt.get('ssh_options')),
-                                               args[1], instances[0].public_ip),
-                                               shell=True)
+    service.proxy(opt.get('ssh_options'), args[1])
 
   elif command == 'exec':
-    (opt, args, cluster) = parse_options_and_config(command, SSH_OPTIONS,
+    (opt, args, service) = parse_options_and_config(command, SSH_OPTIONS,
                                                     ("CMD",), True)
-    instances = cluster.check_running(commands.MASTER, 1)
-    if not instances:
-      sys.exit(1)
-    subprocess.call("ssh %s root@%s '%s'" % (xstr(opt.get('ssh_options')),
-                                             instances[0].public_ip,
-                                             " ".join(args[1:])), shell=True)
+    service.execute(opt.get('ssh_options'), args[1:])
 
   elif command == 'terminate-cluster':
-    (opt, args, cluster) = parse_options_and_config(command, FORCE_OPTIONS)
-    cluster.print_status(commands.ROLES)
-    if not opt["force"] and not _prompt("Terminate all instances?"):
-      print "Not terminating cluster."
-    else:
-      print "Terminating cluster"
-      cluster.terminate()
+    (opt, args, service) = parse_options_and_config(command, FORCE_OPTIONS)
+    service.terminate_cluster(opt["force"])
 
   elif command == 'delete-cluster':
-    (opt, args, cluster) = parse_options_and_config(command, BASIC_OPTIONS)
-    cluster.delete()
+    (opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS)
+    service.delete_cluster()
 
   elif command == 'create-formatted-snapshot':
-    (opt, args, cluster) = parse_options_and_config(command, SNAPSHOT_OPTIONS,
+    (opt, args, service) = parse_options_and_config(command, SNAPSHOT_OPTIONS,
                                                     ("SIZE",))
     size = int(args[1])
     check_options_set(opt, ['availability_zone', 'key_name'])
     ami_ubuntu_intrepid_x86 = 'ami-ec48af85' # use a general AMI
-    Ec2Storage.create_formatted_snapshot(cluster, size,
+    service.create_formatted_snapshot(size,
                                          opt.get('availability_zone'),
                                          ami_ubuntu_intrepid_x86,
                                          opt.get('key_name'),
                                          xstr(opt.get('ssh_options')))
 
   elif command == 'list-storage':
-    (opt, args, cluster) = parse_options_and_config(command, BASIC_OPTIONS)
-    storage = cluster.get_storage()
-    storage.print_status(commands.ROLES)
+    (opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS)
+    service.list_storage()
 
   elif command == 'create-storage':
-    (opt, args, cluster) = parse_options_and_config(command, PLACEMENT_OPTIONS,
+    (opt, args, service) = parse_options_and_config(command, PLACEMENT_OPTIONS,
                                                     ("ROLE", "NUM_INSTANCES",
                                                      "SPEC_FILE"))
-    storage = cluster.get_storage()
     role = args[1]
     number_of_instances = int(args[2])
     spec_file = args[3]
     check_options_set(opt, ['availability_zone'])
-    storage.create(role, number_of_instances, opt.get('availability_zone'),
-                   spec_file)
-    storage.print_status(commands.ROLES)
+    service.create_storage(role, number_of_instances,
+                           opt.get('availability_zone'), spec_file)
 
   elif command == 'attach-storage':
-    (opt, args, cluster) = parse_options_and_config(command, BASIC_OPTIONS,
+    (opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS,
                                                     ("ROLE",))
-    storage = cluster.get_storage()
-    role = args[1]
-    storage.attach(role, cluster.get_instances_in_role(role, 'running'))
-    storage.print_status(commands.ROLES)
+    service.attach_storage(args[1])
 
   elif command == 'delete-storage':
-    (opt, args, cluster) = parse_options_and_config(command, FORCE_OPTIONS)
-    storage = cluster.get_storage()
-    storage.print_status(commands.ROLES)
-    if not opt["force"] and not _prompt("Delete all storage volumes? THIS WILL \
-      PERMANENTLY DELETE ALL DATA"):
-      print "Not deleting storage volumes."
-    else:
-      print "Deleting storage"
-      for role in commands.ROLES:
-        storage.delete(role)
+    (opt, args, service) = parse_options_and_config(command, FORCE_OPTIONS)
+    service.delete_storage(opt["force"])
 
   elif command == 'update-slaves-file':
-    (opt, args, cluster) = parse_options_and_config(command, SSH_OPTIONS)
+    (opt, args, service) = parse_options_and_config(command, SSH_OPTIONS)
     check_options_set(opt, ['private_key'])
     ssh_options = xstr(opt.get('ssh_options'))
-    instances = cluster.check_running(commands.MASTER, 1)
-    if not instances:
-      sys.exit(1)
-    master = instances[0]
-    slaves = cluster.get_instances_in_role(commands.SLAVE)
-    with open('slaves', 'w') as f:
-      for slave in slaves:
-        f.write(slave.public_ip + "\n")
-    subprocess.call('scp %s -r %s root@%s:/etc/hadoop/conf' % \
-                    (ssh_options, 'slaves', master.public_ip), shell=True)
-
-    # Copy private key
-    private_key = opt.get('private_key')
-    subprocess.call('scp %s -r %s root@%s:/root/.ssh/id_rsa' % \
-                    (ssh_options, private_key, master.public_ip), shell=True)
-    for slave in slaves:
-      subprocess.call('scp %s -r %s root@%s:/root/.ssh/id_rsa' % \
-                      (ssh_options, private_key, slave.public_ip), shell=True)
+    config_dir = get_config_dir(opt)
+    service.update_slaves_file(config_dir, ssh_options, opt.get('private_key'))
 
   else:
     print "Unrecognized command '%s'" % command

Copied: hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/cluster.py (from r888072, hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/cluster.py)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/cluster.py?p2=hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/cluster.py&p1=hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/cluster.py&r1=888072&r2=897029&rev=897029&view=diff
==============================================================================
--- hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/cluster.py (original)
+++ hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/cluster.py Thu Jan  7 22:04:37 2010
@@ -70,7 +70,7 @@
     """
     raise Exception("Unimplemented")
 
-  def print_status(self, roles, state_filter="running"):
+  def print_status(self, roles=None, state_filter="running"):
     """
     Print the status of instances in the given roles, filtered by state.
     """
@@ -88,10 +88,10 @@
     else:
       return instances
 
-  def launch_instances(self, role, number, image_id, size_id,
+  def launch_instances(self, roles, number, image_id, size_id,
                        instance_user_data, **kwargs):
     """
-    Launch instances (of the given role) in the cluster.
+    Launch instances (having the given roles) in the cluster.
     Returns a list of IDs for the instances started.
     """
     pass

Copied: hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/providers/ec2.py (from r888072, hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/providers/ec2.py)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/providers/ec2.py?p2=hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/providers/ec2.py&p1=hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/providers/ec2.py&r1=888072&r2=897029&rev=897029&view=diff
==============================================================================
--- hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/providers/ec2.py (original)
+++ hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/providers/ec2.py Thu Jan  7 22:04:37 2010
@@ -85,7 +85,7 @@
     return self.name
 
   def _check_role_name(self, role):
-    if not re.match("^[a-zA-Z0-9_]+$", role):
+    if not re.match("^[a-zA-Z0-9_+]+$", role):
       raise RoleSyntaxException("Invalid role name '%s'" % role)
 
   def _group_name_for_role(self, role):
@@ -95,9 +95,11 @@
     self._check_role_name(role)
     return "%s-%s" % (self.name, role)
 
-  def _get_group_names(self, role):
-    self._check_role_name(role)
-    return [self._get_cluster_group_name(), self._group_name_for_role(role)]
+  def _get_group_names(self, roles):
+    group_names = [self._get_cluster_group_name()]
+    for role in roles:
+      group_names.append(self._group_name_for_role(role))
+    return group_names
 
   def _get_all_group_names(self):
     security_groups = self.ec2Connection.get_all_security_groups()
@@ -111,7 +113,7 @@
     if self.name not in all_group_names:
       return r
     for group in all_group_names:
-      if re.match("^%s(-[a-zA-Z0-9_]+)?$" % self.name, group):
+      if re.match("^%s(-[a-zA-Z0-9_+]+)?$" % self.name, group):
         r.append(group)
     return r
 
@@ -198,25 +200,32 @@
       instance.state, xstr(instance.key_name), instance.instance_type,
       str(instance.launch_time), instance.placement))
 
-  def print_status(self, roles, state_filter="running"):
+  def print_status(self, roles=None, state_filter="running"):
     """
     Print the status of instances in the given roles, filtered by state.
     """
-    for role in roles:
-      for instance in self._get_instances(self._group_name_for_role(role),
+    if not roles:
+      for instance in self._get_instances(self._get_cluster_group_name(),
                                           state_filter):
-        self._print_instance(role, instance)
+        self._print_instance("", instance)
+    else:
+      for role in roles:
+        for instance in self._get_instances(self._group_name_for_role(role),
+                                            state_filter):
+          self._print_instance(role, instance)
 
-  def launch_instances(self, role, number, image_id, size_id,
+  def launch_instances(self, roles, number, image_id, size_id,
                        instance_user_data, **kwargs):
-    self._check_role_name(role)
-
-    self._create_groups(role)
+    for role in roles:
+      self._check_role_name(role)  
+      self._create_groups(role)
+      
     user_data = instance_user_data.read_as_gzip_stream()
+    security_groups = self._get_group_names(roles) + kwargs.get('security_groups', [])
 
     reservation = self.ec2Connection.run_instances(image_id, min_count=number,
       max_count=number, key_name=kwargs.get('key_name', None),
-      security_groups=self._get_group_names(role), user_data=user_data,
+      security_groups=security_groups, user_data=user_data,
       instance_type=size_id, placement=kwargs.get('placement', None))
     return [instance.id for instance in reservation.instances]
 
@@ -372,6 +381,11 @@
         return True
     return False
 
+  def get_roles(self):
+    storage_filename = self._get_storage_filename()
+    volume_manager = JsonVolumeManager(storage_filename)
+    return volume_manager.get_roles()
+  
   def _get_ec2_volumes_dict(self, mountable_volumes):
     volume_ids = [mv.volume_id for mv in sum(mountable_volumes, [])]
     volumes = self.cluster.ec2Connection.get_all_volumes(volume_ids)
@@ -386,7 +400,11 @@
                      volume.status, str(volume.create_time),
                      str(volume.attach_time)))
 
-  def print_status(self, roles):
+  def print_status(self, roles=None):
+    if roles == None:
+      storage_filename = self._get_storage_filename()
+      volume_manager = JsonVolumeManager(storage_filename)
+      roles = volume_manager.get_roles()
     for role in roles:
       mountable_volumes_list = self._get_mountable_volumes(role)
       ec2_volumes = self._get_ec2_volumes_dict(mountable_volumes_list)
@@ -441,20 +459,21 @@
         print "Attaching %s to %s" % (volume.id, instance.id)
         volume.attach(instance.id, mountable_volume.device)
 
-  def delete(self, role):
+  def delete(self, roles=[]):
     storage_filename = self._get_storage_filename()
     volume_manager = JsonVolumeManager(storage_filename)
-    mountable_volumes_list = volume_manager.get_instance_storage_for_role(role)
-    ec2_volumes = self._get_ec2_volumes_dict(mountable_volumes_list)
-    all_available = True
-    for volume in ec2_volumes.itervalues():
-      if volume.status != 'available':
-        all_available = False
-        logger.warning("Volume %s is not available.", volume)
-    if not all_available:
-      logger.warning("Some volumes are still in use for role %s.\
-        Aborting delete.", role)
-      return
-    for volume in ec2_volumes.itervalues():
-      volume.delete()
-    volume_manager.remove_instance_storage_for_role(role)
+    for role in roles:
+      mountable_volumes_list = volume_manager.get_instance_storage_for_role(role)
+      ec2_volumes = self._get_ec2_volumes_dict(mountable_volumes_list)
+      all_available = True
+      for volume in ec2_volumes.itervalues():
+        if volume.status != 'available':
+          all_available = False
+          logger.warning("Volume %s is not available.", volume)
+      if not all_available:
+        logger.warning("Some volumes are still in use for role %s.\
+          Aborting delete.", role)
+        return
+      for volume in ec2_volumes.itervalues():
+        volume.delete()
+      volume_manager.remove_instance_storage_for_role(role)

Copied: hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/storage.py (from r888072, hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/storage.py)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/storage.py?p2=hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/storage.py&p1=hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/storage.py&r1=888072&r2=897029&rev=897029&view=diff
==============================================================================
--- hadoop/common/trunk/src/contrib/cloud/src/py/hadoop/cloud/storage.py (original)
+++ hadoop/common/branches/HADOOP-6194/src/contrib/cloud/src/py/hadoop/cloud/storage.py Thu Jan  7 22:04:37 2010
@@ -81,6 +81,10 @@
 
   def _store(self, obj):
     return json.dump(obj, open(self.filename, "w"), sort_keys=True, indent=2)
+  
+  def get_roles(self):
+    json_dict = self._load()
+    return json_dict.keys()
 
   def add_instance_storage_for_role(self, role, mountable_volumes):
     json_dict = self._load()
@@ -142,7 +146,13 @@
     """
     return False
 
-  def print_status(self, roles):
+  def get_roles(self):
+    """
+    Return a list of roles that have storage defined.
+    """
+    return []
+
+  def print_status(self, roles=None):
     """
     Print the status of storage volumes for the given roles.
     """
@@ -156,8 +166,8 @@
     """
     pass
 
-  def delete(self, role):
+  def delete(self, roles=[]):
     """
-    Permanently delete all the storage for a role.
+    Permanently delete all the storage for the given roles.
     """
     pass

Propchange: hadoop/common/branches/HADOOP-6194/src/contrib/ec2/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jan  7 22:04:37 2010
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/src/contrib/ec2:804966-888120
+/hadoop/common/trunk/src/contrib/ec2:804966-897004
 /hadoop/core/branches/branch-0.19/core/src/contrib/ec2:713112
 /hadoop/core/trunk/src/contrib/ec2:776175-784663

Propchange: hadoop/common/branches/HADOOP-6194/src/docs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jan  7 22:04:37 2010
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/src/docs:804966-888120
+/hadoop/common/trunk/src/docs:804966-897004
 /hadoop/core/branches/HADOOP-4687/core/src/docs:776175-786719
 /hadoop/core/branches/branch-0.19/src/docs:713112

Propchange: hadoop/common/branches/HADOOP-6194/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jan  7 22:04:37 2010
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/src/java:804966-888120
+/hadoop/common/trunk/src/java:804966-897004
 /hadoop/core/branches/branch-0.19/core/src/java:713112
 /hadoop/core/trunk/src/core:776175-785643,785929-786278

Modified: hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/conf/Configuration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/conf/Configuration.java?rev=897029&r1=897028&r2=897029&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/conf/Configuration.java (original)
+++ hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/conf/Configuration.java Thu Jan  7 22:04:37 2010
@@ -27,9 +27,11 @@
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
+import java.io.OutputStreamWriter;
 import java.io.Reader;
 import java.io.Writer;
 import java.net.URL;
+import java.util.AbstractMap;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -47,11 +49,13 @@
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
 
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerException;
 import javax.xml.transform.TransformerFactory;
 import javax.xml.transform.dom.DOMSource;
 import javax.xml.transform.stream.StreamResult;
@@ -67,6 +71,7 @@
 import org.apache.hadoop.util.StringUtils;
 import org.codehaus.jackson.JsonFactory;
 import org.codehaus.jackson.JsonGenerator;
+import org.w3c.dom.Comment;
 import org.w3c.dom.DOMException;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
@@ -153,6 +158,12 @@
   private ArrayList<Object> resources = new ArrayList<Object>();
 
   /**
+   * The value reported as the setting resource when a key is set
+   * by code rather than a file resource.
+   */
+  static final String UNKNOWN_RESOURCE = "Unknown";
+
+  /**
    * List of configuration parameters marked <b>final</b>. 
    */
   private Set<String> finalParameters = new HashSet<String>();
@@ -174,13 +185,7 @@
 
   private static final Map<ClassLoader, Map<String, Class<?>>>
     CACHE_CLASSES = new WeakHashMap<ClassLoader, Map<String, Class<?>>>();
-  
-  /**
-   * Flag to indicate if the storage of resource which updates a key needs 
-   * to be stored for each key
-   */
-  private boolean storeResource;
-  
+
   /**
    * Stores the mapping of key to the resource which modifies or loads 
    * the key most recently
@@ -384,26 +389,10 @@
    */
   public Configuration(boolean loadDefaults) {
     this.loadDefaults = loadDefaults;
+    updatingResource = new HashMap<String, String>();
     synchronized(Configuration.class) {
       REGISTRY.put(this, null);
     }
-    this.storeResource = false;
-  }
-  
-  /**
-   * A new configuration with the same settings and additional facility for
-   * storage of resource to each key which loads or updates 
-   * the key most recently
-   * @param other the configuration from which to clone settings
-   * @param storeResource flag to indicate if the storage of resource to 
-   * each key is to be stored
-   */
-  private Configuration(Configuration other, boolean storeResource) {
-    this(other);
-    this.storeResource = storeResource;
-    if (storeResource) {
-      updatingResource = new HashMap<String, String>();
-    }
   }
   
   /** 
@@ -422,6 +411,8 @@
      if (other.overlay!=null) {
        this.overlay = (Properties)other.overlay.clone();
      }
+
+     this.updatingResource = new HashMap<String, String>(other.updatingResource);
    }
    
     this.finalParameters = new HashSet<String>(other.finalParameters);
@@ -603,6 +594,7 @@
     if (!isDeprecated(name)) {
       getOverlay().setProperty(name, value);
       getProps().setProperty(name, value);
+      updatingResource.put(name, UNKNOWN_RESOURCE);
     }
     else {
       DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name);
@@ -832,6 +824,45 @@
   }
 
   /**
+   * Get the value of the <code>name</code> property as a <code>Pattern</code>.
+   * If no such property is specified, or if the specified value is not a valid
+   * <code>Pattern</code>, then <code>DefaultValue</code> is returned.
+   *
+   * @param name property name
+   * @param defaultValue default value
+   * @return property value as a compiled Pattern, or defaultValue
+   */
+  public Pattern getPattern(String name, Pattern defaultValue) {
+    String valString = get(name);
+    if (null == valString || "".equals(valString)) {
+      return defaultValue;
+    }
+    try {
+      return Pattern.compile(valString);
+    } catch (PatternSyntaxException pse) {
+      LOG.warn("Regular expression '" + valString + "' for property '" +
+               name + "' not valid. Using default", pse);
+      return defaultValue;
+    }
+  }
+
+  /**
+   * Set the given property to <code>Pattern</code>.
+   * If the pattern is passed as null, sets the empty pattern which results in
+   * further calls to getPattern(...) returning the default value.
+   *
+   * @param name property name
+   * @param pattern new value
+   */
+  public void setPattern(String name, Pattern pattern) {
+    if (null == pattern) {
+      set(name, null);
+    } else {
+      set(name, pattern.pattern());
+    }
+  }
+
+  /**
    * A class that represents a set of positive integer ranges. It parses 
    * strings of the form: "2-3,5,7-" where ranges are separated by comma and 
    * the lower/upper bounds are separated by dash. Either the lower or upper 
@@ -1037,7 +1068,139 @@
   public void setStrings(String name, String... values) {
     set(name, StringUtils.arrayToString(values));
   }
- 
+
+  /**
+   * Instantiates a map view over a subset of the entries in
+   * the Configuration. This is instantiated by getMap(), which
+   * binds a prefix of the namespace to the ConfigItemMap. This
+   * mapping reflects changes to the underlying Configuration.
+   *
+   * This map does not support iteration.
+   */
+  protected class ConfigItemMap extends AbstractMap<String, String>
+      implements Map<String, String> {
+
+    private final String prefix;
+
+    public ConfigItemMap(String prefix) {
+      this.prefix = prefix;
+    }
+
+    @Override
+    public boolean containsKey(Object key) {
+      return lookup(key.toString()) != null;
+    }
+
+    @Override
+    public Set<Map.Entry<String, String>> entrySet() {
+      throw new UnsupportedOperationException("unsupported");
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      return o != null && o instanceof ConfigItemMap
+          && prefix.equals(((ConfigItemMap) o).prefix)
+          && Configuration.this == ((ConfigItemMap) o).getConfiguration();
+    }
+
+    private Configuration getConfiguration() {
+      return Configuration.this;
+    }
+
+    @Override
+    public String get(Object key) {
+      if (null == key) {
+        return null;
+      }
+
+      return lookup(key.toString());
+    }
+
+    @Override
+    public int hashCode() {
+      return prefix.hashCode();
+    }
+
+    @Override
+    public String put(String key, String val) {
+      if (null == key) {
+        return null;
+      }
+
+      String ret = get(key);
+      Configuration.this.set(prefix + key, val);
+      return ret;
+    }
+
+    @Override
+    public void putAll(Map<? extends String, ? extends String> m) {
+      for (Map.Entry<? extends String, ? extends String> entry : m.entrySet()) {
+        put(entry.getKey(), entry.getValue());
+      }
+    }
+
+    private String lookup(String subKey) {
+      String configKey = prefix + subKey;
+      Properties props = Configuration.this.getProps();
+      Object val = props.get(configKey);
+      String str = null;
+      if (null != val) {
+        str = substituteVars(val.toString());
+      }
+
+      return str;
+    }
+  }
+
+  /**
+   * Given a string -&gt; string map as a value, embed this in the
+   * Configuration by prepending 'name' to all the keys in the valueMap,
+   * and storing it inside the current Configuration.
+   *
+   * e.g., setMap("foo", { "bar" -&gt; "a", "baz" -&gt; "b" }) would
+   * insert "foo.bar" -&gt; "a" and "foo.baz" -&gt; "b" in this
+   * Configuration.
+   *
+   * @param name the prefix to attach to all keys in the valueMap. This
+   * should not have a trailing "." character.
+   * @param valueMap the map to embed in the Configuration.
+   */
+  public void setMap(String name, Map<String, String> valueMap) {
+    // Store all elements of the map proper.
+    for (Map.Entry<String, String> entry : valueMap.entrySet()) {
+      set(name + "." + entry.getKey(), entry.getValue());
+    }
+  }
+
+  /**
+   * Returns a map containing a view of all configuration properties
+   * whose names begin with "name.*", with the "name." prefix  removed.
+   * e.g., if "foo.bar" -&gt; "a" and "foo.baz" -&gt; "b" are in the
+   * Configuration, getMap("foo") would return { "bar" -&gt; "a",
+   * "baz" -&gt; "b" }.
+   *
+   * Map name deprecation is handled via "prefix deprecation"; the individual
+   * keys created in a configuration by inserting a map do not need to be
+   * individually deprecated -- it is sufficient to deprecate the 'name'
+   * associated with the map and bind that to a new name. e.g., if "foo"
+   * is deprecated for "newfoo," and the configuration contains entries for
+   * "newfoo.a" and "newfoo.b", getMap("foo") will return a map containing
+   * the keys "a" and "b".
+   *
+   * The returned map does not support iteration; it is a lazy view over
+   * the slice of the configuration whose keys begin with 'name'. Updates
+   * to the underlying configuration are reflected in the returned map,
+   * and updates to the map will modify the underlying configuration.
+   *
+   * @param name The prefix of the key names to extract into the output map.
+   * @return a String-&gt;String map that contains all (k, v) pairs
+   * where 'k' begins with 'name.'; the 'name.' prefix is removed in the output.
+   */
+  public Map<String, String> getMap(String name) {
+    String prefix = handleDeprecation(name) + ".";
+    return new ConfigItemMap(prefix);
+  }
+
   /**
    * Load a class by name.
    * 
@@ -1312,10 +1475,8 @@
       loadResources(properties, resources, quietmode);
       if (overlay!= null) {
         properties.putAll(overlay);
-        if (storeResource) {
-          for (Map.Entry<Object,Object> item: overlay.entrySet()) {
-            updatingResource.put((String) item.getKey(), "Unknown");
-          }
+        for (Map.Entry<Object,Object> item: overlay.entrySet()) {
+          updatingResource.put((String) item.getKey(), UNKNOWN_RESOURCE);
         }
       }
     }
@@ -1398,9 +1559,7 @@
       if (finalParameters.contains(oldKey)) {
         finalParameters.remove(oldKey);
       }
-      if (storeResource) {
-        updatingResource.remove(oldKey);
-      }
+      updatingResource.remove(oldKey);
     }
   }
   
@@ -1424,9 +1583,7 @@
         continue;
       }
       properties.setProperty(key, value);
-      if (storeResource) {
-        updatingResource.put(key, updatingResource.get(attr));
-      }
+      updatingResource.put(key, updatingResource.get(attr));
       if (finalParameter) {
         finalParameters.add(key);
       }
@@ -1541,9 +1698,7 @@
           if (value != null) {
             if (!finalParameters.contains(attr)) {
               properties.setProperty(attr, value);
-              if (storeResource) {
-                updatingResource.put(attr, name.toString());
-              }
+              updatingResource.put(attr, name.toString());
             } else {
               LOG.warn(name+":a attempt to override final parameter: "+attr
                      +";  Ignoring.");
@@ -1571,12 +1726,22 @@
   }
 
   /** 
-   * Write out the non-default properties in this configuration to the give
+   * Write out the non-default properties in this configuration to the given
    * {@link OutputStream}.
    * 
    * @param out the output stream to write to.
    */
   public void writeXml(OutputStream out) throws IOException {
+    writeXml(new OutputStreamWriter(out));
+  }
+
+  /** 
+   * Write out the non-default properties in this configuration to the given
+   * {@link Writer}.
+   * 
+   * @param out the writer to write to.
+   */
+  public synchronized void writeXml(Writer out) throws IOException {
     Properties properties = getProps();
     try {
       Document doc =
@@ -1595,7 +1760,12 @@
         }
         Element propNode = doc.createElement("property");
         conf.appendChild(propNode);
-      
+
+        if (updatingResource != null) {
+          Comment commentNode = doc.createComment(
+            "Loaded from " + updatingResource.get(name));
+          propNode.appendChild(commentNode);
+        }
         Element nameNode = doc.createElement("name");
         nameNode.appendChild(doc.createTextNode(name));
         propNode.appendChild(nameNode);
@@ -1612,8 +1782,10 @@
       TransformerFactory transFactory = TransformerFactory.newInstance();
       Transformer transformer = transFactory.newTransformer();
       transformer.transform(source, result);
-    } catch (Exception e) {
-      throw new RuntimeException(e);
+    } catch (TransformerException te) {
+      throw new IOException(te);
+    } catch (ParserConfigurationException pe) {
+      throw new IOException(pe);
     }
   }
 
@@ -1628,26 +1800,26 @@
    * @param out the Writer to write to
    * @throws IOException
    */
-  public static void dumpConfiguration(Configuration conf, 
+  public static void dumpConfiguration(Configuration config,
       Writer out) throws IOException {
-    Configuration config = new Configuration(conf,true);
-    config.reloadConfiguration();
     JsonFactory dumpFactory = new JsonFactory();
     JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out);
     dumpGenerator.writeStartObject();
     dumpGenerator.writeFieldName("properties");
     dumpGenerator.writeStartArray();
     dumpGenerator.flush();
-    for (Map.Entry<Object,Object> item: config.getProps().entrySet()) {
-      dumpGenerator.writeStartObject();
-      dumpGenerator.writeStringField("key", (String) item.getKey());
-      dumpGenerator.writeStringField("value", 
-          config.get((String) item.getKey()));
-      dumpGenerator.writeBooleanField("isFinal",
-          config.finalParameters.contains(item.getKey()));
-      dumpGenerator.writeStringField("resource",
-          config.updatingResource.get(item.getKey()));
-      dumpGenerator.writeEndObject();
+    synchronized (config) {
+      for (Map.Entry<Object,Object> item: config.getProps().entrySet()) {
+        dumpGenerator.writeStartObject();
+        dumpGenerator.writeStringField("key", (String) item.getKey());
+        dumpGenerator.writeStringField("value", 
+                                       config.get((String) item.getKey()));
+        dumpGenerator.writeBooleanField("isFinal",
+                                        config.finalParameters.contains(item.getKey()));
+        dumpGenerator.writeStringField("resource",
+                                       config.updatingResource.get(item.getKey()));
+        dumpGenerator.writeEndObject();
+      }
     }
     dumpGenerator.writeEndArray();
     dumpGenerator.writeEndObject();

Modified: hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java?rev=897029&r1=897028&r2=897029&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java (original)
+++ hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java Thu Jan  7 22:04:37 2010
@@ -205,24 +205,41 @@
     @Override
     protected int readChunk(long pos, byte[] buf, int offset, int len,
         byte[] checksum) throws IOException {
+
       boolean eof = false;
-      if(needChecksum()) {
-        try {
-          long checksumPos = getChecksumFilePos(pos); 
-          if(checksumPos != sums.getPos()) {
-            sums.seek(checksumPos);
-          }
-          sums.readFully(checksum);
-        } catch (EOFException e) {
+      if (needChecksum()) {
+        assert checksum != null; // we have a checksum buffer
+        assert checksum.length % CHECKSUM_SIZE == 0; // it is sane length
+        assert len >= bytesPerSum; // we must read at least one chunk
+
+        final int checksumsToRead = Math.min(
+          len/bytesPerSum, // number of checksums based on len to read
+          checksum.length / CHECKSUM_SIZE); // size of checksum buffer
+        long checksumPos = getChecksumFilePos(pos); 
+        if(checksumPos != sums.getPos()) {
+          sums.seek(checksumPos);
+        }
+
+        int sumLenRead = sums.read(checksum, 0, CHECKSUM_SIZE * checksumsToRead);
+        if (sumLenRead >= 0 && sumLenRead % CHECKSUM_SIZE != 0) {
+          throw new ChecksumException(
+            "Checksum file not a length multiple of checksum size " +
+            "in " + file + " at " + pos + " checksumpos: " + checksumPos +
+            " sumLenread: " + sumLenRead,
+            pos);
+        }
+        if (sumLenRead <= 0) { // we're at the end of the file
           eof = true;
+        } else {
+          // Adjust amount of data to read based on how many checksum chunks we read
+          len = Math.min(len, bytesPerSum * (sumLenRead / CHECKSUM_SIZE));
         }
-        len = bytesPerSum;
       }
       if(pos != datas.getPos()) {
         datas.seek(pos);
       }
       int nread = readFully(datas, buf, offset, len);
-      if( eof && nread > 0) {
+      if (eof && nread > 0) {
         throw new ChecksumException("Checksum error: "+file+" at "+pos, pos);
       }
       return nread;

Modified: hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/ChecksumFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/ChecksumFs.java?rev=897029&r1=897028&r2=897029&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/ChecksumFs.java (original)
+++ hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/ChecksumFs.java Thu Jan  7 22:04:37 2010
@@ -200,21 +200,35 @@
         byte[] checksum) throws IOException {
       boolean eof = false;
       if (needChecksum()) {
-        try {
-          final long checksumPos = getChecksumFilePos(pos); 
-          if (checksumPos != sums.getPos()) {
-            sums.seek(checksumPos);
-          }
-          sums.readFully(checksum);
-        } catch (EOFException e) {
+        assert checksum != null; // we have a checksum buffer
+        assert checksum.length % CHECKSUM_SIZE == 0; // it is sane length
+        assert len >= bytesPerSum; // we must read at least one chunk
+
+        final int checksumsToRead = Math.min(
+          len/bytesPerSum, // number of checksums based on len to read
+          checksum.length / CHECKSUM_SIZE); // size of checksum buffer
+        long checksumPos = getChecksumFilePos(pos); 
+        if(checksumPos != sums.getPos()) {
+          sums.seek(checksumPos);
+        }
+
+        int sumLenRead = sums.read(checksum, 0, CHECKSUM_SIZE * checksumsToRead);
+        if (sumLenRead >= 0 && sumLenRead % CHECKSUM_SIZE != 0) {
+          throw new EOFException("Checksum file not a length multiple of checksum size " +
+                                 "in " + file + " at " + pos + " checksumpos: " + checksumPos +
+                                 " sumLenread: " + sumLenRead );
+        }
+        if (sumLenRead <= 0) { // we're at the end of the file
           eof = true;
+        } else {
+          // Adjust amount of data to read based on how many checksum chunks we read
+          len = Math.min(len, bytesPerSum * (sumLenRead / CHECKSUM_SIZE));
         }
-        len = bytesPerSum;
       }
       if (pos != datas.getPos()) {
         datas.seek(pos);
       }
-      final int nread = readFully(datas, buf, offset, len);
+      int nread = readFully(datas, buf, offset, len);
       if (eof && nread > 0) {
         throw new ChecksumException("Checksum error: "+file+" at "+pos, pos);
       }

Modified: hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java?rev=897029&r1=897028&r2=897029&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java (original)
+++ hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java Thu Jan  7 22:04:37 2010
@@ -126,5 +126,7 @@
   public static final String  HADOOP_JOB_UGI_KEY = "hadoop.job.ugi";
   public static final String  HADOOP_UTIL_HASH_TYPE_KEY = "hadoop.util.hash.type";
   public static final String  HADOOP_UTIL_HASH_TYPE_DEFAULT = "murmur";
+  public static final String  HADOOP_SECURITY_GROUP_MAPPING = "hadoop.security.group.mapping";
+  public static final String  HADOOP_SECURITY_GROUPS_CACHE_SECS = "hadoop.security.groups.cache.secs";
 }
 



Mime
View raw message