Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHTestCase.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHTestCase.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHTestCase.java (original) +++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHTestCase.java Tue Oct 16 00:02:55 2012 @@ -64,6 +64,7 @@ public class TestHTestCase extends HTest public void waitFor() { long start = Time.now(); long waited = waitFor(1000, new Predicate() { + @Override public boolean evaluate() throws Exception { return true; } @@ -78,6 +79,7 @@ public class TestHTestCase extends HTest setWaitForRatio(1); long start = Time.now(); long waited = waitFor(200, new Predicate() { + @Override public boolean evaluate() throws Exception { return false; } @@ -92,6 +94,7 @@ public class TestHTestCase extends HTest setWaitForRatio(2); long start = Time.now(); long waited = waitFor(200, new Predicate() { + @Override public boolean evaluate() throws Exception { return false; } Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHdfsHelper.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHdfsHelper.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHdfsHelper.java (original) +++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHdfsHelper.java Tue Oct 16 00:02:55 2012 @@ -31,6 +31,7 @@ import org.junit.runners.model.Statement public class TestHdfsHelper extends TestDirHelper { + @Override @Test public void dummy() { } Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Oct 16 00:02:55 2012 @@ -1,6 +1,6 @@ Hadoop HDFS Change Log -Trunk (unreleased changes) +Trunk (Unreleased) INCOMPATIBLE CHANGES @@ -8,14 +8,15 @@ Trunk (unreleased changes) NEW FEATURES - HDFS-234. Integration with BookKeeper logging system. (Ivan Kelly - via jitendra) - HDFS-3125. Add JournalService to enable Journal Daemon. (suresh) HDFS-3601. Add BlockPlacementPolicyWithNodeGroup to support block placement with 4-layer network topology. (Junping Du via szetszwo) + HDFS-3077. Implement QuorumJournalManager, a distributed mechanism for + reliably storing HDFS edit logs. See dedicated section below for breakdown + of subtasks. + IMPROVEMENTS HDFS-1620. Rename HdfsConstants -> HdfsServerConstants, FSConstants -> @@ -126,6 +127,26 @@ Trunk (unreleased changes) HDFS-3819. Should check whether invalidate work percentage default value is not greater than 1.0f. (Jing Zhao via jitendra) + HDFS-3844. Add @Override and remove {@inheritdoc} and unnecessary + imports. (Jing Zhao via suresh) + + HDFS-3851. DFSOutputStream class code cleanup. (Jing Zhao via suresh) + + HDFS-2580. NameNode#main(...) can make use of GenericOptionsParser. (harsh) + + HDFS-3880. Use Builder to build RPC server in HDFS. + (Brandon Li vias suresh) + + HDFS-2127. Add a test that ensure AccessControlExceptions contain + a full path. (Stephen Chu via eli) + + HDFS-3995. Use DFSTestUtil.createFile() for file creation and + writing in test cases. (Jing Zhao via suresh) + + HDFS-3735. NameNode WebUI should allow sorting live datanode list by fields + Block Pool Used, Block Pool Used(%) and Failed Volumes. + (Brahma Reddy Battula via suresh) + OPTIMIZATIONS BUG FIXES @@ -200,7 +221,226 @@ Trunk (unreleased changes) HDFS-3827. TestHASafeMode#assertSafemode method should be made static. (Jing Zhao via suresh) -Branch-2 ( Unreleased changes ) + HDFS-3834. Remove unused static fields NAME, DESCRIPTION and Usage from + Command. (Jing Zhao via suresh) + + HDFS-3678. Edit log files are never being purged from 2NN. (atm) + + HADOOP-8158. Interrupting hadoop fs -put from the command line + causes a LeaseExpiredException. (daryn via harsh) + + BREAKDOWN OF HDFS-3077 SUBTASKS + + HDFS-3077. Quorum-based protocol for reading and writing edit logs. + (todd, Brandon Li, and Hari Mankude via todd) + + HDFS-3694. Fix getEditLogManifest to fetch httpPort if necessary (todd) + + HDFS-3692. Support purgeEditLogs() call to remotely purge logs on JNs + (todd) + + HDFS-3693. JNStorage should read its storage info even before a writer + becomes active (todd) + + HDFS-3725. Fix QJM startup when individual JNs have gaps (todd) + + HDFS-3741. Exhaustive failure injection test for skipped RPCs (todd) + + HDFS-3773. TestNNWithQJM fails after HDFS-3741. (atm) + + HDFS-3793. Implement genericized format() in QJM (todd) + + HDFS-3795. QJM: validate journal dir at startup (todd) + + HDFS-3798. Avoid throwing NPE when finalizeSegment() is called on invalid + segment (todd) + + HDFS-3799. QJM: handle empty log segments during recovery (todd) + + HDFS-3797. QJM: add segment txid as a parameter to journal() RPC (todd) + + HDFS-3800. improvements to QJM fault testing (todd) + + HDFS-3823. QJM: TestQJMWithFaults fails occasionally because of missed + setting of HTTP port. (todd and atm) + + HDFS-3826. QJM: Some trivial logging / exception text improvements. (todd + and atm) + + HDFS-3839. QJM: hadoop-daemon.sh should be updated to accept "journalnode" + (eli) + + HDFS-3845. Fixes for edge cases in QJM recovery protocol (todd) + + HDFS-3877. QJM: Provide defaults for dfs.journalnode.*address (eli) + + HDFS-3863. Track last "committed" txid in QJM (todd) + + HDFS-3869. Expose non-file journal manager details in web UI (todd) + + HDFS-3884. Journal format() should reset cached values (todd) + + HDFS-3870. Add metrics to JournalNode (todd) + + HDFS-3891. Make selectInputStreams throw IOE instead of RTE (todd) + + HDFS-3726. If a logger misses an RPC, don't retry that logger until next + segment (todd) + + HDFS-3893. QJM: Make QJM work with security enabled. (atm) + + HDFS-3897. QJM: TestBlockToken fails after HDFS-3893. (atm) + + HDFS-3898. QJM: enable TCP_NODELAY for IPC (todd) + + HDFS-3885. QJM: optimize log sync when JN is lagging behind (todd) + + HDFS-3900. QJM: avoid validating log segments on log rolls (todd) + + HDFS-3901. QJM: send 'heartbeat' messages to JNs even when they are + out-of-sync (todd) + + HDFS-3899. QJM: Add client-side metrics (todd) + + HDFS-3914. QJM: acceptRecovery should abort current segment (todd) + + HDFS-3915. QJM: Failover fails with auth error in secure cluster (todd) + + HDFS-3906. QJM: quorum timeout on failover with large log segment (todd) + + HDFS-3840. JournalNodes log JournalNotFormattedException backtrace error + before being formatted (todd) + + HDFS-3894. QJM: testRecoverAfterDoubleFailures can be flaky due to IPC + client caching (todd) + + HDFS-3926. QJM: Add user documentation for QJM. (atm) + + HDFS-3943. QJM: remove currently-unused md5sum field (todd) + + HDFS-3950. QJM: misc TODO cleanup, improved log messages, etc. (todd) + + HDFS-3955. QJM: Make acceptRecovery() atomic. (todd) + + HDFS-3956. QJM: purge temporary files when no longer within retention + period (todd) + + HDFS-4004. TestJournalNode#testJournal fails because of test case execution + order (Chao Shi via todd) + + HDFS-4017. Unclosed FileInputStream in GetJournalEditServlet + (Chao Shi via todd) + +Release 2.0.3-alpha - Unreleased + + INCOMPATIBLE CHANGES + + NEW FEATURES + + HDFS-2656. Add libwebhdfs, a pure C client based on WebHDFS. + (Jaimin D Jetly and Jing Zhao via szetszwo) + + HDFS-3912. Detect and avoid stale datanodes for writes. + (Jing Zhao via suresh) + + IMPROVEMENTS + + HDFS-3925. Prettify PipelineAck#toString() for printing to a log + (Andrew Wang via todd) + + HDFS-3939. NN RPC address cleanup. (eli) + + HDFS-3373. Change DFSClient input stream socket cache to global static and + add a thread to cleanup expired cache entries. (John George via szetszwo) + + HDFS-3896. Add descriptions for dfs.namenode.rpc-address and + dfs.namenode.servicerpc-address to hdfs-default.xml. (Jeff Lord via atm) + + HDFS-3996. Add debug log removed in HDFS-3873 back. (eli) + + HDFS-3916. libwebhdfs (C client) code cleanups. + (Colin Patrick McCabe via eli) + + HDFS-3813. Log error message if security and WebHDFS are enabled but + principal/keytab are not configured. (Stephen Chu via atm) + + HDFS-3483. Better error message when hdfs fsck is run against a ViewFS + config. (Stephen Fritz via atm) + + HDFS-3682. MiniDFSCluster#init should provide more info when it fails. + (todd via eli) + + HDFS-4008. TestBalancerWithEncryptedTransfer needs a timeout. (eli) + + HDFS-4007. Rehabilitate bit-rotted unit tests under + hadoop-hdfs-project/hadoop-hdfs/src/test/unit/ + (Colin Patrick McCabe via todd) + + HDFS-4041. Hadoop HDFS Maven protoc calls must not depend on external + sh script. (Chris Nauroth via suresh) + + HADOOP-8911. CRLF characters in source and text files. + (Raja Aluri via suresh) + + HDFS-4037. Rename the getReplication() method in BlockCollection to + getBlockReplication(). (szetszwo) + + HDFS-4036. Remove "throw UnresolvedLinkException" from + FSDirectory.unprotectedAddFile(..). (Jing Zhao via szetszwo) + + OPTIMIZATIONS + + BUG FIXES + + HDFS-3919. MiniDFSCluster:waitClusterUp can hang forever. + (Andy Isaacson via eli) + + HDFS-3924. Multi-byte id in HdfsVolumeId. (Andrew Wang via atm) + + HDFS-3936. MiniDFSCluster shutdown races with BlocksMap usage. (eli) + + HDFS-3951. datanode web ui does not work over HTTPS when datanode is started in secure mode. (tucu) + + HDFS-3949. NameNodeRpcServer#join should join on both client and + server RPC servers. (eli) + + HDFS-3932. NameNode Web UI broken if the rpc-address is set to the wildcard. + (Colin Patrick McCabe via eli) + + HDFS-3931. TestDatanodeBlockScanner#testBlockCorruptionPolicy2 is broken. + (Andy Isaacson via eli) + + HDFS-3964. Make NN log of fs.defaultFS debug rather than info. (eli) + + HDFS-3992. Method org.apache.hadoop.hdfs.TestHftpFileSystem.tearDown() + sometimes throws NPEs. (Ivan A. Veselovsky via atm) + + HDFS-3753. Tests don't run with native libraries. + (Colin Patrick McCabe via eli) + + HDFS-4000. TestParallelLocalRead fails with "input ByteBuffers + must be direct buffers". (Colin Patrick McCabe via eli) + + HDFS-3999. HttpFS OPEN operation expects len parameter, it should be length. (tucu) + + HDFS-4006. TestCheckpoint#testSecondaryHasVeryOutOfDateImage + occasionally fails due to unexpected exit. (todd via eli) + + HDFS-4003. test-patch should build the common native libs before + running hdfs tests. (Colin Patrick McCabe via eli) + + HDFS-4018. testMiniDFSClusterWithMultipleNN is missing some + cluster cleanup. (eli) + + HDFS-4020. TestRBWBlockInvalidation may time out. (eli) + + HDFS-4021. Misleading error message when resources are low on the NameNode. + (Christopher Conner via atm) + + HDFS-4044. Duplicate ChecksumType definition in HDFS .proto files. + (Binglin Chang via suresh) + +Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES @@ -228,6 +468,13 @@ Branch-2 ( Unreleased changes ) HDFS-3150. Add option for clients to contact DNs via hostname. (eli) + HDFS-2793. Add an admin command to trigger an edit log roll. (todd) + + HDFS-3703. Datanodes are marked stale if heartbeat is not received in + configured timeout and are selected as the last location to read from. + (Jing Zhao via suresh) + + IMPROVEMENTS HDFS-3390. DFSAdmin should print full stack traces of errors when DEBUG @@ -421,6 +668,21 @@ Branch-2 ( Unreleased changes ) HDFS-3832. Remove protocol methods related to DistributedUpgrade. (suresh) + HDFS-3177. Update DFSClient and DataXceiver to handle different checkum + types in file checksum computation. (Kihwal Lee via szetszwo) + + HDFS-3871. Change NameNodeProxies to use RetryUtils. (Arun C Murthy + via szetszwo) + + HDFS-3887. Remove redundant chooseTarget methods in BlockPlacementPolicy. + (Jing Zhao via szetszwo) + + HDFS-3888. Clean up BlockPlacementPolicyDefault. (Jing Zhao via szetszwo) + + HDFS-3907. Allow multiple users for local block readers. (eli) + + HDFS-3910. DFSTestUtil#waitReplication should timeout. (eli) + OPTIMIZATIONS HDFS-2982. Startup performance suffers when there are many edit log @@ -437,6 +699,8 @@ Branch-2 ( Unreleased changes ) HDFS-2421. Improve the concurrency of SerialNumberMap in NameNode. (Jing Zhao and Weiyan Wang via szetszwo) + HDFS-3866. HttpFS POM should have property where to download tomcat from (zero45 via tucu) + BUG FIXES HDFS-3385. The last block of INodeFileUnderConstruction is not @@ -659,11 +923,64 @@ Branch-2 ( Unreleased changes ) HDFS-3830. test_libhdfs_threaded: use forceNewInstance. (Colin Patrick McCabe via eli) - HDFS-3837. Fix DataNode.recoverBlock findbugs warning. (eli) - HDFS-3835. Long-lived 2NN cannot perform a checkpoint if security is enabled and the NN restarts with outstanding delegation tokens. (atm) + HDFS-3715. Fix TestFileCreation#testFileCreationNamenodeRestart. + (Andrew Whang via eli) + + HDFS-3731. 2.0 release upgrade must handle blocks being written from 1.0. + (Colin Patrick McCabe via eli) + + HDFS-3683. Edit log replay progress indicator shows >100% complete. (Plamen + Jeliazkov via atm) + + HDFS-3856. TestHDFSServerPorts failure is causing surefire fork failure. + (eli) + + HDFS-3860. HeartbeatManager#Monitor may wrongly hold the writelock of + namesystem. (Jing Zhao via atm) + + HDFS-3849. When re-loading the FSImage, we should clear the existing + genStamp and leases. (Colin Patrick McCabe via atm) + + HDFS-3864. NN does not update internal file mtime for OP_CLOSE when reading + from the edit log. (atm) + + HDFS-3837. Fix DataNode.recoverBlock findbugs warning. (eli) + + HDFS-3733. Audit logs should include WebHDFS access. (Andy Isaacson via + eli) + + HDFS-3466. Get HTTP kerberos principal from the web authentication keytab. + (omalley) + + HDFS-3469. start-dfs.sh will start zkfc, but stop-dfs.sh will not stop zkfc similarly. + (Vinay via umamahesh) + + HDFS-1490. TransferFSImage should timeout (Dmytro Molkov and Vinay via todd) + + HDFS-3828. Block Scanner rescans blocks too frequently. + (Andy Isaacson via eli) + + HDFS-3809. Make BKJM use protobufs for all serialization with ZK.(Ivan Kelly via umamahesh) + + HDFS-3895. hadoop-client must include commons-cli (tucu) + + HDFS-2757. Cannot read a local block that's being written to when + using the local read short circuit. (Jean-Daniel Cryans via eli) + + HDFS-3664. BlockManager race when stopping active services. + (Colin Patrick McCabe via eli) + + HDFS-3928. MiniDFSCluster should reset the first ExitException on shutdown. (eli) + + HDFS-3938. remove current limitations from HttpFS docs. (tucu) + + HDFS-3944. Httpfs resolveAuthority() is not resolving host correctly. (tucu) + + HDFS-3972. Trash emptier fails in secure HA cluster. (todd via eli) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) @@ -680,6 +997,12 @@ Branch-2 ( Unreleased changes ) HDFS-3432. TestDFSZKFailoverController tries to fail over too early (todd) + HDFS-3833. TestDFSShell fails on windows due to concurrent file + read/write. (Brandon Li via suresh) + + HDFS-3902. TestDatanodeBlockScanner#testBlockCorruptionPolicy is broken. + (Andy Isaacson via eli) + Release 2.0.0-alpha - 05-23-2012 INCOMPATIBLE CHANGES @@ -763,6 +1086,9 @@ Release 2.0.0-alpha - 05-23-2012 HDFS-3298. Add HdfsDataOutputStream as a public API. (szetszwo) + HDFS-234. Integration with BookKeeper logging system. (Ivan Kelly + via jitendra) + IMPROVEMENTS HDFS-2018. Move all journal stream management code into one place. @@ -958,7 +1284,7 @@ Release 2.0.0-alpha - 05-23-2012 (Brandon Li via szetszwo) HDFS-2617. Replaced Kerberized SSL for image transfer and fsck - with SPNEGO-based solution. (jghoman, tucu, and atm via eli) + with SPNEGO-based solution. (jghoman, omalley, tucu, and atm via eli) HDFS-3365. Enable users to disable socket caching in DFS client configuration (todd) @@ -1490,7 +1816,43 @@ Release 2.0.0-alpha - 05-23-2012 HDFS-3039. Address findbugs and javadoc warnings on branch. (todd via atm) -Release 0.23.3 - UNRELEASED +Release 0.23.5 - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + + HDFS-3829. TestHftpURLTimeouts fails intermittently with JDK7 (Trevor + Robinson via tgraves) + + HDFS-3824. TestHftpDelegationToken fails intermittently with JDK7 (Trevor + Robinson via tgraves) + + HDFS-3224. Bug in check for DN re-registration with different storage ID + (jlowe) + +Release 0.23.4 - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + + HDFS-3831. Failure to renew tokens due to test-sources left in classpath + (jlowe via bobby) + +Release 0.23.3 INCOMPATIBLE CHANGES @@ -1542,6 +1904,15 @@ Release 0.23.3 - UNRELEASED HDFS-3718. Datanode won't shutdown because of runaway DataBlockScanner thread (Kihwal Lee via daryn) + HDFS-3861. Deadlock in DFSClient (Kihwal Lee via daryn) + + HDFS-3873. Hftp assumes security is disabled if token fetch fails (daryn) + + HDFS-3852. TestHftpDelegationToken is broken after HADOOP-8225 (daryn) + + HDFS-3890. filecontext mkdirs doesn't apply umask as expected + (Tom Graves via daryn) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml (original) +++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml Tue Oct 16 00:02:55 2012 @@ -9,6 +9,9 @@ + + + @@ -273,4 +276,18 @@ + + + + + + + + + + + + + + Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/pom.xml?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/pom.xml (original) +++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/pom.xml Tue Oct 16 00:02:55 2012 @@ -35,6 +35,7 @@ http://maven.apache.org/xsd/maven-4.0.0. ../../hadoop-common-project/hadoop-common/src/test/resources/kdc true false + false @@ -188,6 +189,12 @@ http://maven.apache.org/xsd/maven-4.0.0. ${startKdc} ${kdc.resource.dir} + + + listener + org.apache.hadoop.test.TimedOutTestsListener + + @@ -233,6 +240,25 @@ http://maven.apache.org/xsd/maven-4.0.0. + journal + generate-sources + + compile + + + false + ${project.build.directory}/generated-src/main/jsp + ${project.build.directory}/journal-jsp-servlet-definitions.xml + org.apache.hadoop.hdfs.server.journalservice + + ${basedir}/src/main/webapps/journal + + *.jsp + + + + + datanode generate-sources @@ -297,37 +323,14 @@ http://maven.apache.org/xsd/maven-4.0.0. - compile-proto - generate-sources + create-protobuf-generated-sources-directory + initialize run - - PROTO_DIR=src/main/proto - JAVA_DIR=target/generated-sources/java - which cygpath 2> /dev/null - if [ $? = 1 ]; then - IS_WIN=false - else - IS_WIN=true - WIN_PROTO_DIR=`cygpath --windows $PROTO_DIR` - WIN_JAVA_DIR=`cygpath --windows $JAVA_DIR` - fi - mkdir -p $JAVA_DIR 2> /dev/null - for PROTO_FILE in `ls $PROTO_DIR/*.proto 2> /dev/null` - do - if [ "$IS_WIN" = "true" ]; then - protoc -I$WIN_PROTO_DIR --java_out=$WIN_JAVA_DIR $PROTO_FILE - else - protoc -I$PROTO_DIR --java_out=$JAVA_DIR $PROTO_FILE - fi - done - - - - + @@ -342,6 +345,7 @@ http://maven.apache.org/xsd/maven-4.0.0. + @@ -357,6 +361,9 @@ http://maven.apache.org/xsd/maven-4.0.0. + @@ -401,6 +408,81 @@ http://maven.apache.org/xsd/maven-4.0.0. + org.codehaus.mojo + exec-maven-plugin + + + compile-proto + generate-sources + + exec + + + protoc + + -Isrc/main/proto/ + --java_out=target/generated-sources/java + src/main/proto/GetUserMappingsProtocol.proto + src/main/proto/HAZKInfo.proto + src/main/proto/InterDatanodeProtocol.proto + src/main/proto/JournalProtocol.proto + src/main/proto/RefreshAuthorizationPolicyProtocol.proto + src/main/proto/RefreshUserMappingsProtocol.proto + src/main/proto/datatransfer.proto + src/main/proto/hdfs.proto + + + + + compile-proto-datanode + generate-sources + + exec + + + protoc + + -Isrc/main/proto/ + --java_out=target/generated-sources/java + src/main/proto/ClientDatanodeProtocol.proto + src/main/proto/DatanodeProtocol.proto + + + + + compile-proto-namenode + generate-sources + + exec + + + protoc + + -Isrc/main/proto/ + --java_out=target/generated-sources/java + src/main/proto/ClientNamenodeProtocol.proto + src/main/proto/NamenodeProtocol.proto + + + + + compile-proto-qjournal + generate-sources + + exec + + + protoc + + -Isrc/main/proto/ + --java_out=target/generated-sources/java + src/main/proto/QJournalProtocol.proto + + + + + + org.apache.maven.plugins maven-javadoc-plugin @@ -472,7 +554,7 @@ http://maven.apache.org/xsd/maven-4.0.0. - + Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt (original) +++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt Tue Oct 16 00:02:55 2012 @@ -85,8 +85,8 @@ CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/confi add_dual_library(hdfs main/native/libhdfs/exception.c - main/native/libhdfs/hdfs.c main/native/libhdfs/jni_helper.c + main/native/libhdfs/hdfs.c ) target_link_dual_libraries(hdfs ${JAVA_JVM_LIBRARY} @@ -147,4 +147,7 @@ target_link_libraries(test_libhdfs_threa pthread ) +IF(REQUIRE_LIBWEBHDFS) + add_subdirectory(contrib/libwebhdfs) +ENDIF(REQUIRE_LIBWEBHDFS) add_subdirectory(main/native/fuse-dfs) Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml (original) +++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml Tue Oct 16 00:02:55 2012 @@ -89,6 +89,87 @@ http://maven.apache.org/xsd/maven-4.0.0. test + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-source + generate-sources + + add-source + + + + ${project.build.directory}/generated-sources/java + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + false + + + + create-protobuf-generated-sources-directory + initialize + + run + + + + + + + + + + + org.codehaus.mojo + exec-maven-plugin + + + compile-proto + generate-sources + + exec + + + protoc + + -Isrc/main/proto/ + -I../../main/proto + --java_out=target/generated-sources/java + src/main/proto/bkjournal.proto + + + + + + + org.codehaus.mojo + findbugs-maven-plugin + + ${basedir}/dev-support/findbugsExcludeFile.xml + + + + org.apache.rat + apache-rat-plugin + + + dev-support/findbugsExcludeFile.xml + + + + + dist Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java (original) +++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java Tue Oct 16 00:02:55 2012 @@ -70,7 +70,7 @@ class BookKeeperEditLogInputStream exten this.lh = lh; this.firstTxId = metadata.getFirstTxId(); this.lastTxId = metadata.getLastTxId(); - this.logVersion = metadata.getVersion(); + this.logVersion = metadata.getDataLayoutVersion(); this.inProgress = metadata.isInProgress(); if (firstBookKeeperEntry < 0 Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogOutputStream.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogOutputStream.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogOutputStream.java (original) +++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogOutputStream.java Tue Oct 16 00:02:55 2012 @@ -84,7 +84,7 @@ class BookKeeperEditLogOutputStream @Override public void close() throws IOException { setReadyToFlush(); - flushAndSync(); + flushAndSync(true); try { lh.close(); } catch (InterruptedException ie) { @@ -130,7 +130,7 @@ class BookKeeperEditLogOutputStream } @Override - public void flushAndSync() throws IOException { + public void flushAndSync(boolean durable) throws IOException { assert(syncLatch != null); try { syncLatch.await(); Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java (original) +++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java Tue Oct 16 00:02:55 2012 @@ -50,6 +50,11 @@ import java.io.IOException; import java.net.URI; +import org.apache.hadoop.hdfs.protocolPB.PBHelper; +import org.apache.hadoop.contrib.bkjournal.BKJournalProtos.VersionProto; +import com.google.protobuf.TextFormat; +import static com.google.common.base.Charsets.UTF_8; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import com.google.common.annotations.VisibleForTesting; @@ -143,36 +148,17 @@ public class BookKeeperJournalManager im private final int quorumSize; private final String digestpw; private final CountDownLatch zkConnectLatch; - + private final NamespaceInfo nsInfo; private LedgerHandle currentLedger = null; - private int bytesToInt(byte[] b) { - assert b.length >= 4; - return b[0] << 24 | b[1] << 16 | b[2] << 8 | b[3]; - } - - private byte[] intToBytes(int i) { - return new byte[] { - (byte)(i >> 24), - (byte)(i >> 16), - (byte)(i >> 8), - (byte)(i) }; - } - - BookKeeperJournalManager(Configuration conf, URI uri) throws IOException { - this(conf, uri, null); - // TODO(ivank): update BookKeeperJournalManager to do something - // with the NamespaceInfo. This constructor has been added - // for compatibility with the old tests, and may be removed - // when the tests are updated. - } - /** * Construct a Bookkeeper journal manager. */ public BookKeeperJournalManager(Configuration conf, URI uri, NamespaceInfo nsInfo) throws IOException { this.conf = conf; + this.nsInfo = nsInfo; + String zkConnect = uri.getAuthority().replace(";", ","); String zkPath = uri.getPath(); ensembleSize = conf.getInt(BKJM_BOOKKEEPER_ENSEMBLE_SIZE, @@ -202,10 +188,32 @@ public class BookKeeperJournalManager im Stat versionStat = zkc.exists(versionPath, false); if (versionStat != null) { byte[] d = zkc.getData(versionPath, false, versionStat); + VersionProto.Builder builder = VersionProto.newBuilder(); + TextFormat.merge(new String(d, UTF_8), builder); + if (!builder.isInitialized()) { + throw new IOException("Invalid/Incomplete data in znode"); + } + VersionProto vp = builder.build(); + // There's only one version at the moment - assert bytesToInt(d) == BKJM_LAYOUT_VERSION; - } else { - zkc.create(versionPath, intToBytes(BKJM_LAYOUT_VERSION), + assert vp.getLayoutVersion() == BKJM_LAYOUT_VERSION; + + NamespaceInfo readns = PBHelper.convert(vp.getNamespaceInfo()); + + if (nsInfo.getNamespaceID() != readns.getNamespaceID() || + !nsInfo.clusterID.equals(readns.getClusterID()) || + !nsInfo.getBlockPoolID().equals(readns.getBlockPoolID())) { + String err = String.format("Environment mismatch. Running process %s" + +", stored in ZK %s", nsInfo, readns); + LOG.error(err); + throw new IOException(err); + } + } else if (nsInfo.getNamespaceID() > 0) { + VersionProto.Builder builder = VersionProto.newBuilder(); + builder.setNamespaceInfo(PBHelper.convert(nsInfo)) + .setLayoutVersion(BKJM_LAYOUT_VERSION); + byte[] data = TextFormat.printToString(builder.build()).getBytes(UTF_8); + zkc.create(versionPath, data, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); } @@ -214,11 +222,11 @@ public class BookKeeperJournalManager im Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); } prepareBookKeeperEnv(); - bkc = new BookKeeper(new ClientConfiguration(), - zkc); + bkc = new BookKeeper(new ClientConfiguration(), zkc); } catch (KeeperException e) { throw new IOException("Error initializing zk", e); } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); throw new IOException("Interrupted while initializing bk journal manager", ie); } @@ -322,13 +330,14 @@ public class BookKeeperJournalManager im } catch (KeeperException ke) { throw new IOException("Error in zookeeper while creating ledger", ke); } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); throw new IOException("Interrupted creating ledger", ie); } try { String znodePath = inprogressZNode(txId); EditLogLedgerMetadata l = new EditLogLedgerMetadata(znodePath, - HdfsConstants.LAYOUT_VERSION, currentLedger.getId(), txId); + HdfsConstants.LAYOUT_VERSION, currentLedger.getId(), txId); /* Write the ledger metadata out to the inprogress ledger znode * This can fail if for some reason our write lock has * expired (@see WriteLock) and another process has managed to @@ -356,6 +365,7 @@ public class BookKeeperJournalManager im //log & ignore, an IOException will be thrown soon LOG.error("Error closing ledger", bke); } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); LOG.warn("Interrupted while closing ledger", ie); } } @@ -425,6 +435,7 @@ public class BookKeeperJournalManager im } catch (KeeperException e) { throw new IOException("Error finalising ledger", e); } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); throw new IOException("Error finalising ledger", ie); } } @@ -454,6 +465,7 @@ public class BookKeeperJournalManager im } catch (BKException e) { throw new IOException("Could not open ledger for " + fromTxId, e); } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); throw new IOException("Interrupted opening ledger for " + fromTxId, ie); } @@ -567,6 +579,7 @@ public class BookKeeperJournalManager im } catch (KeeperException ke) { throw new IOException("Couldn't get list of inprogress segments", ke); } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); throw new IOException("Interrupted getting list of inprogress segments", ie); } @@ -583,6 +596,7 @@ public class BookKeeperJournalManager im zkc.delete(l.getZkPath(), stat.getVersion()); bkc.deleteLedger(l.getLedgerId()); } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); LOG.error("Interrupted while purging " + l, ie); } catch (BKException bke) { LOG.error("Couldn't delete ledger from bookkeeper", bke); @@ -601,6 +615,7 @@ public class BookKeeperJournalManager im } catch (BKException bke) { throw new IOException("Couldn't close bookkeeper client", bke); } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); throw new IOException("Interrupted while closing journal manager", ie); } } @@ -635,6 +650,7 @@ public class BookKeeperJournalManager im } catch (BKException bke) { throw new IOException("Exception opening ledger for " + l, bke); } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); throw new IOException("Interrupted opening ledger for " + l, ie); } @@ -692,6 +708,7 @@ public class BookKeeperJournalManager im } catch (KeeperException e) { throw new IOException("Exception reading ledger list from zk", e); } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); throw new IOException("Interrupted getting list of ledgers from zk", ie); } Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/CurrentInprogress.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/CurrentInprogress.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/CurrentInprogress.java (original) +++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/CurrentInprogress.java Tue Oct 16 00:02:55 2012 @@ -29,6 +29,10 @@ import org.apache.zookeeper.KeeperExcept import org.apache.zookeeper.ZooDefs.Ids; import org.apache.zookeeper.data.Stat; +import org.apache.hadoop.contrib.bkjournal.BKJournalProtos.CurrentInprogressProto; +import com.google.protobuf.TextFormat; +import static com.google.common.base.Charsets.UTF_8; + /** * Distributed write permission lock, using ZooKeeper. Read the version number * and return the current inprogress node path available in CurrentInprogress @@ -42,29 +46,28 @@ import org.apache.zookeeper.data.Stat; */ class CurrentInprogress { - private static final String CONTENT_DELIMITER = ","; - static final Log LOG = LogFactory.getLog(CurrentInprogress.class); private final ZooKeeper zkc; private final String currentInprogressNode; private volatile int versionNumberForPermission = -1; - private static final int CURRENT_INPROGRESS_LAYOUT_VERSION = -1; private final String hostName = InetAddress.getLocalHost().toString(); CurrentInprogress(ZooKeeper zkc, String lockpath) throws IOException { this.currentInprogressNode = lockpath; this.zkc = zkc; try { - Stat isCurrentInprogressNodeExists = zkc.exists(lockpath, false); + Stat isCurrentInprogressNodeExists = zkc.exists(currentInprogressNode, + false); if (isCurrentInprogressNodeExists == null) { try { - zkc.create(lockpath, null, Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); + zkc.create(currentInprogressNode, null, Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); } catch (NodeExistsException e) { // Node might created by other process at the same time. Ignore it. if (LOG.isDebugEnabled()) { - LOG.debug(lockpath + " already created by other process.", e); + LOG.debug(currentInprogressNode + " already created by other process.", + e); } } } @@ -83,10 +86,13 @@ class CurrentInprogress { * @throws IOException */ void update(String path) throws IOException { - String content = CURRENT_INPROGRESS_LAYOUT_VERSION - + CONTENT_DELIMITER + hostName + CONTENT_DELIMITER + path; + CurrentInprogressProto.Builder builder = CurrentInprogressProto.newBuilder(); + builder.setPath(path).setHostname(hostName); + + String content = TextFormat.printToString(builder.build()); + try { - zkc.setData(this.currentInprogressNode, content.getBytes(), + zkc.setData(this.currentInprogressNode, content.getBytes(UTF_8), this.versionNumberForPermission); } catch (KeeperException e) { throw new IOException("Exception when setting the data " @@ -123,23 +129,12 @@ class CurrentInprogress { } this.versionNumberForPermission = stat.getVersion(); if (data != null) { - String stringData = new String(data); - LOG.info("Read data[layout version number,hostname,inprogressNode path]" - + "= [" + stringData + "] from CurrentInprogress"); - String[] contents = stringData.split(CONTENT_DELIMITER); - assert contents.length == 3 : "As per the current data format, " - + "CurrentInprogress node data should contain 3 fields. " - + "i.e layout version number,hostname,inprogressNode path"; - String layoutVersion = contents[0]; - if (Long.valueOf(layoutVersion) > CURRENT_INPROGRESS_LAYOUT_VERSION) { - throw new IOException( - "Supported layout version of CurrentInprogress node is : " - + CURRENT_INPROGRESS_LAYOUT_VERSION - + " . Layout version of CurrentInprogress node in ZK is : " - + layoutVersion); + CurrentInprogressProto.Builder builder = CurrentInprogressProto.newBuilder(); + TextFormat.merge(new String(data, UTF_8), builder); + if (!builder.isInitialized()) { + throw new IOException("Invalid/Incomplete data in znode"); } - String inprogressNodePath = contents[2]; - return inprogressNodePath; + return builder.build().getPath(); } else { LOG.info("No data available in CurrentInprogress"); } Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/EditLogLedgerMetadata.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/EditLogLedgerMetadata.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/EditLogLedgerMetadata.java (original) +++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/EditLogLedgerMetadata.java Tue Oct 16 00:02:55 2012 @@ -29,6 +29,10 @@ import org.apache.hadoop.hdfs.protocol.H import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.contrib.bkjournal.BKJournalProtos.EditLogLedgerProto; +import com.google.protobuf.TextFormat; +import static com.google.common.base.Charsets.UTF_8; + /** * Utility class for storing the metadata associated * with a single edit log segment, stored in a single ledger @@ -37,8 +41,8 @@ public class EditLogLedgerMetadata { static final Log LOG = LogFactory.getLog(EditLogLedgerMetadata.class); private String zkPath; + private final int dataLayoutVersion; private final long ledgerId; - private final int version; private final long firstTxId; private long lastTxId; private boolean inprogress; @@ -57,21 +61,22 @@ public class EditLogLedgerMetadata { } }; - EditLogLedgerMetadata(String zkPath, int version, + EditLogLedgerMetadata(String zkPath, int dataLayoutVersion, long ledgerId, long firstTxId) { this.zkPath = zkPath; + this.dataLayoutVersion = dataLayoutVersion; this.ledgerId = ledgerId; - this.version = version; this.firstTxId = firstTxId; this.lastTxId = HdfsConstants.INVALID_TXID; this.inprogress = true; } - EditLogLedgerMetadata(String zkPath, int version, long ledgerId, - long firstTxId, long lastTxId) { + EditLogLedgerMetadata(String zkPath, int dataLayoutVersion, + long ledgerId, long firstTxId, + long lastTxId) { this.zkPath = zkPath; + this.dataLayoutVersion = dataLayoutVersion; this.ledgerId = ledgerId; - this.version = version; this.firstTxId = firstTxId; this.lastTxId = lastTxId; this.inprogress = false; @@ -93,14 +98,14 @@ public class EditLogLedgerMetadata { return ledgerId; } - int getVersion() { - return version; - } - boolean isInProgress() { return this.inprogress; } + int getDataLayoutVersion() { + return this.dataLayoutVersion; + } + void finalizeLedger(long newLastTxId) { assert this.lastTxId == HdfsConstants.INVALID_TXID; this.lastTxId = newLastTxId; @@ -111,22 +116,27 @@ public class EditLogLedgerMetadata { throws IOException, KeeperException.NoNodeException { try { byte[] data = zkc.getData(path, false, null); - String[] parts = new String(data).split(";"); - if (parts.length == 3) { - int version = Integer.valueOf(parts[0]); - long ledgerId = Long.valueOf(parts[1]); - long txId = Long.valueOf(parts[2]); - return new EditLogLedgerMetadata(path, version, ledgerId, txId); - } else if (parts.length == 4) { - int version = Integer.valueOf(parts[0]); - long ledgerId = Long.valueOf(parts[1]); - long firstTxId = Long.valueOf(parts[2]); - long lastTxId = Long.valueOf(parts[3]); - return new EditLogLedgerMetadata(path, version, ledgerId, - firstTxId, lastTxId); + + EditLogLedgerProto.Builder builder = EditLogLedgerProto.newBuilder(); + if (LOG.isDebugEnabled()) { + LOG.debug("Reading " + path + " data: " + new String(data, UTF_8)); + } + TextFormat.merge(new String(data, UTF_8), builder); + if (!builder.isInitialized()) { + throw new IOException("Invalid/Incomplete data in znode"); + } + EditLogLedgerProto ledger = builder.build(); + + int dataLayoutVersion = ledger.getDataLayoutVersion(); + long ledgerId = ledger.getLedgerId(); + long firstTxId = ledger.getFirstTxId(); + if (ledger.hasLastTxId()) { + long lastTxId = ledger.getLastTxId(); + return new EditLogLedgerMetadata(path, dataLayoutVersion, + ledgerId, firstTxId, lastTxId); } else { - throw new IOException("Invalid ledger entry, " - + new String(data)); + return new EditLogLedgerMetadata(path, dataLayoutVersion, + ledgerId, firstTxId); } } catch(KeeperException.NoNodeException nne) { throw nne; @@ -140,17 +150,17 @@ public class EditLogLedgerMetadata { void write(ZooKeeper zkc, String path) throws IOException, KeeperException.NodeExistsException { this.zkPath = path; - String finalisedData; - if (inprogress) { - finalisedData = String.format("%d;%d;%d", - version, ledgerId, firstTxId); - } else { - finalisedData = String.format("%d;%d;%d;%d", - version, ledgerId, firstTxId, lastTxId); + + EditLogLedgerProto.Builder builder = EditLogLedgerProto.newBuilder(); + builder.setDataLayoutVersion(dataLayoutVersion) + .setLedgerId(ledgerId).setFirstTxId(firstTxId); + + if (!inprogress) { + builder.setLastTxId(lastTxId); } try { - zkc.create(path, finalisedData.getBytes(), Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); + zkc.create(path, TextFormat.printToString(builder.build()).getBytes(UTF_8), + Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); } catch (KeeperException.NodeExistsException nee) { throw nee; } catch (KeeperException e) { @@ -183,9 +193,9 @@ public class EditLogLedgerMetadata { } EditLogLedgerMetadata ol = (EditLogLedgerMetadata)o; return ledgerId == ol.ledgerId + && dataLayoutVersion == ol.dataLayoutVersion && firstTxId == ol.firstTxId - && lastTxId == ol.lastTxId - && version == ol.version; + && lastTxId == ol.lastTxId; } public int hashCode() { @@ -193,15 +203,15 @@ public class EditLogLedgerMetadata { hash = hash * 31 + (int) ledgerId; hash = hash * 31 + (int) firstTxId; hash = hash * 31 + (int) lastTxId; - hash = hash * 31 + (int) version; + hash = hash * 31 + (int) dataLayoutVersion; return hash; } public String toString() { return "[LedgerId:"+ledgerId + ", firstTxId:" + firstTxId + - ", lastTxId:" + lastTxId + - ", version:" + version + "]"; + ", lastTxId:" + lastTxId + + ", dataLayoutVersion:" + dataLayoutVersion + "]"; } } Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/MaxTxId.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/MaxTxId.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/MaxTxId.java (original) +++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/MaxTxId.java Tue Oct 16 00:02:55 2012 @@ -27,6 +27,10 @@ import org.apache.zookeeper.ZooKeeper; import org.apache.zookeeper.ZooDefs.Ids; import org.apache.zookeeper.data.Stat; +import org.apache.hadoop.contrib.bkjournal.BKJournalProtos.MaxTxIdProto; +import com.google.protobuf.TextFormat; +import static com.google.common.base.Charsets.UTF_8; + /** * Utility class for storing and reading * the max seen txid in zookeeper @@ -55,14 +59,16 @@ class MaxTxId { } synchronized void reset(long maxTxId) throws IOException { - String txidStr = Long.toString(maxTxId); try { + MaxTxIdProto.Builder builder = MaxTxIdProto.newBuilder().setTxId(maxTxId); + + byte[] data = TextFormat.printToString(builder.build()).getBytes(UTF_8); if (currentStat != null) { - currentStat = zkc.setData(path, txidStr.getBytes("UTF-8"), currentStat + currentStat = zkc.setData(path, data, currentStat .getVersion()); } else { - zkc.create(path, txidStr.getBytes("UTF-8"), Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); + zkc.create(path, data, Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); } } catch (KeeperException e) { throw new IOException("Error writing max tx id", e); @@ -77,9 +83,16 @@ class MaxTxId { if (currentStat == null) { return 0; } else { + byte[] bytes = zkc.getData(path, false, currentStat); - String txidString = new String(bytes, "UTF-8"); - return Long.valueOf(txidString); + + MaxTxIdProto.Builder builder = MaxTxIdProto.newBuilder(); + TextFormat.merge(new String(bytes, UTF_8), builder); + if (!builder.isInitialized()) { + throw new IOException("Invalid/Incomplete data in znode"); + } + + return builder.build().getTxId(); } } catch (KeeperException e) { throw new IOException("Error reading the max tx id from zk", e); Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperConfiguration.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperConfiguration.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperConfiguration.java (original) +++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperConfiguration.java Tue Oct 16 00:02:55 2012 @@ -23,6 +23,7 @@ import java.net.InetSocketAddress; import java.net.URI; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.Random; import org.apache.bookkeeper.util.LocalBookKeeper; import org.apache.commons.logging.Log; @@ -42,6 +43,8 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; + public class TestBookKeeperConfiguration { private static final Log LOG = LogFactory .getLog(TestBookKeeperConfiguration.class); @@ -73,6 +76,11 @@ public class TestBookKeeperConfiguration return zkc; } + private NamespaceInfo newNSInfo() { + Random r = new Random(); + return new NamespaceInfo(r.nextInt(), "testCluster", "TestBPID", -1); + } + @BeforeClass public static void setupZooKeeper() throws Exception { // create a ZooKeeper server(dataDir, dataLogDir, port) @@ -137,8 +145,10 @@ public class TestBookKeeperConfiguration bkAvailablePath); Assert.assertNull(bkAvailablePath + " already exists", zkc.exists( bkAvailablePath, false)); - bkjm = new BookKeeperJournalManager(conf, URI.create("bookkeeper://" - + HOSTPORT + "/hdfsjournal-WithBKPath")); + NamespaceInfo nsi = newNSInfo(); + bkjm = new BookKeeperJournalManager(conf, + URI.create("bookkeeper://" + HOSTPORT + "/hdfsjournal-WithBKPath"), + nsi); Assert.assertNotNull("Bookie available path : " + bkAvailablePath + " doesn't exists", zkc.exists(bkAvailablePath, false)); } @@ -152,8 +162,10 @@ public class TestBookKeeperConfiguration Configuration conf = new Configuration(); Assert.assertNull(BK_ROOT_PATH + " already exists", zkc.exists( BK_ROOT_PATH, false)); - new BookKeeperJournalManager(conf, URI.create("bookkeeper://" + HOSTPORT - + "/hdfsjournal-DefaultBKPath")); + NamespaceInfo nsi = newNSInfo(); + bkjm = new BookKeeperJournalManager(conf, + URI.create("bookkeeper://" + HOSTPORT + "/hdfsjournal-DefaultBKPath"), + nsi); Assert.assertNotNull("Bookie available path : " + BK_ROOT_PATH + " doesn't exists", zkc.exists(BK_ROOT_PATH, false)); } Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java (original) +++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java Tue Oct 16 00:02:55 2012 @@ -29,6 +29,7 @@ import org.mockito.Mockito; import java.io.IOException; import java.net.URI; import java.util.List; +import java.util.Random; import org.apache.hadoop.conf.Configuration; @@ -37,6 +38,7 @@ import org.apache.hadoop.hdfs.server.nam import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp; import org.apache.hadoop.hdfs.server.namenode.FSEditLogTestUtil; import org.apache.hadoop.hdfs.server.namenode.JournalManager; +import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.bookkeeper.proto.BookieServer; import org.apache.zookeeper.CreateMode; @@ -78,10 +80,17 @@ public class TestBookKeeperJournalManage zkc.close(); } + private NamespaceInfo newNSInfo() { + Random r = new Random(); + return new NamespaceInfo(r.nextInt(), "testCluster", "TestBPID", -1); + } + @Test public void testSimpleWrite() throws Exception { + NamespaceInfo nsi = newNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-simplewrite")); + BKJMUtil.createJournalURI("/hdfsjournal-simplewrite"), nsi); + EditLogOutputStream out = bkjm.startLogSegment(1); for (long i = 1 ; i <= 100; i++) { FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); @@ -99,8 +108,10 @@ public class TestBookKeeperJournalManage @Test public void testNumberOfTransactions() throws Exception { + NamespaceInfo nsi = newNSInfo(); + BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-txncount")); + BKJMUtil.createJournalURI("/hdfsjournal-txncount"), nsi); EditLogOutputStream out = bkjm.startLogSegment(1); for (long i = 1 ; i <= 100; i++) { FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); @@ -116,8 +127,10 @@ public class TestBookKeeperJournalManage @Test public void testNumberOfTransactionsWithGaps() throws Exception { + NamespaceInfo nsi = newNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-gaps")); + BKJMUtil.createJournalURI("/hdfsjournal-gaps"), nsi); + long txid = 1; for (long i = 0; i < 3; i++) { long start = txid; @@ -151,8 +164,10 @@ public class TestBookKeeperJournalManage @Test public void testNumberOfTransactionsWithInprogressAtEnd() throws Exception { + NamespaceInfo nsi = newNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-inprogressAtEnd")); + BKJMUtil.createJournalURI("/hdfsjournal-inprogressAtEnd"), nsi); + long txid = 1; for (long i = 0; i < 3; i++) { long start = txid; @@ -190,8 +205,10 @@ public class TestBookKeeperJournalManage */ @Test public void testWriteRestartFrom1() throws Exception { + NamespaceInfo nsi = newNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-restartFrom1")); + BKJMUtil.createJournalURI("/hdfsjournal-restartFrom1"), nsi); + long txid = 1; long start = txid; EditLogOutputStream out = bkjm.startLogSegment(txid); @@ -245,11 +262,15 @@ public class TestBookKeeperJournalManage @Test public void testTwoWriters() throws Exception { long start = 1; + NamespaceInfo nsi = newNSInfo(); + BookKeeperJournalManager bkjm1 = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-dualWriter")); + BKJMUtil.createJournalURI("/hdfsjournal-dualWriter"), nsi); + BookKeeperJournalManager bkjm2 = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-dualWriter")); - + BKJMUtil.createJournalURI("/hdfsjournal-dualWriter"), nsi); + + EditLogOutputStream out1 = bkjm1.startLogSegment(start); try { bkjm2.startLogSegment(start); @@ -263,8 +284,11 @@ public class TestBookKeeperJournalManage @Test public void testSimpleRead() throws Exception { + NamespaceInfo nsi = newNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-simpleread")); + BKJMUtil.createJournalURI("/hdfsjournal-simpleread"), + nsi); + final long numTransactions = 10000; EditLogOutputStream out = bkjm.startLogSegment(1); for (long i = 1 ; i <= numTransactions; i++) { @@ -287,8 +311,11 @@ public class TestBookKeeperJournalManage @Test public void testSimpleRecovery() throws Exception { + NamespaceInfo nsi = newNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-simplerecovery")); + BKJMUtil.createJournalURI("/hdfsjournal-simplerecovery"), + nsi); + EditLogOutputStream out = bkjm.startLogSegment(1); for (long i = 1 ; i <= 100; i++) { FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); @@ -334,8 +361,10 @@ public class TestBookKeeperJournalManage conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_QUORUM_SIZE, ensembleSize); long txid = 1; + NamespaceInfo nsi = newNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-allbookiefailure")); + BKJMUtil.createJournalURI("/hdfsjournal-allbookiefailure"), + nsi); EditLogOutputStream out = bkjm.startLogSegment(txid); for (long i = 1 ; i <= 3; i++) { @@ -416,8 +445,12 @@ public class TestBookKeeperJournalManage conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_QUORUM_SIZE, ensembleSize); long txid = 1; + + NamespaceInfo nsi = newNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-onebookiefailure")); + BKJMUtil.createJournalURI("/hdfsjournal-onebookiefailure"), + nsi); + EditLogOutputStream out = bkjm.startLogSegment(txid); for (long i = 1 ; i <= 3; i++) { FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); @@ -464,7 +497,9 @@ public class TestBookKeeperJournalManage @Test public void testEmptyInprogressNode() throws Exception { URI uri = BKJMUtil.createJournalURI("/hdfsjournal-emptyInprogress"); - BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri); + NamespaceInfo nsi = newNSInfo(); + BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, + nsi); EditLogOutputStream out = bkjm.startLogSegment(1); for (long i = 1; i <= 100; i++) { @@ -481,7 +516,7 @@ public class TestBookKeeperJournalManage String inprogressZNode = bkjm.inprogressZNode(101); zkc.setData(inprogressZNode, new byte[0], -1); - bkjm = new BookKeeperJournalManager(conf, uri); + bkjm = new BookKeeperJournalManager(conf, uri, nsi); try { bkjm.recoverUnfinalizedSegments(); fail("Should have failed. There should be no way of creating" @@ -489,7 +524,7 @@ public class TestBookKeeperJournalManage } catch (IOException e) { // correct behaviour assertTrue("Exception different than expected", e.getMessage().contains( - "Invalid ledger entry,")); + "Invalid/Incomplete data in znode")); } finally { bkjm.close(); } @@ -503,7 +538,9 @@ public class TestBookKeeperJournalManage @Test public void testCorruptInprogressNode() throws Exception { URI uri = BKJMUtil.createJournalURI("/hdfsjournal-corruptInprogress"); - BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri); + NamespaceInfo nsi = newNSInfo(); + BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, + nsi); EditLogOutputStream out = bkjm.startLogSegment(1); for (long i = 1; i <= 100; i++) { @@ -521,7 +558,7 @@ public class TestBookKeeperJournalManage String inprogressZNode = bkjm.inprogressZNode(101); zkc.setData(inprogressZNode, "WholeLottaJunk".getBytes(), -1); - bkjm = new BookKeeperJournalManager(conf, uri); + bkjm = new BookKeeperJournalManager(conf, uri, nsi); try { bkjm.recoverUnfinalizedSegments(); fail("Should have failed. There should be no way of creating" @@ -529,8 +566,7 @@ public class TestBookKeeperJournalManage } catch (IOException e) { // correct behaviour assertTrue("Exception different than expected", e.getMessage().contains( - "Invalid ledger entry,")); - + "has no field named")); } finally { bkjm.close(); } @@ -544,7 +580,9 @@ public class TestBookKeeperJournalManage @Test public void testEmptyInprogressLedger() throws Exception { URI uri = BKJMUtil.createJournalURI("/hdfsjournal-emptyInprogressLedger"); - BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri); + NamespaceInfo nsi = newNSInfo(); + BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, + nsi); EditLogOutputStream out = bkjm.startLogSegment(1); for (long i = 1; i <= 100; i++) { @@ -559,7 +597,7 @@ public class TestBookKeeperJournalManage out.close(); bkjm.close(); - bkjm = new BookKeeperJournalManager(conf, uri); + bkjm = new BookKeeperJournalManager(conf, uri, nsi); bkjm.recoverUnfinalizedSegments(); out = bkjm.startLogSegment(101); for (long i = 1; i <= 100; i++) { @@ -581,7 +619,9 @@ public class TestBookKeeperJournalManage public void testRefinalizeAlreadyFinalizedInprogress() throws Exception { URI uri = BKJMUtil .createJournalURI("/hdfsjournal-refinalizeInprogressLedger"); - BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri); + NamespaceInfo nsi = newNSInfo(); + BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, + nsi); EditLogOutputStream out = bkjm.startLogSegment(1); for (long i = 1; i <= 100; i++) { @@ -601,7 +641,7 @@ public class TestBookKeeperJournalManage byte[] inprogressData = zkc.getData(inprogressZNode, false, null); // finalize - bkjm = new BookKeeperJournalManager(conf, uri); + bkjm = new BookKeeperJournalManager(conf, uri, nsi); bkjm.recoverUnfinalizedSegments(); bkjm.close(); @@ -613,7 +653,7 @@ public class TestBookKeeperJournalManage CreateMode.PERSISTENT); // should work fine - bkjm = new BookKeeperJournalManager(conf, uri); + bkjm = new BookKeeperJournalManager(conf, uri, nsi); bkjm.recoverUnfinalizedSegments(); bkjm.close(); } @@ -626,7 +666,10 @@ public class TestBookKeeperJournalManage @Test public void testEditLogFileNotExistsWhenReadingMetadata() throws Exception { URI uri = BKJMUtil.createJournalURI("/hdfsjournal-editlogfile"); - BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri); + NamespaceInfo nsi = newNSInfo(); + BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, + nsi); + try { // start new inprogress log segment with txid=1 // and write transactions till txid=50 Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs (original) +++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs Tue Oct 16 00:02:55 2012 @@ -17,7 +17,7 @@ bin=`which $0` bin=`dirname ${bin}` -bin=`cd "$bin"; pwd` +bin=`cd "$bin" > /dev/null; pwd` DEFAULT_LIBEXEC_DIR="$bin"/../libexec HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} @@ -30,6 +30,7 @@ function print_usage(){ echo " namenode -format format the DFS filesystem" echo " secondarynamenode run the DFS secondary namenode" echo " namenode run the DFS namenode" + echo " journalnode run the DFS journalnode" echo " zkfc run the ZK Failover Controller daemon" echo " datanode run a DFS datanode" echo " dfsadmin run a DFS admin client" @@ -90,6 +91,9 @@ elif [ "$COMMAND" = "datanode" ] ; then else HADOOP_OPTS="$HADOOP_OPTS -server $HADOOP_DATANODE_OPTS" fi +elif [ "$COMMAND" = "journalnode" ] ; then + CLASS='org.apache.hadoop.hdfs.qjournal.server.JournalNode' + HADOOP_OPTS="$HADOOP_OPTS $HADOOP_JOURNALNODE_OPTS" elif [ "$COMMAND" = "dfs" ] ; then CLASS=org.apache.hadoop.fs.FsShell HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh (original) +++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh Tue Oct 16 00:02:55 2012 @@ -61,4 +61,14 @@ if [ -n "$SECONDARY_NAMENODES" ]; then --script "$bin/hdfs" stop secondarynamenode fi +#--------------------------------------------------------- +# ZK Failover controllers, if auto-HA is enabled +AUTOHA_ENABLED=$($HADOOP_PREFIX/bin/hdfs getconf -confKey dfs.ha.automatic-failover.enabled) +if [ "$(echo "$AUTOHA_ENABLED" | tr A-Z a-z)" = "true" ]; then + echo "Stopping ZK Failover Controllers on NN hosts [$NAMENODES]" + "$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \ + --config "$HADOOP_CONF_DIR" \ + --hostnames "$NAMENODES" \ + --script "$bin/hdfs" stop zkfc +fi # eof Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/conf/hadoop-metrics2.properties URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/conf/hadoop-metrics2.properties?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/conf/hadoop-metrics2.properties (original) +++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/conf/hadoop-metrics2.properties Tue Oct 16 00:02:55 2012 @@ -19,7 +19,7 @@ # See javadoc of package-info.java for org.apache.hadoop.metrics2 for details *.sink.file.class=org.apache.hadoop.metrics2.sink.FileSink -# default sampling period +# default sampling period, in seconds *.period=10 # The namenode-metrics.out will contain metrics from all context Modified: hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/libhdfs.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/libhdfs.xml?rev=1398581&r1=1398580&r2=1398581&view=diff ============================================================================== --- hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/libhdfs.xml (original) +++ hadoop/common/branches/MR-3902/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/libhdfs.xml Tue Oct 16 00:02:55 2012 @@ -1,110 +1,110 @@ - - - - - - -
-C API libhdfs -Content-Type -text/html; -utf-8 -
- -
-Overview - -

-libhdfs is a JNI based C API for Hadoop's Distributed File System (HDFS). -It provides C APIs to a subset of the HDFS APIs to manipulate HDFS files and -the filesystem. libhdfs is part of the Hadoop distribution and comes -pre-compiled in ${HADOOP_PREFIX}/libhdfs/libhdfs.so . -

- -
-
-The APIs - -

-The libhdfs APIs are a subset of: hadoop fs APIs. -

-

-The header file for libhdfs describes each API in detail and is available in ${HADOOP_PREFIX}/src/c++/libhdfs/hdfs.h -

-
-
-A Sample Program - - -#include "hdfs.h" - -int main(int argc, char **argv) { - - hdfsFS fs = hdfsConnect("default", 0); - const char* writePath = "/tmp/testfile.txt"; - hdfsFile writeFile = hdfsOpenFile(fs, writePath, O_WRONLY|O_CREAT, 0, 0, 0); - if(!writeFile) { - fprintf(stderr, "Failed to open %s for writing!\n", writePath); - exit(-1); - } - char* buffer = "Hello, World!"; - tSize num_written_bytes = hdfsWrite(fs, writeFile, (void*)buffer, strlen(buffer)+1); - if (hdfsFlush(fs, writeFile)) { - fprintf(stderr, "Failed to 'flush' %s\n", writePath); - exit(-1); - } - hdfsCloseFile(fs, writeFile); -} - -
- -
-How To Link With The Library -

-See the Makefile for hdfs_test.c in the libhdfs source directory (${HADOOP_PREFIX}/src/c++/libhdfs/Makefile) or something like:
-gcc above_sample.c -I${HADOOP_PREFIX}/src/c++/libhdfs -L${HADOOP_PREFIX}/libhdfs -lhdfs -o above_sample -

-
-
-Common Problems -

-The most common problem is the CLASSPATH is not set properly when calling a program that uses libhdfs. -Make sure you set it to all the Hadoop jars needed to run Hadoop itself. Currently, there is no way to -programmatically generate the classpath, but a good bet is to include all the jar files in ${HADOOP_PREFIX} -and ${HADOOP_PREFIX}/lib as well as the right configuration directory containing hdfs-site.xml -

-
-
-Thread Safe -

libdhfs is thread safe.

-
    -
  • Concurrency and Hadoop FS "handles" -
    The Hadoop FS implementation includes a FS handle cache which caches based on the URI of the -namenode along with the user connecting. So, all calls to hdfsConnect will return the same handle but -calls to hdfsConnectAsUser with different users will return different handles. But, since HDFS client -handles are completely thread safe, this has no bearing on concurrency. -
    • -
  • Concurrency and libhdfs/JNI -
    The libhdfs calls to JNI should always be creating thread local storage, so (in theory), libhdfs -should be as thread safe as the underlying calls to the Hadoop FS. -
    • -
-
- - + + + + + + +
+C API libhdfs +Content-Type +text/html; +utf-8 +
+ +
+Overview + +

+libhdfs is a JNI based C API for Hadoop's Distributed File System (HDFS). +It provides C APIs to a subset of the HDFS APIs to manipulate HDFS files and +the filesystem. libhdfs is part of the Hadoop distribution and comes +pre-compiled in ${HADOOP_PREFIX}/libhdfs/libhdfs.so . +

+ +
+
+The APIs + +

+The libhdfs APIs are a subset of: hadoop fs APIs. +

+

+The header file for libhdfs describes each API in detail and is available in ${HADOOP_PREFIX}/src/c++/libhdfs/hdfs.h +

+
+
+A Sample Program + + +#include "hdfs.h" + +int main(int argc, char **argv) { + + hdfsFS fs = hdfsConnect("default", 0); + const char* writePath = "/tmp/testfile.txt"; + hdfsFile writeFile = hdfsOpenFile(fs, writePath, O_WRONLY|O_CREAT, 0, 0, 0); + if(!writeFile) { + fprintf(stderr, "Failed to open %s for writing!\n", writePath); + exit(-1); + } + char* buffer = "Hello, World!"; + tSize num_written_bytes = hdfsWrite(fs, writeFile, (void*)buffer, strlen(buffer)+1); + if (hdfsFlush(fs, writeFile)) { + fprintf(stderr, "Failed to 'flush' %s\n", writePath); + exit(-1); + } + hdfsCloseFile(fs, writeFile); +} + +
+ +
+How To Link With The Library +

+See the Makefile for hdfs_test.c in the libhdfs source directory (${HADOOP_PREFIX}/src/c++/libhdfs/Makefile) or something like:
+gcc above_sample.c -I${HADOOP_PREFIX}/src/c++/libhdfs -L${HADOOP_PREFIX}/libhdfs -lhdfs -o above_sample +

+
+
+Common Problems +

+The most common problem is the CLASSPATH is not set properly when calling a program that uses libhdfs. +Make sure you set it to all the Hadoop jars needed to run Hadoop itself. Currently, there is no way to +programmatically generate the classpath, but a good bet is to include all the jar files in ${HADOOP_PREFIX} +and ${HADOOP_PREFIX}/lib as well as the right configuration directory containing hdfs-site.xml +

+
+
+Thread Safe +

libdhfs is thread safe.

+
    +
  • Concurrency and Hadoop FS "handles" +
    The Hadoop FS implementation includes a FS handle cache which caches based on the URI of the +namenode along with the user connecting. So, all calls to hdfsConnect will return the same handle but +calls to hdfsConnectAsUser with different users will return different handles. But, since HDFS client +handles are completely thread safe, this has no bearing on concurrency. +
    • +
  • Concurrency and libhdfs/JNI +
    The libhdfs calls to JNI should always be creating thread local storage, so (in theory), libhdfs +should be as thread safe as the underlying calls to the Hadoop FS. +
    • +
+
+ +