hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ji...@apache.org
Subject [22/22] git commit: Merge changes from trunk
Date Wed, 17 Sep 2014 00:59:17 GMT
Merge changes from trunk


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/432b8637
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/432b8637
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/432b8637

Branch: refs/heads/HDFS-6584
Commit: 432b863719e1a36459812cdcbe4785e6a19f3f3a
Parents: 2e4033e b6d3230
Author: Jing Zhao <jing@hortonworks.com>
Authored: Tue Sep 16 17:58:39 2014 -0700
Committer: Jing Zhao <jing@hortonworks.com>
Committed: Tue Sep 16 17:58:39 2014 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-auth/pom.xml       |  13 +
 .../server/AuthenticationFilter.java            | 152 ++++--
 .../util/RandomSignerSecretProvider.java        |   4 +-
 .../util/RolloverSignerSecretProvider.java      |   7 +-
 .../util/SignerSecretProvider.java              |   9 +-
 .../util/StringSignerSecretProvider.java        |  15 +-
 .../util/ZKSignerSecretProvider.java            | 506 +++++++++++++++++++
 .../src/site/apt/Configuration.apt.vm           | 148 +++++-
 .../hadoop-auth/src/site/apt/index.apt.vm       |   5 +
 .../server/TestAuthenticationFilter.java        | 117 ++++-
 .../util/TestJaasConfiguration.java             |  55 ++
 .../util/TestRandomSignerSecretProvider.java    |   2 +-
 .../util/TestRolloverSignerSecretProvider.java  |   2 +-
 .../authentication/util/TestSigner.java         |  23 +-
 .../util/TestStringSignerSecretProvider.java    |   9 +-
 .../util/TestZKSignerSecretProvider.java        | 270 ++++++++++
 hadoop-common-project/hadoop-common/CHANGES.txt |  16 +
 .../hadoop-common/src/main/bin/hadoop-config.sh |   1 +
 .../src/main/bin/hadoop-functions.sh            | 127 +++--
 .../hadoop-common/src/main/conf/hadoop-env.sh   |  81 ---
 .../src/main/conf/hadoop-policy.xml             |   9 +
 .../main/conf/hadoop-user-functions.sh.example  |  94 ++++
 .../main/java/org/apache/hadoop/ha/HAAdmin.java |   6 +-
 .../io/nativeio/SharedFileDescriptorFactory.c   |  32 +-
 .../src/site/apt/CommandsManual.apt.vm          |  53 ++
 hadoop-common-project/hadoop-kms/pom.xml        |  25 +-
 .../hadoop/crypto/key/kms/server/MiniKMS.java   |  47 +-
 .../test/resources/mini-kms-acls-default.xml    | 135 +++++
 .../hadoop/fs/http/server/TestHttpFSServer.java |   8 +-
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |  20 +
 hadoop-hdfs-project/hadoop-hdfs/pom.xml         |  13 +
 .../hadoop-hdfs/src/main/bin/hdfs-config.sh     |   7 +-
 .../java/org/apache/hadoop/hdfs/DFSClient.java  |   5 +-
 .../org/apache/hadoop/hdfs/DFSOutputStream.java |  40 +-
 .../hadoop/hdfs/protocol/ClientProtocol.java    |   4 +-
 .../hadoop/hdfs/protocol/EncryptionZone.java    |  18 +-
 .../hdfs/protocol/EncryptionZoneIterator.java   |  20 +-
 .../hdfs/protocol/EncryptionZoneWithId.java     |  81 ---
 .../protocol/EncryptionZoneWithIdIterator.java  |  53 --
 .../datatransfer/DataTransferProtoUtil.java     |  44 +-
 .../hdfs/protocol/datatransfer/Receiver.java    | 123 +++--
 .../hdfs/protocol/datatransfer/Sender.java      |  26 +-
 ...tNamenodeProtocolServerSideTranslatorPB.java |   6 +-
 .../ClientNamenodeProtocolTranslatorPB.java     |  14 +-
 .../apache/hadoop/hdfs/protocolPB/PBHelper.java |  12 +-
 .../web/resources/DatanodeWebHdfsMethods.java   |  13 +-
 .../server/namenode/EncryptionZoneManager.java  |  24 +-
 .../hdfs/server/namenode/FSDirectory.java       |   6 +-
 .../hdfs/server/namenode/FSNamesystem.java      |  10 +-
 .../hdfs/server/namenode/NameNodeRpcServer.java |   6 +-
 .../src/main/proto/datatransfer.proto           |   8 +
 .../hadoop-hdfs/src/main/proto/encryption.proto |   6 +-
 .../hadoop/hdfs/TestDFSClientFailover.java      |   5 +-
 .../apache/hadoop/hdfs/TestEncryptionZones.java |  62 ++-
 .../hadoop/hdfs/TestEncryptionZonesWithKMS.java |  56 ++
 .../blockmanagement/TestBlockManager.java       |  56 +-
 .../server/datanode/SimulatedFSDataset.java     |   7 +-
 .../hdfs/tools/TestDFSHAAdminMiniCluster.java   |  20 -
 .../org/apache/hadoop/tracing/TestTracing.java  |  52 +-
 .../TestTracingShortCircuitLocalRead.java       |  97 ++++
 hadoop-mapreduce-project/bin/mapred-config.sh   |   7 +-
 hadoop-project/pom.xml                          |  25 +
 .../org/apache/hadoop/fs/s3a/Constants.java     |  36 +-
 .../org/apache/hadoop/fs/s3a/S3AFileSystem.java |  46 +-
 .../apache/hadoop/fs/s3a/S3AOutputStream.java   |   6 +-
 hadoop-yarn-project/CHANGES.txt                 |  15 +
 .../hadoop-yarn/bin/yarn-config.sh              |  11 +-
 .../hadoop-yarn/conf/yarn-env.sh                |  52 +-
 .../hadoop/yarn/conf/YarnConfiguration.java     |  14 +
 .../applications/distributedshell/Client.java   |  18 +-
 .../TestDSSleepingAppMaster.java                |  58 +++
 .../distributedshell/TestDistributedShell.java  |  76 +++
 .../src/main/resources/yarn-default.xml         |  10 +
 .../ApplicationHistoryClientService.java        |  15 +-
 .../authorize/TimelinePolicyProvider.java       |  44 ++
 .../hadoop-yarn-server-nodemanager/pom.xml      |  14 +-
 .../util/CgroupsLCEResourcesHandler.java        |  29 +-
 .../util/TestCgroupsLCEResourcesHandler.java    | 165 +++++-
 78 files changed, 2892 insertions(+), 634 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 0e13e83,1a8262f..9b980dc
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@@ -38,16 -40,19 +40,21 @@@ import org.apache.hadoop.fs.Path
  import org.apache.hadoop.hdfs.DFSConfigKeys;
  import org.apache.hadoop.hdfs.DFSTestUtil;
  import org.apache.hadoop.hdfs.HdfsConfiguration;
 +import org.apache.hadoop.hdfs.StorageType;
+ import org.apache.hadoop.hdfs.MiniDFSCluster;
  import org.apache.hadoop.hdfs.protocol.Block;
  import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
  import org.apache.hadoop.hdfs.protocol.HdfsConstants;
  import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
+ import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
  import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+ import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
  import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
  import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+ import org.apache.hadoop.ipc.RemoteException;
  import org.apache.hadoop.net.NetworkTopology;
 +import org.junit.Assert;
+ import org.apache.hadoop.test.GenericTestUtils;
  import org.junit.Before;
  import org.junit.Test;
  import org.mockito.Mockito;
@@@ -601,19 -606,53 +608,66 @@@ public class TestBlockManager 
          new BlockListAsLongs(null, null));
      assertEquals(1, ds.getBlockReportCount());
    }
+   
 -  
+   /**
+    * Tests that a namenode doesn't choose a datanode with full disks to 
+    * store blocks.
+    * @throws Exception
+    */
+   @Test
+   public void testStorageWithRemainingCapacity() throws Exception {
+     final Configuration conf = new HdfsConfiguration();
+     final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+     FileSystem fs = FileSystem.get(conf);
+     Path file1 = null;
+     try {
+       cluster.waitActive();
+       final FSNamesystem namesystem = cluster.getNamesystem();
+       final String poolId = namesystem.getBlockPoolId();
+       final DatanodeRegistration nodeReg =
+         DataNodeTestUtils.getDNRegistrationForBP(cluster.getDataNodes().
+         		get(0), poolId);
+       final DatanodeDescriptor dd = NameNodeAdapter.getDatanode(namesystem,
+     		  nodeReg);
+       // By default, MiniDFSCluster will create 1 datanode with 2 storages.
+       // Assigning 64k for remaining storage capacity and will 
+       //create a file with 100k.
+       for(DatanodeStorageInfo storage:  dd.getStorageInfos()) { 
+     	  storage.setUtilizationForTesting(65536, 0, 65536, 0);
+       }
+       //sum of the remaining capacity of both the storages
+       dd.setRemaining(131072);
+       file1 = new Path("testRemainingStorage.dat");
+       try {
+         DFSTestUtil.createFile(fs, file1, 102400, 102400, 102400, (short)1,
+         		0x1BAD5EED);
+       }
+       catch (RemoteException re) {
+     	  GenericTestUtils.assertExceptionContains("nodes instead of "
+     	  		+ "minReplication", re);
+       }
+     }
+     finally {
+       // Clean up
+       assertTrue(fs.exists(file1));
+       fs.delete(file1, true);
+       assertTrue(!fs.exists(file1));
+       cluster.shutdown();
+     }
+   }
 -}
  
 +  @Test
 +  public void testUseDelHint() {
 +    DatanodeStorageInfo delHint = new DatanodeStorageInfo(
 +        DFSTestUtil.getLocalDatanodeDescriptor(), new DatanodeStorage("id"));
 +    List<DatanodeStorageInfo> moreThan1Racks = Arrays.asList(delHint);
 +    List<StorageType> excessTypes = new ArrayList<StorageType>();
 +
 +    excessTypes.add(StorageType.DEFAULT);
 +    Assert.assertTrue(BlockManager.useDelHint(true, delHint, null,
 +        moreThan1Racks, excessTypes));
 +    excessTypes.add(StorageType.SSD);
 +    Assert.assertFalse(BlockManager.useDelHint(true, delHint, null,
 +        moreThan1Racks, excessTypes));
 +  }
- }
++}


Mime
View raw message