hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject [4/4] git commit: Merge trunk into HDFS-6581
Date Sun, 07 Sep 2014 21:47:38 GMT
Merge trunk into HDFS-6581

Conflicts:
	hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
	hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb8284d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb8284d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb8284d5

Branch: refs/heads/HDFS-6581
Commit: eb8284d50e1aa9f196556ed20b4b5e3f330e65fe
Parents: 31bbeaf a23144f
Author: arp <arp@apache.org>
Authored: Sun Sep 7 14:46:46 2014 -0700
Committer: arp <arp@apache.org>
Committed: Sun Sep 7 14:46:46 2014 -0700

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |   6 +
 .../hadoop/hdfs/protocol/HdfsConstants.java     |   2 +-
 .../server/datanode/ReplicaBeingWritten.java    |  12 +-
 .../hdfs/server/datanode/ReplicaInPipeline.java |  33 ++-
 .../hdfs/server/datanode/ReplicaInfo.java       |   7 +
 .../server/datanode/fsdataset/FsVolumeSpi.java  |  13 +-
 .../datanode/fsdataset/impl/BlockPoolSlice.java |   6 +-
 .../datanode/fsdataset/impl/FsDatasetImpl.java  |  15 +-
 .../datanode/fsdataset/impl/FsVolumeImpl.java   |  58 +++-
 .../apache/hadoop/cli/TestCryptoAdminCLI.java   |   4 +-
 .../org/apache/hadoop/hdfs/TestDFSUtil.java     |   4 +-
 .../apache/hadoop/hdfs/TestEncryptionZones.java |  12 +-
 .../hadoop/hdfs/TestReservedRawPaths.java       |   3 +-
 .../server/datanode/TestDirectoryScanner.java   |   8 +
 .../fsdataset/impl/TestRbwSpaceReservation.java | 288 +++++++++++++++++++
 .../fsdataset/impl/TestWriteToReplica.java      |   2 +-
 hadoop-yarn-project/CHANGES.txt                 |   3 +
 .../yarn/webapp/util/TestWebAppUtils.java       |   4 +-
 18 files changed, 448 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb8284d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb8284d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb8284d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
index e141cd7,cba23c3..8ebf2b4
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
@@@ -46,6 -46,14 +46,17 @@@ public interface FsVolumeSpi 
    
    public StorageType getStorageType();
  
 +  /** Returns true if the volume is NOT backed by persistent storage. */
 +  public boolean isTransientStorage();
- }
++
+   /**
+    * Reserve disk space for an RBW block so a writer does not run out of
+    * space before the block is full.
+    */
+   public void reserveSpaceForRbw(long bytesToReserve);
+ 
+   /**
+    * Release disk space previously reserved for RBW block.
+    */
+   public void releaseReservedSpace(long bytesToRelease);
 -}
++}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb8284d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb8284d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 606b067,4511f21..b243f27
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@@ -920,30 -872,12 +920,30 @@@ class FsDatasetImpl implements FsDatase
        " and thus cannot be created.");
      }
      // create a new block
 -    FsVolumeImpl v = volumes.getNextVolume(storageType, b.getNumBytes());
 -    // create a rbw file to hold block in the designated volume
 +    FsVolumeImpl v;
 +    while (true) {
 +      try {
 +        if (allowLazyPersist) {
 +          // First try to place the block on a transient volume.
 +          v = volumes.getNextTransientVolume(b.getNumBytes());
 +        } else {
 +          v = volumes.getNextVolume(storageType, b.getNumBytes());
 +        }
 +      } catch (DiskOutOfSpaceException de) {
 +        if (allowLazyPersist) {
 +          allowLazyPersist = false;
 +          continue;
 +        }
 +        throw de;
 +      }
 +      break;
 +    }
 +    // create an rbw file to hold block in the designated volume
      File f = v.createRbwFile(b.getBlockPoolId(), b.getLocalBlock());
      ReplicaBeingWritten newReplicaInfo = new ReplicaBeingWritten(b.getBlockId(), 
-         b.getGenerationStamp(), v, f.getParentFile());
+         b.getGenerationStamp(), v, f.getParentFile(), b.getNumBytes());
      volumeMap.add(b.getBlockPoolId(), newReplicaInfo);
 +
      return newReplicaInfo;
    }
    
@@@ -1079,8 -1013,9 +1079,8 @@@
      // create a temporary file to hold block in the designated volume
      File f = v.createTmpFile(b.getBlockPoolId(), b.getLocalBlock());
      ReplicaInPipeline newReplicaInfo = new ReplicaInPipeline(b.getBlockId(), 
-         b.getGenerationStamp(), v, f.getParentFile());
+         b.getGenerationStamp(), v, f.getParentFile(), 0);
      volumeMap.add(b.getBlockPoolId(), newReplicaInfo);
 -    
      return newReplicaInfo;
    }
  
@@@ -1144,15 -1079,11 +1144,16 @@@
              " for block " + replicaInfo);
        }
  
-       File dest = v.addBlock(bpid, replicaInfo, f);
+       File dest = v.addFinalizedBlock(
+           bpid, replicaInfo, f, replicaInfo.getBytesReserved());
        newReplicaInfo = new FinalizedReplica(replicaInfo, v, dest.getParentFile());
 +
 +      if (v.isTransientStorage()) {
 +        lazyWriteReplicaTracker.addReplica(bpid, replicaInfo.getBlockId(), v);
 +      }
      }
      volumeMap.add(bpid, newReplicaInfo);
 +
      return newReplicaInfo;
    }
  

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb8284d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb8284d5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
index ae147e9,bc50eaa..89649aa
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
@@@ -426,9 -426,12 +426,17 @@@ public class TestDirectoryScanner 
      }
  
      @Override
 +    public boolean isTransientStorage() {
 +      return false;
 +    }
++
++    @Override
+     public void reserveSpaceForRbw(long bytesToReserve) {
+     }
+ 
+     @Override
+     public void releaseReservedSpace(long bytesToRelease) {
+     }
    }
  
    private final static TestFsVolumeSpi TEST_VOLUME = new TestFsVolumeSpi();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb8284d5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
----------------------------------------------------------------------


Mime
View raw message